From 8a979cd8210d29a168089589c695ea790224a1e8 Mon Sep 17 00:00:00 2001 From: Juro Oravec Date: Fri, 3 Oct 2025 09:34:18 +0200 Subject: [PATCH] ci: check supported versions once a week (#1419) --- ...abot.yml => maint-automate-dependabot.yml} | 0 .../workflows/maint-supported-versions.yml | 36 ++ .../workflows/{docs.yml => release-docs.yml} | 0 .../{publish-to-pypi.yml => release-pypi.yml} | 0 .pre-commit-config.yaml | 16 +- docs/community/development.md | 15 +- scripts/supported_versions.py | 524 +++++++++++++++++- 7 files changed, 558 insertions(+), 33 deletions(-) rename .github/workflows/{automate-dependabot.yml => maint-automate-dependabot.yml} (100%) create mode 100644 .github/workflows/maint-supported-versions.yml rename .github/workflows/{docs.yml => release-docs.yml} (100%) rename .github/workflows/{publish-to-pypi.yml => release-pypi.yml} (100%) diff --git a/.github/workflows/automate-dependabot.yml b/.github/workflows/maint-automate-dependabot.yml similarity index 100% rename from .github/workflows/automate-dependabot.yml rename to .github/workflows/maint-automate-dependabot.yml diff --git a/.github/workflows/maint-supported-versions.yml b/.github/workflows/maint-supported-versions.yml new file mode 100644 index 00000000..1cc1a55d --- /dev/null +++ b/.github/workflows/maint-supported-versions.yml @@ -0,0 +1,36 @@ +name: Check supported versions + +on: + schedule: + # Run every Sunday at 2:00 AM UTC + - cron: '0 2 * * 0' + workflow_dispatch: # Allow manual triggering + +permissions: + contents: read + issues: write + +jobs: + check-versions: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: "pip" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements-dev.txt + + - name: Check supported versions + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python scripts/supported_versions.py check diff --git a/.github/workflows/docs.yml b/.github/workflows/release-docs.yml similarity index 100% rename from .github/workflows/docs.yml rename to .github/workflows/release-docs.yml diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/release-pypi.yml similarity index 100% rename from .github/workflows/publish-to-pypi.yml rename to .github/workflows/release-pypi.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 267d207c..af2c922e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ repos: -- repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.12.9 +- repo: local hooks: - # Run the linter. - - id: ruff-check - args: [ --fix ] - # Run the formatter. - - id: ruff-format + # See https://stackoverflow.com/questions/70778806 + - id: tox + name: tox + entry: .venv/bin/tox + args: ["-e", "mypy,ruff"] + language: system + pass_filenames: false diff --git a/docs/community/development.md b/docs/community/development.md index 93168e29..220bbffe 100644 --- a/docs/community/development.md +++ b/docs/community/development.md @@ -310,14 +310,21 @@ Head over to [Dev guides](./devguides/dependency_mgmt.md) for a deep dive into h ### Updating supported versions -The `scripts/supported_versions.py` script can be used to update the supported versions. +The `scripts/supported_versions.py` script manages the supported Python and Django versions for the project. + +The script runs automatically via GitHub Actions once a week to check for version updates. If changes are detected, it creates a GitHub issue with the necessary updates. See the [`maint-supported-versions.yml`](https://github.com/django-components/django-components/blob/master/.github/workflows/maint-supported-versions.yml) workflow. + +You can also run the script manually: ```sh -python scripts/supported_versions.py +# Check if versions need updating +python scripts/supported_versions.py check + +# Generate configuration snippets for manual updates +python scripts/supported_versions.py generate ``` -This will check the current versions of Django and Python, and will print to the terminal -all the places that need updating and what to set them to. +The `generate` command will print to the terminal all the places that need updating and what to set them to. ### Updating link references diff --git a/scripts/supported_versions.py b/scripts/supported_versions.py index b276e0f2..f0f061d3 100644 --- a/scripts/supported_versions.py +++ b/scripts/supported_versions.py @@ -1,14 +1,73 @@ -# ruff: noqa: T201, S310 +# ruff: noqa: BLE001, PLW2901, RUF001, S310, T201 +""" +This script manages the info about supported Python and Django versions. + +The script fetches the latest supported version information from official sources: +- Python versions from https://devguide.python.org/versions/ +- Django versions and compatibility matrix from https://docs.djangoproject.com/ + +Commands: + generate: Generates instructions for updating various files (tox.ini, pyproject.toml, + GitHub Actions, documentation) based on current supported versions + + check: Compares the current compatibility table in `docs/overview/compatibility.md` + with the latest official version information. If differences are found, + creates a GitHub issue to track the needed updates. + +Usage: + python scripts/supported_versions.py generate + python scripts/supported_versions.py check + + # For GitHub issue creation (check command): + GITHUB_TOKEN=your_token python scripts/supported_versions.py check + +Files updated by this script: +- docs/overview/compatibility.md (compatibility table) +- tox.ini (test environments) +- pyproject.toml (Python classifiers) +- .github/workflows/tests.yml (CI matrix) +- docs/community/development.md (development setup) +""" + +import argparse +import json +import os import re +import sys import textwrap from collections import defaultdict -from typing import Any, Callable, Dict, List, Tuple +from pathlib import Path +from typing import Any, Callable, Dict, List, NamedTuple, Tuple from urllib import request Version = Tuple[int, ...] VersionMapping = Dict[Version, List[Version]] +class DjangoVersionChanges(NamedTuple): + added: List[Version] + removed: List[Version] + + +class VersionDifferences(NamedTuple): + added_python_versions: List[Version] + removed_python_versions: List[Version] + changed_django_versions: Dict[Version, DjangoVersionChanges] + has_changes: bool + + +###################################### +# GET DATA FROM OFFICIAL SOURCES +###################################### + + +HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; django-components version script)"} + + +def filter_dict(d: Dict, filter_fn: Callable[[Any], bool]) -> Dict: + return dict(filter(filter_fn, d.items())) + + def cut_by_content(content: str, cut_from: str, cut_to: str) -> str: return content.split(cut_from)[1].split(cut_to)[0] @@ -18,7 +77,8 @@ def keys_from_content(content: str) -> List[str]: def get_python_supported_version(url: str) -> List[Version]: - with request.urlopen(url) as response: + req = request.Request(url, headers=HEADERS) + with request.urlopen(req) as response: response_content = response.read() content = response_content.decode("utf-8") @@ -39,7 +99,8 @@ def get_python_supported_version(url: str) -> List[Version]: def get_django_to_python_versions(url: str) -> VersionMapping: - with request.urlopen(url) as response: + req = request.Request(url, headers=HEADERS) + with request.urlopen(req) as response: response_content = response.read() content = response_content.decode("utf-8") @@ -69,7 +130,8 @@ def get_django_to_python_versions(url: str) -> VersionMapping: def get_django_supported_versions(url: str) -> List[Tuple[int, ...]]: """Extract Django versions from the HTML content, e.g. `5.0` or `4.2`""" - with request.urlopen(url) as response: + req = request.Request(url, headers=HEADERS) + with request.urlopen(req) as response: response_content = response.read() content = response_content.decode("utf-8") @@ -94,7 +156,8 @@ def get_django_supported_versions(url: str) -> List[Tuple[int, ...]]: def get_latest_version(url: str) -> Version: - with request.urlopen(url) as response: + req = request.Request(url, headers=HEADERS) + with request.urlopen(req) as response: response_content = response.read() content = response_content.decode("utf-8") @@ -117,6 +180,28 @@ def build_python_to_django(django_to_python: VersionMapping, latest_version: Ver return python_to_django +def get_python_to_django() -> VersionMapping: + """Get the Python to Django version mapping as extracted from the websites.""" + django_to_python = get_django_to_python_versions("https://docs.djangoproject.com/en/dev/faq/install/") + django_supported_versions = get_django_supported_versions("https://www.djangoproject.com/download/") + latest_version = get_latest_version("https://www.djangoproject.com/download/") + + supported_django_to_python = filter_dict(django_to_python, lambda item: item[0] in django_supported_versions) + python_to_django = build_python_to_django(supported_django_to_python, latest_version) + # NOTE: Uncomment the below if you want to include only those Python versions + # that are still actively supported. Otherwise, we include all Python versions + # that are compatible with supported Django versions. + # active_python = get_python_supported_version("https://devguide.python.org/versions/") + # python_to_django = filter_dict(python_to_django, lambda item: item[0] in active_python) + + return python_to_django + + +###################################### +# GENERATE COMMAND +###################################### + + def env_format(version_tuple: Version, divider: str = "") -> str: return divider.join(str(num) for num in version_tuple) @@ -222,26 +307,15 @@ def build_pyenv(python_to_django: VersionMapping) -> str: def build_ci_python_versions(python_to_django: VersionMapping) -> str: # Outputs python-version, like: ['3.8', '3.9', '3.10', '3.11', '3.12'] lines = [ - f"'{env_format(python_version, divider='.')}'" for python_version, django_versions in python_to_django.items() + f"'{env_format(python_version, divider='.')}'" for python_version, _django_versions in python_to_django.items() ] lines_formatted = " " * 8 + f"python-version: [{', '.join(lines)}]" return lines_formatted -def filter_dict(d: Dict, filter_fn: Callable[[Any], bool]) -> Dict: - return dict(filter(filter_fn, d.items())) - - -def main() -> None: - active_python = get_python_supported_version("https://devguide.python.org/versions/") - django_to_python = get_django_to_python_versions("https://docs.djangoproject.com/en/dev/faq/install/") - django_supported_versions = get_django_supported_versions("https://www.djangoproject.com/download/") - latest_version = get_latest_version("https://www.djangoproject.com/download/") - - supported_django_to_python = filter_dict(django_to_python, lambda item: item[0] in django_supported_versions) - python_to_django = build_python_to_django(supported_django_to_python, latest_version) - - python_to_django = filter_dict(python_to_django, lambda item: item[0] in active_python) +def command_generate() -> None: + print("šŸ”„ Fetching latest version information...") + python_to_django = get_python_to_django() tox_envlist = build_tox_envlist(python_to_django) print("Add this to tox.ini:\n") @@ -285,5 +359,413 @@ def main() -> None: print() +###################################### +# CHECK COMMAND +###################################### + + +def parse_compatibility_markdown(file_path: Path) -> VersionMapping: + """ + Extract compatibility table from markdown file with following format: + + ``` + | Python version | Django version | + |----------------|----------------| + | 3.9 | 4.2 | + | 3.10 | 4.2, 5.1, 5.2 | + | 3.11 | 4.2, 5.1, 5.2 | + | 3.12 | 4.2, 5.1, 5.2 | + | 3.13 | 5.1, 5.2 | + ``` + """ + try: + with file_path.open(encoding="utf-8") as f: + content = f.read() + except FileNotFoundError: + print(f"Error: Could not find compatibility file at {file_path}") + sys.exit(1) + + # Find the table section + lines = content.split("\n") + table_start = -1 + table_end = -1 + + for i, line in enumerate(lines): + # Search for the table headers line + # `| Python version | Django version |` + if re.search(r"\|\s*Python\s+version\s*\|\s*Django\s+version\s*\|", line, re.IGNORECASE): + table_start = i + 2 # Skip header and separator line + # Search for the end of the table + elif table_start != -1 and (line.strip() == "" or not line.startswith("|")): + table_end = i + break + + if table_start == -1: + print("Error: Could not find compatibility table in markdown file") + sys.exit(1) + + if table_end == -1: + # If the end of the table is not found, use the last line of the file + table_end = len(lines) + + # Parse table rows + # `| 3.10 | 4.2, 5.1, 5.2 |` + python_to_django: VersionMapping = {} + for i in range(table_start, table_end): + # Skip empty and non-table lines + line = lines[i].strip() + if not line or not line.startswith("|"): + continue + + # Split by | and clean up + parts = [part.strip() for part in line.split("|")[1:-1]] # Remove empty first/last + if len(parts) != 2: + raise ValueError(f"Unexpected table row: {line}") + + python_version_str = parts[0].strip() + django_versions_str = parts[1].strip() + + try: + python_version = version_to_tuple(python_version_str) + django_versions = [] + for version_str in django_versions_str.split(","): + version_str = version_str.strip() + if version_str: + django_versions.append(version_to_tuple(version_str)) + + if django_versions: + python_to_django[python_version] = django_versions + except ValueError as e: + raise ValueError(f"Invalid version string in table row '{line}': {e}") from e + + return python_to_django + + +def compare_version_mappings(current: VersionMapping, expected: VersionMapping) -> VersionDifferences: + current_pythons = set(current.keys()) + expected_pythons = set(expected.keys()) + + # Find added/removed Python versions + added_pythons = expected_pythons - current_pythons + removed_pythons = current_pythons - expected_pythons + + added_python_versions = sorted(added_pythons) + removed_python_versions = sorted(removed_pythons) + + # Find changed Django versions for existing Python versions + changed_django_versions = {} + common_pythons = current_pythons & expected_pythons + for python_version in common_pythons: + current_djangos = set(current[python_version]) + expected_djangos = set(expected[python_version]) + + if current_djangos != expected_djangos: + changed_django_versions[python_version] = DjangoVersionChanges( + added=sorted(expected_djangos - current_djangos), + removed=sorted(current_djangos - expected_djangos), + ) + + # Check if there are any changes + has_changes = bool(added_pythons) or bool(removed_pythons) or bool(changed_django_versions) + + return VersionDifferences( + added_python_versions=added_python_versions, + removed_python_versions=removed_python_versions, + changed_django_versions=changed_django_versions, + has_changes=has_changes, + ) + + +def command_check(repo_owner: str = "django-components", repo_name: str = "django-components") -> None: + """Check if supported versions need updating and create GitHub issue if needed""" + print("šŸ”„ Checking supported versions...") + + # Get current versions from markdown + compatibility_file = Path("docs/overview/compatibility.md") + try: + current_python_to_django = parse_compatibility_markdown(compatibility_file) + print(f"šŸ“– Parsed current versions from {compatibility_file}") + except Exception as e: + print(f"āŒ Error parsing compatibility file: {e}") + sys.exit(1) + + # Get expected versions from official sources + try: + expected_python_to_django = get_python_to_django() + print("šŸŒ Fetched expected versions from official sources") + except Exception as e: + print(f"āŒ Error fetching expected versions: {e}") + sys.exit(1) + + # Compare versions + differences = compare_version_mappings(current_python_to_django, expected_python_to_django) + + if not differences.has_changes: + print("āœ… Supported versions are up to date!") + return + + print("āš ļø Supported versions need updating!") + + # Print differences + if differences.added_python_versions: + print("āž• Added Python versions:") + for version in differences.added_python_versions: + print(f" - Python {env_format(version, divider='.')}") + + if differences.removed_python_versions: + print("āž– Removed Python versions:") + for version in differences.removed_python_versions: + print(f" - Python {env_format(version, divider='.')}") + + if differences.changed_django_versions: + print("šŸ”„ Changed Django version support:") + for python_version, changes in differences.changed_django_versions.items(): + python_str = env_format(python_version, divider=".") + print(f" Python {python_str}:") + for django_version in changes.added: + django_str = env_format(django_version, divider=".") + print(f" āœ… Added Django {django_str}") + for django_version in changes.removed: + django_str = env_format(django_version, divider=".") + print(f" āŒ Removed Django {django_str}") + + # Create GitHub issue + github_token = os.environ.get("GITHUB_TOKEN") + + if not github_token: + print("\nāš ļø GITHUB_TOKEN environment variable not set.") + print("Set GITHUB_TOKEN to create GitHub issues automatically.") + print("Run `python scripts/supported_versions.py generate` to get updated configurations.") + return + + # Generate issue title and body + title = create_github_issue_title(differences) + body = generate_issue_body(differences, current_python_to_django, expected_python_to_django) + + # Check for existing issues + print("šŸ” Checking for existing issues...") + if check_existing_github_issue(title, repo_owner, repo_name, github_token): + print("ā„¹ļø Similar issue already exists. Skipping issue creation.") + return + + # Create the issue + print(f"šŸ“ Creating GitHub issue: {title}") + success = create_github_issue(title, body, repo_owner, repo_name, github_token) + + if not success: + print("āŒ Failed to create GitHub issue") + sys.exit(1) + + +###################################### +# CHECK COMMAND - GITHUB ISSUE +###################################### + + +def create_github_issue_title(differences: VersionDifferences) -> str: + """ + Generate a GitHub issue title based on version differences + + We rely on this title to find the issue in the future to avoid duplicates. + """ + parts = [] + + if differences.added_python_versions: + for version in differences.added_python_versions: + version_str = env_format(version, divider=".") + parts.append(f"Add Python {version_str}") + + if differences.removed_python_versions: + for version in differences.removed_python_versions: + version_str = env_format(version, divider=".") + parts.append(f"Remove Python {version_str}") + + # Check for Django version changes + for python_version, changes in differences.changed_django_versions.items(): + python_str = env_format(python_version, divider=".") + if changes.added: + for django_version in changes.added: + django_str = env_format(django_version, divider=".") + parts.append(f"Add Django {django_str} for Python {python_str}") + if changes.removed: + for django_version in changes.removed: + django_str = env_format(django_version, divider=".") + parts.append(f"Remove Django {django_str} for Python {python_str}") + + if not parts: + return "[maint] Update supported versions" + + # Create a concise title + if len(parts) == 1: + return f"[maint] {parts[0]} to supported versions" + return f"[maint] Update supported versions ({len(parts)} changes)" + + +def check_existing_github_issue(title: str, repo_owner: str, repo_name: str, token: str) -> bool: + """Check if a GitHub issue with similar title already exists""" + # Search for issues with similar titles + search_url = "https://api.github.com/search/issues" + repo_name = f"{repo_owner}/{repo_name}" + params = { + "q": f'repo:{repo_name} is:issue "{title}"'.replace(" ", "%20"), + "sort": "created", + "order": "desc", + } + + headers = {"Authorization": f"token {token}", "Accept": "application/vnd.github.v3+json", **HEADERS} + + try: + # Build query string manually + query_string = "&".join([f"{k}={v}" for k, v in params.items()]) + full_url = f"{search_url}?{query_string}" + + req = request.Request(full_url, headers=headers) + with request.urlopen(req) as response: + data = json.loads(response.read().decode("utf-8")) + + # Check if any existing issues match our pattern + for issue in data.get("items", []): + issue_title = issue["title"].lower() + if "supported versions" in issue_title and ("[maint]" in issue_title or "maint" in issue_title): + return True + + return False + except Exception as e: + print(f"Warning: Could not check existing issues: {e}") + return False + + +def create_github_issue(title: str, body: str, repo_owner: str, repo_name: str, token: str) -> bool: + url = f"https://api.github.com/repos/{repo_owner}/{repo_name}/issues" + + data = {"title": title, "body": body, "labels": ["maintenance", "dependencies"]} + + headers = { + "Authorization": f"token {token}", + "Accept": "application/vnd.github.v3+json", + "Content-Type": "application/json", + **HEADERS, + } + + try: + req = request.Request(url, data=json.dumps(data).encode("utf-8"), headers=headers, method="POST") + with request.urlopen(req) as response: + if response.status == 201: + issue_data = json.loads(response.read().decode("utf-8")) + print(f"āœ… Created GitHub issue: {issue_data['html_url']}") + return True + print(f"āŒ Failed to create issue. Status: {response.status}") + return False + except Exception as e: + print(f"āŒ Error creating GitHub issue: {e}") + return False + + +def generate_issue_body(differences: VersionDifferences, _current: VersionMapping, expected: VersionMapping) -> str: + body = "## Supported versions need updating\n\n" + body += ( + "The supported Python/Django version combinations have changed and " + "need to be updated in the documentation.\n\n" + ) + + if differences.added_python_versions: + body += "### Added Python versions\n" + for version in differences.added_python_versions: + version_str = env_format(version, divider=".") + body += f"- Python {version_str}\n" + body += "\n" + + if differences.removed_python_versions: + body += "### Removed Python versions\n" + for version in differences.removed_python_versions: + version_str = env_format(version, divider=".") + body += f"- Python {version_str}\n" + body += "\n" + + if differences.changed_django_versions: + body += "### Changed Django version support\n" + for python_version, changes in differences.changed_django_versions.items(): + python_str = env_format(python_version, divider=".") + body += f"**Python {python_str}:**\n" + if changes.added: + for django_version in changes.added: + django_str = env_format(django_version, divider=".") + body += f"- āœ… Added Django {django_str}\n" + if changes.removed: + for django_version in changes.removed: + django_str = env_format(django_version, divider=".") + body += f"- āŒ Removed Django {django_str}\n" + body += "\n" + + body += "### Expected compatibility table\n\n" + body += build_readme(expected) + body += "\n\n" + + body += "### Files to update\n" + body += "- `docs/overview/compatibility.md`\n" + body += "- `tox.ini`\n" + body += "- `pyproject.toml`\n" + body += "- `.github/workflows/tests.yml`\n\n" + + body += "Run `python scripts/supported_versions.py generate` to get the updated configurations." + + return body + + +###################################### +# MAIN +###################################### + + +def create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Manage supported Python/Django version combinations", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=textwrap.dedent(""" + Commands: + generate Generate configuration for current supported versions + check Check if versions need updating and create GitHub issue + + Environment variables: + GITHUB_TOKEN Required for 'check' command to create GitHub issues + + Examples: + python scripts/supported_versions.py generate + GITHUB_TOKEN=your_token python scripts/supported_versions.py check + """), + ) + + parser.add_argument("command", choices=["generate", "check"], help="Command to execute") + + parser.add_argument( + "--repo-owner", default="django-components", help="GitHub repository owner (default: django-components)" + ) + + parser.add_argument( + "--repo-name", default="django-components", help="GitHub repository name (default: django-components)" + ) + + return parser + + +def main() -> None: + parser = create_parser() + args = parser.parse_args() + + try: + if args.command == "generate": + command_generate() + elif args.command == "check": + command_check(args.repo_owner, args.repo_name) + else: + parser.error(f"Invalid command: {args.command}") + except KeyboardInterrupt: + print("\nāŒ Interrupted by user") + sys.exit(1) + except Exception as e: + print(f"āŒ Unexpected error: {e}") + sys.exit(1) + + if __name__ == "__main__": main()