From b0b68a5601d2c00e6427348de1d662471ba35a18 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Tue, 25 Jun 2024 13:34:02 -0400 Subject: [PATCH] Migrate release workflow to `cargo-dist` (#9559) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary This PR migrates our release workflow to [`cargo-dist`](https://github.com/axodotdev/cargo-dist). The primary motivation here is that we want to ship dedicated installers for Ruff that work across platforms, and `cargo-dist` gives us those installers out-of-the-box. The secondary motivation is that `cargo-dist` formalizes some of the patterns that we've built up over time in our own release process. At a high level: - The `release.yml` file is generated by `cargo-dist` with `cargo dist generate`. It doesn't contain any modifications vis-a-vis the generated file. (If it's edited out of band from generation, the release fails.) - Our customizations are inserted as custom steps within the `cargo-dist` workflow. Specifically, `build-binaries` builds the wheels and packages them into binaries (as on `main`), while `build-docker.yml` builds the Docker image. `publish-pypi.yml` publishes the wheels to PyPI. This is effectively our `release.yaml` (on `main`), broken down into individual workflows rather than steps within a single workflow. ### Changes from `main` The workflow is _nearly_ unchanged. We kick off a release manually via the GitHub Action by providing a tag. If the tag doesn't match the `Cargo.toml`, the release fails. If the tag matches an already-existing release, the release fails. The release proceeds by (in order): 0. Doing some upfront validation via `cargo-dist`. 1. Creating the wheels and archives. 2. Building and pushing the Docker image. 3. Publishing to PyPI (if it's not a "dry run"). 4. Creating the GitHub Release (if it's not a "dry run"). 5. Notifying `ruff-pre-commit` (if it's not a "dry run"). There are a few changes in the workflow as compared to `main`: - **We no longer validate the SHA** (just the tag). It's not an input to the job. The Axo team is considering whether / how to support this. - **Releases are now published directly** (rather than as draft). Again, the Axo team is considering whether / how to support this. The downside of drafts is that the URLs aren't stable, so the installers don't work _as long as the release is in draft_. This is fine for our workflow. It seems like the Axo team will add it. - Releases already contain the latest entry from the changelog (we don't need to copy it over). This "Just Works", which is nice, though we'll still want to edit them to add contributors. There are also a few **breaking changes** for consumers of the binaries: - **We no longer include the version tag in the file name**. This enables users to install via `/latest` URLs on GitHub, and is part of the cargo-dist paradigm. - **Archives now include an extra level of nesting,** which you can remove with `--strip-components=1` when untarring. Here's an example release that I created -- I omitted all the artifacts since I was just testing a workflow, so none of the installers or links work, but it gives you a sense for what the release looks like: https://github.com/charliermarsh/cargodisttest/releases/tag/0.1.13. ### Test Plan I ran a successful release to completion last night, and installed Ruff via the installer: ![Screenshot 2024-01-17 at 12 12 53 AM](https://github.com/astral-sh/ruff/assets/1309177/a5334466-2ca3-4279-a453-e912a0805df2) ![Screenshot 2024-01-17 at 12 12 48 AM](https://github.com/astral-sh/ruff/assets/1309177/63ac969e-69a1-488c-8367-4cb783526ca7) The piece I'm least confident about is the Docker push. We build the image, but the push fails in my test repo since I haven't wired up the credentials. --- .../{release.yaml => build-binaries.yml} | 339 ++++++------------ .github/workflows/build-docker.yml | 68 ++++ .github/workflows/notify-dependents.yml | 29 ++ .github/workflows/publish-pypi.yml | 34 ++ .github/workflows/release.yml | 246 +++++++++++++ .prettierignore | 2 + Cargo.toml | 57 +++ _typos.toml | 3 +- crates/ruff/Cargo.toml | 2 +- 9 files changed, 542 insertions(+), 238 deletions(-) rename .github/workflows/{release.yaml => build-binaries.yml} (57%) create mode 100644 .github/workflows/build-docker.yml create mode 100644 .github/workflows/notify-dependents.yml create mode 100644 .github/workflows/publish-pypi.yml create mode 100644 .github/workflows/release.yml create mode 100644 .prettierignore diff --git a/.github/workflows/release.yaml b/.github/workflows/build-binaries.yml similarity index 57% rename from .github/workflows/release.yaml rename to .github/workflows/build-binaries.yml index e3a092b32d..91ccc529dc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/build-binaries.yml @@ -1,21 +1,23 @@ -name: "[ruff] Release" +# Build ruff on all platforms. +# +# Generates both wheels (for PyPI) and archived binaries (for GitHub releases). +# +# Assumed to run as a subworkflow of .github/workflows/release.yml; specifically, as a local +# artifacts job within `cargo-dist`. +name: "Build binaries" on: - workflow_dispatch: + workflow_call: inputs: - tag: - description: "The version to tag, without the leading 'v'. If omitted, will initiate a dry run (no uploads)." - type: string - sha: - description: "The full sha of the commit to be released. If omitted, the latest commit on the default branch will be used." - default: "" + plan: + required: true type: string pull_request: paths: - # When we change pyproject.toml, we want to ensure that the maturin builds still work + # When we change pyproject.toml, we want to ensure that the maturin builds still work. - pyproject.toml # And when we change this workflow itself... - - .github/workflows/release.yaml + - .github/workflows/build-binaries.yml concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,6 +25,7 @@ concurrency: env: PACKAGE_NAME: ruff + MODULE_NAME: ruff PYTHON_VERSION: "3.11" CARGO_INCREMENTAL: 0 CARGO_NET_RETRY: 10 @@ -31,11 +34,12 @@ env: jobs: sdist: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -49,8 +53,8 @@ jobs: - name: "Test sdist" run: | pip install dist/${{ env.PACKAGE_NAME }}-*.tar.gz --force-reinstall - ruff --help - python -m ruff --help + ${{ env.MODULE_NAME }} --help + python -m ${{ env.MODULE_NAME }} --help - name: "Upload sdist" uses: actions/upload-artifact@v4 with: @@ -58,11 +62,12 @@ jobs: path: dist macos-x86_64: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: macos-12 steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -74,11 +79,6 @@ jobs: with: target: x86_64 args: --release --locked --out dist - - name: "Test wheel - x86_64" - run: | - pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall - ruff --help - python -m ruff --help - name: "Upload wheels" uses: actions/upload-artifact@v4 with: @@ -86,23 +86,29 @@ jobs: path: dist - name: "Archive binary" run: | - ARCHIVE_FILE=ruff-${{ inputs.tag }}-x86_64-apple-darwin.tar.gz - tar czvf $ARCHIVE_FILE -C target/x86_64-apple-darwin/release ruff + TARGET=x86_64-apple-darwin + ARCHIVE_NAME=ruff-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/ruff $ARCHIVE_NAME/ruff + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 - name: "Upload binary" uses: actions/upload-artifact@v4 with: - name: binaries-macos-x86_64 + name: artifacts-macos-x86_64 path: | *.tar.gz *.sha256 macos-aarch64: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: macos-14 steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -126,18 +132,24 @@ jobs: path: dist - name: "Archive binary" run: | - ARCHIVE_FILE=ruff-${{ inputs.tag }}-aarch64-apple-darwin.tar.gz - tar czvf $ARCHIVE_FILE -C target/aarch64-apple-darwin/release ruff + TARGET=aarch64-apple-darwin + ARCHIVE_NAME=ruff-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/ruff $ARCHIVE_NAME/ruff + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 - name: "Upload binary" uses: actions/upload-artifact@v4 with: - name: binaries-aarch64-apple-darwin + name: artifacts-aarch64-apple-darwin path: | *.tar.gz *.sha256 windows: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: windows-latest strategy: matrix: @@ -151,7 +163,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -171,8 +183,8 @@ jobs: shell: bash run: | python -m pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall - ruff --help - python -m ruff --help + ${{ env.MODULE_NAME }} --help + python -m ${{ env.MODULE_NAME }} --help - name: "Upload wheels" uses: actions/upload-artifact@v4 with: @@ -181,18 +193,19 @@ jobs: - name: "Archive binary" shell: bash run: | - ARCHIVE_FILE=ruff-${{ inputs.tag }}-${{ matrix.platform.target }}.zip + ARCHIVE_FILE=ruff-${{ matrix.platform.target }}.zip 7z a $ARCHIVE_FILE ./target/${{ matrix.platform.target }}/release/ruff.exe sha256sum $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 - name: "Upload binary" uses: actions/upload-artifact@v4 with: - name: binaries-${{ matrix.platform.target }} + name: artifacts-${{ matrix.platform.target }} path: | *.zip *.sha256 linux: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: ubuntu-latest strategy: matrix: @@ -202,7 +215,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -219,27 +232,36 @@ jobs: if: ${{ startsWith(matrix.target, 'x86_64') }} run: | pip install dist/${{ env.PACKAGE_NAME }}-*.whl --force-reinstall - ruff --help - python -m ruff --help + ${{ env.MODULE_NAME }} --help + python -m ${{ env.MODULE_NAME }} --help - name: "Upload wheels" uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.target }} path: dist - name: "Archive binary" + shell: bash run: | - ARCHIVE_FILE=ruff-${{ inputs.tag }}-${{ matrix.target }}.tar.gz - tar czvf $ARCHIVE_FILE -C target/${{ matrix.target }}/release ruff + set -euo pipefail + + TARGET=${{ matrix.target }} + ARCHIVE_NAME=ruff-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/ruff $ARCHIVE_NAME/ruff + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 - name: "Upload binary" uses: actions/upload-artifact@v4 with: - name: binaries-${{ matrix.target }} + name: artifacts-${{ matrix.target }} path: | *.tar.gz *.sha256 linux-cross: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: ubuntu-latest strategy: matrix: @@ -261,11 +283,13 @@ jobs: arch: ppc64 # see https://github.com/astral-sh/ruff/issues/10073 maturin_docker_options: -e JEMALLOC_SYS_WITH_LG_PAGE=16 + - target: arm-unknown-linux-musleabihf + arch: arm steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -282,8 +306,8 @@ jobs: if: matrix.platform.arch != 'ppc64' name: Test wheel with: - arch: ${{ matrix.platform.arch }} - distro: ubuntu20.04 + arch: ${{ matrix.platform.arch == 'arm' && 'armv6' || matrix.platform.arch }} + distro: ${{ matrix.platform.arch == 'arm' && 'bullseye' || 'ubuntu20.04' }} githubToken: ${{ github.token }} install: | apt-get update @@ -298,19 +322,28 @@ jobs: name: wheels-${{ matrix.platform.target }} path: dist - name: "Archive binary" + shell: bash run: | - ARCHIVE_FILE=ruff-${{ inputs.tag }}-${{ matrix.platform.target }}.tar.gz - tar czvf $ARCHIVE_FILE -C target/${{ matrix.platform.target }}/release ruff + set -euo pipefail + + TARGET=${{ matrix.platform.target }} + ARCHIVE_NAME=ruff-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/ruff $ARCHIVE_NAME/ruff + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 - name: "Upload binary" uses: actions/upload-artifact@v4 with: - name: binaries-${{ matrix.platform.target }} + name: artifacts-${{ matrix.platform.target }} path: | *.tar.gz *.sha256 musllinux: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: ubuntu-latest strategy: matrix: @@ -320,7 +353,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -343,26 +376,35 @@ jobs: apk add python3 python -m venv .venv .venv/bin/pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links dist/ --force-reinstall - .venv/bin/ruff check --help + .venv/bin/${{ env.MODULE_NAME }} --help - name: "Upload wheels" uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.target }} path: dist - name: "Archive binary" + shell: bash run: | - ARCHIVE_FILE=ruff-${{ inputs.tag }}-${{ matrix.target }}.tar.gz - tar czvf $ARCHIVE_FILE -C target/${{ matrix.target }}/release ruff + set -euo pipefail + + TARGET=${{ matrix.target }} + ARCHIVE_NAME=ruff-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/ruff $ARCHIVE_NAME/ruff + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 - name: "Upload binary" uses: actions/upload-artifact@v4 with: - name: binaries-${{ matrix.target }} + name: artifacts-${{ matrix.target }} path: | *.tar.gz *.sha256 musllinux-cross: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-build') }} runs-on: ubuntu-latest strategy: matrix: @@ -376,7 +418,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ inputs.sha }} + submodules: recursive - uses: actions/setup-python@v5 with: python-version: ${{ env.PYTHON_VERSION }} @@ -400,204 +442,29 @@ jobs: run: | python -m venv .venv .venv/bin/pip3 install ${{ env.PACKAGE_NAME }} --no-index --find-links dist/ --force-reinstall - .venv/bin/ruff check --help + .venv/bin/${{ env.MODULE_NAME }} --help - name: "Upload wheels" uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.platform.target }} path: dist - name: "Archive binary" + shell: bash run: | - ARCHIVE_FILE=ruff-${{ inputs.tag }}-${{ matrix.platform.target }}.tar.gz - tar czvf $ARCHIVE_FILE -C target/${{ matrix.platform.target }}/release ruff + set -euo pipefail + + TARGET=${{ matrix.platform.target }} + ARCHIVE_NAME=ruff-$TARGET + ARCHIVE_FILE=$ARCHIVE_NAME.tar.gz + + mkdir -p $ARCHIVE_NAME + cp target/$TARGET/release/ruff $ARCHIVE_NAME/ruff + tar czvf $ARCHIVE_FILE $ARCHIVE_NAME shasum -a 256 $ARCHIVE_FILE > $ARCHIVE_FILE.sha256 - name: "Upload binary" uses: actions/upload-artifact@v4 with: - name: binaries-${{ matrix.platform.target }} + name: artifacts-${{ matrix.platform.target }} path: | *.tar.gz *.sha256 - - validate-tag: - name: Validate tag - runs-on: ubuntu-latest - # If you don't set an input tag, it's a dry run (no uploads). - if: ${{ inputs.tag }} - steps: - - uses: actions/checkout@v4 - with: - ref: main # We checkout the main branch to check for the commit - - name: Check main branch - if: ${{ inputs.sha }} - run: | - # Fetch the main branch since a shallow checkout is used by default - git fetch origin main --unshallow - if ! git branch --contains ${{ inputs.sha }} | grep -E '(^|\s)main$'; then - echo "The specified sha is not on the main branch" >&2 - exit 1 - fi - - name: Check tag consistency - run: | - # Switch to the commit we want to release - git checkout ${{ inputs.sha }} - version=$(grep "version = " pyproject.toml | sed -e 's/version = "\(.*\)"/\1/g') - if [ "${{ inputs.tag }}" != "${version}" ]; then - echo "The input tag does not match the version from pyproject.toml:" >&2 - echo "${{ inputs.tag }}" >&2 - echo "${version}" >&2 - exit 1 - else - echo "Releasing ${version}" - fi - - upload-release: - name: Upload to PyPI - runs-on: ubuntu-latest - needs: - - macos-aarch64 - - macos-x86_64 - - windows - - linux - - linux-cross - - musllinux - - musllinux-cross - - validate-tag - # If you don't set an input tag, it's a dry run (no uploads). - if: ${{ inputs.tag }} - environment: - name: release - permissions: - # For pypi trusted publishing - id-token: write - steps: - - uses: actions/download-artifact@v4 - with: - pattern: wheels-* - path: wheels - merge-multiple: true - - name: Publish to PyPi - uses: pypa/gh-action-pypi-publish@release/v1 - with: - skip-existing: true - packages-dir: wheels - verbose: true - - tag-release: - name: Tag release - runs-on: ubuntu-latest - needs: upload-release - # If you don't set an input tag, it's a dry run (no uploads). - if: ${{ inputs.tag }} - permissions: - # For git tag - contents: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - name: git tag - run: | - git config user.email "hey@astral.sh" - git config user.name "Ruff Release CI" - git tag -m "v${{ inputs.tag }}" "v${{ inputs.tag }}" - # If there is duplicate tag, this will fail. The publish to pypi action will have been a noop (due to skip - # existing), so we make a non-destructive exit here - git push --tags - - publish-release: - name: Publish to GitHub - runs-on: ubuntu-latest - needs: tag-release - # If you don't set an input tag, it's a dry run (no uploads). - if: ${{ inputs.tag }} - permissions: - # For GitHub release publishing - contents: write - steps: - - uses: actions/download-artifact@v4 - with: - pattern: binaries-* - path: binaries - merge-multiple: true - - name: "Publish to GitHub" - uses: softprops/action-gh-release@v2 - with: - draft: true - files: binaries/* - tag_name: v${{ inputs.tag }} - - docker-publish: - # This action doesn't need to wait on any other task, it's easy to re-tag if something failed and we're validating - # the tag here also - name: Push Docker image ghcr.io/astral-sh/ruff - runs-on: ubuntu-latest - environment: - name: release - permissions: - # For the docker push - packages: write - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - - uses: docker/setup-buildx-action@v3 - - - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v5 - with: - images: ghcr.io/astral-sh/ruff - - - name: Check tag consistency - # Unlike validate-tag we don't check if the commit is on the main branch, but it seems good enough since we can - # change docker tags - if: ${{ inputs.tag }} - run: | - version=$(grep "version = " pyproject.toml | sed -e 's/version = "\(.*\)"/\1/g') - if [ "${{ inputs.tag }}" != "${version}" ]; then - echo "The input tag does not match the version from pyproject.toml:" >&2 - echo "${{ inputs.tag }}" >&2 - echo "${version}" >&2 - exit 1 - else - echo "Releasing ${version}" - fi - - - name: "Build and push Docker image" - uses: docker/build-push-action@v6 - with: - context: . - platforms: linux/amd64,linux/arm64 - # Reuse the builder - cache-from: type=gha - cache-to: type=gha,mode=max - push: ${{ inputs.tag != '' }} - tags: ghcr.io/astral-sh/ruff:latest,ghcr.io/astral-sh/ruff:${{ inputs.tag || 'dry-run' }} - labels: ${{ steps.meta.outputs.labels }} - - # After the release has been published, we update downstream repositories - # This is separate because if this fails the release is still fine, we just need to do some manual workflow triggers - update-dependents: - name: Update dependents - runs-on: ubuntu-latest - needs: publish-release - steps: - - name: "Update pre-commit mirror" - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.RUFF_PRE_COMMIT_PAT }} - script: | - github.rest.actions.createWorkflowDispatch({ - owner: 'astral-sh', - repo: 'ruff-pre-commit', - workflow_id: 'main.yml', - ref: 'main', - }) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000000..88e5503e31 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,68 @@ +# Build and publish a Docker image. +# +# Assumed to run as a subworkflow of .github/workflows/release.yml; specifically, as a local +# artifacts job within `cargo-dist`. +# +# TODO(charlie): Ideally, the publish step would happen as a publish job within `cargo-dist`, but +# sharing the built image as an artifact between jobs is challenging. +name: "[ruff] Build Docker image" + +on: + workflow_call: + inputs: + plan: + required: true + type: string + pull_request: + paths: + - .github/workflows/build-docker.yml + +jobs: + docker-publish: + name: Build Docker image (ghcr.io/astral-sh/ruff) + runs-on: ubuntu-latest + environment: + name: release + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ghcr.io/astral-sh/ruff + + - name: Check tag consistency + if: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }} + run: | + version=$(grep "version = " pyproject.toml | sed -e 's/version = "\(.*\)"/\1/g') + if [ "${{ fromJson(inputs.plan).announcement_tag }}" != "${version}" ]; then + echo "The input tag does not match the version from pyproject.toml:" >&2 + echo "${{ fromJson(inputs.plan).announcement_tag }}" >&2 + echo "${version}" >&2 + exit 1 + else + echo "Releasing ${version}" + fi + + - name: "Build and push Docker image" + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + # Reuse the builder + cache-from: type=gha + cache-to: type=gha,mode=max + push: ${{ inputs.plan != '' && !fromJson(inputs.plan).announcement_tag_is_implicit }} + tags: ghcr.io/astral-sh/ruff:latest,ghcr.io/astral-sh/ruff:${{ (inputs.plan != '' && fromJson(inputs.plan).announcement_tag) || 'dry-run' }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/notify-dependents.yml b/.github/workflows/notify-dependents.yml new file mode 100644 index 0000000000..54ddbb19ab --- /dev/null +++ b/.github/workflows/notify-dependents.yml @@ -0,0 +1,29 @@ +# Notify downstream repositories of a new release. +# +# Assumed to run as a subworkflow of .github/workflows/release.yml; specifically, as a post-announce +# job within `cargo-dist`. +name: "[ruff] Notify dependents" + +on: + workflow_call: + inputs: + plan: + required: true + type: string + +jobs: + update-dependents: + name: Notify dependents + runs-on: ubuntu-latest + steps: + - name: "Update pre-commit mirror" + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.RUFF_PRE_COMMIT_PAT }} + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'astral-sh', + repo: 'ruff-pre-commit', + workflow_id: 'main.yml', + ref: 'main', + }) diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 0000000000..4e250f24e0 --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,34 @@ +# Publish a release to PyPI. +# +# Assumed to run as a subworkflow of .github/workflows/release.yml; specifically, as a publish job +# within `cargo-dist`. +name: "[ruff] Publish to PyPI" + +on: + workflow_call: + inputs: + plan: + required: true + type: string + +jobs: + pypi-publish: + name: Upload to PyPI + runs-on: ubuntu-latest + environment: + name: release + permissions: + # For PyPI's trusted publishing. + id-token: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: wheels + merge-multiple: true + - name: Publish to PyPi + uses: pypa/gh-action-pypi-publish@release/v1 + with: + skip-existing: true + packages-dir: wheels + verbose: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..f0de4cd9c7 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,246 @@ +# Copyright 2022-2024, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with cargo-dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a GitHub Release +# +# Note that the GitHub Release will be created with a generated +# title/body based on your changelogs. + +name: Release + +permissions: + contents: write + +# This task will run whenever you workflow_dispatch with a tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't cargo-dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (cargo-dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However, GitHub +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + workflow_dispatch: + inputs: + tag: + description: Release Tag + required: true + default: dry-run + type: string + +jobs: + # Run 'cargo dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: ubuntu-latest + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ (inputs.tag != 'dry-run' && inputs.tag) || '' }} + tag-flag: ${{ inputs.tag && inputs.tag != 'dry-run' && format('--tag={0}', inputs.tag) || '' }} + publishing: ${{ inputs.tag && inputs.tag != 'dry-run' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + # we specify bash to get pipefail; it guards against the `curl` command + # failing. otherwise `sh` won't catch that `curl` returned non-0 + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.14.0/cargo-dist-installer.sh | sh" + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + cargo dist ${{ (inputs.tag && inputs.tag != 'dry-run' && format('host --steps=create --tag={0}', inputs.tag)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "cargo dist ran successfully" + cat plan-dist-manifest.json + echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + name: artifacts-plan-dist-manifest + path: plan-dist-manifest.json + + custom-build-binaries: + needs: + - plan + if: ${{ needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload' || inputs.tag == 'dry-run' }} + uses: ./.github/workflows/build-binaries.yml + with: + plan: ${{ needs.plan.outputs.val }} + secrets: inherit + + custom-build-docker: + needs: + - plan + if: ${{ needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload' || inputs.tag == 'dry-run' }} + uses: ./.github/workflows/build-docker.yml + with: + plan: ${{ needs.plan.outputs.val }} + secrets: inherit + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - custom-build-binaries + - custom-build-docker + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.14.0/cargo-dist-installer.sh | sh" + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: cargo-dist + shell: bash + run: | + cargo dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "cargo dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v4 + with: + name: artifacts-build-global + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + # Determines if we should publish/announce + host: + needs: + - plan + - custom-build-binaries + - custom-build-docker + - build-global-artifacts + # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.custom-build-binaries.result == 'skipped' || needs.custom-build-binaries.result == 'success') && (needs.custom-build-docker.result == 'skipped' || needs.custom-build-docker.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-20.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install cargo-dist + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.14.0/cargo-dist-installer.sh | sh" + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + # This is a harmless no-op for GitHub Releases, hosting for that happens in "announce" + - id: host + shell: bash + run: | + cargo dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v4 + with: + # Overwrite the previous copy + name: artifacts-dist-manifest + path: dist-manifest.json + + custom-publish-pypi: + needs: + - plan + - host + if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} + uses: ./.github/workflows/publish-pypi.yml + with: + plan: ${{ needs.plan.outputs.val }} + secrets: inherit + # publish jobs get escalated permissions + permissions: + id-token: write + packages: write + + # Create a GitHub Release while uploading all files to it + announce: + needs: + - plan + - host + - custom-publish-pypi + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' && (needs.custom-publish-pypi.result == 'skipped' || needs.custom-publish-pypi.result == 'success') }} + runs-on: "ubuntu-20.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: "Download GitHub Artifacts" + uses: actions/download-artifact@v4 + with: + pattern: artifacts-* + path: artifacts + merge-multiple: true + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create GitHub Release + uses: ncipollo/release-action@v1 + with: + tag: ${{ needs.plan.outputs.tag }} + name: ${{ fromJson(needs.host.outputs.val).announcement_title }} + body: ${{ fromJson(needs.host.outputs.val).announcement_github_body }} + prerelease: ${{ fromJson(needs.host.outputs.val).announcement_is_prerelease }} + artifacts: "artifacts/*" + + custom-notify-dependents: + needs: + - plan + - announce + uses: ./.github/workflows/notify-dependents.yml + with: + plan: ${{ needs.plan.outputs.val }} + secrets: inherit diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000..0e8c9be149 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,2 @@ +# Auto-generated by `cargo-dist`. +.github/workflows/release.yml diff --git a/Cargo.toml b/Cargo.toml index 645ee958a4..c768abf013 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -219,3 +219,60 @@ opt-level = 1 [profile.profiling] inherits = "release" debug = 1 + +# The profile that 'cargo dist' will build with. +[profile.dist] +inherits = "release" + +# Config for 'cargo dist' +[workspace.metadata.dist] +# The preferred cargo-dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.14.0" +# CI backends to support +ci = ["github"] +# The installers to generate for each app +installers = ["shell", "powershell"] +# The archive format to use for windows builds (defaults .zip) +windows-archive = ".zip" +# The archive format to use for non-windows builds (defaults .tar.xz) +unix-archive = ".tar.gz" +# Target platforms to build apps for (Rust target-triple syntax) +targets = [ + "aarch64-apple-darwin", + "aarch64-pc-windows-msvc", + "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", + "arm-unknown-linux-musleabihf", + "armv7-unknown-linux-gnueabihf", + "armv7-unknown-linux-musleabihf", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "i686-unknown-linux-musl", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", +] +# Whether to auto-include files like READMEs, LICENSEs, and CHANGELOGs (default true) +auto-includes = false +# Whether cargo-dist should create a Github Release or use an existing draft +create-release = true +# Publish jobs to run in CI +pr-run-mode = "skip" +# Whether CI should trigger releases with dispatches instead of tag pushes +dispatch-releases = true +# Whether CI should include auto-generated code to build local artifacts +build-local-artifacts = false +# Local artifacts jobs to run in CI +local-artifacts-jobs = ["./build-binaries", "./build-docker"] +# Publish jobs to run in CI +publish-jobs = ["./publish-pypi"] +# Announcement jobs to run in CI +post-announce-jobs = ["./notify-dependents"] +# Skip checking whether the specified configuration files are up to date +#allow-dirty = ["ci"] +# Whether to install an updater program +install-updater = false diff --git a/_typos.toml b/_typos.toml index 60d6225822..cdaa1c3f58 100644 --- a/_typos.toml +++ b/_typos.toml @@ -16,5 +16,6 @@ jod = "jod" # e.g., `jod-thread` [default] extend-ignore-re = [ # Line ignore with trailing "spellchecker:disable-line" - "(?Rm)^.*#\\s*spellchecker:disable-line$" + "(?Rm)^.*#\\s*spellchecker:disable-line$", + "LICENSEs", ] diff --git a/crates/ruff/Cargo.toml b/crates/ruff/Cargo.toml index dbd57b7c72..59fafbc123 100644 --- a/crates/ruff/Cargo.toml +++ b/crates/ruff/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "ruff" version = "0.4.10" -publish = false +publish = true authors = { workspace = true } edition = { workspace = true } rust-version = { workspace = true }