From d862147ec3fd8855453a4a7d9adf3f6d871d9502 Mon Sep 17 00:00:00 2001 From: Robbie McKinstry Date: Wed, 24 Jun 2026 16:33:10 -0400 Subject: [PATCH] ci: two-tier CI + gating Linux dist build + rolling edge prerelease MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modernize the GitHub Actions setup to match the current Wack pattern (keystore/aviary/hare/metricstore), adapted for a binary (cargo-dist) release instead of a Helm/Docker release. on-push.yml (fast tier) - Run only `format` + `clippy` instead of the full `cargo make ci-flow`, for fast PR feedback. Inline jobs on wack/gh-actions/setup-rust@trunk, replacing the monolithic validate.yml reusable workflow. - Exclude `gh-readonly-queue/**` from push triggers so on-push's fast "⚡ PR Ready" can't satisfy the merge queue's required check before on-merge's comprehensive one finishes. on-merge.yml (comprehensive tier) - `format` + `build` (`cargo make ci-flow`, which already bundles coverage) gate the merge queue. - Add a gating cross-platform release build: `dist-plan` reads the build matrix from `dist plan` at runtime (so it tracks dist-workspace.toml), filtered to Linux targets; `dist-build` builds all Linux dist targets on every merge candidate. macOS builds stay on tags (release.yml) to keep 10x-billed macOS runner spend off the merge queue. - Add `publish-edge`: runs last in the merge group (after format, build, and dist-build are green), reuses the validated dist-build artifacts, and force-updates a rolling `edge` prerelease pointed at the candidate commit. It gates the merge (in pr-ready's needs) with retry on the release upload. `edge` carries Linux binaries; macOS ships with tags. release.yml / dist - Bump cargo-dist 0.29.0 -> 0.32.0 and regenerate release.yml (checkout@v6, upload-artifact@v7, download-artifact@v8, attest@v4). No CROSS_REPO_PAT is configured: multitool's only git dependency (multitool-rust-sdk) is public. The dev-only Dockerfile is removed; it was never published and nothing referenced it. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/on-merge.yml | 228 +++++++++++++++++++++++++++++++-- .github/workflows/on-push.yml | 76 ++++++++--- .github/workflows/release.yml | 50 ++++---- Dockerfile | 9 -- dist-workspace.toml | 2 +- 5 files changed, 299 insertions(+), 66 deletions(-) delete mode 100644 Dockerfile diff --git a/.github/workflows/on-merge.yml b/.github/workflows/on-merge.yml index 1ea3eb6..9244b4d 100644 --- a/.github/workflows/on-merge.yml +++ b/.github/workflows/on-merge.yml @@ -1,33 +1,233 @@ -name: "On Merge" +name: "On Merge | Validate Build" on: - # Use a merge queue to gate the creation and storage - # of these Docker images. + # The merge queue gates trunk; this is the comprehensive tier. merge_group: - # Allow this job to be executed manually from the GH UI. + # Allow this workflow to be executed manually from the GH UI. workflow_dispatch: +env: + CARGO_TERM_COLOR: always + +permissions: + contents: read + jobs: + # Single required-check context for the merge queue. Keep this job's name + # ("⚡ PR Ready") identical to the one in on-push.yml. publish-edge is in the + # needs list so the merge waits for the rolling edge prerelease to be + # published before the candidate fast-forwards onto trunk. pr-ready: if: always() name: "⚡ PR Ready" runs-on: ubuntu-22.04 needs: - - "build" + - "format" + - "build" + - "dist-plan" + - "dist-build" + - "publish-edge" steps: - - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} run: | echo "One or more dependent jobs failed, was skipped, or was cancelled. All jobs must pass for the PR to be ready." exit 1 - run: echo "OK" - - # This job installs Cargo Make and Cargo Nextest before running - # the CI workflow using Cargo Make. Most of the time, it should - # restore Cargo Make and other dependencies from cache. + + # Fast-fail formatting gate (cheap; surfaces a formatting miss before the + # heavier build job finishes). See on-push.yml for why we use setup-rust + # without forcing a toolchain and without a CROSS_REPO_PAT. + format: + name: Check Formatting + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + - name: Setup Rust + uses: "wack/gh-actions/setup-rust@trunk" + - name: Check formatting + run: cargo make check-format + + # Full validation. multitool's `ci-flow` bundles pre-build checks, format, + # clippy, build, docs, the nextest suite, AND llvm-cov coverage in one task, + # so there is no separate coverage job (unlike the Helm-based Wack repos that + # split coverage out). build: name: Validate Rust Build - uses: "wack/gh-actions/.github/workflows/validate.yml@trunk" + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v5 + - name: Setup Rust + uses: "wack/gh-actions/setup-rust@trunk" + with: + # ci-flow runs the suite with cargo-nextest and collects coverage with + # cargo-llvm-cov, so both must be installed alongside cargo-make. + shared-key: rust-ci + extra-tools: cargo-nextest,cargo-llvm-cov + - name: Cargo Make + run: cargo make ci-flow + + # ─────────────────────────────────────────────────────────────────────────── + # Gating release build. Every merge-queue candidate is built for all Linux + # dist targets before it can land, so trunk never holds a commit that fails to + # produce a release artifact. The macOS targets are intentionally excluded + # here to keep (10×-billed) macOS runner spend off every merge — they're built + # and published only on a version tag, via release.yml. Consequently the + # rolling `edge` prerelease published below carries Linux binaries only; macOS + # binaries ship with tagged releases. + # + # The matrix is read at runtime from `dist plan` (then filtered to Linux), so + # it tracks dist-workspace.toml automatically — no hand-maintained target list. + # ─────────────────────────────────────────────────────────────────────────── + dist-plan: + name: Plan release build + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + submodules: recursive + - name: Install dist + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.32.0/cargo-dist-installer.sh | sh" + - id: plan + shell: bash + run: | + # Tagless plan (no GitHub Release is created); we only need the build + # matrix that describes each target's runner and dist invocation. + # Filter to Linux targets so the merge queue never pays for macOS + # builds — those run on tags (release.yml) with the full target set. + dist plan --output-format=json > plan-dist-manifest.json + echo "matrix=$(jq -c '{include: [.ci.github.artifacts_matrix.include[] | select([.targets[] | contains("linux")] | any)]}' plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + + dist-build: + name: "Build (${{ join(matrix.targets, ', ') }})" + needs: dist-plan + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.dist-plan.outputs.matrix) }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout repository + uses: actions/checkout@v5 + with: + submodules: recursive + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: Install OpenSSL and MUSL build dependencies + if: ${{ startsWith(matrix.runner, 'ubuntu') }} + run: | + sudo apt-get update + sudo apt-get install -yq openssl libssl-dev musl-tools perl make + - name: Install dist + run: ${{ matrix.install_dist.run }} + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - id: build + name: Build artifacts + shell: bash + run: | + dist build --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" + # Collect the artifact paths dist just produced so publish-edge can + # reuse them without rebuilding. + echo "paths<> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: edge-artifacts-${{ join(matrix.targets, '_') }} + path: ${{ steps.build.outputs.paths }} + retention-days: 1 + if-no-files-found: error + + # ─────────────────────────────────────────────────────────────────────────── + # Publish the validated artifacts to the rolling `edge` prerelease. Because it + # depends on format + build + dist-build, it runs LAST in the merge group — + # only after every other check is green — so it publishes a candidate that has + # already cleared validation and is the commit about to fast-forward onto + # trunk. It reuses dist-build's (Linux-only) artifacts (no rebuild), so `edge` + # carries Linux binaries; macOS binaries ship with tagged releases. It is in + # pr-ready's needs, so the merge waits for it. + # + # The publish steps only run on merge_group, not workflow_dispatch, so a manual + # dispatch (used for inspection) validates without touching the edge release — + # but the job still succeeds, keeping pr-ready green. + # ─────────────────────────────────────────────────────────────────────────── + publish-edge: + name: Publish edge prerelease + needs: + - format + - build + - dist-build + runs-on: ubuntu-22.04 permissions: - id-token: "write" - contents: "read" - attestations: "write" + contents: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + EDGE_TAG: edge + steps: + - name: Download built artifacts + if: ${{ github.event_name == 'merge_group' }} + uses: actions/download-artifact@v8 + with: + pattern: edge-artifacts-* + path: edge/ + merge-multiple: true + - name: Generate checksums + if: ${{ github.event_name == 'merge_group' }} + working-directory: edge + run: | + shopt -s nullglob + files=(*) + if [ ${#files[@]} -eq 0 ]; then + echo "No artifacts were downloaded; refusing to publish an empty edge release." >&2 + exit 1 + fi + sha256sum "${files[@]}" > SHA256SUMS + cat SHA256SUMS + - name: Update rolling edge prerelease + if: ${{ github.event_name == 'merge_group' }} + env: + # github.sha is the merge-queue candidate that fast-forwards onto trunk + # when this run passes — the exact commit we want edge to point at. + RELEASE_COMMIT: ${{ github.sha }} + run: | + # Recreate the rolling prerelease so its assets and target commit are + # always the latest trunk candidate. Delete + recreate (rather than + # uploading over the top) guarantees stale per-platform assets from a + # previous build never linger. + gh release delete "$EDGE_TAG" --repo "$GITHUB_REPOSITORY" --cleanup-tag --yes 2>/dev/null || true + # Retry create/upload so a transient API hiccup doesn't wedge the queue. + n=0 + until [ "$n" -ge 3 ]; do + if gh release create "$EDGE_TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$RELEASE_COMMIT" \ + --prerelease \ + --title "Edge (latest trunk build)" \ + --notes "Rolling pre-release built from the most recent trunk commit (${RELEASE_COMMIT}). Updated automatically on every merge; not a stable release." \ + edge/*; then + break + fi + n=$((n + 1)) + echo "gh release create failed (attempt ${n}); retrying in 10s..." >&2 + sleep 10 + done + if [ "$n" -ge 3 ]; then + echo "Failed to publish edge release after 3 attempts." >&2 + exit 1 + fi diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml index 7096fd2..9d08709 100644 --- a/.github/workflows/on-push.yml +++ b/.github/workflows/on-push.yml @@ -1,37 +1,77 @@ -name: "On Push" +name: "On Push – Validate Build" on: push: - branches-ignore: - - trunk + # Skip the default branch (the merge queue validates trunk commits) and the + # merge queue's own temporary branches. on-push and on-merge share the + # "⚡ PR Ready" required-check name; if on-push ran on a gh-readonly-queue/** + # branch, its fast PR-time gate (format + clippy) could satisfy the merge + # queue's required check before on-merge.yml's comprehensive gate (build + + # coverage + dist-build + publish-edge) finished — letting a failing commit + # merge. The merge_group event in on-merge.yml is what gates the queue. + branches: + - '**' + - '!trunk' + - '!gh-readonly-queue/**' concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + CARGO_TERM_COLOR: always + +permissions: + contents: read + jobs: - # Do not change this job's name without also changing "pr-ready"'s - # job name in "on-merge.yml". These jobs must have the same name. - # See the README for more details. + # Do not change this job's name without also changing "pr-ready"'s job name + # in "on-merge.yml". Both workflows back the same "⚡ PR Ready" required-check + # context, so the name must stay identical in both. pr-ready: if: always() name: "⚡ PR Ready" runs-on: ubuntu-22.04 needs: - - "build" + - "format" + - "clippy" steps: - - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} + - if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} run: | echo "One or more dependent jobs failed, was skipped, or was cancelled. All jobs must pass for the PR to be ready." exit 1 - run: echo "OK" - # This job installs Cargo Make and Cargo Nextest before running - # the CI workflow using Cargo Make. Most of the time, it should - # restore Cargo Make and other dependencies from cache. - build: - name: Validate Rust Build - uses: "wack/gh-actions/.github/workflows/validate.yml@trunk" - permissions: - id-token: "write" - contents: "read" - attestations: "write" + + # Fast formatting gate. The full test suite + coverage run only in the merge + # queue (on-merge.yml) so PR authors get quick feedback here. + # + # We use wack/gh-actions/setup-rust (which pins to dtolnay's stable as the + # rustup *default*) rather than forcing a toolchain: the repo's + # rust-toolchain.toml (1.96.0) takes precedence over the default, so cargo + # commands run on the pinned toolchain — matching local dev exactly. No + # CROSS_REPO_PAT is configured because multitool's only git dependency + # (multitool-rust-sdk) is a public repo. + format: + name: Check Formatting + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v5 + - name: Setup Rust + uses: "wack/gh-actions/setup-rust@trunk" + - name: Check formatting + run: cargo make check-format + + # Clippy runs on PR branches so lint failures surface here, not only in the + # merge queue. Same canonical command the merge queue's ci-flow runs: + # `cargo make clippy` == clippy --all-targets --workspace --locked -D warnings. + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v5 + - name: Setup Rust + uses: "wack/gh-actions/setup-rust@trunk" + - name: Clippy + run: cargo make clippy diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dbe11bd..e593cf6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,9 +15,7 @@ name: Release permissions: - "attestations": "write" "contents": "write" - "id-token": "write" # This task will run whenever you push a git tag that looks like a version # like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. @@ -58,7 +56,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false submodules: recursive @@ -66,9 +64,9 @@ jobs: # we specify bash to get pipefail; it guards against the `curl` command # failing. otherwise `sh` won't catch that `curl` returned non-0 shell: bash - run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.29.0/cargo-dist-installer.sh | sh" + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.32.0/cargo-dist-installer.sh | sh" - name: Cache dist - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: cargo-dist-cache path: ~/.cargo/bin/dist @@ -84,7 +82,7 @@ jobs: cat plan-dist-manifest.json echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" - name: "Upload dist-manifest.json" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: artifacts-plan-dist-manifest path: plan-dist-manifest.json @@ -114,11 +112,15 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + permissions: + "attestations": "write" + "contents": "read" + "id-token": "write" steps: - name: enable windows longpaths run: | git config --global core.longpaths true - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false submodules: recursive @@ -138,7 +140,7 @@ jobs: run: ${{ matrix.install_dist.run }} # Get the dist-manifest - name: Fetch local artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: artifacts-* path: target/distrib/ @@ -152,7 +154,7 @@ jobs: dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json echo "dist ran successfully" - name: Attest - uses: actions/attest-build-provenance@v2 + uses: actions/attest@v4 with: subject-path: "target/distrib/*${{ join(matrix.targets, ', ') }}*" - id: cargo-dist @@ -169,7 +171,7 @@ jobs: cp dist-manifest.json "$BUILD_MANIFEST_NAME" - name: "Upload artifacts" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: artifacts-build-local-${{ join(matrix.targets, '_') }} path: | @@ -186,19 +188,19 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false submodules: recursive - name: Install cached dist - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: cargo-dist-cache path: ~/.cargo/bin/ - run: chmod +x ~/.cargo/bin/dist # Get all the local artifacts for the global tasks to use (for e.g. checksums) - name: Fetch local artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: artifacts-* path: target/distrib/ @@ -216,7 +218,7 @@ jobs: cp dist-manifest.json "$BUILD_MANIFEST_NAME" - name: "Upload artifacts" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: artifacts-build-global path: | @@ -228,27 +230,27 @@ jobs: - plan - build-local-artifacts - build-global-artifacts - # Only run if we're "publishing", and only if local and global didn't fail (skipped is fine) - if: ${{ always() && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} runs-on: "ubuntu-22.04" outputs: val: ${{ steps.host.outputs.manifest }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false submodules: recursive - name: Install cached dist - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: name: cargo-dist-cache path: ~/.cargo/bin/ - run: chmod +x ~/.cargo/bin/dist # Fetch artifacts from scratch-storage - name: Fetch artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: artifacts-* path: target/distrib/ @@ -261,14 +263,14 @@ jobs: cat dist-manifest.json echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" - name: "Upload dist-manifest.json" - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: # Overwrite the previous copy name: artifacts-dist-manifest path: dist-manifest.json # Create a GitHub Release while uploading all files to it - name: "Download GitHub Artifacts" - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: artifacts-* path: artifacts @@ -301,14 +303,14 @@ jobs: GITHUB_EMAIL: "admin+bot@axo.dev" if: ${{ !fromJson(needs.plan.outputs.val).announcement_is_prerelease || fromJson(needs.plan.outputs.val).publish_prereleases }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: true repository: "wack/homebrew-tap" token: ${{ secrets.HOMEBREW_TAP_TOKEN }} # So we have access to the formula - name: Fetch homebrew formulae - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v8 with: pattern: artifacts-* path: Formula/ @@ -348,7 +350,7 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false submodules: recursive diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 86db9be..0000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -# This Dockerfile defines a development environment for working with -# MultiTool. It's meant to serve as a jumping off point for contributors -# who may not have the right tools installed for development. - -FROM rust:1.86.0-slim-bookworm - -RUN apt-get update && \ - apt-get install libssl-dev pkg-config -yq && \ - cargo install cargo-make cargo-nextest diff --git a/dist-workspace.toml b/dist-workspace.toml index 8c30bbf..a48716c 100644 --- a/dist-workspace.toml +++ b/dist-workspace.toml @@ -4,7 +4,7 @@ members = ["cargo:."] # Config for 'dist' [dist] # The preferred dist version to use in CI (Cargo.toml SemVer syntax) -cargo-dist-version = "0.29.0" +cargo-dist-version = "0.32.0" # CI backends to support ci = "github" # Whether to enable GitHub Attestations