diff --git a/.github/workflows/release-kbagent.yml b/.github/workflows/release-kbagent.yml index d251578f..97f0aa0c 100644 --- a/.github/workflows/release-kbagent.yml +++ b/.github/workflows/release-kbagent.yml @@ -1,9 +1,9 @@ # Release & distribution pipeline for kbagent — see docs/adr/0003-release-distribution-cicd.md # # Ships a SELF-CONTAINED NATIVE BINARY (no Python / no uv at the user's end), packaged -# for brew / apt / dnf / apk / chocolatey / winget — the same UX the legacy Go `kbc` +# for brew / apt / dnf / chocolatey / winget — the same UX the legacy Go `kbc` # delivered. The Python app is frozen with PyInstaller (proven: `env -i kbagent -# --version` works), then packaged with nfpm (deb/rpm/apk) and wrapped for Homebrew / +# --version` works), then packaged with nfpm (deb/rpm) and wrapped for Homebrew / # Chocolatey / WinGet. GoReleaser is NOT used (it builds Go); nothing depends on the # deprecated keboola-as-code repo. # @@ -56,9 +56,14 @@ jobs: outputs: VERSION: ${{ steps.v.outputs.VERSION }} # "true" for a pre-release tag (has a -suffix, e.g. v0.0.1-dev.1). Pre-releases - # build + package + sign + GitHub pre-release ONLY; all external publishing - # (PyPI, S3 repo index, Homebrew, Chocolatey, WinGet) is skipped. + # build + package + sign, publish to S3 under the dev prefix (PUBLISH_PREFIX), and + # run test-install + a GitHub pre-release. The user-facing publishers (PyPI, + # Homebrew, Chocolatey, WinGet) are skipped — only real tags reach those. IS_PRERELEASE: ${{ steps.v.outputs.IS_PRERELEASE }} + # S3 prefix to publish under: prod (`keboola-cli2`) for real releases, a separate + # `keboola-cli2-dev` for pre-releases so a dev tag can exercise the full S3 publish + # + client install test without touching the prod apt/rpm repos. + PUBLISH_PREFIX: ${{ steps.v.outputs.PUBLISH_PREFIX }} steps: - id: v env: @@ -74,7 +79,10 @@ jobs: # Pre-release = any a/b/rc/.dev/-suffix marker (.post is a real release). if printf '%s' "$VERSION" | grep -Eq '(a|b|rc)[0-9]+$|\.dev[0-9]+$|-'; then PRE=true; else PRE=false; fi echo "IS_PRERELEASE=$PRE" >> "$GITHUB_OUTPUT" - echo "version=$VERSION prerelease=$PRE" + # Pre-releases publish to a separate dev prefix (see PUBLISH_PREFIX output above). + if [ "$PRE" = true ]; then PUBLISH_PREFIX="${S3_PREFIX}-dev"; else PUBLISH_PREFIX="${S3_PREFIX}"; fi + echo "PUBLISH_PREFIX=$PUBLISH_PREFIX" >> "$GITHUB_OUTPUT" + echo "version=$VERSION prerelease=$PRE prefix=$PUBLISH_PREFIX" # Reuse the per-PR gates against the tagged commit, plus changelog-check. gate: @@ -210,11 +218,11 @@ jobs: dist/*.sha256 dist/${{ env.BIN_NAME }}* - # ── Package the Linux binaries into deb/rpm/apk with nfpm (language-agnostic). + # ── Package the Linux binaries into deb/rpm with nfpm (language-agnostic). package-linux: needs: [version, freeze] runs-on: ubuntu-latest - environment: release # so DEB/RPM/APK signing keys resolve + environment: release # so DEB/RPM signing keys resolve env: VERSION: ${{ needs.version.outputs.VERSION }} steps: @@ -228,19 +236,17 @@ jobs: with: { pattern: bin-linux-*, path: artifacts } - name: Write package signing keys env: - # DEB/RPM are signed with GPG keys; APK with an abuild RSA key. + # DEB/RPM are signed with GPG keys. DEB_KEY_PRIVATE: ${{ secrets.DEB_KEY_PRIVATE }} RPM_KEY_PRIVATE: ${{ secrets.RPM_KEY_PRIVATE }} - APK_KEY_PRIVATE: ${{ secrets.APK_KEY_PRIVATE }} run: | mkdir -p /tmp/keys - for k in DEB_KEY_PRIVATE RPM_KEY_PRIVATE APK_KEY_PRIVATE; do + for k in DEB_KEY_PRIVATE RPM_KEY_PRIVATE; do [ -n "${!k}" ] || { echo "::error::$k is empty — refusing to build unsigned/garbage-signed packages"; exit 1; } done printf '%s' "$DEB_KEY_PRIVATE" > /tmp/keys/deb.key && chmod 600 /tmp/keys/deb.key printf '%s' "$RPM_KEY_PRIVATE" > /tmp/keys/rpm.key && chmod 600 /tmp/keys/rpm.key - printf '%s' "$APK_KEY_PRIVATE" > /tmp/keys/apk.key && chmod 600 /tmp/keys/apk.key - - name: Build deb/rpm/apk for each arch + - name: Build deb/rpm for each arch run: bash build/package/linux/build_packages.sh "$VERSION" artifacts - uses: actions/upload-artifact@v4 with: { name: linux-packages, path: dist/* } @@ -264,7 +270,7 @@ jobs: - name: Collect release assets run: | mkdir -p release - find artifacts -type f \( -name '*.zip' -o -name '*.sha256' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' \) -exec cp {} release/ \; + find artifacts -type f \( -name '*.zip' -o -name '*.sha256' -o -name '*.deb' -o -name '*.rpm' \) -exec cp {} release/ \; ls -al release/ - uses: softprops/action-gh-release@v2 with: @@ -303,11 +309,19 @@ jobs: echo "::error::wheel(s) still missing 10 min after dispatching release.yml for v${VERSION} -- check that run" exit 1 - # ── Publish to S3 (cli-dist.keboola.com/keboola-cli2/) and index the apt/rpm/apk + # ── Publish to S3 (cli-dist.keboola.com/keboola-cli2/) and index the apt/rpm # repos so `apt-get install keboola-cli2` works. Real; needs AWS + GPG secrets. publish-s3: needs: [version, freeze, package-linux] - if: needs.version.outputs.IS_PRERELEASE == 'false' && startsWith(github.ref, 'refs/tags/') + # Runs on every tag. Pre-releases publish under the dev prefix (PUBLISH_PREFIX), so a + # dev tag exercises the full publish + client install test without touching prod repos. + if: startsWith(github.ref, 'refs/tags/') + # Serialize per prefix: the wipe + index steps mutate a shared repo prefix, so two + # concurrent tag runs (e.g. two dev tags) could delete/overwrite each other mid-publish. + # Queue (don't cancel) so an in-flight publish always finishes intact. + concurrency: + group: publish-s3-${{ needs.version.outputs.PUBLISH_PREFIX }} + cancel-in-progress: false runs-on: ubuntu-latest environment: release permissions: @@ -315,6 +329,7 @@ jobs: id-token: write # AWS OIDC (configure-aws-credentials assumes the role) env: VERSION: ${{ needs.version.outputs.VERSION }} + PUBLISH_PREFIX: ${{ needs.version.outputs.PUBLISH_PREFIX }} steps: - uses: actions/checkout@v5 - uses: actions/download-artifact@v4 @@ -323,18 +338,26 @@ jobs: with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} + - name: Wipe dev prefix (pre-release only) + # Dev tags reuse the keboola-cli2-dev prefix; clear it so each dev run is a clean + # slate and version resolution isn't confused by leftover packages from a prior + # dev tag. NEVER runs for the prod prefix (guarded on the -dev suffix). + if: needs.version.outputs.IS_PRERELEASE == 'true' + run: | + case "$PUBLISH_PREFIX" in + *-dev) aws s3 rm --recursive "s3://${AWS_BUCKET_NAME}/${PUBLISH_PREFIX}/" || true ;; + *) echo "refusing to wipe non-dev prefix '$PUBLISH_PREFIX'"; exit 1 ;; + esac - name: Upload versioned assets run: | - mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' -o -name '*.sha256' \) -exec cp {} up/ \; - aws s3 sync up/ "s3://${AWS_BUCKET_NAME}/${S3_PREFIX}/v${VERSION}/" - - name: Index apt/rpm/apk repos (signed) + mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.sha256' \) -exec cp {} up/ \; + aws s3 sync up/ "s3://${AWS_BUCKET_NAME}/${PUBLISH_PREFIX}/v${VERSION}/" + - name: Index apt/rpm repos (signed) env: # No DEB_KEY_PUBLIC: the apt keyring is derived (dearmored) from DEB_KEY_PRIVATE. DEB_KEY_PRIVATE: ${{ secrets.DEB_KEY_PRIVATE }} RPM_KEY_PUBLIC: ${{ secrets.RPM_KEY_PUBLIC }} - APK_KEY_PRIVATE: ${{ secrets.APK_KEY_PRIVATE }} - APK_KEY_PUBLIC: ${{ secrets.APK_KEY_PUBLIC }} - run: bash build/package/linux/index.sh "${AWS_BUCKET_NAME}" "${S3_PREFIX}" + run: bash build/package/linux/index.sh "${AWS_BUCKET_NAME}" "${PUBLISH_PREFIX}" # ── Homebrew: render the formula from the template and push to the kbagent-owned tap. homebrew: @@ -411,28 +434,47 @@ jobs: powershell -Command '$s = Get-AuthenticodeSignature wingetcreate.exe; if ($s.Status -ne "Valid" -or $s.SignerCertificate.Subject -notmatch "Microsoft") { Write-Error "wingetcreate.exe signature not valid/Microsoft: $($s.Status)"; exit 1 }' ./wingetcreate.exe update -v "$VERSION" -u "$url" -t "$WINGET_TOKEN" Keboola.KeboolaCLI2 -s - # ── Smoke-test the real install paths end to end. + # ── Smoke-test the real install paths end to end. Runs on every tag (incl. dev tags, + # against PUBLISH_PREFIX); the Homebrew leg is prod-only since the tap isn't pushed + # for pre-releases. always() so it still runs when homebrew is skipped on a dev tag. test-install: needs: [version, publish-s3, homebrew] + if: always() && needs.publish-s3.result == 'success' runs-on: ubuntu-latest env: VERSION: ${{ needs.version.outputs.VERSION }} + PREFIX: ${{ needs.version.outputs.PUBLISH_PREFIX }} steps: - name: apt (Ubuntu) run: | - docker run --rm ubuntu bash -c ' + docker run --rm -e PREFIX -e VERSION ubuntu bash -c ' set -eo pipefail apt-get update -y && apt-get install -y wget ca-certificates gnupg - wget -P /etc/apt/trusted.gpg.d https://cli-dist.keboola.com/${{ env.S3_PREFIX }}/deb/keboola.gpg - echo "deb https://cli-dist.keboola.com/${{ env.S3_PREFIX }}/deb /" > /etc/apt/sources.list.d/keboola.list + wget -P /etc/apt/trusted.gpg.d "https://cli-dist.keboola.com/${PREFIX}/deb/keboola.gpg" + echo "deb https://cli-dist.keboola.com/${PREFIX}/deb /" > /etc/apt/sources.list.d/keboola.list apt-get update && apt-get install -y keboola-cli2 - kbagent --version | grep -q "${{ env.VERSION }}" + kbagent --version | grep -q "${VERSION}" + ' + - name: rpm (Fedora) + if: ${{ !cancelled() }} # run even if an earlier leg failed, so one run shows all results + run: | + docker run --rm -e PREFIX -e VERSION fedora bash -c ' + set -eo pipefail + rpm --import "https://cli-dist.keboola.com/${PREFIX}/rpm/keboola.gpg" + printf "%s\n" "[keboola]" "name=keboola" \ + "baseurl=https://cli-dist.keboola.com/${PREFIX}/rpm" \ + "enabled=1" "gpgcheck=1" \ + "gpgkey=https://cli-dist.keboola.com/${PREFIX}/rpm/keboola.gpg" \ + > /etc/yum.repos.d/keboola.repo + dnf install -y keboola-cli2 + kbagent --version | grep -q "${VERSION}" ' - name: Homebrew (Linux) + if: ${{ !cancelled() && needs.version.outputs.IS_PRERELEASE == 'false' && needs.homebrew.result == 'success' }} run: | - docker run --rm homebrew/brew bash -c ' + docker run --rm -e VERSION homebrew/brew bash -c ' set -e brew tap keboola/keboola-cli2 https://github.com/keboola/homebrew-keboola-cli2 brew install keboola-cli2 - kbagent --version | grep -q "${{ env.VERSION }}" + kbagent --version | grep -q "${VERSION}" ' diff --git a/build/package/linux/build_packages.sh b/build/package/linux/build_packages.sh index a6404df6..9815ce44 100755 --- a/build/package/linux/build_packages.sh +++ b/build/package/linux/build_packages.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Build deb/rpm/apk for each Linux arch from the frozen binaries, using nfpm. +# Build deb/rpm for each Linux arch from the frozen binaries, using nfpm. +# (No apk: the glibc-linked PyInstaller binary won't run on musl Alpine.) # nfpm does not reliably expand ${...} in its config, so we render it with envsubst. # Usage: build_packages.sh [artifacts-dir] set -euo pipefail @@ -17,7 +18,7 @@ for arch in amd64 arm64; do export VERSION PKG_ARCH="$arch" BIN_PATH="$BIN" envsubst '${VERSION} ${PKG_ARCH} ${BIN_PATH}' < build/package/nfpm.yaml > /tmp/nfpm.yaml - for fmt in deb rpm apk; do + for fmt in deb rpm; do nfpm package -f /tmp/nfpm.yaml -p "$fmt" -t "dist/keboola-cli2_${VERSION}_linux_${arch}.${fmt}" done done diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index 15e0ddfd..d0c3ef1c 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash -# Build/refresh the SIGNED apt(deb), yum(rpm) and apk repositories on the CLI dist +# Build/refresh the SIGNED apt(deb) and yum(rpm) repositories on the CLI dist # S3 bucket so `apt-get install keboola-cli2` (etc.) works out of the box. +# (No apk: the frozen binary is glibc-linked and won't run on musl Alpine.) # # Signing keys (separate per format): # DEB_KEY_PRIVATE — GPG, signs the apt repo. The public keyring apt downloads is # derived from it (dearmored binary), so there's no DEB_KEY_PUBLIC. # RPM_KEY_PUBLIC — public half of the SEPARATE rpm signing key (nfpm signs the rpm # packages with RPM_KEY_PRIVATE); published for yum clients. -# APK_KEY_PRIVATE / APK_KEY_PUBLIC — abuild RSA keypair, signs the apk index # Requires AWS creds already configured (OIDC). -# Usage: index.sh → s3:////{deb,rpm,apk}/ +# Usage: index.sh → s3:////{deb,rpm}/ set -euo pipefail BUCKET="$1" PREFIX="$2" @@ -17,15 +17,13 @@ WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT # clean up temp dir + any key material on exit/failure if [ -z "${DEB_KEY_PRIVATE:-}" ]; then - # publish-s3 only runs on real (non-pre-release) tags, where the repo + key must - # exist for the downstream test-install job. Fail loudly rather than silently - # skipping and leaving test-install to fail with an obscure root cause. - echo "::error::DEB_KEY_PRIVATE not set — cannot sign/index the apt repo for a real release." + # Fail loudly: the key is required in the `release` environment for any tag (publish-s3 + # runs on every tag now — real under the prod prefix, pre-release under the dev prefix). + echo "::error::DEB_KEY_PRIVATE not set — signing environment is misconfigured." exit 1 fi sudo apt-get update -y -# Only deb/rpm tooling on the host; the apk index is built in Alpine (see index_apk). sudo apt-get install -y dpkg-dev apt-utils createrepo-c gnupg # Import GPG signing key (deb + rpm metadata). @@ -43,11 +41,24 @@ publish_repo() { find . -path ./.git -prune -o -name "*.$fmt" -exec cp {} "$dir/" \; ( cd "$dir" && "$indexer" ) [ -n "$pub_content" ] && printf '%s' "$pub_content" > "$dir/$pub_file" - aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" + # Index/metadata files (Packages, Release, repodata/*, keyrings) sit at STABLE paths + # that are overwritten each release, so they must NOT be cached long by CloudFront + # (default_ttl is 24h) or clients install stale versions. Upload them no-cache; the + # packages themselves have version-unique names, so cache them immutably. + aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" --exclude "*.$fmt" --cache-control "no-cache" + aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" --exclude "*" --include "*.$fmt" --cache-control "public, max-age=31536000, immutable" } index_deb() { - dpkg-scanpackages . /dev/null > Packages && gzip -kf Packages + # -m/--multiversion: keep ALL packages, not just the newest per name. Without it + # dpkg-scanpackages dedups on name+version IGNORING architecture, so the amd64 and + # arm64 debs collide and one is dropped ("is repeat; ignored"), leaving apt on the + # other arch with no install candidate. -m emits a stanza per file (both arches). + # Strip the leading "./" from Filename: apt would fetch /deb/./pkg.deb, and S3 + # (behind CloudFront) treats the literal "/./" as a missing key -> 404. A clean + # Filename: pkg.deb yields /deb/pkg.deb, which exists. + dpkg-scanpackages -m . /dev/null | sed -E 's|^Filename: \./|Filename: |' > Packages + gzip -kf Packages apt-ftparchive release . > Release gpg --batch --yes --default-key "$KEYID" -abs -o Release.gpg Release gpg --batch --yes --default-key "$KEYID" --clearsign -o InRelease Release @@ -56,26 +67,9 @@ index_deb() { gpg --export "$KEYID" > keboola.gpg } index_rpm() { createrepo_c .; } -index_apk() { - printf '%s' "$APK_KEY_PRIVATE" > "$WORK/apk_index.rsa" && chmod 600 "$WORK/apk_index.rsa" - # apk/abuild-sign are Alpine-only (not in Ubuntu apt), so sign the index in Alpine. - # $PWD is the per-format work dir (publish_repo cd's into it); mount it + the key. - docker run --rm -v "$PWD:/work" -v "$WORK/apk_index.rsa:/key.rsa:ro" -w /work alpine:3 \ - sh -ceu 'apk add --no-cache abuild >/dev/null - apk index -o APKINDEX.tar.gz ./*.apk - abuild-sign -k /key.rsa APKINDEX.tar.gz' -} # deb: index_deb writes its own (dearmored) keboola.gpg, so pass no pub-key content. publish_repo deb index_deb "" "" publish_repo rpm index_rpm keboola.gpg "${RPM_KEY_PUBLIC:-}" -if [ -z "${APK_KEY_PRIVATE:-}" ]; then - # No apk signing key configured — the apk index is genuinely opt-out, so skip it. - echo "::warning::APK_KEY_PRIVATE not set — skipping apk index (deb/rpm done)." -else - # Key set → apk publishing intended. If Docker is somehow absent (it isn't on - # ubuntu-latest), index_apk's `docker run` fails loud under set -e — fine. - publish_repo apk index_apk keboola.rsa.pub "${APK_KEY_PUBLIC:-}" -fi -echo "Repositories indexed and published under s3://$BUCKET/$PREFIX/{deb,rpm,apk}/" +echo "Repositories indexed and published under s3://$BUCKET/$PREFIX/{deb,rpm}/" diff --git a/build/package/nfpm.yaml b/build/package/nfpm.yaml index 22ec8a51..2ef96e55 100644 --- a/build/package/nfpm.yaml +++ b/build/package/nfpm.yaml @@ -1,4 +1,5 @@ -# nfpm config — packages the frozen `kbagent` binary into deb / rpm / apk. +# nfpm config — packages the frozen `kbagent` binary into deb / rpm. +# (No apk: the glibc-linked PyInstaller binary won't run on musl Alpine.) # # nfpm (https://nfpm.goreleaser.com) is a standalone, language-agnostic packager. # We do NOT use goreleaser (it builds Go); the binary is produced by PyInstaller in @@ -33,6 +34,3 @@ deb: rpm: signature: key_file: /tmp/keys/rpm.key -apk: - signature: - key_file: /tmp/keys/apk.key