From 2a6fcd1181481db438f2724dc067dd3bbb3ce426 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 23 Jun 2026 20:42:33 +0200 Subject: [PATCH 1/8] fix(release): apk index --allow-untrusted for self-signed packages The .apk packages are signed by our own key via nfpm (apk.signature in nfpm.yaml). The throwaway alpine:3 container used to build the index only trusts Alpine's official keys, so 'apk index' rejected our packages with 'UNTRUSTED signature' (exit 99). Indexing only reads package metadata, so pass --allow-untrusted to skip the trust check; the index is still signed by abuild-sign and clients still verify packages against the published key. Co-Authored-By: Claude Opus 4.8 (1M context) --- build/package/linux/index.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index 15e0ddfd..39b319e4 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -60,9 +60,11 @@ index_apk() { printf '%s' "$APK_KEY_PRIVATE" > "$WORK/apk_index.rsa" && chmod 600 "$WORK/apk_index.rsa" # apk/abuild-sign are Alpine-only (not in Ubuntu apt), so sign the index in Alpine. # $PWD is the per-format work dir (publish_repo cd's into it); mount it + the key. + # --allow-untrusted: the .apk packages are signed by our own key (nfpm), which the + # throwaway container doesn't trust; indexing only reads metadata, so skip the check. docker run --rm -v "$PWD:/work" -v "$WORK/apk_index.rsa:/key.rsa:ro" -w /work alpine:3 \ sh -ceu 'apk add --no-cache abuild >/dev/null - apk index -o APKINDEX.tar.gz ./*.apk + apk index --allow-untrusted -o APKINDEX.tar.gz ./*.apk abuild-sign -k /key.rsa APKINDEX.tar.gz' } From e67f8f00c588f02069c3427af281378fdd6ae6e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 23 Jun 2026 20:47:27 +0200 Subject: [PATCH 2/8] fix(release): consistent 'keboola' apk key naming + trust pubkey in container MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mirrors the proven keboola-as-code apk setup. Three names must agree for both apk index (build time) and apk add (client side) to verify: - nfpm.yaml: add apk.signature.key_name=keboola so packages are signed as .SIGN.RSA.keboola.rsa.pub (was unset → nfpm default, mismatched the published pubkey). - index.sh: import APK_KEY_PUBLIC as /etc/apk/keys/keboola.rsa.pub inside the Alpine container so 'apk index' trusts the self-signed packages (fixes the 'UNTRUSTED signature' exit-99 crash) WITHOUT --allow-untrusted; name the abuild-sign key keboola.rsa so the index signature matches the published keboola.rsa.pub clients download. Replaces the earlier --allow-untrusted band-aid, which would have let indexing pass but left clients unable to verify (index signed .SIGN.RSA.key.rsa.pub vs published keboola.rsa.pub). Co-Authored-By: Claude Opus 4.8 (1M context) --- build/package/linux/index.sh | 23 +++++++++++++++-------- build/package/nfpm.yaml | 4 ++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index 39b319e4..8895cfa5 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -57,15 +57,22 @@ index_deb() { } index_rpm() { createrepo_c .; } index_apk() { - printf '%s' "$APK_KEY_PRIVATE" > "$WORK/apk_index.rsa" && chmod 600 "$WORK/apk_index.rsa" - # apk/abuild-sign are Alpine-only (not in Ubuntu apt), so sign the index in Alpine. - # $PWD is the per-format work dir (publish_repo cd's into it); mount it + the key. - # --allow-untrusted: the .apk packages are signed by our own key (nfpm), which the - # throwaway container doesn't trust; indexing only reads metadata, so skip the check. - docker run --rm -v "$PWD:/work" -v "$WORK/apk_index.rsa:/key.rsa:ro" -w /work alpine:3 \ + # Everything agrees on the name "keboola": nfpm signs each package as + # .SIGN.RSA.keboola.rsa.pub (apk.key_name in nfpm.yaml), so importing the pubkey as + # /etc/apk/keys/keboola.rsa.pub makes the container trust the packages (no + # --allow-untrusted), and signing the index with keboola.rsa yields a signature + # clients verify against the published keboola.rsa.pub. apk/abuild-sign are + # Alpine-only (not in Ubuntu apt), so index + sign inside Alpine. + printf '%s' "$APK_KEY_PRIVATE" > "$WORK/keboola.rsa" && chmod 600 "$WORK/keboola.rsa" + printf '%s' "$APK_KEY_PUBLIC" > "$WORK/keboola.rsa.pub" + docker run --rm \ + -v "$PWD:/work" \ + -v "$WORK/keboola.rsa:/keboola.rsa:ro" \ + -v "$WORK/keboola.rsa.pub:/etc/apk/keys/keboola.rsa.pub:ro" \ + -w /work alpine:3 \ sh -ceu 'apk add --no-cache abuild >/dev/null - apk index --allow-untrusted -o APKINDEX.tar.gz ./*.apk - abuild-sign -k /key.rsa APKINDEX.tar.gz' + apk index -o APKINDEX.tar.gz ./*.apk + abuild-sign -k /keboola.rsa APKINDEX.tar.gz' } # deb: index_deb writes its own (dearmored) keboola.gpg, so pass no pub-key content. diff --git a/build/package/nfpm.yaml b/build/package/nfpm.yaml index 22ec8a51..9bbc5a65 100644 --- a/build/package/nfpm.yaml +++ b/build/package/nfpm.yaml @@ -35,4 +35,8 @@ rpm: key_file: /tmp/keys/rpm.key apk: signature: + # key_name must match the pubkey filename clients trust (keboola.rsa.pub) and the + # abuild-sign key in build/package/linux/index.sh — see that file. nfpm signs the + # package as .SIGN.RSA..rsa.pub, so all three must agree on "keboola". + key_name: keboola key_file: /tmp/keys/apk.key From 9226826f012343dbb66dfbf6a6e1f7fd95ceb756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 23 Jun 2026 21:09:36 +0200 Subject: [PATCH 3/8] fix(release): per-arch apk repo layout + dev-tag pipeline gating + apk/rpm install tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The apk path failed at successive stages because it had never run end-to-end. Fixes the remaining gaps so a dev tag can validate the whole chain: - index.sh: build the apk repo PER-ARCH (apk//APKINDEX.tar.gz; amd64->x86_64, arm64->aarch64) — apk clients fetch //APKINDEX.tar.gz, so the previous flat index was never installable. Import the pubkey into the container so packages are trusted (no --allow-untrusted); publish keboola.rsa.pub at the apk root. - release-kbagent.yml: pre-releases now publish under a separate dev prefix (keboola-cli2-dev) and run publish-s3 + test-install, so a dev tag exercises the real signing + client install without touching prod repos. Homebrew/Chocolatey/ WinGet/PyPI stay prod-only. - test-install: add apk (Alpine) and rpm (Fedora) client-install legs that trust the published keys and verify --version; brew leg is prod-only. This is the CI gate that would have caught every apk failure before a tag was cut. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release-kbagent.yml | 59 ++++++++++++++++++++++----- build/package/linux/index.sh | 40 ++++++++++++------ 2 files changed, 75 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release-kbagent.yml b/.github/workflows/release-kbagent.yml index d251578f..c54e4ddc 100644 --- a/.github/workflows/release-kbagent.yml +++ b/.github/workflows/release-kbagent.yml @@ -59,6 +59,10 @@ jobs: # build + package + sign + GitHub pre-release ONLY; all external publishing # (PyPI, S3 repo index, Homebrew, Chocolatey, WinGet) is skipped. IS_PRERELEASE: ${{ steps.v.outputs.IS_PRERELEASE }} + # S3 prefix to publish under: prod (`keboola-cli2`) for real releases, a separate + # `keboola-cli2-dev` for pre-releases so a dev tag can exercise the full S3 publish + # + client install test without touching the prod apt/rpm/apk repos. + PUBLISH_PREFIX: ${{ steps.v.outputs.PUBLISH_PREFIX }} steps: - id: v env: @@ -74,7 +78,9 @@ jobs: # Pre-release = any a/b/rc/.dev/-suffix marker (.post is a real release). if printf '%s' "$VERSION" | grep -Eq '(a|b|rc)[0-9]+$|\.dev[0-9]+$|-'; then PRE=true; else PRE=false; fi echo "IS_PRERELEASE=$PRE" >> "$GITHUB_OUTPUT" - echo "version=$VERSION prerelease=$PRE" + # Pre-releases publish to a separate dev prefix (see PUBLISH_PREFIX output above). + if [ "$PRE" = true ]; then echo "PUBLISH_PREFIX=${S3_PREFIX}-dev"; else echo "PUBLISH_PREFIX=${S3_PREFIX}"; fi >> "$GITHUB_OUTPUT" + echo "version=$VERSION prerelease=$PRE prefix=${S3_PREFIX}$([ "$PRE" = true ] && echo -dev)" # Reuse the per-PR gates against the tagged commit, plus changelog-check. gate: @@ -307,7 +313,9 @@ jobs: # repos so `apt-get install keboola-cli2` works. Real; needs AWS + GPG secrets. publish-s3: needs: [version, freeze, package-linux] - if: needs.version.outputs.IS_PRERELEASE == 'false' && startsWith(github.ref, 'refs/tags/') + # Runs on every tag. Pre-releases publish under the dev prefix (PUBLISH_PREFIX), so a + # dev tag exercises the full publish + client install test without touching prod repos. + if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest environment: release permissions: @@ -315,6 +323,7 @@ jobs: id-token: write # AWS OIDC (configure-aws-credentials assumes the role) env: VERSION: ${{ needs.version.outputs.VERSION }} + PUBLISH_PREFIX: ${{ needs.version.outputs.PUBLISH_PREFIX }} steps: - uses: actions/checkout@v5 - uses: actions/download-artifact@v4 @@ -326,7 +335,7 @@ jobs: - name: Upload versioned assets run: | mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' -o -name '*.sha256' \) -exec cp {} up/ \; - aws s3 sync up/ "s3://${AWS_BUCKET_NAME}/${S3_PREFIX}/v${VERSION}/" + aws s3 sync up/ "s3://${AWS_BUCKET_NAME}/${PUBLISH_PREFIX}/v${VERSION}/" - name: Index apt/rpm/apk repos (signed) env: # No DEB_KEY_PUBLIC: the apt keyring is derived (dearmored) from DEB_KEY_PRIVATE. @@ -334,7 +343,7 @@ jobs: RPM_KEY_PUBLIC: ${{ secrets.RPM_KEY_PUBLIC }} APK_KEY_PRIVATE: ${{ secrets.APK_KEY_PRIVATE }} APK_KEY_PUBLIC: ${{ secrets.APK_KEY_PUBLIC }} - run: bash build/package/linux/index.sh "${AWS_BUCKET_NAME}" "${S3_PREFIX}" + run: bash build/package/linux/index.sh "${AWS_BUCKET_NAME}" "${PUBLISH_PREFIX}" # ── Homebrew: render the formula from the template and push to the kbagent-owned tap. homebrew: @@ -411,28 +420,56 @@ jobs: powershell -Command '$s = Get-AuthenticodeSignature wingetcreate.exe; if ($s.Status -ne "Valid" -or $s.SignerCertificate.Subject -notmatch "Microsoft") { Write-Error "wingetcreate.exe signature not valid/Microsoft: $($s.Status)"; exit 1 }' ./wingetcreate.exe update -v "$VERSION" -u "$url" -t "$WINGET_TOKEN" Keboola.KeboolaCLI2 -s - # ── Smoke-test the real install paths end to end. + # ── Smoke-test the real install paths end to end. Runs on every tag (incl. dev tags, + # against PUBLISH_PREFIX); the Homebrew leg is prod-only since the tap isn't pushed + # for pre-releases. always() so it still runs when homebrew is skipped on a dev tag. test-install: needs: [version, publish-s3, homebrew] + if: always() && needs.publish-s3.result == 'success' runs-on: ubuntu-latest env: VERSION: ${{ needs.version.outputs.VERSION }} + PREFIX: ${{ needs.version.outputs.PUBLISH_PREFIX }} steps: - name: apt (Ubuntu) run: | - docker run --rm ubuntu bash -c ' + docker run --rm -e PREFIX -e VERSION ubuntu bash -c ' set -eo pipefail apt-get update -y && apt-get install -y wget ca-certificates gnupg - wget -P /etc/apt/trusted.gpg.d https://cli-dist.keboola.com/${{ env.S3_PREFIX }}/deb/keboola.gpg - echo "deb https://cli-dist.keboola.com/${{ env.S3_PREFIX }}/deb /" > /etc/apt/sources.list.d/keboola.list + wget -P /etc/apt/trusted.gpg.d "https://cli-dist.keboola.com/${PREFIX}/deb/keboola.gpg" + echo "deb https://cli-dist.keboola.com/${PREFIX}/deb /" > /etc/apt/sources.list.d/keboola.list apt-get update && apt-get install -y keboola-cli2 - kbagent --version | grep -q "${{ env.VERSION }}" + kbagent --version | grep -q "${VERSION}" + ' + - name: apk (Alpine) + run: | + docker run --rm -e PREFIX -e VERSION alpine sh -c ' + set -eu + wget -O /etc/apk/keys/keboola.rsa.pub "https://cli-dist.keboola.com/${PREFIX}/apk/keboola.rsa.pub" + echo "https://cli-dist.keboola.com/${PREFIX}/apk" >> /etc/apk/repositories + apk update + apk add keboola-cli2 + kbagent --version | grep -q "${VERSION}" + ' + - name: rpm (Fedora) + run: | + docker run --rm -e PREFIX -e VERSION fedora bash -c ' + set -eo pipefail + rpm --import "https://cli-dist.keboola.com/${PREFIX}/rpm/keboola.gpg" + printf "%s\n" "[keboola]" "name=keboola" \ + "baseurl=https://cli-dist.keboola.com/${PREFIX}/rpm" \ + "enabled=1" "gpgcheck=1" \ + "gpgkey=https://cli-dist.keboola.com/${PREFIX}/rpm/keboola.gpg" \ + > /etc/yum.repos.d/keboola.repo + dnf install -y keboola-cli2 + kbagent --version | grep -q "${VERSION}" ' - name: Homebrew (Linux) + if: needs.version.outputs.IS_PRERELEASE == 'false' && needs.homebrew.result == 'success' run: | - docker run --rm homebrew/brew bash -c ' + docker run --rm -e VERSION homebrew/brew bash -c ' set -e brew tap keboola/keboola-cli2 https://github.com/keboola/homebrew-keboola-cli2 brew install keboola-cli2 - kbagent --version | grep -q "${{ env.VERSION }}" + kbagent --version | grep -q "${VERSION}" ' diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index 8895cfa5..a13e2ac0 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -56,23 +56,39 @@ index_deb() { gpg --export "$KEYID" > keboola.gpg } index_rpm() { createrepo_c .; } -index_apk() { - # Everything agrees on the name "keboola": nfpm signs each package as - # .SIGN.RSA.keboola.rsa.pub (apk.key_name in nfpm.yaml), so importing the pubkey as - # /etc/apk/keys/keboola.rsa.pub makes the container trust the packages (no - # --allow-untrusted), and signing the index with keboola.rsa yields a signature - # clients verify against the published keboola.rsa.pub. apk/abuild-sign are - # Alpine-only (not in Ubuntu apt), so index + sign inside Alpine. + +# apk has its own publisher (not publish_repo): repos are PER-ARCH — clients fetch +# //APKINDEX.tar.gz — so packages + signed index live under apk// +# (deb arch name -> apk arch name). Everything agrees on the name "keboola": nfpm signs +# each package as .SIGN.RSA.keboola.rsa.pub (apk.key_name in nfpm.yaml); importing the +# pubkey as /etc/apk/keys/keboola.rsa.pub makes the container trust the packages (no +# --allow-untrusted); signing each index with keboola.rsa yields a signature clients +# verify against the published keboola.rsa.pub. apk/abuild-sign are Alpine-only (not in +# Ubuntu apt), so index + sign inside Alpine. +publish_apk() { + local root="$WORK/apk" printf '%s' "$APK_KEY_PRIVATE" > "$WORK/keboola.rsa" && chmod 600 "$WORK/keboola.rsa" printf '%s' "$APK_KEY_PUBLIC" > "$WORK/keboola.rsa.pub" + local pair deb_arch apk_arch + for pair in amd64:x86_64 arm64:aarch64; do + deb_arch="${pair%%:*}"; apk_arch="${pair##*:}" + mkdir -p "$root/$apk_arch" + aws s3 sync "s3://$BUCKET/$PREFIX/apk/$apk_arch/" "$root/$apk_arch/" --exclude '*' --include '*.apk' || true + find . -path ./.git -prune -o -name "*_linux_${deb_arch}.apk" -exec cp {} "$root/$apk_arch/" \; + done + # Publish the pubkey at the apk root; clients install it into /etc/apk/keys/keboola.rsa.pub. + printf '%s' "$APK_KEY_PUBLIC" > "$root/keboola.rsa.pub" docker run --rm \ - -v "$PWD:/work" \ + -v "$root:/work" \ -v "$WORK/keboola.rsa:/keboola.rsa:ro" \ -v "$WORK/keboola.rsa.pub:/etc/apk/keys/keboola.rsa.pub:ro" \ -w /work alpine:3 \ sh -ceu 'apk add --no-cache abuild >/dev/null - apk index -o APKINDEX.tar.gz ./*.apk - abuild-sign -k /keboola.rsa APKINDEX.tar.gz' + for d in */; do + ls "$d"*.apk >/dev/null 2>&1 || continue + ( cd "$d" && apk index -o APKINDEX.tar.gz ./*.apk && abuild-sign -k /keboola.rsa APKINDEX.tar.gz ) + done' + aws s3 sync "$root/" "s3://$BUCKET/$PREFIX/apk/" } # deb: index_deb writes its own (dearmored) keboola.gpg, so pass no pub-key content. @@ -82,9 +98,7 @@ if [ -z "${APK_KEY_PRIVATE:-}" ]; then # No apk signing key configured — the apk index is genuinely opt-out, so skip it. echo "::warning::APK_KEY_PRIVATE not set — skipping apk index (deb/rpm done)." else - # Key set → apk publishing intended. If Docker is somehow absent (it isn't on - # ubuntu-latest), index_apk's `docker run` fails loud under set -e — fine. - publish_repo apk index_apk keboola.rsa.pub "${APK_KEY_PUBLIC:-}" + publish_apk fi echo "Repositories indexed and published under s3://$BUCKET/$PREFIX/{deb,rpm,apk}/" From a67a3dbf4d147ab5ae96e48d4340dda8de783235 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 23 Jun 2026 21:23:44 +0200 Subject: [PATCH 4/8] fix(release): dpkg-scanpackages -m for multi-arch deb repo; independent test legs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit dpkg-scanpackages dedups on package name+version IGNORING architecture, so the same-version amd64+arm64 debs collided ('is repeat; ignored') and only one arch landed in Packages — apt on the other arch had no candidate. -m/--multiversion emits a stanza per file. Also make the apk/rpm/brew test-install legs run with !cancelled() so one run surfaces all client-install results instead of aborting at the first failing leg. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release-kbagent.yml | 4 +++- build/package/linux/index.sh | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-kbagent.yml b/.github/workflows/release-kbagent.yml index c54e4ddc..b99d8b54 100644 --- a/.github/workflows/release-kbagent.yml +++ b/.github/workflows/release-kbagent.yml @@ -442,6 +442,7 @@ jobs: kbagent --version | grep -q "${VERSION}" ' - name: apk (Alpine) + if: ${{ !cancelled() }} # run even if an earlier leg failed, so one run shows all results run: | docker run --rm -e PREFIX -e VERSION alpine sh -c ' set -eu @@ -452,6 +453,7 @@ jobs: kbagent --version | grep -q "${VERSION}" ' - name: rpm (Fedora) + if: ${{ !cancelled() }} # run even if an earlier leg failed, so one run shows all results run: | docker run --rm -e PREFIX -e VERSION fedora bash -c ' set -eo pipefail @@ -465,7 +467,7 @@ jobs: kbagent --version | grep -q "${VERSION}" ' - name: Homebrew (Linux) - if: needs.version.outputs.IS_PRERELEASE == 'false' && needs.homebrew.result == 'success' + if: ${{ !cancelled() && needs.version.outputs.IS_PRERELEASE == 'false' && needs.homebrew.result == 'success' }} run: | docker run --rm -e VERSION homebrew/brew bash -c ' set -e diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index a13e2ac0..5bbe2927 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -47,7 +47,11 @@ publish_repo() { } index_deb() { - dpkg-scanpackages . /dev/null > Packages && gzip -kf Packages + # -m/--multiversion: keep ALL packages, not just the newest per name. Without it + # dpkg-scanpackages dedups on name+version IGNORING architecture, so the amd64 and + # arm64 debs collide and one is dropped ("is repeat; ignored"), leaving apt on the + # other arch with no install candidate. -m emits a stanza per file (both arches). + dpkg-scanpackages -m . /dev/null > Packages && gzip -kf Packages apt-ftparchive release . > Release gpg --batch --yes --default-key "$KEYID" -abs -o Release.gpg Release gpg --batch --yes --default-key "$KEYID" --clearsign -o InRelease Release From be3fc6fe159e64e38dd79a83f765cc61865682ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 23 Jun 2026 21:38:41 +0200 Subject: [PATCH 5/8] fix(release): clean deb Filename (drop ./), apk -.apk naming, wipe dev prefix Two install-time 404s, both unrelated to apk signing (which works): - deb: dpkg-scanpackages emits 'Filename: ./pkg.deb', so apt fetches /deb/./pkg.deb; S3/CloudFront treats the literal /./ as a missing key -> 404. Strip the leading ./. - apk: clients fetch //-.apk, but nfpm names the file keboola-cli2__linux_.apk -> 404. Rename to apk's convention on copy. Also wipe the keboola-cli2-dev prefix on pre-release runs so dev tags start clean and version resolution isn't confused by leftovers (guarded to the -dev suffix; never prod). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release-kbagent.yml | 10 ++++++++++ build/package/linux/index.sh | 15 +++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-kbagent.yml b/.github/workflows/release-kbagent.yml index b99d8b54..1b1f3552 100644 --- a/.github/workflows/release-kbagent.yml +++ b/.github/workflows/release-kbagent.yml @@ -332,6 +332,16 @@ jobs: with: role-to-assume: ${{ secrets.AWS_ROLE_ARN }} aws-region: ${{ env.AWS_REGION }} + - name: Wipe dev prefix (pre-release only) + # Dev tags reuse the keboola-cli2-dev prefix; clear it so each dev run is a clean + # slate and version resolution isn't confused by leftover packages from a prior + # dev tag. NEVER runs for the prod prefix (guarded on the -dev suffix). + if: needs.version.outputs.IS_PRERELEASE == 'true' + run: | + case "$PUBLISH_PREFIX" in + *-dev) aws s3 rm --recursive "s3://${AWS_BUCKET_NAME}/${PUBLISH_PREFIX}/" || true ;; + *) echo "refusing to wipe non-dev prefix '$PUBLISH_PREFIX'"; exit 1 ;; + esac - name: Upload versioned assets run: | mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' -o -name '*.sha256' \) -exec cp {} up/ \; diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index 5bbe2927..d1ca5cf7 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -51,7 +51,11 @@ index_deb() { # dpkg-scanpackages dedups on name+version IGNORING architecture, so the amd64 and # arm64 debs collide and one is dropped ("is repeat; ignored"), leaving apt on the # other arch with no install candidate. -m emits a stanza per file (both arches). - dpkg-scanpackages -m . /dev/null > Packages && gzip -kf Packages + # Strip the leading "./" from Filename: apt would fetch /deb/./pkg.deb, and S3 + # (behind CloudFront) treats the literal "/./" as a missing key -> 404. A clean + # Filename: pkg.deb yields /deb/pkg.deb, which exists. + dpkg-scanpackages -m . /dev/null | sed -E 's|^Filename: \./|Filename: |' > Packages + gzip -kf Packages apt-ftparchive release . > Release gpg --batch --yes --default-key "$KEYID" -abs -o Release.gpg Release gpg --batch --yes --default-key "$KEYID" --clearsign -o InRelease Release @@ -78,7 +82,14 @@ publish_apk() { deb_arch="${pair%%:*}"; apk_arch="${pair##*:}" mkdir -p "$root/$apk_arch" aws s3 sync "s3://$BUCKET/$PREFIX/apk/$apk_arch/" "$root/$apk_arch/" --exclude '*' --include '*.apk' || true - find . -path ./.git -prune -o -name "*_linux_${deb_arch}.apk" -exec cp {} "$root/$apk_arch/" \; + # apk clients fetch //-.apk, so the file on disk must be + # named that way — NOT nfpm's deb-style keboola-cli2__linux_.apk (which + # apk index records as P/V but the client then 404s on). Rename on copy. + local f base ver + for f in $(find . -path ./.git -prune -o -name "*_linux_${deb_arch}.apk" -print); do + base="${f##*/}"; ver="${base#keboola-cli2_}"; ver="${ver%_linux_${deb_arch}.apk}" + cp "$f" "$root/$apk_arch/keboola-cli2-${ver}.apk" + done done # Publish the pubkey at the apk root; clients install it into /etc/apk/keys/keboola.rsa.pub. printf '%s' "$APK_KEY_PUBLIC" > "$root/keboola.rsa.pub" From 797cfb9487a2827ffc76e0ecde6f382418dfa97f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 24 Jun 2026 07:00:47 +0200 Subject: [PATCH 6/8] fix(release): no-cache repo metadata so CloudFront serves fresh indexes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dist domain is CloudFront-fronted with default_ttl 24h and S3 uploads carry no Cache-Control, so the stable index paths (deb/Packages, apk//APKINDEX.tar.gz, rpm/repodata/*) were cached ~24h — clients installed the PREVIOUS version after a publish (dnf pulled dev2 when dev3 was live). Upload index/metadata with Cache-Control: no-cache (effective TTL ~min_ttl=1s) and packages as immutable (version-unique names). Needs only s3:PutObject (already granted) — no CloudFront invalidation permission required. Co-Authored-By: Claude Opus 4.8 (1M context) --- build/package/linux/index.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index d1ca5cf7..d45bd1d7 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -43,7 +43,12 @@ publish_repo() { find . -path ./.git -prune -o -name "*.$fmt" -exec cp {} "$dir/" \; ( cd "$dir" && "$indexer" ) [ -n "$pub_content" ] && printf '%s' "$pub_content" > "$dir/$pub_file" - aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" + # Index/metadata files (Packages, Release, repodata/*, keyrings) sit at STABLE paths + # that are overwritten each release, so they must NOT be cached long by CloudFront + # (default_ttl is 24h) or clients install stale versions. Upload them no-cache; the + # packages themselves have version-unique names, so cache them immutably. + aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" --exclude "*.$fmt" --cache-control "no-cache" + aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" --exclude "*" --include "*.$fmt" --cache-control "public, max-age=31536000, immutable" } index_deb() { @@ -103,7 +108,10 @@ publish_apk() { ls "$d"*.apk >/dev/null 2>&1 || continue ( cd "$d" && apk index -o APKINDEX.tar.gz ./*.apk && abuild-sign -k /keboola.rsa APKINDEX.tar.gz ) done' - aws s3 sync "$root/" "s3://$BUCKET/$PREFIX/apk/" + # As in publish_repo: APKINDEX.tar.gz + pubkey live at stable paths -> no-cache so + # CloudFront doesn't serve a stale index; the .apk packages are version-named -> immutable. + aws s3 sync "$root/" "s3://$BUCKET/$PREFIX/apk/" --exclude "*.apk" --cache-control "no-cache" + aws s3 sync "$root/" "s3://$BUCKET/$PREFIX/apk/" --exclude "*" --include "*.apk" --cache-control "public, max-age=31536000, immutable" } # deb: index_deb writes its own (dearmored) keboola.gpg, so pass no pub-key content. From 492e6d4d2f4fdc20540056bc09b040b1a4ff578e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 24 Jun 2026 07:35:21 +0200 Subject: [PATCH 7/8] =?UTF-8?q?fix(release):=20drop=20apk=20packaging=20?= =?UTF-8?q?=E2=80=94=20glibc=20binary=20can't=20run=20on=20musl=20Alpine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The apk repo signed/indexed/installed correctly, but the PyInstaller binary is glibc-linked (interpreter /lib64/ld-linux-x86-64.so.2) and won't execute on musl Alpine ('sh: kbagent: not found' after a successful install). Rather than ship a package that can't run on its target, drop apk: remove it from nfpm formats, build_packages.sh, the index.sh publisher, the package-linux/publish-s3 APK key wiring, and the apk test-install leg. deb/rpm/brew/choco/winget unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release-kbagent.yml | 39 ++++++------------ build/package/linux/build_packages.sh | 5 ++- build/package/linux/index.sh | 59 ++------------------------- build/package/nfpm.yaml | 10 +---- 4 files changed, 21 insertions(+), 92 deletions(-) diff --git a/.github/workflows/release-kbagent.yml b/.github/workflows/release-kbagent.yml index 1b1f3552..b8ebe817 100644 --- a/.github/workflows/release-kbagent.yml +++ b/.github/workflows/release-kbagent.yml @@ -1,9 +1,9 @@ # Release & distribution pipeline for kbagent — see docs/adr/0003-release-distribution-cicd.md # # Ships a SELF-CONTAINED NATIVE BINARY (no Python / no uv at the user's end), packaged -# for brew / apt / dnf / apk / chocolatey / winget — the same UX the legacy Go `kbc` +# for brew / apt / dnf / chocolatey / winget — the same UX the legacy Go `kbc` # delivered. The Python app is frozen with PyInstaller (proven: `env -i kbagent -# --version` works), then packaged with nfpm (deb/rpm/apk) and wrapped for Homebrew / +# --version` works), then packaged with nfpm (deb/rpm) and wrapped for Homebrew / # Chocolatey / WinGet. GoReleaser is NOT used (it builds Go); nothing depends on the # deprecated keboola-as-code repo. # @@ -61,7 +61,7 @@ jobs: IS_PRERELEASE: ${{ steps.v.outputs.IS_PRERELEASE }} # S3 prefix to publish under: prod (`keboola-cli2`) for real releases, a separate # `keboola-cli2-dev` for pre-releases so a dev tag can exercise the full S3 publish - # + client install test without touching the prod apt/rpm/apk repos. + # + client install test without touching the prod apt/rpm repos. PUBLISH_PREFIX: ${{ steps.v.outputs.PUBLISH_PREFIX }} steps: - id: v @@ -216,11 +216,11 @@ jobs: dist/*.sha256 dist/${{ env.BIN_NAME }}* - # ── Package the Linux binaries into deb/rpm/apk with nfpm (language-agnostic). + # ── Package the Linux binaries into deb/rpm with nfpm (language-agnostic). package-linux: needs: [version, freeze] runs-on: ubuntu-latest - environment: release # so DEB/RPM/APK signing keys resolve + environment: release # so DEB/RPM signing keys resolve env: VERSION: ${{ needs.version.outputs.VERSION }} steps: @@ -234,19 +234,17 @@ jobs: with: { pattern: bin-linux-*, path: artifacts } - name: Write package signing keys env: - # DEB/RPM are signed with GPG keys; APK with an abuild RSA key. + # DEB/RPM are signed with GPG keys. DEB_KEY_PRIVATE: ${{ secrets.DEB_KEY_PRIVATE }} RPM_KEY_PRIVATE: ${{ secrets.RPM_KEY_PRIVATE }} - APK_KEY_PRIVATE: ${{ secrets.APK_KEY_PRIVATE }} run: | mkdir -p /tmp/keys - for k in DEB_KEY_PRIVATE RPM_KEY_PRIVATE APK_KEY_PRIVATE; do + for k in DEB_KEY_PRIVATE RPM_KEY_PRIVATE; do [ -n "${!k}" ] || { echo "::error::$k is empty — refusing to build unsigned/garbage-signed packages"; exit 1; } done printf '%s' "$DEB_KEY_PRIVATE" > /tmp/keys/deb.key && chmod 600 /tmp/keys/deb.key printf '%s' "$RPM_KEY_PRIVATE" > /tmp/keys/rpm.key && chmod 600 /tmp/keys/rpm.key - printf '%s' "$APK_KEY_PRIVATE" > /tmp/keys/apk.key && chmod 600 /tmp/keys/apk.key - - name: Build deb/rpm/apk for each arch + - name: Build deb/rpm for each arch run: bash build/package/linux/build_packages.sh "$VERSION" artifacts - uses: actions/upload-artifact@v4 with: { name: linux-packages, path: dist/* } @@ -270,7 +268,7 @@ jobs: - name: Collect release assets run: | mkdir -p release - find artifacts -type f \( -name '*.zip' -o -name '*.sha256' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' \) -exec cp {} release/ \; + find artifacts -type f \( -name '*.zip' -o -name '*.sha256' -o -name '*.deb' -o -name '*.rpm' \) -exec cp {} release/ \; ls -al release/ - uses: softprops/action-gh-release@v2 with: @@ -309,7 +307,7 @@ jobs: echo "::error::wheel(s) still missing 10 min after dispatching release.yml for v${VERSION} -- check that run" exit 1 - # ── Publish to S3 (cli-dist.keboola.com/keboola-cli2/) and index the apt/rpm/apk + # ── Publish to S3 (cli-dist.keboola.com/keboola-cli2/) and index the apt/rpm # repos so `apt-get install keboola-cli2` works. Real; needs AWS + GPG secrets. publish-s3: needs: [version, freeze, package-linux] @@ -344,15 +342,13 @@ jobs: esac - name: Upload versioned assets run: | - mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' -o -name '*.sha256' \) -exec cp {} up/ \; + mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.sha256' \) -exec cp {} up/ \; aws s3 sync up/ "s3://${AWS_BUCKET_NAME}/${PUBLISH_PREFIX}/v${VERSION}/" - - name: Index apt/rpm/apk repos (signed) + - name: Index apt/rpm repos (signed) env: # No DEB_KEY_PUBLIC: the apt keyring is derived (dearmored) from DEB_KEY_PRIVATE. DEB_KEY_PRIVATE: ${{ secrets.DEB_KEY_PRIVATE }} RPM_KEY_PUBLIC: ${{ secrets.RPM_KEY_PUBLIC }} - APK_KEY_PRIVATE: ${{ secrets.APK_KEY_PRIVATE }} - APK_KEY_PUBLIC: ${{ secrets.APK_KEY_PUBLIC }} run: bash build/package/linux/index.sh "${AWS_BUCKET_NAME}" "${PUBLISH_PREFIX}" # ── Homebrew: render the formula from the template and push to the kbagent-owned tap. @@ -451,17 +447,6 @@ jobs: apt-get update && apt-get install -y keboola-cli2 kbagent --version | grep -q "${VERSION}" ' - - name: apk (Alpine) - if: ${{ !cancelled() }} # run even if an earlier leg failed, so one run shows all results - run: | - docker run --rm -e PREFIX -e VERSION alpine sh -c ' - set -eu - wget -O /etc/apk/keys/keboola.rsa.pub "https://cli-dist.keboola.com/${PREFIX}/apk/keboola.rsa.pub" - echo "https://cli-dist.keboola.com/${PREFIX}/apk" >> /etc/apk/repositories - apk update - apk add keboola-cli2 - kbagent --version | grep -q "${VERSION}" - ' - name: rpm (Fedora) if: ${{ !cancelled() }} # run even if an earlier leg failed, so one run shows all results run: | diff --git a/build/package/linux/build_packages.sh b/build/package/linux/build_packages.sh index a6404df6..9815ce44 100755 --- a/build/package/linux/build_packages.sh +++ b/build/package/linux/build_packages.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# Build deb/rpm/apk for each Linux arch from the frozen binaries, using nfpm. +# Build deb/rpm for each Linux arch from the frozen binaries, using nfpm. +# (No apk: the glibc-linked PyInstaller binary won't run on musl Alpine.) # nfpm does not reliably expand ${...} in its config, so we render it with envsubst. # Usage: build_packages.sh [artifacts-dir] set -euo pipefail @@ -17,7 +18,7 @@ for arch in amd64 arm64; do export VERSION PKG_ARCH="$arch" BIN_PATH="$BIN" envsubst '${VERSION} ${PKG_ARCH} ${BIN_PATH}' < build/package/nfpm.yaml > /tmp/nfpm.yaml - for fmt in deb rpm apk; do + for fmt in deb rpm; do nfpm package -f /tmp/nfpm.yaml -p "$fmt" -t "dist/keboola-cli2_${VERSION}_linux_${arch}.${fmt}" done done diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index d45bd1d7..0165b99a 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash -# Build/refresh the SIGNED apt(deb), yum(rpm) and apk repositories on the CLI dist +# Build/refresh the SIGNED apt(deb) and yum(rpm) repositories on the CLI dist # S3 bucket so `apt-get install keboola-cli2` (etc.) works out of the box. +# (No apk: the frozen binary is glibc-linked and won't run on musl Alpine.) # # Signing keys (separate per format): # DEB_KEY_PRIVATE — GPG, signs the apt repo. The public keyring apt downloads is # derived from it (dearmored binary), so there's no DEB_KEY_PUBLIC. # RPM_KEY_PUBLIC — public half of the SEPARATE rpm signing key (nfpm signs the rpm # packages with RPM_KEY_PRIVATE); published for yum clients. -# APK_KEY_PRIVATE / APK_KEY_PUBLIC — abuild RSA keypair, signs the apk index # Requires AWS creds already configured (OIDC). -# Usage: index.sh → s3:////{deb,rpm,apk}/ +# Usage: index.sh → s3:////{deb,rpm}/ set -euo pipefail BUCKET="$1" PREFIX="$2" @@ -25,7 +25,6 @@ if [ -z "${DEB_KEY_PRIVATE:-}" ]; then fi sudo apt-get update -y -# Only deb/rpm tooling on the host; the apk index is built in Alpine (see index_apk). sudo apt-get install -y dpkg-dev apt-utils createrepo-c gnupg # Import GPG signing key (deb + rpm metadata). @@ -70,58 +69,8 @@ index_deb() { } index_rpm() { createrepo_c .; } -# apk has its own publisher (not publish_repo): repos are PER-ARCH — clients fetch -# //APKINDEX.tar.gz — so packages + signed index live under apk// -# (deb arch name -> apk arch name). Everything agrees on the name "keboola": nfpm signs -# each package as .SIGN.RSA.keboola.rsa.pub (apk.key_name in nfpm.yaml); importing the -# pubkey as /etc/apk/keys/keboola.rsa.pub makes the container trust the packages (no -# --allow-untrusted); signing each index with keboola.rsa yields a signature clients -# verify against the published keboola.rsa.pub. apk/abuild-sign are Alpine-only (not in -# Ubuntu apt), so index + sign inside Alpine. -publish_apk() { - local root="$WORK/apk" - printf '%s' "$APK_KEY_PRIVATE" > "$WORK/keboola.rsa" && chmod 600 "$WORK/keboola.rsa" - printf '%s' "$APK_KEY_PUBLIC" > "$WORK/keboola.rsa.pub" - local pair deb_arch apk_arch - for pair in amd64:x86_64 arm64:aarch64; do - deb_arch="${pair%%:*}"; apk_arch="${pair##*:}" - mkdir -p "$root/$apk_arch" - aws s3 sync "s3://$BUCKET/$PREFIX/apk/$apk_arch/" "$root/$apk_arch/" --exclude '*' --include '*.apk' || true - # apk clients fetch //-.apk, so the file on disk must be - # named that way — NOT nfpm's deb-style keboola-cli2__linux_.apk (which - # apk index records as P/V but the client then 404s on). Rename on copy. - local f base ver - for f in $(find . -path ./.git -prune -o -name "*_linux_${deb_arch}.apk" -print); do - base="${f##*/}"; ver="${base#keboola-cli2_}"; ver="${ver%_linux_${deb_arch}.apk}" - cp "$f" "$root/$apk_arch/keboola-cli2-${ver}.apk" - done - done - # Publish the pubkey at the apk root; clients install it into /etc/apk/keys/keboola.rsa.pub. - printf '%s' "$APK_KEY_PUBLIC" > "$root/keboola.rsa.pub" - docker run --rm \ - -v "$root:/work" \ - -v "$WORK/keboola.rsa:/keboola.rsa:ro" \ - -v "$WORK/keboola.rsa.pub:/etc/apk/keys/keboola.rsa.pub:ro" \ - -w /work alpine:3 \ - sh -ceu 'apk add --no-cache abuild >/dev/null - for d in */; do - ls "$d"*.apk >/dev/null 2>&1 || continue - ( cd "$d" && apk index -o APKINDEX.tar.gz ./*.apk && abuild-sign -k /keboola.rsa APKINDEX.tar.gz ) - done' - # As in publish_repo: APKINDEX.tar.gz + pubkey live at stable paths -> no-cache so - # CloudFront doesn't serve a stale index; the .apk packages are version-named -> immutable. - aws s3 sync "$root/" "s3://$BUCKET/$PREFIX/apk/" --exclude "*.apk" --cache-control "no-cache" - aws s3 sync "$root/" "s3://$BUCKET/$PREFIX/apk/" --exclude "*" --include "*.apk" --cache-control "public, max-age=31536000, immutable" -} - # deb: index_deb writes its own (dearmored) keboola.gpg, so pass no pub-key content. publish_repo deb index_deb "" "" publish_repo rpm index_rpm keboola.gpg "${RPM_KEY_PUBLIC:-}" -if [ -z "${APK_KEY_PRIVATE:-}" ]; then - # No apk signing key configured — the apk index is genuinely opt-out, so skip it. - echo "::warning::APK_KEY_PRIVATE not set — skipping apk index (deb/rpm done)." -else - publish_apk -fi -echo "Repositories indexed and published under s3://$BUCKET/$PREFIX/{deb,rpm,apk}/" +echo "Repositories indexed and published under s3://$BUCKET/$PREFIX/{deb,rpm}/" diff --git a/build/package/nfpm.yaml b/build/package/nfpm.yaml index 9bbc5a65..2ef96e55 100644 --- a/build/package/nfpm.yaml +++ b/build/package/nfpm.yaml @@ -1,4 +1,5 @@ -# nfpm config — packages the frozen `kbagent` binary into deb / rpm / apk. +# nfpm config — packages the frozen `kbagent` binary into deb / rpm. +# (No apk: the glibc-linked PyInstaller binary won't run on musl Alpine.) # # nfpm (https://nfpm.goreleaser.com) is a standalone, language-agnostic packager. # We do NOT use goreleaser (it builds Go); the binary is produced by PyInstaller in @@ -33,10 +34,3 @@ deb: rpm: signature: key_file: /tmp/keys/rpm.key -apk: - signature: - # key_name must match the pubkey filename clients trust (keboola.rsa.pub) and the - # abuild-sign key in build/package/linux/index.sh — see that file. nfpm signs the - # package as .SIGN.RSA..rsa.pub, so all three must agree on "keboola". - key_name: keboola - key_file: /tmp/keys/apk.key From 42329fd918feccb03c58f41240523291fc929302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 24 Jun 2026 07:55:40 +0200 Subject: [PATCH 8/8] =?UTF-8?q?fix(release):=20address=20review=20?= =?UTF-8?q?=E2=80=94=20concurrency=20guard,=20accurate=20comments,=20clear?= =?UTF-8?q?er=20debug=20line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add a concurrency group to publish-s3 keyed by PUBLISH_PREFIX (cancel-in-progress: false) so concurrent tag runs can't wipe/overwrite a shared repo prefix mid-publish (Copilot). - Correct the now-stale comments: the DEB_KEY guard in index.sh and the version-job IS_PRERELEASE doc both said publish-s3 is real-tags-only; it now runs on every tag (dev prefix for pre-releases) (Padak NIT-1, Copilot). - Replace the command-substitution debug echo with a precomputed PUBLISH_PREFIX var (Padak NIT-2). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/release-kbagent.yml | 16 ++++++++++++---- build/package/linux/index.sh | 7 +++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release-kbagent.yml b/.github/workflows/release-kbagent.yml index b8ebe817..97f0aa0c 100644 --- a/.github/workflows/release-kbagent.yml +++ b/.github/workflows/release-kbagent.yml @@ -56,8 +56,9 @@ jobs: outputs: VERSION: ${{ steps.v.outputs.VERSION }} # "true" for a pre-release tag (has a -suffix, e.g. v0.0.1-dev.1). Pre-releases - # build + package + sign + GitHub pre-release ONLY; all external publishing - # (PyPI, S3 repo index, Homebrew, Chocolatey, WinGet) is skipped. + # build + package + sign, publish to S3 under the dev prefix (PUBLISH_PREFIX), and + # run test-install + a GitHub pre-release. The user-facing publishers (PyPI, + # Homebrew, Chocolatey, WinGet) are skipped — only real tags reach those. IS_PRERELEASE: ${{ steps.v.outputs.IS_PRERELEASE }} # S3 prefix to publish under: prod (`keboola-cli2`) for real releases, a separate # `keboola-cli2-dev` for pre-releases so a dev tag can exercise the full S3 publish @@ -79,8 +80,9 @@ jobs: if printf '%s' "$VERSION" | grep -Eq '(a|b|rc)[0-9]+$|\.dev[0-9]+$|-'; then PRE=true; else PRE=false; fi echo "IS_PRERELEASE=$PRE" >> "$GITHUB_OUTPUT" # Pre-releases publish to a separate dev prefix (see PUBLISH_PREFIX output above). - if [ "$PRE" = true ]; then echo "PUBLISH_PREFIX=${S3_PREFIX}-dev"; else echo "PUBLISH_PREFIX=${S3_PREFIX}"; fi >> "$GITHUB_OUTPUT" - echo "version=$VERSION prerelease=$PRE prefix=${S3_PREFIX}$([ "$PRE" = true ] && echo -dev)" + if [ "$PRE" = true ]; then PUBLISH_PREFIX="${S3_PREFIX}-dev"; else PUBLISH_PREFIX="${S3_PREFIX}"; fi + echo "PUBLISH_PREFIX=$PUBLISH_PREFIX" >> "$GITHUB_OUTPUT" + echo "version=$VERSION prerelease=$PRE prefix=$PUBLISH_PREFIX" # Reuse the per-PR gates against the tagged commit, plus changelog-check. gate: @@ -314,6 +316,12 @@ jobs: # Runs on every tag. Pre-releases publish under the dev prefix (PUBLISH_PREFIX), so a # dev tag exercises the full publish + client install test without touching prod repos. if: startsWith(github.ref, 'refs/tags/') + # Serialize per prefix: the wipe + index steps mutate a shared repo prefix, so two + # concurrent tag runs (e.g. two dev tags) could delete/overwrite each other mid-publish. + # Queue (don't cancel) so an in-flight publish always finishes intact. + concurrency: + group: publish-s3-${{ needs.version.outputs.PUBLISH_PREFIX }} + cancel-in-progress: false runs-on: ubuntu-latest environment: release permissions: diff --git a/build/package/linux/index.sh b/build/package/linux/index.sh index 0165b99a..d0c3ef1c 100755 --- a/build/package/linux/index.sh +++ b/build/package/linux/index.sh @@ -17,10 +17,9 @@ WORK=$(mktemp -d) trap 'rm -rf "$WORK"' EXIT # clean up temp dir + any key material on exit/failure if [ -z "${DEB_KEY_PRIVATE:-}" ]; then - # publish-s3 only runs on real (non-pre-release) tags, where the repo + key must - # exist for the downstream test-install job. Fail loudly rather than silently - # skipping and leaving test-install to fail with an obscure root cause. - echo "::error::DEB_KEY_PRIVATE not set — cannot sign/index the apt repo for a real release." + # Fail loudly: the key is required in the `release` environment for any tag (publish-s3 + # runs on every tag now — real under the prod prefix, pre-release under the dev prefix). + echo "::error::DEB_KEY_PRIVATE not set — signing environment is misconfigured." exit 1 fi