fix(release): make deb/rpm install paths correct, gate them in CI, drop apk (musl-incompatible)

.github/workflows/release-kbagent.yml

@@ -1,9 +1,9 @@
 # Release & distribution pipeline for kbagent — see docs/adr/0003-release-distribution-cicd.md
 #
 # Ships a SELF-CONTAINED NATIVE BINARY (no Python / no uv at the user's end), packaged
-# for brew / apt / dnf / apk / chocolatey / winget — the same UX the legacy Go `kbc`
+# for brew / apt / dnf / chocolatey / winget — the same UX the legacy Go `kbc`
 # delivered. The Python app is frozen with PyInstaller (proven: `env -i kbagent
-# --version` works), then packaged with nfpm (deb/rpm/apk) and wrapped for Homebrew /
+# --version` works), then packaged with nfpm (deb/rpm) and wrapped for Homebrew /
 # Chocolatey / WinGet. GoReleaser is NOT used (it builds Go); nothing depends on the
 # deprecated keboola-as-code repo.
 #
@@ -56,9 +56,14 @@ jobs:
     outputs:
       VERSION: ${{ steps.v.outputs.VERSION }}
       # "true" for a pre-release tag (has a -suffix, e.g. v0.0.1-dev.1). Pre-releases
-      # build + package + sign + GitHub pre-release ONLY; all external publishing
-      # (PyPI, S3 repo index, Homebrew, Chocolatey, WinGet) is skipped.
+      # build + package + sign, publish to S3 under the dev prefix (PUBLISH_PREFIX), and
+      # run test-install + a GitHub pre-release. The user-facing publishers (PyPI,
+      # Homebrew, Chocolatey, WinGet) are skipped — only real tags reach those.
       IS_PRERELEASE: ${{ steps.v.outputs.IS_PRERELEASE }}
+      # S3 prefix to publish under: prod (`keboola-cli2`) for real releases, a separate
+      # `keboola-cli2-dev` for pre-releases so a dev tag can exercise the full S3 publish
+      # + client install test without touching the prod apt/rpm repos.
+      PUBLISH_PREFIX: ${{ steps.v.outputs.PUBLISH_PREFIX }}
     steps:
       - id: v
         env:
@@ -74,7 +79,10 @@ jobs:
           # Pre-release = any a/b/rc/.dev/-suffix marker (.post is a real release).
           if printf '%s' "$VERSION" | grep -Eq '(a|b|rc)[0-9]+$|\.dev[0-9]+$|-'; then PRE=true; else PRE=false; fi
           echo "IS_PRERELEASE=$PRE" >> "$GITHUB_OUTPUT"
-          echo "version=$VERSION prerelease=$PRE"
+          # Pre-releases publish to a separate dev prefix (see PUBLISH_PREFIX output above).
+          if [ "$PRE" = true ]; then PUBLISH_PREFIX="${S3_PREFIX}-dev"; else PUBLISH_PREFIX="${S3_PREFIX}"; fi
+          echo "PUBLISH_PREFIX=$PUBLISH_PREFIX" >> "$GITHUB_OUTPUT"
+          echo "version=$VERSION prerelease=$PRE prefix=$PUBLISH_PREFIX"
 
   # Reuse the per-PR gates against the tagged commit, plus changelog-check.
   gate:
@@ -210,11 +218,11 @@ jobs:
             dist/*.sha256
             dist/${{ env.BIN_NAME }}*
 
-  # ── Package the Linux binaries into deb/rpm/apk with nfpm (language-agnostic).
+  # ── Package the Linux binaries into deb/rpm with nfpm (language-agnostic).
   package-linux:
     needs: [version, freeze]
     runs-on: ubuntu-latest
-    environment: release # so DEB/RPM/APK signing keys resolve
+    environment: release # so DEB/RPM signing keys resolve
     env:
       VERSION: ${{ needs.version.outputs.VERSION }}
     steps:
@@ -228,19 +236,17 @@ jobs:
         with: { pattern: bin-linux-*, path: artifacts }
       - name: Write package signing keys
         env:
-          # DEB/RPM are signed with GPG keys; APK with an abuild RSA key.
+          # DEB/RPM are signed with GPG keys.
           DEB_KEY_PRIVATE: ${{ secrets.DEB_KEY_PRIVATE }}
           RPM_KEY_PRIVATE: ${{ secrets.RPM_KEY_PRIVATE }}
-          APK_KEY_PRIVATE: ${{ secrets.APK_KEY_PRIVATE }}
         run: |
           mkdir -p /tmp/keys
-          for k in DEB_KEY_PRIVATE RPM_KEY_PRIVATE APK_KEY_PRIVATE; do
+          for k in DEB_KEY_PRIVATE RPM_KEY_PRIVATE; do
             [ -n "${!k}" ] || { echo "::error::$k is empty — refusing to build unsigned/garbage-signed packages"; exit 1; }
           done
           printf '%s' "$DEB_KEY_PRIVATE" > /tmp/keys/deb.key && chmod 600 /tmp/keys/deb.key
           printf '%s' "$RPM_KEY_PRIVATE" > /tmp/keys/rpm.key && chmod 600 /tmp/keys/rpm.key
-          printf '%s' "$APK_KEY_PRIVATE" > /tmp/keys/apk.key && chmod 600 /tmp/keys/apk.key
-      - name: Build deb/rpm/apk for each arch
+      - name: Build deb/rpm for each arch
         run: bash build/package/linux/build_packages.sh "$VERSION" artifacts
       - uses: actions/upload-artifact@v4
         with: { name: linux-packages, path: dist/* }
@@ -264,7 +270,7 @@ jobs:
       - name: Collect release assets
         run: |
           mkdir -p release
-          find artifacts -type f \( -name '*.zip' -o -name '*.sha256' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' \) -exec cp {} release/ \;
+          find artifacts -type f \( -name '*.zip' -o -name '*.sha256' -o -name '*.deb' -o -name '*.rpm' \) -exec cp {} release/ \;
           ls -al release/
       - uses: softprops/action-gh-release@v2
         with:
@@ -303,18 +309,27 @@ jobs:
           echo "::error::wheel(s) still missing 10 min after dispatching release.yml for v${VERSION} -- check that run"
           exit 1
 
-  # ── Publish to S3 (cli-dist.keboola.com/keboola-cli2/) and index the apt/rpm/apk
+  # ── Publish to S3 (cli-dist.keboola.com/keboola-cli2/) and index the apt/rpm
   #    repos so `apt-get install keboola-cli2` works. Real; needs AWS + GPG secrets.
   publish-s3:
     needs: [version, freeze, package-linux]
-    if: needs.version.outputs.IS_PRERELEASE == 'false' && startsWith(github.ref, 'refs/tags/')
+    # Runs on every tag. Pre-releases publish under the dev prefix (PUBLISH_PREFIX), so a
+    # dev tag exercises the full publish + client install test without touching prod repos.
+    if: startsWith(github.ref, 'refs/tags/')
+    # Serialize per prefix: the wipe + index steps mutate a shared repo prefix, so two
+    # concurrent tag runs (e.g. two dev tags) could delete/overwrite each other mid-publish.
+    # Queue (don't cancel) so an in-flight publish always finishes intact.
+    concurrency:
+      group: publish-s3-${{ needs.version.outputs.PUBLISH_PREFIX }}
+      cancel-in-progress: false
     runs-on: ubuntu-latest
     environment: release
     permissions:
       contents: read # actions/checkout
       id-token: write # AWS OIDC (configure-aws-credentials assumes the role)
     env:
       VERSION: ${{ needs.version.outputs.VERSION }}
+      PUBLISH_PREFIX: ${{ needs.version.outputs.PUBLISH_PREFIX }}
     steps:
       - uses: actions/checkout@v5
       - uses: actions/download-artifact@v4
@@ -323,18 +338,26 @@ jobs:
         with:
           role-to-assume: ${{ secrets.AWS_ROLE_ARN }}
           aws-region: ${{ env.AWS_REGION }}
+      - name: Wipe dev prefix (pre-release only)
+        # Dev tags reuse the keboola-cli2-dev prefix; clear it so each dev run is a clean
+        # slate and version resolution isn't confused by leftover packages from a prior
+        # dev tag. NEVER runs for the prod prefix (guarded on the -dev suffix).
+        if: needs.version.outputs.IS_PRERELEASE == 'true'
+        run: |
+          case "$PUBLISH_PREFIX" in
+            *-dev) aws s3 rm --recursive "s3://${AWS_BUCKET_NAME}/${PUBLISH_PREFIX}/" || true ;;
+            *) echo "refusing to wipe non-dev prefix '$PUBLISH_PREFIX'"; exit 1 ;;
+          esac
       - name: Upload versioned assets
         run: |
-          mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.apk' -o -name '*.sha256' \) -exec cp {} up/ \;
-          aws s3 sync up/ "s3://${AWS_BUCKET_NAME}/${S3_PREFIX}/v${VERSION}/"
-      - name: Index apt/rpm/apk repos (signed)
+          mkdir -p up && find artifacts -type f \( -name '*.zip' -o -name '*.deb' -o -name '*.rpm' -o -name '*.sha256' \) -exec cp {} up/ \;
+          aws s3 sync up/ "s3://${AWS_BUCKET_NAME}/${PUBLISH_PREFIX}/v${VERSION}/"
+      - name: Index apt/rpm repos (signed)
         env:
           # No DEB_KEY_PUBLIC: the apt keyring is derived (dearmored) from DEB_KEY_PRIVATE.
           DEB_KEY_PRIVATE: ${{ secrets.DEB_KEY_PRIVATE }}
           RPM_KEY_PUBLIC: ${{ secrets.RPM_KEY_PUBLIC }}
-          APK_KEY_PRIVATE: ${{ secrets.APK_KEY_PRIVATE }}
-          APK_KEY_PUBLIC: ${{ secrets.APK_KEY_PUBLIC }}
-        run: bash build/package/linux/index.sh "${AWS_BUCKET_NAME}" "${S3_PREFIX}"
+        run: bash build/package/linux/index.sh "${AWS_BUCKET_NAME}" "${PUBLISH_PREFIX}"
 
   # ── Homebrew: render the formula from the template and push to the kbagent-owned tap.
   homebrew:
@@ -411,28 +434,47 @@ jobs:
           powershell -Command '$s = Get-AuthenticodeSignature wingetcreate.exe; if ($s.Status -ne "Valid" -or $s.SignerCertificate.Subject -notmatch "Microsoft") { Write-Error "wingetcreate.exe signature not valid/Microsoft: $($s.Status)"; exit 1 }'
           ./wingetcreate.exe update -v "$VERSION" -u "$url" -t "$WINGET_TOKEN" Keboola.KeboolaCLI2 -s
 
-  # ── Smoke-test the real install paths end to end.
+  # ── Smoke-test the real install paths end to end. Runs on every tag (incl. dev tags,
+  #    against PUBLISH_PREFIX); the Homebrew leg is prod-only since the tap isn't pushed
+  #    for pre-releases. always() so it still runs when homebrew is skipped on a dev tag.
   test-install:
     needs: [version, publish-s3, homebrew]
+    if: always() && needs.publish-s3.result == 'success'
     runs-on: ubuntu-latest
     env:
       VERSION: ${{ needs.version.outputs.VERSION }}
+      PREFIX: ${{ needs.version.outputs.PUBLISH_PREFIX }}
     steps:
       - name: apt (Ubuntu)
         run: |
-          docker run --rm ubuntu bash -c '
+          docker run --rm -e PREFIX -e VERSION ubuntu bash -c '
             set -eo pipefail
             apt-get update -y && apt-get install -y wget ca-certificates gnupg
-            wget -P /etc/apt/trusted.gpg.d https://cli-dist.keboola.com/${{ env.S3_PREFIX }}/deb/keboola.gpg
-            echo "deb https://cli-dist.keboola.com/${{ env.S3_PREFIX }}/deb /" > /etc/apt/sources.list.d/keboola.list
+            wget -P /etc/apt/trusted.gpg.d "https://cli-dist.keboola.com/${PREFIX}/deb/keboola.gpg"
+            echo "deb https://cli-dist.keboola.com/${PREFIX}/deb /" > /etc/apt/sources.list.d/keboola.list
             apt-get update && apt-get install -y keboola-cli2
-            kbagent --version | grep -q "${{ env.VERSION }}"
+            kbagent --version | grep -q "${VERSION}"
+          '
+      - name: rpm (Fedora)
+        if: ${{ !cancelled() }}   # run even if an earlier leg failed, so one run shows all results
+        run: |
+          docker run --rm -e PREFIX -e VERSION fedora bash -c '
+            set -eo pipefail
+            rpm --import "https://cli-dist.keboola.com/${PREFIX}/rpm/keboola.gpg"
+            printf "%s\n" "[keboola]" "name=keboola" \
+              "baseurl=https://cli-dist.keboola.com/${PREFIX}/rpm" \
+              "enabled=1" "gpgcheck=1" \
+              "gpgkey=https://cli-dist.keboola.com/${PREFIX}/rpm/keboola.gpg" \
+              > /etc/yum.repos.d/keboola.repo
+            dnf install -y keboola-cli2
+            kbagent --version | grep -q "${VERSION}"
           '
       - name: Homebrew (Linux)
+        if: ${{ !cancelled() && needs.version.outputs.IS_PRERELEASE == 'false' && needs.homebrew.result == 'success' }}
         run: |
-          docker run --rm homebrew/brew bash -c '
+          docker run --rm -e VERSION homebrew/brew bash -c '
             set -e
             brew tap keboola/keboola-cli2 https://github.com/keboola/homebrew-keboola-cli2
             brew install keboola-cli2
-            kbagent --version | grep -q "${{ env.VERSION }}"
+            kbagent --version | grep -q "${VERSION}"
           '

build/package/linux/build_packages.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Build deb/rpm/apk for each Linux arch from the frozen binaries, using nfpm.
+# Build deb/rpm for each Linux arch from the frozen binaries, using nfpm.
+# (No apk: the glibc-linked PyInstaller binary won't run on musl Alpine.)
 # nfpm does not reliably expand ${...} in its config, so we render it with envsubst.
 # Usage: build_packages.sh <version> [artifacts-dir]
 set -euo pipefail
@@ -17,7 +18,7 @@ for arch in amd64 arm64; do
 
   export VERSION PKG_ARCH="$arch" BIN_PATH="$BIN"
   envsubst '${VERSION} ${PKG_ARCH} ${BIN_PATH}' < build/package/nfpm.yaml > /tmp/nfpm.yaml
-  for fmt in deb rpm apk; do
+  for fmt in deb rpm; do
     nfpm package -f /tmp/nfpm.yaml -p "$fmt" -t "dist/keboola-cli2_${VERSION}_linux_${arch}.${fmt}"
   done
 done

build/package/linux/index.sh

@@ -1,31 +1,29 @@
 #!/usr/bin/env bash
-# Build/refresh the SIGNED apt(deb), yum(rpm) and apk repositories on the CLI dist
+# Build/refresh the SIGNED apt(deb) and yum(rpm) repositories on the CLI dist
 # S3 bucket so `apt-get install keboola-cli2` (etc.) works out of the box.
+# (No apk: the frozen binary is glibc-linked and won't run on musl Alpine.)
 #
 # Signing keys (separate per format):
 #   DEB_KEY_PRIVATE   — GPG, signs the apt repo. The public keyring apt downloads is
 #                       derived from it (dearmored binary), so there's no DEB_KEY_PUBLIC.
 #   RPM_KEY_PUBLIC    — public half of the SEPARATE rpm signing key (nfpm signs the rpm
 #                       packages with RPM_KEY_PRIVATE); published for yum clients.
-#   APK_KEY_PRIVATE / APK_KEY_PUBLIC   — abuild RSA keypair, signs the apk index
 # Requires AWS creds already configured (OIDC).
-# Usage: index.sh <s3-bucket> <prefix>   →  s3://<bucket>/<prefix>/{deb,rpm,apk}/
+# Usage: index.sh <s3-bucket> <prefix>   →  s3://<bucket>/<prefix>/{deb,rpm}/
 set -euo pipefail
 BUCKET="$1"
 PREFIX="$2"
 WORK=$(mktemp -d)
 trap 'rm -rf "$WORK"' EXIT  # clean up temp dir + any key material on exit/failure
 
 if [ -z "${DEB_KEY_PRIVATE:-}" ]; then
-  # publish-s3 only runs on real (non-pre-release) tags, where the repo + key must
-  # exist for the downstream test-install job. Fail loudly rather than silently
-  # skipping and leaving test-install to fail with an obscure root cause.
-  echo "::error::DEB_KEY_PRIVATE not set — cannot sign/index the apt repo for a real release."
+  # Fail loudly: the key is required in the `release` environment for any tag (publish-s3
+  # runs on every tag now — real under the prod prefix, pre-release under the dev prefix).
+  echo "::error::DEB_KEY_PRIVATE not set — signing environment is misconfigured."
   exit 1
 fi
 
 sudo apt-get update -y
-# Only deb/rpm tooling on the host; the apk index is built in Alpine (see index_apk).
 sudo apt-get install -y dpkg-dev apt-utils createrepo-c gnupg
 
 # Import GPG signing key (deb + rpm metadata).
@@ -43,11 +41,24 @@ publish_repo() {
   find . -path ./.git -prune -o -name "*.$fmt" -exec cp {} "$dir/" \;
   ( cd "$dir" && "$indexer" )
   [ -n "$pub_content" ] && printf '%s' "$pub_content" > "$dir/$pub_file"
-  aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/"
+  # Index/metadata files (Packages, Release, repodata/*, keyrings) sit at STABLE paths
+  # that are overwritten each release, so they must NOT be cached long by CloudFront
+  # (default_ttl is 24h) or clients install stale versions. Upload them no-cache; the
+  # packages themselves have version-unique names, so cache them immutably.
+  aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" --exclude "*.$fmt" --cache-control "no-cache"
+  aws s3 sync "$dir/" "s3://$BUCKET/$PREFIX/$fmt/" --exclude "*" --include "*.$fmt" --cache-control "public, max-age=31536000, immutable"
 }
 
 index_deb() {
-  dpkg-scanpackages . /dev/null > Packages && gzip -kf Packages
+  # -m/--multiversion: keep ALL packages, not just the newest per name. Without it
+  # dpkg-scanpackages dedups on name+version IGNORING architecture, so the amd64 and
+  # arm64 debs collide and one is dropped ("is repeat; ignored"), leaving apt on the
+  # other arch with no install candidate. -m emits a stanza per file (both arches).
+  # Strip the leading "./" from Filename: apt would fetch <repo>/deb/./pkg.deb, and S3
+  # (behind CloudFront) treats the literal "/./" as a missing key -> 404. A clean
+  # Filename: pkg.deb yields <repo>/deb/pkg.deb, which exists.
+  dpkg-scanpackages -m . /dev/null | sed -E 's|^Filename: \./|Filename: |' > Packages
+  gzip -kf Packages
   apt-ftparchive release . > Release
   gpg --batch --yes --default-key "$KEYID" -abs -o Release.gpg Release
   gpg --batch --yes --default-key "$KEYID" --clearsign -o InRelease Release
@@ -56,26 +67,9 @@ index_deb() {
   gpg --export "$KEYID" > keboola.gpg
 }
 index_rpm() { createrepo_c .; }
-index_apk() {
-  printf '%s' "$APK_KEY_PRIVATE" > "$WORK/apk_index.rsa" && chmod 600 "$WORK/apk_index.rsa"
-  # apk/abuild-sign are Alpine-only (not in Ubuntu apt), so sign the index in Alpine.
-  # $PWD is the per-format work dir (publish_repo cd's into it); mount it + the key.
-  docker run --rm -v "$PWD:/work" -v "$WORK/apk_index.rsa:/key.rsa:ro" -w /work alpine:3 \
-    sh -ceu 'apk add --no-cache abuild >/dev/null
-             apk index -o APKINDEX.tar.gz ./*.apk
-             abuild-sign -k /key.rsa APKINDEX.tar.gz'
-}
 
 # deb: index_deb writes its own (dearmored) keboola.gpg, so pass no pub-key content.
 publish_repo deb index_deb "" ""
 publish_repo rpm index_rpm keboola.gpg "${RPM_KEY_PUBLIC:-}"
-if [ -z "${APK_KEY_PRIVATE:-}" ]; then
-  # No apk signing key configured — the apk index is genuinely opt-out, so skip it.
-  echo "::warning::APK_KEY_PRIVATE not set — skipping apk index (deb/rpm done)."
-else
-  # Key set → apk publishing intended. If Docker is somehow absent (it isn't on
-  # ubuntu-latest), index_apk's `docker run` fails loud under set -e — fine.
-  publish_repo apk index_apk keboola.rsa.pub "${APK_KEY_PUBLIC:-}"
-fi
 
-echo "Repositories indexed and published under s3://$BUCKET/$PREFIX/{deb,rpm,apk}/"
+echo "Repositories indexed and published under s3://$BUCKET/$PREFIX/{deb,rpm}/"

build/package/nfpm.yaml

@@ -1,4 +1,5 @@
-# nfpm config — packages the frozen `kbagent` binary into deb / rpm / apk.
+# nfpm config — packages the frozen `kbagent` binary into deb / rpm.
+# (No apk: the glibc-linked PyInstaller binary won't run on musl Alpine.)
 #
 # nfpm (https://nfpm.goreleaser.com) is a standalone, language-agnostic packager.
 # We do NOT use goreleaser (it builds Go); the binary is produced by PyInstaller in
@@ -33,6 +34,3 @@ deb:
 rpm:
   signature:
     key_file: /tmp/keys/rpm.key
-apk:
-  signature:
-    key_file: /tmp/keys/apk.key