prebuilt #29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: prebuilt | |
| on: | |
| release: | |
| types: [published] | |
| workflow_dispatch: | |
| inputs: | |
| tag: | |
| description: | |
| 'Release tag to upload assets to (e.g., v3.51.0-build1). Leave blank | |
| to use package.json' | |
| required: false | |
| workflow_run: | |
| workflows: ['create-release'] | |
| types: [completed] | |
| jobs: | |
| ensure-release: | |
| if: | |
| ${{ github.event_name != 'workflow_run' || | |
| github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| tag: ${{ steps.ver.outputs.tag }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Resolve tag | |
| id: ver | |
| shell: bash | |
| run: | | |
| if [ "${{ github.event_name }}" = "release" ]; then | |
| TAG='${{ github.event.release.tag_name }}' | |
| elif [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.tag }}" ]; then | |
| TAG='${{ github.event.inputs.tag }}' | |
| else | |
| VER=$(node -p "require('./package.json').version") | |
| TAG="v${VER#v}" | |
| fi | |
| echo "tag=$TAG" >> $GITHUB_OUTPUT | |
| echo "Resolved tag: $TAG" | |
| - name: Ensure GitHub release exists | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| TAG='${{ steps.ver.outputs.tag }}' | |
| if gh release view "$TAG" -R "$GITHUB_REPOSITORY" >/dev/null 2>&1; then | |
| echo "Release $TAG already exists" | |
| else | |
| gh release create "$TAG" --title "$TAG" --notes "Automated prebuilt assets" -R "$GITHUB_REPOSITORY" | |
| fi | |
| build-matrix: | |
| needs: ensure-release | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-22.04, macos-13, macos-14, windows-2022] | |
| arch: [x64, arm64] | |
| exclude: | |
| # Avoid duplicate uploads and unsupported combos | |
| - os: ubuntu-22.04 | |
| arch: arm64 # built in prebuild-linux-arm | |
| - os: macos-13 | |
| arch: arm64 # macOS 13 runner is x64 | |
| - os: macos-14 | |
| arch: x64 # macOS 14 runner is arm64 | |
| - os: windows-2022 | |
| arch: arm64 # no Windows arm64 runner | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Install toolchain (Linux) | |
| if: startsWith(matrix.os, 'ubuntu') | |
| run: | |
| sudo apt-get update && sudo apt-get install -y build-essential git | |
| make | |
| - name: Install toolchain (macOS) | |
| if: startsWith(matrix.os, 'macos') | |
| run: echo 'Using Xcode toolchain' | |
| - name: Install toolchain (Windows) | |
| if: startsWith(matrix.os, 'windows') | |
| shell: bash | |
| run: | | |
| choco install -y make mingw | |
| # Ensure MinGW is on PATH for subsequent bash steps | |
| echo "C:/tools/mingw64/bin" >> $GITHUB_PATH | |
| echo "C:/ProgramData/chocolatey/lib/mingw/tools/install/mingw64/bin" >> $GITHUB_PATH | |
| - name: Clone sqlite-vec | |
| run: git clone --depth 1 https://github.com/asg017/sqlite-vec.git | |
| - name: Vendor and build loadable | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| pushd sqlite-vec | |
| # Use gcc on Windows where 'cc' may be missing | |
| if [ "$RUNNER_OS" = "Windows" ]; then MAKE_CC="CC=gcc"; else MAKE_CC=""; fi | |
| if [ -x ./scripts/vendor.sh ]; then ./scripts/vendor.sh; fi | |
| make loadable ${MAKE_CC} | |
| popd | |
| - name: Find artifact | |
| id: find_art | |
| shell: bash | |
| run: | | |
| set -e | |
| ART="" | |
| for f in \ | |
| sqlite-vec/dist/*.so sqlite-vec/dist/*.dylib sqlite-vec/dist/*.dll \ | |
| sqlite-vec/sqlite-vec.*.so sqlite-vec/sqlite-vec.*.dylib sqlite-vec/sqlite-vec.*.dll \ | |
| sqlite-vec/sqlite-vec.so sqlite-vec/sqlite-vec.dylib sqlite-vec/sqlite-vec.dll; do | |
| if [ -f "$f" ]; then ART="$f"; break; fi | |
| done | |
| if [ -z "$ART" ]; then echo "::error::No sqlite-vec artifact found"; exit 1; fi | |
| echo "artifact=$ART" >> $GITHUB_OUTPUT | |
| - name: Upload release asset | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| shell: bash | |
| run: | | |
| set -e | |
| if [ "${{ matrix.os }}" = "windows-2022" ]; then EXT=dll; elif [[ "${{ matrix.os }}" == macos* ]]; then EXT=dylib; else EXT=so; fi | |
| if [ "${{ matrix.arch }}" = "arm64" ]; then ARCH=arm64; else ARCH=x64; fi | |
| if [[ "${{ matrix.os }}" == ubuntu* ]]; then LIBC=gnu; TRIPLE=linux-$ARCH-$LIBC; fi | |
| if [[ "${{ matrix.os }}" == macos* ]]; then TRIPLE=darwin-$ARCH; fi | |
| if [[ "${{ matrix.os }}" == windows* ]]; then TRIPLE=win32-$ARCH; fi | |
| NAME=sqlite-vec-$TRIPLE.$EXT | |
| echo "Uploading $NAME" | |
| # Rename by copying to a temp filepath so the asset name is exact even on old gh versions | |
| TMPFILE=$(mktemp) | |
| rm -f "$TMPFILE" && cp "${{ steps.find_art.outputs.artifact }}" "$NAME" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME" --clobber -R "$GITHUB_REPOSITORY" | |
| # Create checksum file for the uploaded asset name | |
| HASH=$(node -e "const fs=require('fs');const c=require('crypto');const d=fs.readFileSync(process.argv[1]);process.stdout.write(c.createHash('sha256').update(d).digest('hex'));" "$NAME") | |
| printf "%s %s" "$HASH" "$NAME" > "$NAME.sha256" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME.sha256" --clobber -R "$GITHUB_REPOSITORY" | |
| # Verify asset is present on the release (tolerate eventual consistency) | |
| TAG='${{ needs.ensure-release.outputs.tag }}' | |
| DIRECT_URL="https://github.com/$GITHUB_REPOSITORY/releases/download/$TAG/$NAME" | |
| ok="false" | |
| for i in $(seq 1 30); do | |
| CODE=$(curl -sI "$DIRECT_URL" | awk 'NR==1{print $2}') || true | |
| if [ "$CODE" = "200" ] || [ "$CODE" = "302" ]; then ok="true"; break; fi | |
| # Also check the assets list as a fallback view | |
| NAMES=$(gh release view "$TAG" --json assets -q '.assets[].name' -R "$GITHUB_REPOSITORY" || true) | |
| if echo "$NAMES" | grep -Fqx "$NAME"; then ok="true"; break; fi | |
| echo "Asset $NAME not visible yet (HTTP $CODE); retry $i/30"; sleep 10 | |
| done | |
| if [ "$ok" != "true" ]; then | |
| echo "Asset still missing; attempting re-upload of $NAME" | |
| gh release upload "$TAG" "${{ steps.find_art.outputs.artifact }}#${NAME}" --clobber -R "$GITHUB_REPOSITORY" || true | |
| # Final tries after re-upload | |
| for i in $(seq 1 6); do | |
| CODE=$(curl -sI "$DIRECT_URL" | awk 'NR==1{print $2}') || true | |
| if [ "$CODE" = "200" ] || [ "$CODE" = "302" ]; then ok="true"; break; fi | |
| echo "Post re-upload wait; retry $i/6"; sleep 10 | |
| done | |
| fi | |
| if [ "$ok" != "true" ]; then | |
| echo "::error::Missing asset after upload attempts: $NAME ($DIRECT_URL)"; | |
| echo "Release assets:"; gh release view "$TAG" --json assets -q '.assets[].name' -R "$GITHUB_REPOSITORY" || true | |
| exit 1 | |
| fi | |
| prebuild-alpine: | |
| name: Prebuild on alpine | |
| runs-on: ubuntu-latest | |
| needs: [ensure-release, build-matrix] | |
| container: node:20-alpine | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | |
| apk add build-base git python3 py3-setuptools bash gettext curl unzip | |
| github-cli --update-cache | |
| - run: git clone --depth 1 https://github.com/asg017/sqlite-vec.git | |
| - name: Patch sqlite-vec for Alpine (musl) | |
| shell: bash | |
| run: | | |
| set -euxo pipefail | |
| pushd sqlite-vec | |
| # Quiet broken unzip invocation in vendor.sh (harmless but noisy) | |
| if [ -f scripts/vendor.sh ]; then | |
| sed -i 's/^unzip -d$/# patched: remove empty unzip -d/' scripts/vendor.sh || true | |
| fi | |
| # Avoid redefining stdint types on musl which breaks compilation | |
| sed -i \ | |
| -e 's/^typedef u_int8_t uint8_t;/\/\/ patched: skip musl typedef uint8_t/' \ | |
| -e 's/^typedef u_int16_t uint16_t;/\/\/ patched: skip musl typedef uint16_t/' \ | |
| -e 's/^typedef u_int64_t uint64_t;/\/\/ patched: skip musl typedef uint64_t/' \ | |
| sqlite-vec.c || true | |
| popd | |
| - run: | | |
| set -eux; cd sqlite-vec; [ -f ./scripts/vendor.sh ] && bash ./scripts/vendor.sh || true; make loadable | |
| - name: Upload release asset (alpine x64) | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| shell: sh | |
| run: | | |
| set -e | |
| ART="" | |
| for f in sqlite-vec/dist/*.so sqlite-vec/sqlite-vec*.so; do [ -f "$f" ] && ART="$f" && break; done | |
| [ -n "$ART" ] || (echo "No artifact" && exit 1) | |
| NAME=sqlite-vec-linux-x64-musl.so | |
| echo "Uploading $NAME" | |
| cp "$ART" "$NAME" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME" --clobber -R "$GITHUB_REPOSITORY" | |
| HASH=$(node -e "const fs=require('fs');const c=require('crypto');const d=fs.readFileSync(process.argv[1]);process.stdout.write(c.createHash('sha256').update(d).digest('hex'));" "$NAME") | |
| printf "%s %s" "$HASH" "$NAME" > "$NAME.sha256" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME.sha256" --clobber -R "$GITHUB_REPOSITORY" | |
| # Verify asset exists (HEAD the direct URL with retries) | |
| TAG='${{ needs.ensure-release.outputs.tag }}' | |
| DIRECT_URL="https://github.com/$GITHUB_REPOSITORY/releases/download/$TAG/$NAME" | |
| ok=false | |
| for i in 1 2 3 4 5 6 7 8 9 10; do | |
| CODE=$(curl -sI "$DIRECT_URL" | awk 'NR==1{print $2}') || true | |
| if [ "$CODE" = "200" ] || [ "$CODE" = "302" ]; then ok=true; break; fi | |
| echo "Waiting for $NAME (HTTP $CODE) ($i/10)"; sleep 6 | |
| done | |
| $ok || { echo "::error::Missing asset after upload: $NAME"; exit 1; } | |
| prebuild-alpine-arm: | |
| name: Prebuild on alpine (arm) | |
| runs-on: ubuntu-latest | |
| needs: [ensure-release, build-matrix] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: [arm/v7, arm64] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - name: Build (alpine ${{ matrix.arch }}) | |
| run: | | |
| docker run --rm -v $PWD:/w --entrypoint /bin/sh --platform linux/${{ matrix.arch }} node:20-alpine -c ' | |
| set -eux; | |
| apk add build-base git python3 py3-setuptools bash gettext curl unzip --update-cache; | |
| cd /w; git clone --depth 1 https://github.com/asg017/sqlite-vec.git; | |
| # Patch sqlite-vec for musl typedefs and vendor.sh | |
| cd sqlite-vec; | |
| if [ -f scripts/vendor.sh ]; then sed -i "s/^unzip -d$/# patched: remove empty unzip -d/" scripts/vendor.sh || true; fi; | |
| sed -i \ | |
| -e "s/^typedef u_int8_t uint8_t;/\/\/ patched: skip musl typedef uint8_t/" \ | |
| -e "s/^typedef u_int16_t uint16_t;/\/\/ patched: skip musl typedef uint16_t/" \ | |
| -e "s/^typedef u_int64_t uint64_t;/\/\/ patched: skip musl typedef uint64_t/" \ | |
| sqlite-vec.c || true; | |
| [ -f ./scripts/vendor.sh ] && bash ./scripts/vendor.sh || true; make loadable; | |
| ' | |
| - name: Upload release asset (alpine arm) | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -e | |
| ART="" | |
| for f in sqlite-vec/dist/*.so sqlite-vec/sqlite-vec*.so; do [ -f "$f" ] && ART="$f" && break; done | |
| if [ "${{ matrix.arch }}" = "arm/v7" ]; then ARCH=armv7; else ARCH=arm64; fi | |
| NAME=sqlite-vec-linux-$ARCH-musl.so | |
| cp "$ART" "$NAME" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME" --clobber -R "$GITHUB_REPOSITORY" | |
| HASH=$(node -e "const fs=require('fs');const c=require('crypto');const d=fs.readFileSync(process.argv[1]);process.stdout.write(c.createHash('sha256').update(d).digest('hex'));" "$NAME") | |
| printf "%s %s" "$HASH" "$NAME" > "$NAME.sha256" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME.sha256" --clobber -R "$GITHUB_REPOSITORY" | |
| # Verify asset exists (HEAD the direct URL with retries) | |
| TAG='${{ needs.ensure-release.outputs.tag }}' | |
| DIRECT_URL="https://github.com/$GITHUB_REPOSITORY/releases/download/$TAG/$NAME" | |
| ok=false | |
| for i in 1 2 3 4 5 6 7 8 9 10; do | |
| CODE=$(curl -sI "$DIRECT_URL" | awk 'NR==1{print $2}') || true | |
| if [ "$CODE" = "200" ] || [ "$CODE" = "302" ]; then ok=true; break; fi | |
| echo "Waiting for $NAME (HTTP $CODE) ($i/10)"; sleep 6 | |
| done | |
| $ok || { echo "::error::Missing asset after upload: $NAME"; exit 1; } | |
| prebuild-linux-arm: | |
| name: Prebuild on Linux (arm64) | |
| runs-on: ubuntu-latest | |
| needs: [ensure-release, build-matrix] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - name: Build manylinux (arm64) | |
| run: | | |
| docker run --rm -v $PWD:/w --entrypoint /bin/sh --platform linux/arm64 node:20-bullseye -c ' | |
| set -eux; | |
| apt-get update && apt-get install -y git make g++ gcc gettext-base; | |
| cd /w; git clone --depth 1 https://github.com/asg017/sqlite-vec.git; | |
| cd sqlite-vec; [ -x ./scripts/vendor.sh ] && ./scripts/vendor.sh || true; make loadable; | |
| ' | |
| - name: Upload release asset (linux arm64 gnu) | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| set -e | |
| ART="" | |
| for f in sqlite-vec/dist/*.so sqlite-vec/sqlite-vec*.so; do [ -f "$f" ] && ART="$f" && break; done | |
| NAME=sqlite-vec-linux-arm64-gnu.so | |
| cp "$ART" "$NAME" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME" --clobber -R "$GITHUB_REPOSITORY" | |
| HASH=$(node -e "const fs=require('fs');const c=require('crypto');const d=fs.readFileSync(process.argv[1]);process.stdout.write(c.createHash('sha256').update(d).digest('hex'));" "$NAME") | |
| printf "%s %s" "$HASH" "$NAME" > "$NAME.sha256" | |
| gh release upload "${{ needs.ensure-release.outputs.tag }}" "$NAME.sha256" --clobber -R "$GITHUB_REPOSITORY" | |
| # Verify asset exists (HEAD the direct URL with retries) | |
| TAG='${{ needs.ensure-release.outputs.tag }}' | |
| DIRECT_URL="https://github.com/$GITHUB_REPOSITORY/releases/download/$TAG/$NAME" | |
| ok=false | |
| for i in 1 2 3 4 5 6 7 8 9 10; do | |
| CODE=$(curl -sI "$DIRECT_URL" | awk 'NR==1{print $2}') || true | |
| if [ "$CODE" = "200" ] || [ "$CODE" = "302" ]; then ok=true; break; fi | |
| echo "Waiting for $NAME (HTTP $CODE) ($i/10)"; sleep 6 | |
| done | |
| $ok || { echo "::error::Missing asset after upload: $NAME"; exit 1; } | |
| permissions: | |
| contents: write |