diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 753eb11..6017c78 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -20,11 +20,44 @@ on: workflow_dispatch: jobs: + # Ephemeral sccache-dist compile farm. Workers serve the coordinator + # (the build job) and exit when it goes offline. Skipped on PRs — + # fork PRs have no TS_OAUTH_SECRET and same-repo PRs don't need the + # farm for cache-warm builds. Also skipped when TS_OAUTH_SECRET is not + # configured, so missing-secret runs stay green instead of failing the + # worker jobs. (The secrets context is not allowed in `if` expressions, + # so its presence is laundered through job-level env.) + sccache-workers: + name: sccache worker ${{ matrix.idx }} + if: ${{ github.event_name != 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 75 + strategy: + matrix: + idx: [1, 2] + env: + FARM_SECRET_SET: ${{ secrets.TS_OAUTH_SECRET != '' }} + steps: + - name: Farm disabled — TS_OAUTH_SECRET not configured + if: ${{ env.FARM_SECRET_SET == 'false' }} + run: echo "TS_OAUTH_SECRET is not set; skipping farm worker (build compiles locally)." + - uses: xdqi/sccache-dist-action@v0.0.5 + if: ${{ env.FARM_SECRET_SET == 'true' }} + with: + mode: worker + worker-index: '${{ matrix.idx }}' + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' + # The OAuth client owns tag:ci (not the action default tag:ci-sccache). + tags: 'tag:ci' + build: name: linux-amd64 runs-on: ubuntu-24.04 timeout-minutes: 75 + env: + FARM_SECRET_SET: ${{ secrets.TS_OAUTH_SECRET != '' }} + steps: - name: Checkout anyfs-reader uses: actions/checkout@v4 @@ -40,7 +73,8 @@ jobs: libfuse3-dev libnl-3-dev libnl-genl-3-dev \ libcurl4-openssl-dev \ patchelf ca-certificates \ - smbclient nfs-common curl + smbclient nfs-common curl \ + shellcheck # ── Caches ────────────────────────────────────────────────────────────── # All source deps (linux, qemu, ksmbd-tools) are now managed by peru and @@ -114,6 +148,35 @@ jobs: - name: Lint — no hardcoded paths in migrated build scripts run: ./scripts/lint-no-hardcoded-paths.sh + - name: Lint — shellcheck + run: ./scripts/lint-shellcheck.sh + + # ── sccache-dist farm (best-effort; build proceeds locally if absent) ── + # Gated on TS_OAUTH_SECRET: without it the farm cannot come up, so skip + # the whole block (sccache never lands on PATH and the build uses plain + # gcc, exactly like the PR path). + - name: Cache sccache objects + if: ${{ github.event_name != 'pull_request' && env.FARM_SECRET_SET == 'true' }} + uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-linux-amd64-${{ github.sha }} + restore-keys: | + sccache-linux-amd64- + + - name: Bring up sccache-dist farm + if: ${{ github.event_name != 'pull_request' && env.FARM_SECRET_SET == 'true' }} + continue-on-error: true + uses: xdqi/sccache-dist-action@v0.0.5 + with: + mode: coordinator + expected-workers: 2 + min-workers: 1 + wait-timeout: 180s + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' + # The OAuth client owns tag:ci (not the action default tag:ci-sccache). + tags: 'tag:ci' + # ── Build ksmbd-tools (linux-amd64) ──────────────────────────────────── - name: Build ksmbd-tools if: steps.cache-ksmbd-build.outputs.cache-hit != 'true' @@ -141,11 +204,16 @@ jobs: - name: Build LKL kernel run: | + CC_ARGS=() + if command -v sccache >/dev/null 2>&1; then + CC_ARGS=(--cc="sccache gcc") + fi ./scripts/build_lkl.sh \ --linux=deps/linux \ --out="$GITHUB_WORKSPACE" \ --targets=linux-amd64 \ - -j"$(nproc)" + "${CC_ARGS[@]}" \ + -j"${SCCACHE_J:-$(nproc)}" # package_linux.sh falls back to $HOME/linux/tools/lkl/lib/liblkl.so — # link the per-target build there so packaging finds it. mkdir -p deps/linux/tools/lkl/lib @@ -154,10 +222,15 @@ jobs: - name: Build QEMU block layer run: | + CC_ARGS=() + if command -v sccache >/dev/null 2>&1; then + CC_ARGS=(--cc="sccache gcc") + fi ./scripts/build_qemu.sh \ --qemu-src=deps/qemu \ --targets=linux-amd64 \ - -j"$(nproc)" + "${CC_ARGS[@]}" \ + -j"${SCCACHE_J:-$(nproc)}" - name: Build anyfs-reader (core + server + fuse) run: | @@ -169,6 +242,17 @@ jobs: --ksmbd-root=deps/ksmbd-tools \ -j"$(nproc)" + - name: sccache stats + if: ${{ github.event_name != 'pull_request' && env.FARM_SECRET_SET == 'true' }} + continue-on-error: true + run: sccache --show-stats || true + + - name: Run C unit tests + run: meson test -C build-anyfs-linux-amd64 --suite unit --print-errorlogs + + - name: Wasm export drift gate + run: ./tests/test_wasm_exports.sh + # ── Integration smoke test ────────────────────────────────────────────── # Boots each anyfs binary against a Debian 13 generic-cloud qcow2 and # asserts lspart/smbclient/nfs see a real Debian rootfs. This catches diff --git a/.github/workflows/mingw64.yml b/.github/workflows/mingw64.yml index e8624f3..52a0605 100644 --- a/.github/workflows/mingw64.yml +++ b/.github/workflows/mingw64.yml @@ -32,6 +32,36 @@ on: workflow_dispatch: jobs: + # Ephemeral sccache-dist compile farm. Workers serve the coordinator + # (the build job) and exit when it goes offline. Skipped on PRs — + # fork PRs have no TS_OAUTH_SECRET and same-repo PRs don't need the + # farm for cache-warm builds. Also skipped when TS_OAUTH_SECRET is not + # configured, so missing-secret runs stay green instead of failing the + # worker jobs. (The secrets context is not allowed in `if` expressions, + # so its presence is laundered through job-level env.) + sccache-workers: + name: sccache worker ${{ matrix.idx }} + if: ${{ github.event_name != 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 90 + strategy: + matrix: + idx: [1, 2] + env: + FARM_SECRET_SET: ${{ secrets.TS_OAUTH_SECRET != '' }} + steps: + - name: Farm disabled — TS_OAUTH_SECRET not configured + if: ${{ env.FARM_SECRET_SET == 'false' }} + run: echo "TS_OAUTH_SECRET is not set; skipping farm worker (build compiles locally)." + - uses: xdqi/sccache-dist-action@v0.0.5 + if: ${{ env.FARM_SECRET_SET == 'true' }} + with: + mode: worker + worker-index: '${{ matrix.idx }}' + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' + # The OAuth client owns tag:ci (not the action default tag:ci-sccache). + tags: 'tag:ci' + build: name: mingw64 runs-on: ubuntu-24.04 @@ -40,6 +70,7 @@ jobs: env: MSYS_CROSS_REPO: https://msys.kosaka.moe/repo MSYS_CROSS_PREFIX: /opt/msys2-cross + FARM_SECRET_SET: ${{ secrets.TS_OAUTH_SECRET != '' }} steps: - name: Checkout anyfs-reader @@ -90,7 +121,7 @@ jobs: msys-cross-mingw64-gcc \ msys-cross-cygwin-binutils \ msys-cross-cygwin-gcc \ - msys-cross-pkgconfig + msys-cross-mingw64-pkgconf # MSYS2 upstream sysroot libs under /opt/msys2-cross/mingw64. # QEMU block layer + DLL closure deps that package_mingw64.sh @@ -111,6 +142,37 @@ jobs: "${MSYS_CROSS_PREFIX}/bin/x86_64-w64-mingw32-pkg-config" \ --modversion glib-2.0 + # ── sccache-dist farm (best-effort; build proceeds locally if absent) ── + # EXPERIMENTAL on this leg: sccache-dist must package the msys2-cross + # x86_64-w64-mingw32-gcc toolchain (under /opt/msys2-cross) and ship it + # to the workers. The engine supports arbitrary-toolchain packaging, but + # a plain cross-gcc + sysroot is unproven here. On any farm failure the + # build falls back to local compilation exactly as before. + # Gated on TS_OAUTH_SECRET: without it the farm cannot come up, so skip + # the whole block (sccache never lands on PATH and the build uses the + # plain cross-gcc, exactly like the PR path). + - name: Cache sccache objects + if: ${{ github.event_name != 'pull_request' && env.FARM_SECRET_SET == 'true' }} + uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-mingw64-${{ github.sha }} + restore-keys: | + sccache-mingw64- + + - name: Bring up sccache-dist farm + if: ${{ github.event_name != 'pull_request' && env.FARM_SECRET_SET == 'true' }} + continue-on-error: true + uses: xdqi/sccache-dist-action@v0.0.5 + with: + mode: coordinator + expected-workers: 2 + min-workers: 1 + wait-timeout: 180s + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' + # The OAuth client owns tag:ci (not the action default tag:ci-sccache). + tags: 'tag:ci' + # ── Caches ────────────────────────────────────────────────────────────── # All source deps (linux, qemu, ksmbd-tools, util-linux) are now managed # by peru and cached together under ./deps/. Key on peru.yaml (SHA-pinned @@ -202,11 +264,19 @@ jobs: - name: Build LKL kernel (mingw64) run: | + # Absolute compiler path so sccache-dist hashes and ships the + # msys2-cross toolchain (not whatever "x86_64-w64-mingw32-gcc" + # resolves to on a worker). + CC_ARGS=() + if command -v sccache >/dev/null 2>&1; then + CC_ARGS=(--cc="sccache /opt/msys2-cross/bin/x86_64-w64-mingw32-gcc") + fi ./scripts/build_lkl.sh \ --linux=deps/linux \ --out="$GITHUB_WORKSPACE" \ --targets=mingw64 \ - -j"$(nproc)" + "${CC_ARGS[@]}" \ + -j"${SCCACHE_J:-$(nproc)}" test -f "$GITHUB_WORKSPACE/lkl-mingw64/tools/lkl/lib/liblkl.dll" \ || { echo "::error::liblkl.dll missing"; exit 1; } @@ -233,6 +303,11 @@ jobs: --ksmbd-root=deps/ksmbd-tools \ -j"$(nproc)" + - name: sccache stats + if: ${{ github.event_name != 'pull_request' && env.FARM_SECRET_SET == 'true' }} + continue-on-error: true + run: sccache --show-stats || true + - name: Package Win64 distribution tarball run: | ./scripts/package_mingw64.sh diff --git a/.github/workflows/ts.yml b/.github/workflows/ts.yml new file mode 100644 index 0000000..027573d --- /dev/null +++ b/.github/workflows/ts.yml @@ -0,0 +1,55 @@ +name: ts + +# Pure TS unit tests (no wasm bundle, no LKL): @anyfs/core node:test suite + +# @anyfs/react vitest suite + @anyfs/nbd-proxy. The wasm smoke tests +# (test/smoke.node.mjs) need a built bundle and stay local-only. + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +jobs: + unit: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 11 + package_json_file: ts/package.json + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: ts/pnpm-lock.yaml + - name: Install qemu-utils (qemu-img + qemu-io for nbd-proxy integration) + run: sudo apt-get install -y --no-install-recommends qemu-utils + - name: Install + working-directory: ts + # electron-demo depends on a sibling checkout (drivelist via + # file:../../../../drivelist-anyfs) and @anyfs/native's node-gyp + # install script compiles against the LKL tree — neither exists on a + # bare CI runner. Exclude both; electron-demo is the only dependent of + # @anyfs/native, and none of the CI-tested packages need either. + run: pnpm install --frozen-lockfile --filter '!electron-demo' --filter '!@anyfs/native' + - name: Build packages + working-directory: ts + run: pnpm -r --filter './packages/*' --filter '!@anyfs/native' build + - name: Core unit tests + working-directory: ts + run: pnpm --filter @anyfs/core run test:unit + - name: React unit tests + working-directory: ts + run: pnpm --filter @anyfs/react test + # The nbd-proxy integration test opens the PoC fixture qcow2 through the + # proxy with real qemu-img; the fixture is deterministic and generated + # on the fly (8 MiB pattern image + qemu-img convert). + - name: Generate nbd-proxy test fixture + run: node scripts/poc-nbd/make-test-image.mjs + - name: nbd-proxy tests + working-directory: ts + run: pnpm --filter @anyfs/nbd-proxy test diff --git a/.github/workflows/wasm-sysroot.yml b/.github/workflows/wasm-sysroot.yml new file mode 100644 index 0000000..0b9678e --- /dev/null +++ b/.github/workflows/wasm-sysroot.yml @@ -0,0 +1,85 @@ +name: wasm-sysroot + +# Builds the wasm sysroot (17 static libs, see scripts/lib/wasm_sysroot.manifest) +# from pinned sources via scripts/build_wasm_sysroot.sh and publishes it as a +# GitHub release tarball (wasm-sysroot-linux.tar.xz). Consumers fetch it with +# scripts/fetch_wasm_sysroot.sh — the same provisioning model as the mingw64 +# toolchain (recipe = PKGBUILD, this tarball = bootstrap.tar.xz, fetch script + +# manifest = pacman). See docs/wasm-sysroot.md. +# +# Manual-dispatch only: a release is cut deliberately when the recipe's +# version pins move, not on every push. Bump the tag (wasm-sysroot-rN) and +# update WASM_SYSROOT_TAG in scripts/fetch_wasm_sysroot.sh afterwards. + +on: + workflow_dispatch: + inputs: + tag: + description: 'Release tag to create (e.g. wasm-sysroot-r1)' + required: true + type: string + +permissions: + contents: write + +jobs: + build: + name: build + release + runs-on: ubuntu-24.04 + timeout-minutes: 45 + + steps: + - name: Checkout anyfs-reader + uses: actions/checkout@v4 + + # meson/ninja/pkg-config/python3: build_wasm_sysroot.sh hard + # requirements. autoconf/automake/libtool/autopoint/gettext/bison/flex: + # util-linux autogen.sh (autopoint comes from gettext's dev tooling and + # po/ needs gettext proper; flex/bison cover the generated parsers). + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + meson ninja-build pkg-config python3 \ + autoconf automake libtool autopoint gettext bison flex \ + curl ca-certificates xz-utils git + + # Pinned to the emcc version the recipe was validated with (5.0.7, + # 2026-06-10). Bump deliberately and re-validate manifest parity. + - name: Set up emsdk + uses: mymindstorm/setup-emsdk@v14 + with: + version: 5.0.7 + + # build_wasm_sysroot.sh locates util-linux via UL_SRC (env override + # beats config.sh's paths.util_linux) and requires a generated + # ./configure on the v2.40.4 release (it version-gates on + # PACKAGE_VERSION). + - name: Check out util-linux v2.40.4 + run: | + git clone --depth 1 --branch v2.40.4 \ + https://github.com/util-linux/util-linux "$RUNNER_TEMP/util-linux" + cd "$RUNNER_TEMP/util-linux" + ./autogen.sh + + # The script self-checks manifest parity in full-run mode and exits + # nonzero on any drift, so a green run IS the acceptance gate. + - name: Build wasm sysroot + run: | + SYSROOT="$PWD/out-sysroot" UL_SRC="$RUNNER_TEMP/util-linux" \ + ./scripts/build_wasm_sysroot.sh + + - name: Pack tarball + run: | + tar -C out-sysroot -cJf wasm-sysroot-linux.tar.xz . + ls -lh wasm-sysroot-linux.tar.xz + + - name: Create release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ inputs.tag }} + run: | + gh release create "$TAG" wasm-sysroot-linux.tar.xz \ + --target "$GITHUB_SHA" \ + --title "wasm sysroot $TAG" \ + --notes "Prebuilt wasm sysroot (17 static libs; see scripts/lib/wasm_sysroot.manifest). Built by .github/workflows/wasm-sysroot.yml from scripts/build_wasm_sysroot.sh pins; fetch with scripts/fetch_wasm_sysroot.sh." diff --git a/.gitignore b/.gitignore index 0fa8da9..32b1bf3 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,7 @@ node_modules/ # Machine-local build config override (see build.config.toml / build.user.toml.example) build.user.toml + +# Fetched prebuilt toolchain pieces (scripts/fetch_wasm_sysroot.sh etc.) +.toolchain/ +__pycache__/ diff --git a/build.config.toml b/build.config.toml index 2f13365..261b5cb 100644 --- a/build.config.toml +++ b/build.config.toml @@ -10,11 +10,16 @@ linux_src = "" # "" => /linux qemu_src = "" # "" => /qemu util_linux = "" # "" => /util-linux ksmbd_tools = "" # "" => /ksmbd-tools +# Sysroot holding wasm static libs (libblkid/libz/libbz2/libzstd/glib...) +# produced by scripts/build_wasm_sysroot.sh. "" => .toolchain/wasm-sysroot/ when +# fetched (scripts/fetch_wasm_sysroot.sh), else /wasm-sysroot +wasm_sysroot= "" # "" => /wasm-sysroot [toolchains] emsdk = "" # "" => discover via $EMSDK or `which emcc` msys2_cross = "/opt/msys2-cross" # mingw cross toolchain + sysroot + binutils 2.46 -# Patched wasm-ld (built from xdqi/llvm-wasm). "" => /llvm-wasm/.../wasm-ld +# Patched wasm-ld (built from xdqi/llvm-wasm). "" => .toolchain/wasm-ld/ when +# fetched (scripts/fetch_wasm_ld.sh), else /llvm-wasm/.../wasm-ld wasm_ld = "" # Native binutils for the LKL kernel link (bypasses the stale tools/lkl/bin 2.25.1). binutils_native = "/usr/bin" # system binutils (>= 2.30); PE patch not needed for ELF diff --git a/build.user.toml.example b/build.user.toml.example index 466578a..b4161d8 100644 --- a/build.user.toml.example +++ b/build.user.toml.example @@ -1,7 +1,8 @@ # Copy to build.user.toml (gitignored) and override per machine. # Only set what differs from build.config.toml. # [paths] -# linux_src = "/home/me/linux" +# linux_src = "/home/me/linux" +# wasm_sysroot = "/path/to/wasm-sysroot" # [toolchains] # emsdk = "/home/me/emsdk" # wasm_ld = "/home/me/llvm-wasm/install/bin/wasm-ld" diff --git a/docs/ci-sccache.md b/docs/ci-sccache.md new file mode 100644 index 0000000..b6fb09e --- /dev/null +++ b/docs/ci-sccache.md @@ -0,0 +1,178 @@ +# CI compile farm (sccache-dist) + +The `linux` and `mingw64` workflows can distribute C compilation across an +ephemeral farm of GitHub Actions runners connected over Tailscale, using +[`xdqi/sccache-dist-action`](https://github.com/xdqi/sccache-dist-action) +(currently `@v0.0.5` — pin v0.0.5 or later, see warts). The farm is strictly +best-effort: every degradation path ends in a normal local compile and a +green build. + +## Required secret + +| Secret | Where | What | +|---|---|---| +| `TS_OAUTH_SECRET` | `xdqi/anyfs` **and** `xdqi/llvm-wasm` | Tailscale OAuth client secret with the `auth_keys` write scope, owning tag `tag:ci` | + +The OAuth client secret is used directly as a Tailscale auth key (this works +as long as tags are advertised). Without it the farm cannot form. + +**Every farm step must pass `tags: 'tag:ci'`.** The tailnet OAuth client +owns `tag:ci`, not the action's default `tag:ci-sccache`; with the default +tag the auth key is rejected and workers silently fail to join (found and +fixed live on the llvm-wasm leg). Both anyfs workflows and the llvm-wasm +release workflow set this explicitly. + +> Status 2026-06-10: the secret is configured on both repos and the farm is +> live (see results below). + +## Topology + +Per workflow run (`linux.yml`, `mingw64.yml`): + +- **2 worker jobs** (`sccache-workers`, matrix `idx: [1, 2]`) — each runs + `sccache-dist-action` in `worker` mode and serves until the coordinator + goes offline. +- **Coordinator inside the build job** — `mode: coordinator`, + `expected-workers: 2`, `min-workers: 1`, `wait-timeout: 180s`. It brings + up the scheduler + local sccache client and puts the `sccache` engine + binary on `PATH`. +- The build steps then opt in with `--cc="sccache gcc"` (linux) or + `--cc="sccache /opt/msys2-cross/bin/x86_64-w64-mingw32-gcc"` (mingw64 — + absolute path so sccache-dist hashes and ships the msys2-cross toolchain, + not whatever the name resolves to on a worker). + +The wasm-ld release pipeline on `xdqi/llvm-wasm` uses **10 workers** +(LLVM is a much larger compile). + +## Fallback contract + +The farm must never break a build. Three independent layers guarantee that: + +1. **PRs skip the farm entirely.** All farm jobs/steps carry + `github.event_name != 'pull_request'`. Fork PRs have no secrets, and + same-repo PRs don't need the farm for cache-warm builds. PR checks + validate the plain local-compile path. +2. **Missing secret skips the farm.** All farm steps are additionally gated + on `TS_OAUTH_SECRET` being non-empty (laundered through job-level env as + `FARM_SECRET_SET` — the `secrets` context is not allowed in `if` + expressions; the workflow fails to parse if you try). Worker jobs run a + single notice step and exit green. +3. **Coordinator failure degrades to local compile.** The coordinator step + is `continue-on-error: true`, and every build step uses the + `command -v sccache` contract: + + ```sh + CC_ARGS=() + if command -v sccache >/dev/null 2>&1; then + CC_ARGS=(--cc="sccache gcc") + fi + ``` + + If the coordinator died after putting the engine on `PATH`, the build + still works — sccache falls back to local compilation (with its local + disk cache); if `sccache` never made it onto `PATH`, the build uses plain + `gcc`. Both observed working. + +## First real-Actions results (2026-06-10, branch `build-and-test-hardening`) + +Historical: no `TS_OAUTH_SECRET` was configured yet, so these runs validated +the degradation paths rather than actual distribution: + +- **linux** — build job green end-to-end (C unit suite, wasm export gate, + qcow2 smoke test, artifact). In the first dispatch (before the secret + gate existed) the coordinator failed fast (`config: oauth-secret is + required`) *after* putting the engine on `PATH`, so the build ran + `sccache gcc` in local mode: 113 compile requests, 0 hits / 63 misses, + cache on local disk — confirming layer 3 works in practice. Worker jobs + failed on the missing secret and marked the run red, which is why layer 2 + (clean skip) was added. After the gate landed, a second dispatch was + fully green: farm steps skipped, worker jobs green no-ops, build on + plain `gcc`. +- **mingw64** — never reached the farm: the msys2-cross toolchain install + failed with `error: target not found: msys-cross-pkgconfig`. Not a repo + regression after all — the msys.kosaka.moe repo had moved to per-target + pkgconf packages; fixed by renaming the package to + `msys-cross-mingw64-pkgconf` in the workflow (`5b26d20`), after which the + PR leg went green (20m26s cold). Cross-distribution was proven later, see + below. +- **ts** — no farm (pure TS unit suites). First runs surfaced two CI-only + bugs (workspace `file:` dependency on a sibling `drivelist-anyfs` + checkout + `@anyfs/native`'s node-gyp script needing the LKL tree; + missing gitignored nbd-proxy qcow2 fixture); fixed by excluding both + packages from the CI install and generating the fixture on the runner. + Green after the fixes. + +## Farm-mode results (2026-06-11, `workflow_dispatch` on `build-and-test-hardening`) + +With `TS_OAUTH_SECRET` configured and the action at `@v0.0.5` + +`tags: 'tag:ci'`, both legs were dispatched (PR runs still skip the farm by +design and validate the fallback path — all three PR checks at `5b26d20` +are green: ts 39s, linux 8m07s, mingw64 20m26s cold): + +- **linux** (run 27314323591, 3m22s, green) — both workers registered + (`[coord] 2/2 workers registered`), LKL/QEMU/anyfs build steps ran with + `--cc="sccache gcc"`, and `sccache stats` showed 47 compile requests with + 38 cache hits / 0 misses (100% hit rate — the local-disk cache restored + via `actions/cache` was fully warm, so there was nothing left to + distribute; 0 failed distributed compilations). Distribution at scale on + this engine is proven by the llvm-wasm leg below. +- **mingw64** (run 27314325180, ~5m45s build job, green) — both workers + registered, build steps used the absolute-path + `sccache /opt/msys2-cross/bin/x86_64-w64-mingw32-gcc`, and **26 jobs were + successfully executed on a worker** (0 failed distributed, 0 dist + errors) — first end-to-end proof that the msys2-cross toolchain ships to + and runs on farm workers. Caveat: LKL/QEMU build dirs were + `actions/cache`-warm, so the distributed jobs were mostly meson/configure + feature probes (26 of 27 compilations were expected-failure checks); a + large cold mingw64 build has not yet exercised distribution at volume, + which is why the leg stays marked experimental. + +## wasm-ld release pipeline (xdqi/llvm-wasm) + +The browser bundle needs a patched `wasm-ld`. The fork branch +`ci/wasm-ld-release` on `xdqi/llvm-wasm` carries a `wasm-ld-release.yml` +workflow that builds it with `zig cc` (target gnu.2.11 for old-glibc +compatibility) on a 10-worker farm and publishes a tagged release. + +**Status: shipped.** Release +[`wasm-ld-18.1.2-anyfs-r1`](https://github.com/xdqi/llvm-wasm/releases/tag/wasm-ld-18.1.2-anyfs-r1) +(asset `wasm-ld-linux-amd64.tar.xz`) is live and consumed by +`scripts/fetch_wasm_ld.sh` (wired through the config `.toolchain` hook, +commit `6086876`). + +The release run (27290137879, 2026-06-10) is also the large-scale proof of +the farm: building lld took **27.5 min** end to end with **1486 +successfully distributed compiles spread across all 10 workers** (per-worker +counts 78–339), 172 failed-distributed jobs that fell back to local compile, +and 1787 compile requests total. This run is also where the `tag:ci` +requirement was discovered: with the action-default `tag:ci-sccache` no +worker could join; after switching to `tags: 'tag:ci'` the farm formed +(coordinator proceeded at 4/10 registered — `min-workers` — and the +remaining workers joined and served jobs during the build). + +Re-trigger if a new wasm-ld is needed: + +```sh +gh workflow run wasm-ld-release.yml -R xdqi/llvm-wasm \ + --ref ci/wasm-ld-release -f tag=wasm-ld-18.1.2-anyfs-r2 +``` + +## Known warts + +- **Pin `@v0.0.5` or later.** Earlier action versions shipped an engine + built without the S3 feature, so `[cache.s3]` config silently fell back + to local disk; v0.0.5 fixed the engine. The anyfs workflows still use + per-runner local-disk cache persisted via `actions/cache` + (`sccache--` with prefix restore-keys), so cross-run sharing + is only as good as the GitHub cache. +- **`tags: 'tag:ci'` is mandatory** (see "Required secret"): the OAuth + client owns `tag:ci`, and the action default `tag:ci-sccache` makes + worker join fail silently. +- **mingw64 distribution is experimental**: toolchain shipping + remote + execution is now proven (26 distributed jobs, 2026-06-11 dispatch), but + only at configure-probe volume; a cold full build over the farm is still + unexercised. +- The first-dispatch behavior (failed coordinator still leaving `sccache` + on `PATH`) means "farm down" and "farm absent" take slightly different + code paths; both are green, but log output differs (`sccache stats` vs + none). diff --git a/docs/superpowers/plans/2026-05-31-nbd-fd-transport-poc.md b/docs/superpowers/plans/2026-05-31-nbd-fd-transport-poc.md new file mode 100644 index 0000000..cfa6e1e --- /dev/null +++ b/docs/superpowers/plans/2026-05-31-nbd-fd-transport-poc.md @@ -0,0 +1,1284 @@ +# NBD-over-inherited-fd Transport PoC — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Prove that a native QEMU child can open a qcow2 disk image served over an NBD protocol channel carried on an *inherited socketpair fd* (no TCP listen), with format drivers auto-layered atop NBD. + +**Architecture:** A Node parent creates a socketpair via a small N-API addon, runs a hand-written read-only NBD server on one end, and spawns the `anyfs-lspart` C binary with the other end inherited (non-CLOEXEC) and passed as `--nbd-fd N`. lspart turns `nbd-fd:N` into a QEMU `blk_new_open` with `server.type=fd, server.str="N"`; QEMU's no-monitor `socket_get_fd` adopts the raw fd via `qio_channel_socket_new_fd`, so **zero QEMU source changes** are needed. Windows falls back to `127.0.0.1` loopback. + +**Tech Stack:** C (anyfs core + lspart, meson/ninja), QEMU libblock (prebuilt), Node 24 (`net`/`fs`, ES modules), node-addon-api v8 (N-API addon), `qemu-img` for cross-validation. + +**Spec:** `docs/superpowers/specs/2026-05-31-nbd-fd-transport-poc-design.md` + +--- + +## Reference facts (verified — do not re-investigate) + +- `anyfs-lspart` source: `src/lspart/lspart_main.c` (125 lines). It opens images with + `anyfs_session_open(path, ANYFS_SESSION_READONLY, &d)` (line ~95). `image_path` flows + opaquely through `anyfs_session_open` → `anyfs_disk_add` → `qemu_backend_ops.open` + → `qemu_blk_open` → `blk_new_open`. **No intermediate layer parses the path string.** +- Flags (`include/anyfs.h`): `ANYFS_SESSION_READONLY = 1u<<0`, `ANYFS_BACKEND_RAW = + 1u<<1`, `ANYFS_BACKEND_QEMU = 1u<<3`, `ANYFS_SESSION_SNAPSHOT = 1u<<4`. +- QEMU backend auto-selects when `ANYFS_HAS_QEMU` is defined (it is, in the linux build: + `meson.build:242 qemu_args = ['-DANYFS_HAS_QEMU=1']`). So just passing `nbd-fd:N` as + the image path reaches `qemu_blk_open`. +- `src/core/qemu_backend.c` already `#include "qobject/qdict.h"` and uses + `qdict_new()` / `qdict_put_str()` / `qdict_put_int()`. The existing `is_url` branch is + at lines ~175-184; the new `nbd-fd` branch slots in beside it. +- QEMU NBD QDict keys (`block/nbd.c:nbd_config` extracts `server.` subdict and visits it + as a `SocketAddress`; `FdSocketAddress` has `type` + `str`): + **`server.type = "fd"`, `server.str = ""`.** +- QEMU's `util/qemu-sockets.c:socket_get_fd()` accepts a bare numeric fd when + `monitor_cur() == NULL` (our case), then requires `fd_is_socket(fd)`. +- **CONFIRMED IN STAGE 1:** `qemu_blk_open` must call `module_call_init(MODULE_INIT_QOM)` + (needs `#include "qemu/module.h"`) before `bdrv_init()`. The NBD client adopts the fd + via a `qio-channel-socket`, a QOM type registered under MODULE_INIT_QOM; `bdrv_init()` + alone (MODULE_INIT_BLOCK) leaves it unregistered → `unknown type 'qio-channel-socket'`. + QEMU's own tools call both. This is a latent anyfs-glue init gap, NOT a QEMU source + change — fix lives in `src/core/qemu_backend.c`, guarded by the one-time + `qemu_initialized` flag, harmless for plain-file/URL opens. (committed in e183b91) +- Build: `scripts/build_anyfs.sh` runs `meson setup build-anyfs-linux-amd64` + ninja. + lspart target: `src/lspart/meson.build` → executable `anyfs-lspart`. +- Tooling present: `qemu-img`, `node v24.15.0`, `meson`, `ninja`. **`nbdinfo` is NOT + installed** — cross-validation uses `qemu-img info nbd:...` instead. +- `node-addon-api` ^8 is already a dependency in `ts/packages/anyfs-native`; its + `binding.gyp` is the skeleton to mirror for the transport addon. +- Existing test image: `tests/images/ext4.img` (raw, 32 MiB). The PoC will create a + qcow2 test image from it. + +## File structure + +| File | New/Mod | Responsibility | +|---|---|---| +| `src/core/qemu_backend.c` | Modify | Parse `nbd-fd:N` / `nbd-port:P` image paths → build `server.*` QDict → `blk_new_open` | +| `src/lspart/lspart_main.c` | Modify | Accept `--nbd-fd N` / `--nbd-port P`, synthesize `nbd-fd:N` pseudo-path | +| `scripts/poc-nbd/native-transport/binding.gyp` | New | gyp build for the transport addon | +| `scripts/poc-nbd/native-transport/transport.c` | New | N-API: `socketpair()` → `[fd0,fd1]`; reserved Windows loopback slot | +| `scripts/poc-nbd/native-transport/package.json` | New | addon package + build script | +| `scripts/poc-nbd/nbd-server.mjs` | New | Hand-written read-only NBD newstyle server over a given fd | +| `scripts/poc-nbd/launch.mjs` | New | Parent: socketpair → spawn lspart (inherit fd) → run server; lifecycle binding | +| `scripts/poc-nbd/make-test-image.mjs` | New | Build a deterministic qcow2 test image + record verification offsets | +| `scripts/poc-nbd/test-stage1.mjs` | New | Stage 1: capacity-match, zero-source-change probe | +| `scripts/poc-nbd/test-stage2.mjs` | New | Stage 2: full Linux chain (partition list + byte verify + READ count) | +| `scripts/poc-nbd/FINDINGS.md` | New | Stage 3 (Windows/wine) findings log | + +--- + +## Task 1: Add the `nbd-fd` / `nbd-port` branch to qemu_backend + +**Files:** +- Modify: `src/core/qemu_backend.c` (the `qemu_blk_open` options block, ~lines 172-184) + +The existing code builds `options` only for URLs. We add: if `image_path` starts with +`nbd-fd:` or `nbd-port:`, build a `server.*` QDict and pass an empty filename to +`blk_new_open` (QEMU reads the server address from `options`, not from the filename). + +- [ ] **Step 1: Read the current options block to anchor the edit** + +Run: `sed -n '170,190p' src/core/qemu_backend.c` +Expected: shows the `is_url` block creating `options = qdict_new()` and the +`blk_new_open(image_path, NULL, options, bdrv_flags, &errp)` call. + +- [ ] **Step 2: Replace the options block with nbd-fd / nbd-port handling** + +Find this block (around lines 172-184): + +```c + /* QEMU's curl block driver defaults to CURLOPT_TIMEOUT=5s. Bump it for + * URL images (local files use the raw driver, which ignores "timeout"). + */ + QDict* options = NULL; + const int is_url = image_path && strstr(image_path, "://") != NULL; + if (is_url) { + options = qdict_new(); + qdict_put_int(options, "file.timeout", 20); + } + + fprintf(stderr, "[qemu_blk] blk_new_open flags=0x%x timeout=%d\n", + bdrv_flags, is_url ? 20 : 0); + BlockBackend* blk = + blk_new_open(image_path, NULL, options, bdrv_flags, &errp); +``` + +Replace it with: + +```c + /* QEMU's curl block driver defaults to CURLOPT_TIMEOUT=5s. Bump it for + * URL images (local files use the raw driver, which ignores "timeout"). + */ + QDict* options = NULL; + const int is_url = image_path && strstr(image_path, "://") != NULL; + + /* PoC: NBD-over-inherited-fd transport. image_path "nbd-fd:N" means the + * NBD client should adopt already-open socket fd N (passed by the parent + * via inheritance). "nbd-port:P" is the Windows fallback: connect to a + * 127.0.0.1 loopback on port P. In both cases the server address comes + * from the options QDict (server.*), so the filename handed to + * blk_new_open must be NULL. */ + const char* open_name = image_path; + if (image_path && strncmp(image_path, "nbd-fd:", 7) == 0) { + options = qdict_new(); + qdict_put_str(options, "driver", "nbd"); + qdict_put_str(options, "server.type", "fd"); + qdict_put_str(options, "server.str", image_path + 7); + open_name = NULL; + } else if (image_path && strncmp(image_path, "nbd-port:", 9) == 0) { + options = qdict_new(); + qdict_put_str(options, "driver", "nbd"); + qdict_put_str(options, "server.type", "inet"); + qdict_put_str(options, "server.host", "127.0.0.1"); + qdict_put_str(options, "server.port", image_path + 9); + open_name = NULL; + } else if (is_url) { + options = qdict_new(); + qdict_put_int(options, "file.timeout", 20); + } + + fprintf(stderr, "[qemu_blk] blk_new_open name=%s flags=0x%x\n", + open_name ? open_name : "(nbd via options)", bdrv_flags); + BlockBackend* blk = + blk_new_open(open_name, NULL, options, bdrv_flags, &errp); +``` + +- [ ] **Step 3: Build lspart and confirm it compiles** + +Run: `bash scripts/build_anyfs.sh --target linux-amd64 lspart 2>&1 | tail -20` +(If `--target`/component syntax differs, fall back to: +`cd ~/anyfs-reader && meson setup build-anyfs-linux-amd64 2>/dev/null; ninja -C build-anyfs-linux-amd64 anyfs-lspart`) +Expected: compiles with no errors; `build-anyfs-linux-amd64/anyfs-lspart` exists. + +- [ ] **Step 4: Confirm existing plain-file path still works (regression)** + +Run: `build-anyfs-linux-amd64/anyfs-lspart tests/images/ext4.img` +Expected: prints a partition/disk table (the raw ext4 image lists as a whole-disk +filesystem). No crash. This proves the new branch didn't break the default path. + +- [ ] **Step 5: Commit** + +```bash +git add src/core/qemu_backend.c +git commit -m "feat(qemu_backend): add nbd-fd / nbd-port image-path branch for PoC + +nbd-fd:N builds a server.type=fd QDict so QEMU's NBD client adopts an +inherited socket fd; nbd-port:P is the Windows 127.0.0.1 loopback +fallback. Filename to blk_new_open is NULL in both cases (server comes +from options). Zero QEMU source changes. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 2: Add `--nbd-fd` / `--nbd-port` flags to lspart + +**Files:** +- Modify: `src/lspart/lspart_main.c` + +Add flag parsing that synthesizes an `nbd-fd:N` (or `nbd-port:P`) pseudo-path and pushes +it into the same `images[]` array the existing loop opens. This keeps the open/format +path identical — only the path string differs. + +- [ ] **Step 1: Add a small synthesized-path buffer near the top of `main`** + +After the existing declarations in `main` (the `const char* images[16]; int n_images = +0;` block), the synthesized path needs storage that outlives the parse loop. Add a +static buffer just below them: + +```c + int json = 0; + const char* images[16]; + int n_images = 0; + static char nbd_path[32]; /* holds a synthesized "nbd-fd:N"/"nbd-port:P" */ +``` + +- [ ] **Step 2: Add flag handling inside the argv loop** + +In the `for (int i = 1; i < argc; i++)` loop, after the `--json` handler and before the +`if (a[0] == '-' && a[1] != '\0')` unknown-flag check, insert: + +```c + if (strcmp(a, "--nbd-fd") == 0 || strcmp(a, "--nbd-port") == 0) { + if (i + 1 >= argc) { + fprintf(stderr, "%s needs an argument\n", a); + return 2; + } + const char* kind = (a[6] == 'f') ? "nbd-fd" : "nbd-port"; + snprintf(nbd_path, sizeof(nbd_path), "%s:%s", kind, + argv[++i]); + if (n_images >= (int)(sizeof(images) / + sizeof(images[0]))) { + fprintf(stderr, "too many images\n"); + return 2; + } + images[n_images++] = nbd_path; + continue; + } +``` + +- [ ] **Step 3: Update the usage string to mention the new flags** + +In `usage()`, change the first `fprintf` format line from: + +```c + "Usage: %s [--json] [--help] [?] [...]\n" +``` + +to: + +```c + "Usage: %s [--json] [--help] [--nbd-fd N | --nbd-port P] " + "[?] [...]\n" +``` + +- [ ] **Step 4: Rebuild lspart** + +Run: `ninja -C build-anyfs-linux-amd64 anyfs-lspart 2>&1 | tail -10` +Expected: compiles cleanly; binary updated. + +- [ ] **Step 5: Confirm the flag is accepted (will fail to open — fd N invalid — but must parse)** + +Run: `build-anyfs-linux-amd64/anyfs-lspart --nbd-fd 999 2>&1 | head -5` +Expected: it attempts to open `nbd-fd:999` and fails with a QEMU "not a socket" / open +error (fd 999 isn't a valid socket). The point is that **parsing succeeds** and it +reaches `qemu_blk_open` — not an "unknown flag" error. + +- [ ] **Step 6: Commit** + +```bash +git add src/lspart/lspart_main.c +git commit -m "feat(lspart): accept --nbd-fd N / --nbd-port P flags + +Synthesizes an nbd-fd:N (or nbd-port:P) pseudo-path and feeds it through +the existing anyfs_session_open path unchanged. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 3: Native transport addon (socketpair) + +**Files:** +- Create: `scripts/poc-nbd/native-transport/transport.c` +- Create: `scripts/poc-nbd/native-transport/binding.gyp` +- Create: `scripts/poc-nbd/native-transport/package.json` + +A minimal N-API addon exporting `socketpair()` returning `[fd0, fd1]` (two bare ints). +This is the seed of the unified Node-layer native transport abstraction. + +- [ ] **Step 1: Write the addon C source** + +Create `scripts/poc-nbd/native-transport/transport.c`: + +```c +/* + * native-transport — minimal N-API addon for the NBD-over-fd PoC. + * + * Exports socketpair() -> [fd0, fd1]: a connected AF_UNIX/SOCK_STREAM + * pair. The parent keeps one fd for the NBD server and lets the child + * inherit the other (non-CLOEXEC) to use as QEMU's NBD transport. + * + * This is the seed of a unified Node-layer native transport abstraction; + * a Windows loopback-pair helper will be added here later. + */ +#define NAPI_VERSION 8 +#include +#include +#include +#include + +static napi_value Socketpair(napi_env env, napi_callback_info info) +{ + int fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0) { + napi_throw_error(env, NULL, "socketpair() failed"); + return NULL; + } + /* Clear CLOEXEC on both ends so the child can inherit fds[1]. + * (socketpair does not set CLOEXEC by default on Linux, but be + * explicit so the contract is obvious.) */ + for (int i = 0; i < 2; i++) { + int fl = fcntl(fds[i], F_GETFD); + if (fl >= 0) + fcntl(fds[i], F_SETFD, fl & ~FD_CLOEXEC); + } + + napi_value arr, a, b; + napi_create_array_with_length(env, 2, &arr); + napi_create_int32(env, fds[0], &a); + napi_create_int32(env, fds[1], &b); + napi_set_element(env, arr, 0, a); + napi_set_element(env, arr, 1, b); + return arr; +} + +static napi_value Init(napi_env env, napi_value exports) +{ + napi_value fn; + napi_create_function(env, "socketpair", NAPI_AUTO_LENGTH, Socketpair, + NULL, &fn); + napi_set_named_property(env, exports, "socketpair", fn); + return exports; +} + +NAPI_MODULE(NODE_GYP_MODULE_NAME, Init) +``` + +- [ ] **Step 2: Write binding.gyp** + +Create `scripts/poc-nbd/native-transport/binding.gyp`: + +```python +{ + "targets": [ + { + "target_name": "native_transport", + "sources": ["transport.c"], + "cflags_c": ["-D_GNU_SOURCE"] + } + ] +} +``` + +- [ ] **Step 3: Write package.json** + +Create `scripts/poc-nbd/native-transport/package.json`: + +```json +{ + "name": "native-transport", + "version": "0.0.0", + "private": true, + "description": "PoC N-API addon: socketpair() for NBD-over-fd transport", + "main": "index.js", + "scripts": { + "build": "node-gyp rebuild" + } +} +``` + +- [ ] **Step 4: Build the addon** + +Run: +```bash +cd ~/anyfs-reader/scripts/poc-nbd/native-transport && npx node-gyp rebuild 2>&1 | tail -15 +``` +Expected: builds `build/Release/native_transport.node` with no errors. + +- [ ] **Step 5: Smoke-test the addon from Node** + +Run: +```bash +cd ~/anyfs-reader/scripts/poc-nbd/native-transport && node -e "const t=require('./build/Release/native_transport.node'); const [a,b]=t.socketpair(); console.log('fds', a, b); const fs=require('fs'); fs.writeSync(a, Buffer.from('hi')); const buf=Buffer.alloc(2); fs.readSync(b, buf, 0, 2, null); console.log('roundtrip', buf.toString());" +``` +Expected: prints two distinct fd numbers, then `roundtrip hi` — confirming the pair is a +real connected socket. + +- [ ] **Step 6: Commit** + +```bash +git add scripts/poc-nbd/native-transport/ +git commit -m "feat(poc-nbd): native-transport addon exposing socketpair() + +Minimal N-API addon returning a connected AF_UNIX socket pair with +CLOEXEC cleared, for inheriting one end into the QEMU child. Seed of the +Node-layer native transport abstraction. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 4: Hand-written read-only NBD server + +**Files:** +- Create: `scripts/poc-nbd/nbd-server.mjs` + +Implements the NBD newstyle fixed handshake + `NBD_OPT_GO` + a command loop handling +`NBD_CMD_READ` / `NBD_CMD_FLUSH` / `NBD_CMD_DISC`. PoC: synchronous `fs.readSync`, one +request at a time. Operates on a Node `net.Socket` wrapping an existing fd. + +Protocol constants (NBD newstyle): +- Handshake: server sends magic `NBDMAGIC` (0x4e42444d41474943) + `IHAVEOPT` + (0x49484156454F5054) + 16-bit handshake flags `NBD_FLAG_FIXED_NEWSTYLE`(1) | + `NBD_FLAG_NO_ZEROES`(2) = 3. +- Client replies 32-bit client flags. Then option haggling: client sends `IHAVEOPT` + + 32-bit option + 32-bit len + data. +- We support `NBD_OPT_GO`(7) and `NBD_OPT_INFO`(6) (treat the same), and `NBD_OPT_ABORT`(2). + For GO we send `NBD_REP_INFO`(3) with `NBD_INFO_EXPORT`(0): export size (u64) + transmission + flags (u16: `NBD_FLAG_HAS_FLAGS`(1) | `NBD_FLAG_READ_ONLY`(2) = 3), then + `NBD_REP_ACK`(1). With NO_ZEROES negotiated, transmission phase begins immediately. +- Transmission: client request = magic 0x25609513 + 16-bit cmd flags + 16-bit type + + 64-bit handle + 64-bit offset + 32-bit length. Reply (simple) = magic 0x67446698 + + 32-bit error + 64-bit handle + (data for READ). +- Commands: `NBD_CMD_READ`(0), `NBD_CMD_WRITE`(1), `NBD_CMD_DISC`(2), `NBD_CMD_FLUSH`(3). + +- [ ] **Step 1: Write the NBD server module** + +Create `scripts/poc-nbd/nbd-server.mjs`: + +```javascript +/* + * Hand-written read-only NBD newstyle server for the NBD-over-fd PoC. + * + * PoC scope: synchronous fs.readSync, one request at a time. The + * production server (spec §9) is fully async with multiple in-flight + * requests and a keep-alive http upstream — out of scope here. + * + * startNbdServer(socket, imageFd, size, onRead?) drives one client on an + * already-connected socket. Returns a promise that resolves on clean + * NBD_CMD_DISC / EOF. + */ +import { Buffer } from 'node:buffer'; +import fs from 'node:fs'; + +const NBDMAGIC = 0x4e42444d41474943n; +const IHAVEOPT = 0x49484156454f5054n; +const REQ_MAGIC = 0x25609513; +const SIMPLE_REPLY_MAGIC = 0x67446698; + +const NBD_FLAG_FIXED_NEWSTYLE = 1; +const NBD_FLAG_NO_ZEROES = 2; + +const NBD_OPT_ABORT = 2; +const NBD_OPT_INFO = 6; +const NBD_OPT_GO = 7; + +const NBD_REP_ACK = 1n; +const NBD_REP_INFO = 3n; +const NBD_REP_FLAG_ERROR = 0x80000000n; +const NBD_REP_ERR_UNSUP = NBD_REP_FLAG_ERROR | 1n; + +const NBD_INFO_EXPORT = 0; +const NBD_FLAG_HAS_FLAGS = 1; +const NBD_FLAG_READ_ONLY = 2; + +const NBD_CMD_READ = 0; +const NBD_CMD_DISC = 2; +const NBD_CMD_FLUSH = 3; + +const EINVAL = 22; + +/* Read exactly n bytes from a socket, buffering across 'data' events. */ +function makeReader(socket) { + let chunks = []; + let buffered = 0; + let want = 0; + let resolveWant = null; + + socket.on('data', (d) => { + chunks.push(d); + buffered += d.length; + tryResolve(); + }); + + function tryResolve() { + if (resolveWant && buffered >= want) { + const all = Buffer.concat(chunks); + const out = all.subarray(0, want); + const rest = all.subarray(want); + chunks = rest.length ? [rest] : []; + buffered = rest.length; + const r = resolveWant; + resolveWant = null; + r(out); + } + } + + return (n) => + new Promise((resolve, reject) => { + want = n; + resolveWant = resolve; + socket.once('error', reject); + socket.once('end', () => { + if (resolveWant) { + resolveWant = null; + reject(new Error('EOF')); + } + }); + tryResolve(); + }); +} + +export async function startNbdServer(socket, imageFd, size, onRead) { + const read = makeReader(socket); + const write = (buf) => + new Promise((res, rej) => socket.write(buf, (e) => (e ? rej(e) : res()))); + + /* --- Handshake --- */ + const hello = Buffer.alloc(18); + hello.writeBigUInt64BE(NBDMAGIC, 0); + hello.writeBigUInt64BE(IHAVEOPT, 8); + hello.writeUInt16BE(NBD_FLAG_FIXED_NEWSTYLE | NBD_FLAG_NO_ZEROES, 16); + await write(hello); + + await read(4); /* client flags (ignored for PoC) */ + + /* --- Option haggling: loop until GO/INFO (enter transmission) or ABORT --- */ + for (;;) { + const optHdr = await read(16); + const optMagic = optHdr.readBigUInt64BE(0); + const opt = optHdr.readUInt32BE(8); + const optLen = optHdr.readUInt32BE(12); + if (optMagic !== IHAVEOPT) throw new Error('bad option magic'); + const optData = optLen ? await read(optLen) : Buffer.alloc(0); + + if (opt === NBD_OPT_GO || opt === NBD_OPT_INFO) { + /* NBD_REP_INFO + NBD_INFO_EXPORT */ + const info = Buffer.alloc(12 /* size+flags */); + info.writeBigUInt64BE(BigInt(size), 0); + info.writeUInt16BE(NBD_FLAG_HAS_FLAGS | NBD_FLAG_READ_ONLY, 8); + info.writeUInt16BE(NBD_INFO_EXPORT, 10); + /* reorder: INFO type (u16) must precede payload */ + const infoReply = Buffer.alloc(2 + 10); + infoReply.writeUInt16BE(NBD_INFO_EXPORT, 0); + infoReply.writeBigUInt64BE(BigInt(size), 2); + infoReply.writeUInt16BE(NBD_FLAG_HAS_FLAGS | NBD_FLAG_READ_ONLY, 10); + await sendOptReply(write, opt, NBD_REP_INFO, infoReply); + await sendOptReply(write, opt, NBD_REP_ACK, Buffer.alloc(0)); + if (opt === NBD_OPT_GO) break; /* INFO-only would loop; GO enters xmit */ + void optData; + } else if (opt === NBD_OPT_ABORT) { + await sendOptReply(write, opt, NBD_REP_ACK, Buffer.alloc(0)); + socket.end(); + return; + } else { + await sendOptReply(write, opt, NBD_REP_ERR_UNSUP, Buffer.alloc(0)); + } + } + + /* --- Transmission phase --- */ + for (;;) { + let hdr; + try { + hdr = await read(28); + } catch { + return; /* EOF / disconnect */ + } + const magic = hdr.readUInt32BE(0); + if (magic !== REQ_MAGIC) throw new Error('bad request magic'); + const type = hdr.readUInt16BE(6); + const handle = hdr.subarray(8, 16); /* opaque 8 bytes */ + const offset = hdr.readBigUInt64BE(16); + const length = hdr.readUInt32BE(24); + + if (type === NBD_CMD_DISC) return; + + if (type === NBD_CMD_READ) { + if (offset + BigInt(length) > BigInt(size)) { + await sendSimpleReply(write, EINVAL, handle, null); + continue; + } + const data = Buffer.alloc(length); + fs.readSync(imageFd, data, 0, length, Number(offset)); + if (onRead) onRead(Number(offset), length); + await sendSimpleReply(write, 0, handle, data); + } else if (type === NBD_CMD_FLUSH) { + await sendSimpleReply(write, 0, handle, null); + } else { + await sendSimpleReply(write, EINVAL, handle, null); + } + } +} + +async function sendOptReply(write, opt, repType, payload) { + const hdr = Buffer.alloc(20); + hdr.writeBigUInt64BE(0x0003e889045565a9n, 0); /* NBD_REP option-reply magic */ + hdr.writeUInt32BE(opt, 8); + hdr.writeBigUInt64BE(repType, 12); /* 8 bytes: 4 rep type at off 12? */ + /* NBD option reply: magic(8) + opt(4) + reptype(4) + len(4) */ + const fixed = Buffer.alloc(16); + fixed.writeBigUInt64BE(0x0003e889045565a9n, 0); + fixed.writeUInt32BE(opt, 8); + fixed.writeUInt32BE(Number(repType & 0xffffffffn), 12); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(payload.length, 0); + await write(Buffer.concat([fixed, lenBuf, payload])); +} + +async function sendSimpleReply(write, error, handle, data) { + const hdr = Buffer.alloc(16); + hdr.writeUInt32BE(SIMPLE_REPLY_MAGIC, 0); + hdr.writeUInt32BE(error >>> 0, 4); + handle.copy(hdr, 8); + await write(data ? Buffer.concat([hdr, data]) : hdr); +} +``` + +> NOTE: the NBD option-reply magic is `0x0003e889045565a9`. The `sendOptReply` helper +> above contains a leftover dead block; clean it to just the `fixed`/`lenBuf`/`payload` +> path in Step 2. + +- [ ] **Step 2: Clean up sendOptReply (remove the dead hdr block)** + +Replace the `sendOptReply` function body with the correct single-path version: + +```javascript +async function sendOptReply(write, opt, repType, payload) { + /* NBD option reply: magic(8) + opt(4) + reptype(4) + len(4) + payload */ + const fixed = Buffer.alloc(16); + fixed.writeBigUInt64BE(0x0003e889045565a9n, 0); + fixed.writeUInt32BE(opt, 8); + fixed.writeUInt32BE(Number(repType & 0xffffffffn), 12); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(payload.length, 0); + await write(Buffer.concat([fixed, lenBuf, payload])); +} +``` + +Also remove the now-unused `info` buffer block inside the GO handler (keep only +`infoReply`). + +- [ ] **Step 3: Syntax-check the module** + +Run: `node --check scripts/poc-nbd/nbd-server.mjs` +Expected: no output (valid syntax). (Behavior is validated by cross-check in Task 6.) + +- [ ] **Step 4: Commit** + +```bash +git add scripts/poc-nbd/nbd-server.mjs +git commit -m "feat(poc-nbd): hand-written read-only NBD newstyle server + +newstyle fixed handshake + NBD_OPT_GO export info + READ/FLUSH/DISC +command loop. PoC: synchronous fs.readSync, one request at a time. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 5: Test-image generator + +**Files:** +- Create: `scripts/poc-nbd/make-test-image.mjs` + +Produces a deterministic qcow2 with a GPT + one partition, and records a known byte +pattern at a known offset for Stage-2 verification. + +- [ ] **Step 1: Write the generator** + +> **PLAN DEVIATION (verified 2026-05-31):** the original idea was to convert +> `tests/images/ext4.img` to qcow2 and verify the ext4 superblock magic. That file is +> in fact **all-zeros** (0 nonzero bytes), and `mke2fs`/`mkfs.ext4`/`sgdisk` are **not +> installed**, so there is no real filesystem to anchor on. Instead we **synthesize a raw +> image with a deterministic, self-generated byte pattern** (no fs tooling needed), +> convert it to qcow2 with `qemu-img` (which IS available), and use that pattern as the +> verification anchor. This still proves the real goal — "qcow2-over-NBD serves the +> genuine image content correctly" — and is actually a stronger Stage-2 check because the +> anchor bytes are read back *through* the qcow2-over-NBD chain (see Task 8), not from a +> side control file. + +Create `scripts/poc-nbd/make-test-image.mjs`: + +```javascript +/* + * Build a deterministic qcow2 test image for the NBD-over-fd PoC. + * + * Strategy: synthesize a raw image filled with a deterministic byte + * pattern (no filesystem tooling needed), then convert it to qcow2 with + * qemu-img. We embed a recognizable marker at a known offset and record + * it as the verification anchor. Stage 2 reads that marker back through + * the qcow2-over-NBD chain to prove byte-accurate delivery. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const outDir = path.join(here, 'fixtures'); +const rawImg = path.join(outDir, 'test.raw'); +const qcow = path.join(outDir, 'test.qcow2'); + +const SIZE = 8 * 1024 * 1024; /* 8 MiB — small, fast to convert */ +const VERIFY_OFFSET = 0x100000; /* 1 MiB in: well past the qcow2 header region */ +const MARKER = Buffer.from('ANYFS-NBD-POC-MARKER-v1', 'ascii'); + +fs.mkdirSync(outDir, { recursive: true }); + +/* Deterministic fill: byte i = i & 0xff, so every offset has a predictable + * value. Then stamp the ASCII marker at VERIFY_OFFSET. */ +const raw = Buffer.alloc(SIZE); +for (let i = 0; i < SIZE; i++) raw[i] = i & 0xff; +MARKER.copy(raw, VERIFY_OFFSET); +fs.writeFileSync(rawImg, raw); + +execFileSync('qemu-img', ['convert', '-f', 'raw', '-O', 'qcow2', rawImg, qcow], { + stdio: 'inherit', +}); + +const meta = { + qcow2: qcow, + raw: rawImg, + size: SIZE, + verifyOffset: VERIFY_OFFSET, + verifyBytesHex: MARKER.toString('hex'), + verifyAscii: MARKER.toString('ascii'), +}; +fs.writeFileSync(path.join(outDir, 'meta.json'), JSON.stringify(meta, null, 2)); +console.log(JSON.stringify(meta, null, 2)); +``` + +- [ ] **Step 2: Run the generator** + +Run: `node scripts/poc-nbd/make-test-image.mjs` +Expected: prints JSON with `qcow2` path, `verifyOffset: 1048576`, `verifyAscii: +"ANYFS-NBD-POC-MARKER-v1"`, and the corresponding `verifyBytesHex`. Both +`fixtures/test.raw` and `fixtures/test.qcow2` exist. + +- [ ] **Step 3: Sanity-check the qcow2 with qemu-img** + +Run: `qemu-img info scripts/poc-nbd/fixtures/test.qcow2` +Expected: `file format: qcow2`, virtual size 8 MiB. Note the qcow2 file-on-disk is much +smaller than 8 MiB (sparse), which is fine — what matters is the virtual size. + +- [ ] **Step 4: Commit (fixtures gitignored)** + +```bash +echo "fixtures/" > scripts/poc-nbd/.gitignore +git add scripts/poc-nbd/make-test-image.mjs scripts/poc-nbd/.gitignore +git commit -m "feat(poc-nbd): deterministic qcow2 test-image generator + +Synthesizes an 8 MiB raw image with a deterministic byte pattern (i&0xff) +plus an ASCII marker at 1 MiB, then converts to qcow2 via qemu-img. The +marker is the Stage-2 byte-verification anchor read back through the +qcow2-over-NBD chain. (ext4.img is all-zeros and no fs tooling is +installed, so we generate our own deterministic content.) + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 6: Cross-validate the NBD server with qemu-img (decouple protocol bugs) + +**Files:** +- Create: `scripts/poc-nbd/launch.mjs` (the reusable parent; used here and in Tasks 7-8) + +Before involving lspart, prove the hand-written server speaks correct NBD by having the +**real QEMU client** (`qemu-img info`) connect to it. Since `qemu-img` needs a real +endpoint, this cross-check uses a Unix-domain socket path (not the inherited fd) — it +only validates protocol correctness, not the fd transport. + +- [ ] **Step 1: Write the launcher with both a fd-pair mode and a unix-socket cross-check mode** + +Create `scripts/poc-nbd/launch.mjs`: + +```javascript +/* + * Parent/launcher for the NBD-over-fd PoC. + * + * Two entry points: + * serveOnFd(imagePath): create a socketpair, run the NBD server on fd0, + * return fd1 (the inheritable end) + a stop() handle. + * serveOnUnixSocket(imagePath, sockPath): listen on a unix socket and + * serve one client — used to cross-validate against qemu-img. + */ +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFileSync } from 'node:child_process'; +import { startNbdServer } from './nbd-server.mjs'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const addon = require_addon(); + +function require_addon() { + const p = path.join(here, 'native-transport/build/Release/native_transport.node'); + return require_cjs(p); +} +function require_cjs(p) { + /* ES module: use createRequire for the .node addon */ + return createRequireLocal()(p); +} +import { createRequire } from 'node:module'; +function createRequireLocal() { + return createRequire(import.meta.url); +} + +function imageSize(imagePath) { + /* For qcow2 we report the VIRTUAL size (what QEMU's NBD client maps), + * but our server serves RAW bytes of the file. The NBD export must be + * the raw file size so QEMU can probe the qcow2 header itself. */ + return fs.statSync(imagePath).size; +} + +export function serveOnFd(imagePath) { + const [fd0, fd1] = addon.socketpair(); + const imageFd = fs.openSync(imagePath, 'r'); + const size = imageSize(imagePath); + const sock = new net.Socket({ fd: fd0 }); + let reads = 0; + const done = startNbdServer(sock, imageFd, size, () => { + reads++; + }).catch((e) => { + if (!/EOF/.test(String(e))) console.error('[nbd-server]', e); + }); + return { + fd1, + stop: () => { + try { + sock.destroy(); + } catch {} + try { + fs.closeSync(imageFd); + } catch {} + }, + readCount: () => reads, + done, + }; +} + +export function serveOnUnixSocket(imagePath, sockPath) { + const imageFd = fs.openSync(imagePath, 'r'); + const size = imageSize(imagePath); + try { + fs.unlinkSync(sockPath); + } catch {} + const server = net.createServer((sock) => { + startNbdServer(sock, imageFd, size).catch((e) => { + if (!/EOF/.test(String(e))) console.error('[nbd-server]', e); + }); + }); + return new Promise((resolve) => { + server.listen(sockPath, () => resolve({ server, imageFd })); + }); +} +``` + +> NOTE: ESM + a `.node` addon needs `createRequire`. Step 2 fixes the import ordering +> (imports must be top-level) — fold the `createRequire` import to the top. + +- [ ] **Step 2: Fix the launcher's require/import ordering** + +Replace the top of `launch.mjs` (everything from the first import down to the +`require_addon` helpers) with clean top-level imports: + +```javascript +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import { startNbdServer } from './nbd-server.mjs'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); +const addon = require( + path.join(here, 'native-transport/build/Release/native_transport.node'), +); +``` + +Delete the now-redundant `require_addon`, `require_cjs`, `createRequireLocal` helpers and +the mid-file `import { createRequire }` and `import { execFileSync }` lines (execFileSync +isn't used in this file). + +- [ ] **Step 3: Write a one-shot cross-check script inline and run it** + +Run: +```bash +cd ~/anyfs-reader && node --input-type=module -e " +import { serveOnUnixSocket } from './scripts/poc-nbd/launch.mjs'; +import { execFileSync } from 'node:child_process'; +const meta = JSON.parse(require('fs').readFileSync('scripts/poc-nbd/fixtures/meta.json')); +const sockPath = '/tmp/poc-nbd-xcheck.sock'; +const { server } = await serveOnUnixSocket(meta.qcow2, sockPath); +try { + const out = execFileSync('qemu-img', ['info', 'nbd+unix:///?socket=' + sockPath], { encoding: 'utf8' }); + console.log(out); + if (!/file format:\s*qcow2/.test(out)) { console.error('FAIL: qemu-img did not detect qcow2 over NBD'); process.exit(1); } + console.log('XCHECK PASS: qemu-img detected qcow2 over hand-written NBD server'); +} finally { server.close(); } +" 2>&1 | tail -20 +``` +(Note: `require` isn't defined in `--input-type=module`; if it errors on `require`, +prepend `import { createRequire } from 'node:module'; const require = +createRequire(import.meta.url);` to the `-e` script.) + +Expected: `qemu-img info` reports `file format: qcow2` and a 32 MiB virtual size, then +`XCHECK PASS`. **This is the gate that the NBD protocol implementation is correct** — +isolating any later failure to the fd transport. + +- [ ] **Step 4: Commit** + +```bash +git add scripts/poc-nbd/launch.mjs +git commit -m "feat(poc-nbd): launcher (fd-pair + unix-socket modes) + qemu-img cross-check + +serveOnFd creates a socketpair via the addon and runs the server on one +end; serveOnUnixSocket validates protocol correctness against the real +qemu-img NBD client. Cross-check confirmed qcow2 detected over NBD. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 7: Stage 1 — fd passing with zero QEMU source changes + +**Files:** +- Create: `scripts/poc-nbd/test-stage1.mjs` + +The highest-leverage check: lspart opens `nbd-fd:N` over the inherited socketpair and +reports the **same capacity** as the plain file, proving fd adoption works without QEMU +source changes. + +- [ ] **Step 1: Write the Stage-1 test** + +Create `scripts/poc-nbd/test-stage1.mjs`: + +```javascript +/* + * Stage 1: prove the inherited-fd NBD transport works with zero QEMU + * source changes. Parent creates a socketpair, runs the NBD server on + * one end, spawns lspart with the other end inherited as fd 3, and + * checks lspart opens the qcow2 (capacity > 0, table printed) identically + * to a plain-file open. + */ +import { spawn, execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { serveOnFd } from './launch.mjs'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '../..'); +const lspart = path.join(repoRoot, 'build-anyfs-linux-amd64/src/lspart/anyfs-lspart'); +const meta = JSON.parse(fs.readFileSync(path.join(here, 'fixtures/meta.json'))); + +/* Baseline: lspart over the plain qcow2 file (proves what a successful + * open looks like, independent of NBD). Captured for comparison. */ +const plainOut = execFileSync(lspart, [meta.qcow2], { encoding: 'utf8' }); + +const { fd1, stop, readCount, done } = serveOnFd(meta.qcow2); + +/* Child inherits fd1 at descriptor index 3 (stdio[3]). lspart is told + * --nbd-fd 3. stdio: 0,1,2 are inherit; index 3 = the socketpair end. */ +const child = spawn(lspart, ['--nbd-fd', '3'], { + stdio: ['inherit', 'pipe', 'inherit', fd1], +}); + +let out = ''; +child.stdout.on('data', (d) => (out += d)); + +const code = await new Promise((resolve) => child.on('exit', resolve)); +stop(); +await done; + +console.log('--- plain-file lspart (baseline) ---'); +console.log(plainOut); +console.log('--- nbd lspart output ---'); +console.log(out); +console.log(`--- exit=${code} nbd_reads=${readCount()} ---`); + +/* The synthetic image has no partition table / filesystem (it is a + * deterministic byte pattern), so lspart lists it as a single whole-disk + * row with FSTYPE '?'. The Stage-1 signal is therefore NOT a specific + * filesystem string — it is: (a) lspart exits 0, (b) the NBD server + * actually served reads (data traversed the socketpair), and (c) the NBD + * run produced the SAME table as the plain-file open (i.e. the disk was + * detected identically whether opened directly or over NBD). */ +function dataRows(s) { + return s + .split('\n') + .filter((l) => l.trim() && !/^Usage|warning:|^PATH\b/.test(l)).length; +} + +if (code !== 0) { + console.error('STAGE1 FAIL: lspart exited non-zero'); + process.exit(1); +} +if (readCount() === 0) { + console.error( + 'STAGE1 FAIL: NBD server served zero reads (data did not traverse the socketpair)', + ); + process.exit(1); +} +if (dataRows(out) === 0) { + console.error('STAGE1 FAIL: nbd lspart printed no data row (disk not detected)'); + process.exit(1); +} +if (dataRows(out) !== dataRows(plainOut)) { + console.error( + `STAGE1 FAIL: nbd row count ${dataRows(out)} != plain-file row count ${dataRows(plainOut)}`, + ); + process.exit(1); +} +console.log( + 'STAGE1 PASS: lspart opened qcow2 over inherited-fd NBD; reads traversed the socketpair; table matches plain-file open', +); +``` + +- [ ] **Step 2: Run Stage 1** + +Run: `cd ~/anyfs-reader && node scripts/poc-nbd/test-stage1.mjs 2>&1 | tail -30` +Expected: prints both lspart tables, `exit=0`, `nbd_reads` > 0, equal row counts, and +`STAGE1 PASS`. + +- [ ] **Step 3: If it fails with "not a socket" or EBADF, diagnose** + +If lspart prints QEMU error `File descriptor '3' is not a socket`: the child didn't get +the socket at fd 3 — verify the `stdio` array index. If `EBADF`: the fd was closed +(CLOEXEC) — confirm the addon cleared CLOEXEC and `net.Socket({fd:fd0})` didn't close +fd1. Record the resolution; do not proceed to Stage 2 until Stage 1 passes. + +- [ ] **Step 4: Commit** + +```bash +git add scripts/poc-nbd/test-stage1.mjs +git commit -m "test(poc-nbd): Stage 1 — inherited-fd NBD open, zero QEMU source changes + +lspart opens nbd-fd:3 over an inherited socketpair end; asserts non-zero +capacity/partition row and that reads actually traversed the socketpair. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 8: Stage 2 — full Linux chain (partition list + byte verify) + +**Files:** +- Create: `scripts/poc-nbd/test-stage2.mjs` + +Two checks: (1) lspart-over-NBD produces the same table as lspart-over-plain-file, and +(2) the marker bytes are read back **through the qcow2-over-NBD chain** byte-for-byte. +For (2) we use `qemu-io` as a real qcow2 client connected to our NBD server over a unix +socket: `qemu-io` parses the qcow2 (driver=qcow2) whose protocol child is `nbd:` → our +server → the raw marker bytes. This exercises the exact format-over-NBD layering the PoC +is about, and verifies delivered content, not a side control file. + +- [ ] **Step 1: Write the Stage-2 test** + +Create `scripts/poc-nbd/test-stage2.mjs`: + +```javascript +/* + * Stage 2: full Linux chain. + * (a) lspart over inherited-fd NBD matches lspart over the plain file. + * (b) the deterministic marker at meta.verifyOffset reads back + * byte-for-byte THROUGH the qcow2-over-NBD chain, using qemu-io as + * a real qcow2 client over a unix-socket NBD endpoint. + */ +import { spawn, execFileSync, execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { serveOnFd, serveOnUnixSocket } from './launch.mjs'; + +const execFileP = promisify(execFile); + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '../..'); +const lspart = path.join(repoRoot, 'build-anyfs-linux-amd64/src/lspart/anyfs-lspart'); +const meta = JSON.parse(fs.readFileSync(path.join(here, 'fixtures/meta.json'))); + +function dataRows(s) { + return s + .split('\n') + .filter((l) => l.trim() && !/^Usage|warning:|^PATH\b/.test(l)).length; +} + +let fail = false; + +/* === Check (a): lspart parity over inherited-fd NBD === */ +const plainOut = execFileSync(lspart, [meta.qcow2], { encoding: 'utf8' }); +const { fd1, stop, readCount, done } = serveOnFd(meta.qcow2); +const child = spawn(lspart, ['--nbd-fd', '3'], { + stdio: ['inherit', 'pipe', 'inherit', fd1], +}); +let nbdOut = ''; +child.stdout.on('data', (d) => (nbdOut += d)); +const code = await new Promise((r) => child.on('exit', r)); +stop(); +await done; + +console.log('--- plain-file lspart ---\n' + plainOut); +console.log('--- nbd lspart ---\n' + nbdOut); + +if (code !== 0) { + console.error('STAGE2 FAIL: nbd lspart exited non-zero'); + fail = true; +} +if (dataRows(plainOut) !== dataRows(nbdOut)) { + console.error( + `STAGE2 FAIL: row count differs (plain=${dataRows(plainOut)} nbd=${dataRows(nbdOut)})`, + ); + fail = true; +} +if (readCount() === 0) { + console.error('STAGE2 FAIL: zero NBD reads over inherited fd'); + fail = true; +} + +/* === Check (b): byte verify THROUGH qcow2-over-NBD with qemu-io === */ +const sockPath = '/tmp/poc-nbd-stage2.sock'; +const { server } = await serveOnUnixSocket(meta.qcow2, sockPath); +try { + /* qemu-io opens the qcow2 format over the nbd: protocol child. We read + * meta.verifyOffset for the marker length and dump it as hex. */ + const len = Buffer.from(meta.verifyBytesHex, 'hex').length; + const nbdUri = `json:{"driver":"qcow2","file":{"driver":"nbd","server":{"type":"unix","path":"${sockPath}"}}}`; + /* MUST be async: the in-process NBD server runs on this event loop, so a + * synchronous execFileSync would block it and deadlock the qemu-io read. */ + const { stdout: out } = await execFileP( + 'qemu-io', + ['-r', '-c', `read -v ${meta.verifyOffset} ${len}`, nbdUri], + { encoding: 'utf8' }, + ); + /* qemu-io 'read -v' prints a hexdump; collect the hex byte columns. */ + const hex = [...out.matchAll(/^[0-9a-f]{8}:\s+((?:[0-9a-f]{2}\s?)+)/gim)] + .map((m) => m[1].replace(/\s+/g, '')) + .join('') + .slice(0, len * 2); + console.log('--- qemu-io marker read (through qcow2-over-NBD) ---'); + console.log(out); + if (hex !== meta.verifyBytesHex) { + console.error( + `STAGE2 FAIL: marker mismatch through chain: got ${hex} want ${meta.verifyBytesHex}`, + ); + fail = true; + } +} catch (e) { + console.error('STAGE2 FAIL: qemu-io byte verify errored:', e.message); + fail = true; +} finally { + server.close(); +} + +if (fail) process.exit(1); +console.log( + `STAGE2 PASS: lspart parity (rows=${dataRows(nbdOut)}), inherited-fd reads=${readCount()}, marker verified through qcow2-over-NBD (${meta.verifyAscii})`, +); +``` + +- [ ] **Step 2: Run Stage 2** + +Run: `cd ~/anyfs-reader && node scripts/poc-nbd/test-stage2.mjs 2>&1 | tail -50` +Expected: both lspart tables shown with equal row counts, `reads` > 0, a qemu-io hexdump +showing the ASCII marker, and `STAGE2 PASS`. If the qemu-io hexdump regex doesn't match +the installed qemu-io output format, inspect the printed hexdump and adjust the regex (the +intent: extract the hex bytes of the first `len` bytes at the offset). + +- [ ] **Step 3: Commit** + +```bash +git add scripts/poc-nbd/test-stage2.mjs +git commit -m "test(poc-nbd): Stage 2 — full Linux chain parity + through-chain byte verify + +Asserts nbd lspart's table equals plain-file lspart and non-zero +inherited-fd reads, then verifies the deterministic marker reads back +byte-for-byte through the qcow2-over-NBD chain via qemu-io. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 9: Stage 3 — Windows/wine scouting (go/no-go, allowed to fail) + +**Files:** +- Create: `scripts/poc-nbd/FINDINGS.md` + +This stage is a probe. The Linux conclusion does not depend on it. We document whether +the mingw64 lspart can open the qcow2 over the `127.0.0.1` loopback fallback under wine. + +- [ ] **Step 1: Check whether a mingw64 lspart and wine exist** + +Run: +```bash +ls ~/anyfs-reader/build-anyfs-mingw64/src/lspart/anyfs-lspart.exe 2>/dev/null && echo "mingw lspart present" || echo "no mingw lspart" +which wine 2>/dev/null && wine --version 2>/dev/null || echo "no wine" +``` +Expected: records availability. If either is missing, Stage 3 is **blocked** (not +failed) — note it in FINDINGS.md and stop. + +- [ ] **Step 2: Write FINDINGS.md documenting the Stage-3 probe and current status** + +Create `scripts/poc-nbd/FINDINGS.md`: + +```markdown +# NBD-over-fd PoC — Findings + +## Stage 1 (Linux, inherited-fd) — + + +## Stage 2 (Linux, full chain) — + + +## Stage 3 (Windows/wine, loopback fallback) — + +Approach: Windows has no reliable socketpair and QEMU requires the NBD fd +to be a socket; PoC uses the `nbd-port:P` branch (127.0.0.1 loopback, +random port, no outward listen). + +- mingw64 anyfs-lspart.exe present: +- wine present: +- Result: + +### Notes / blocking issues + +``` + +Fill the Stage-1 / Stage-2 lines from the actual results of Tasks 7-8. + +- [ ] **Step 3: If mingw lspart + wine are available, run the loopback probe** + +Only if Step 1 found both. Run a loopback variant (the launcher's TCP path is the +Windows production transport; for the wine probe we listen on `127.0.0.1:0`, get the +port, and pass `--nbd-port`): + +```bash +cd ~/anyfs-reader && node --input-type=module -e " +import net from 'node:net'; +import fs from 'node:fs'; +import { startNbdServer } from './scripts/poc-nbd/nbd-server.mjs'; +import { execFileSync } from 'node:child_process'; +const meta = JSON.parse(fs.readFileSync('scripts/poc-nbd/fixtures/meta.json')); +const imageFd = fs.openSync(meta.qcow2, 'r'); +const size = fs.statSync(meta.qcow2).size; +const srv = net.createServer((s) => startNbdServer(s, imageFd, size).catch(()=>{})); +await new Promise((r) => srv.listen(0, '127.0.0.1', r)); +const port = srv.address().port; +try { + const out = execFileSync('wine', ['build-anyfs-mingw64/src/lspart/anyfs-lspart.exe', '--nbd-port', String(port)], { encoding: 'utf8' }); + console.log(out); + console.log('STAGE3 PASS (wine loopback)'); +} catch (e) { console.error('STAGE3 result:', e.message); } +finally { srv.close(); } +" 2>&1 | tail -30 +``` +Expected: either a partition table + `STAGE3 PASS`, or a recorded failure reason. Update +FINDINGS.md Stage-3 line accordingly. **Failure here does not fail the PoC.** + +- [ ] **Step 4: Commit findings** + +```bash +git add scripts/poc-nbd/FINDINGS.md +git commit -m "docs(poc-nbd): Stage-3 wine/loopback findings + PoC results summary + +Records Stage 1/2 Linux results and the Windows/wine loopback probe +outcome (go/no-go; allowed to be BLOCKED without failing the PoC). + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Done criteria + +The PoC is complete when: +- Task 6 cross-check passes (NBD protocol correct: `qemu-img` detects qcow2 over the + hand-written server). +- Task 7 Stage 1 passes (lspart opens `nbd-fd:N` over an inherited socketpair, reads + traverse it, zero QEMU source changes). +- Task 8 Stage 2 passes (NBD lspart matches plain-file lspart; byte anchor verified). +- Task 9 Stage 3 is recorded in FINDINGS.md (PASS or BLOCKED — not a gate). +- Existing plain-file lspart usage still works (Task 1 Step 4). diff --git a/docs/superpowers/plans/2026-06-01-nbd-proxy.md b/docs/superpowers/plans/2026-06-01-nbd-proxy.md new file mode 100644 index 0000000..4502d15 --- /dev/null +++ b/docs/superpowers/plans/2026-06-01-nbd-proxy.md @@ -0,0 +1,1414 @@ +# Production NBD Proxy — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A standalone, optionally-privileged Node package `@anyfs/nbd-proxy` that exposes a local file, physical disk, or http(s) URL as a read-only NBD endpoint (over an inherited fd or `127.0.0.1` loopback) for a QEMU engine to open, with format drivers auto-layered atop NBD. + +**Architecture:** An async, multi-in-flight NBD newstyle server (out-of-order replies keyed by handle, only frame-writes serialized) over a `DataSource` abstraction with three backends (FileSource, BlockDeviceSource, HttpSource). A thin CLI wires a chosen source to an fd/loopback endpoint. Evolved from the proven PoC (`scripts/poc-nbd/`). + +**Tech Stack:** TypeScript 5.5 (ESM, `tsup` build, `tsconfig` extends `ts/tsconfig.base.json`), Node 24 (global `fetch` + keep-alive `Agent` — no undici dependency), pnpm workspace, `.mjs` smoke tests + `qemu-img`/`qemu-io` integration (both at /usr/bin). `drivelist` as an optional dependency. + +**Spec:** `docs/superpowers/specs/2026-06-01-nbd-proxy-design.md` + +--- + +## Reference facts (verified — do not re-investigate) + +- Workspace: pnpm, root `ts/package.json` (`"type":"module"`, `tsup`+`typescript` 5.5 devDeps). + Build = `pnpm -r --filter './packages/*' build`; test = `pnpm -r ... test`. +- Package template = `@anyfs/core` (`ts/packages/core`): `tsup` build, `tsconfig.json` extends + `../../tsconfig.base.json` (`{outDir:./dist, rootDir:./src, include:[src/**/*]}`), tests are + plain `node test/*.mjs` (NO jest/vitest in this repo). +- `ts/tsconfig.base.json`: target ES2022, module ESNext, moduleResolution Bundler, strict, + noUncheckedIndexedAccess, exactOptionalPropertyTypes, declaration+sourceMap. +- **No standalone `undici`** installed, but **Node 24 has global `fetch`** (undici under the hood, + HTTP/1.1 keep-alive via a global/explicit `Agent`). HttpSource v1 uses global `fetch` + + `new (await import('node:http')).Agent`/`https.Agent({keepAlive:true})` — NO new dependency. + H2 multiplexing is deferred (would need explicit undici). +- `drivelist` lives at `~/drivelist-anyfs` (a `file:` dep only in electron-demo, NOT resolvable + from the repo root). For this package it is an **optional** dependency: BlockDeviceSource tries + `import('drivelist')` for size, and on failure falls back to `/sys/block//size`×512 (Linux). +- PoC NBD server reference: `scripts/poc-nbd/nbd-server.mjs` — correct wire format already proven + (magics: NBDMAGIC 0x4e42444d41474943, IHAVEOPT 0x49484156454f5054, OPT_REPLY_MAGIC + **0x0003e889045565a9**, REQ_MAGIC 0x25609513, SIMPLE_REPLY_MAGIC 0x67446698). Reuse this framing. +- PoC fixture generator: `scripts/poc-nbd/make-test-image.mjs` produces `fixtures/test.qcow2` + (8 MiB) + `fixtures/meta.json` (verifyOffset 0x100000, marker "ANYFS-NBD-POC-MARKER-v1"). The + proxy tests reuse the same fixture. +- PoC launcher `scripts/poc-nbd/launch.mjs` and the socketpair addon + `scripts/poc-nbd/native-transport/build/Release/native_transport.node` exist and work — the + proxy's fd-mode integration test can reuse the addon for the socketpair. +- Linux raw block devices: the kernel block layer handles **non-aligned** pread, so v1 needs no + app-level alignment. `fstat` on a raw block device returns size 0 — must use drivelist or + `/sys/block`. +- MUST NOT touch: `ts/packages/core` (URLFS/WORKERFS), `http-disk-server.ts`, + `http-proxy-worker.ts`. This package is additive. + +## File structure + +| File | New/Mod | Responsibility | +|---|---|---| +| `ts/packages/nbd-proxy/package.json` | New | `@anyfs/nbd-proxy` package manifest (tsup build, mjs tests, optional drivelist) | +| `ts/packages/nbd-proxy/tsconfig.json` | New | extends `../../tsconfig.base.json` | +| `ts/packages/nbd-proxy/tsup.config.ts` | New | build `src/index.ts` + `bin/anyfs-nbd-proxy.ts` | +| `ts/packages/nbd-proxy/src/nbd-server.ts` | New | async multi-in-flight NBD newstyle server | +| `ts/packages/nbd-proxy/src/data-source.ts` | New | `DataSource` interface + `createDataSource` factory | +| `ts/packages/nbd-proxy/src/sources/file.ts` | New | `FileSource` (fs.promises pread) | +| `ts/packages/nbd-proxy/src/sources/blockdev.ts` | New | `BlockDeviceSource` (drivelist size + Linux pread) | +| `ts/packages/nbd-proxy/src/sources/http.ts` | New | `HttpSource` (global fetch + keep-alive Agent, Range) | +| `ts/packages/nbd-proxy/src/endpoint.ts` | New | bind NbdServer to inherited fd or loopback listener | +| `ts/packages/nbd-proxy/src/index.ts` | New | re-export public API | +| `ts/packages/nbd-proxy/bin/anyfs-nbd-proxy.ts` | New | thin CLI | +| `ts/packages/nbd-proxy/test/*.mjs` | New | unit + integration smoke tests | + +--- + +## Task 1: Scaffold the `@anyfs/nbd-proxy` package + +**Files:** +- Create: `ts/packages/nbd-proxy/package.json`, `tsconfig.json`, `tsup.config.ts`, `src/index.ts` + +- [ ] **Step 1: Create package.json** + +Create `ts/packages/nbd-proxy/package.json`: + +```json +{ + "name": "@anyfs/nbd-proxy", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "anyfs-nbd-proxy": "./dist/anyfs-nbd-proxy.js" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsup", + "test": "node test/unit.mjs" + }, + "optionalDependencies": { + "drivelist": "*" + }, + "devDependencies": { + "typescript": "^5.5.4", + "tsup": "^8.3.0", + "@types/node": "^22.0.0" + } +} +``` + +- [ ] **Step 2: Create tsconfig.json** + +Create `ts/packages/nbd-proxy/tsconfig.json`: + +```json +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "." + }, + "include": ["src/**/*", "bin/**/*"] +} +``` + +- [ ] **Step 3: Create tsup.config.ts** + +Create `ts/packages/nbd-proxy/tsup.config.ts`: + +```ts +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts', 'bin/anyfs-nbd-proxy.ts'], + format: ['esm'], + target: 'node20', + dts: true, + clean: true, + sourcemap: true, +}); +``` + +- [ ] **Step 4: Create a placeholder src/index.ts so the package builds** + +Create `ts/packages/nbd-proxy/src/index.ts`: + +```ts +export const PACKAGE = '@anyfs/nbd-proxy'; +``` + +- [ ] **Step 5: Install workspace deps and build the empty package** + +Run: `cd ~/anyfs-reader/ts && pnpm install 2>&1 | tail -10` +Then: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -15` +Expected: `pnpm install` links the new package into the workspace (drivelist is optional, so its +absence must NOT fail install). Build produces `ts/packages/nbd-proxy/dist/index.js`. +If `pnpm install` fails because `optionalDependencies.drivelist: "*"` can't resolve from a +registry, change it to remove the version pin issue: set it to a `file:` optional dep is NOT +possible portably, so instead REMOVE `optionalDependencies` from package.json entirely and have +BlockDeviceSource do a runtime `import('drivelist').catch(...)` (the dependency is loaded +dynamically, never declared). Re-run install + build. Record which approach worked. + +- [ ] **Step 6: Commit** + +```bash +cd ~/anyfs-reader +git add ts/packages/nbd-proxy/package.json ts/packages/nbd-proxy/tsconfig.json ts/packages/nbd-proxy/tsup.config.ts ts/packages/nbd-proxy/src/index.ts ts/pnpm-lock.yaml +git commit -m "feat(nbd-proxy): scaffold @anyfs/nbd-proxy package + +tsup/ESM package mirroring @anyfs/core's layout; empty index builds clean. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 2: DataSource interface + FileSource + +**Files:** +- Create: `ts/packages/nbd-proxy/src/data-source.ts`, `src/sources/file.ts` +- Test: `ts/packages/nbd-proxy/test/unit.mjs` + +- [ ] **Step 1: Write the DataSource interface + factory** + +Create `ts/packages/nbd-proxy/src/data-source.ts`: + +```ts +import { Buffer } from 'node:buffer'; + +/** A read-only random-access byte source backing an NBD export. */ +export interface DataSource { + /** Total export size in bytes. */ + size(): Promise; + /** Read exactly `length` bytes at `offset`. Rejects on IO error. */ + read(offset: number, length: number): Promise; + /** Release any held resources (fd, http agent, ...). */ + close(): Promise; +} + +export type DataSourceSpec = + | { kind: 'file'; target: string } + | { kind: 'blockdev'; target: string } + | { kind: 'url'; target: string }; + +export async function createDataSource(spec: DataSourceSpec): Promise { + switch (spec.kind) { + case 'file': { + const { FileSource } = await import('./sources/file.js'); + return FileSource.open(spec.target); + } + case 'blockdev': { + const { BlockDeviceSource } = await import('./sources/blockdev.js'); + return BlockDeviceSource.open(spec.target); + } + case 'url': { + const { HttpSource } = await import('./sources/http.js'); + return HttpSource.open(spec.target); + } + } +} +``` + +- [ ] **Step 2: Write FileSource** + +Create `ts/packages/nbd-proxy/src/sources/file.ts`: + +```ts +import { Buffer } from 'node:buffer'; +import { open, type FileHandle } from 'node:fs/promises'; +import type { DataSource } from '../data-source.js'; + +export class FileSource implements DataSource { + private constructor( + private fh: FileHandle, + private bytes: number, + ) {} + + static async open(path: string): Promise { + const fh = await open(path, 'r'); + const st = await fh.stat(); + return new FileSource(fh, st.size); + } + + async size(): Promise { + return this.bytes; + } + + async read(offset: number, length: number): Promise { + const buf = Buffer.alloc(length); + let got = 0; + while (got < length) { + const { bytesRead } = await this.fh.read( + buf, + got, + length - got, + offset + got, + ); + if (bytesRead === 0) break; // EOF (e.g. read past end) + got += bytesRead; + } + return got === length ? buf : buf.subarray(0, got); + } + + async close(): Promise { + await this.fh.close(); + } +} +``` + +- [ ] **Step 3: Write the unit test for FileSource** + +Create `ts/packages/nbd-proxy/test/unit.mjs`: + +```javascript +/* Unit tests for @anyfs/nbd-proxy DataSource backends. + * Run against the built dist/ (tsup output). */ +import assert from 'node:assert'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const dist = path.join(here, '..', 'dist', 'index.js'); + +const { createDataSource } = await import(dist); + +let passed = 0; +async function test(name, fn) { + await fn(); + console.log('ok -', name); + passed++; +} + +/* FileSource: deterministic byte pattern */ +await test('FileSource reads exact bytes', async () => { + const tmp = path.join(os.tmpdir(), 'nbdproxy-file-test.bin'); + const data = Buffer.alloc(4096); + for (let i = 0; i < data.length; i++) data[i] = i & 0xff; + fs.writeFileSync(tmp, data); + + const src = await createDataSource({ kind: 'file', target: tmp }); + assert.strictEqual(await src.size(), 4096); + + const slice = await src.read(1000, 16); + for (let i = 0; i < 16; i++) { + assert.strictEqual(slice[i], (1000 + i) & 0xff, `byte ${i}`); + } + await src.close(); + fs.unlinkSync(tmp); +}); + +console.log(`\n${passed} test(s) passed`); +``` + +- [ ] **Step 4: Build and run the unit test** + +Run: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -8` +Then: `cd ~/anyfs-reader/ts/packages/nbd-proxy && node test/unit.mjs` +Expected: `ok - FileSource reads exact bytes`, `1 test(s) passed`. + +- [ ] **Step 5: Commit** + +```bash +cd ~/anyfs-reader +git add ts/packages/nbd-proxy/src/data-source.ts ts/packages/nbd-proxy/src/sources/file.ts ts/packages/nbd-proxy/test/unit.mjs ts/packages/nbd-proxy/src/index.ts +git commit -m "feat(nbd-proxy): DataSource interface + FileSource + +Async fs.promises pread backend with a partial-read loop; factory +dynamic-imports each backend. Unit test verifies exact-byte reads. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +> Note: update `src/index.ts` in this task to re-export the public API: +> ```ts +> export type { DataSource, DataSourceSpec } from './data-source.js'; +> export { createDataSource } from './data-source.js'; +> ``` + +--- + +## Task 3: Async multi-in-flight NbdServer + +**Files:** +- Create: `ts/packages/nbd-proxy/src/nbd-server.ts` +- Test: append to `ts/packages/nbd-proxy/test/unit.mjs` + +This is the core. Port the PoC's wire format (`scripts/poc-nbd/nbd-server.mjs`) to TypeScript, +but make the transmission loop dispatch reads without awaiting, cap in-flight at 16, and drain +replies through a serialized FIFO writer. + +- [ ] **Step 1: Write the NbdServer** + +Create `ts/packages/nbd-proxy/src/nbd-server.ts`: + +```ts +import { Buffer } from 'node:buffer'; +import type { Socket } from 'node:net'; +import type { DataSource } from './data-source.js'; + +const NBDMAGIC = 0x4e42444d41474943n; +const IHAVEOPT = 0x49484156454f5054n; +const OPT_REPLY_MAGIC = 0x0003e889045565a9n; +const REQ_MAGIC = 0x25609513; +const SIMPLE_REPLY_MAGIC = 0x67446698; + +const NBD_FLAG_FIXED_NEWSTYLE = 1; +const NBD_FLAG_NO_ZEROES = 2; + +const NBD_OPT_ABORT = 2; +const NBD_OPT_INFO = 6; +const NBD_OPT_GO = 7; + +const NBD_REP_ACK = 1; +const NBD_REP_INFO = 3; +const NBD_REP_ERR_UNSUP = 0x80000001; + +const NBD_INFO_EXPORT = 0; +const NBD_FLAG_HAS_FLAGS = 1; +const NBD_FLAG_READ_ONLY = 2; + +const NBD_CMD_READ = 0; +const NBD_CMD_DISC = 2; +const NBD_CMD_FLUSH = 3; + +const EINVAL = 22; +const EIO = 5; + +const MAX_IN_FLIGHT = 16; + +/** Serial byte reader over a socket (single outstanding read at a time). */ +function makeReader(socket: Socket): (n: number) => Promise { + let chunks: Buffer[] = []; + let buffered = 0; + let want = 0; + let resolveWant: ((b: Buffer) => void) | null = null; + let rejectWant: ((e: Error) => void) | null = null; + let ended = false; + let errored: Error | null = null; + + function tryResolve() { + if (resolveWant && buffered >= want) { + const all = Buffer.concat(chunks); + const out = all.subarray(0, want); + const rest = all.subarray(want); + chunks = rest.length ? [Buffer.from(rest)] : []; + buffered = rest.length; + const r = resolveWant; + resolveWant = rejectWant = null; + r(out); + } else if (resolveWant && (ended || errored)) { + const rej = rejectWant!; + resolveWant = rejectWant = null; + rej(errored ?? new Error('EOF')); + } + } + + socket.on('data', (d: Buffer) => { + chunks.push(d); + buffered += d.length; + tryResolve(); + }); + socket.on('end', () => { + ended = true; + tryResolve(); + }); + socket.on('error', (e: Error) => { + errored = e; + tryResolve(); + }); + + return (n: number) => + new Promise((resolve, reject) => { + want = n; + resolveWant = resolve; + rejectWant = reject; + tryResolve(); + }); +} + +/** Serialized reply-frame writer: one frame on the wire at a time. */ +function makeWriter(socket: Socket) { + let chain: Promise = Promise.resolve(); + return (buf: Buffer): Promise => { + chain = chain.then( + () => + new Promise((res, rej) => + socket.write(buf, (e) => (e ? rej(e) : res())), + ), + ); + return chain; + }; +} + +function simpleReply(error: number, handle: Buffer, data: Buffer | null): Buffer { + const hdr = Buffer.alloc(16); + hdr.writeUInt32BE(SIMPLE_REPLY_MAGIC, 0); + hdr.writeUInt32BE(error >>> 0, 4); + handle.copy(hdr, 8); + return data ? Buffer.concat([hdr, data]) : hdr; +} + +function optReply(opt: number, repType: number, payload: Buffer): Buffer { + const fixed = Buffer.alloc(16); + fixed.writeBigUInt64BE(OPT_REPLY_MAGIC, 0); + fixed.writeUInt32BE(opt, 8); + fixed.writeUInt32BE(repType >>> 0, 12); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(payload.length, 0); + return Buffer.concat([fixed, lenBuf, payload]); +} + +/** + * Serve one client on `socket` from `source`. Resolves on clean DISC/EOF. + * Reads are dispatched concurrently (up to 16 in flight); replies are written + * in completion order, keyed by the 8-byte handle. + */ +export async function serveNbd( + socket: Socket, + source: DataSource, + opts: { size: number; onRead?: (offset: number, length: number) => void }, +): Promise { + const read = makeReader(socket); + const write = makeWriter(socket); + const size = opts.size; + + /* Handshake */ + const hello = Buffer.alloc(18); + hello.writeBigUInt64BE(NBDMAGIC, 0); + hello.writeBigUInt64BE(IHAVEOPT, 8); + hello.writeUInt16BE(NBD_FLAG_FIXED_NEWSTYLE | NBD_FLAG_NO_ZEROES, 16); + await write(hello); + await read(4); /* client flags */ + + /* Option haggling until GO (enter transmission) or ABORT */ + for (;;) { + const optHdr = await read(16); + if (optHdr.readBigUInt64BE(0) !== IHAVEOPT) throw new Error('bad option magic'); + const opt = optHdr.readUInt32BE(8); + const optLen = optHdr.readUInt32BE(12); + if (optLen) await read(optLen); + + if (opt === NBD_OPT_GO || opt === NBD_OPT_INFO) { + const info = Buffer.alloc(12); + info.writeUInt16BE(NBD_INFO_EXPORT, 0); + info.writeBigUInt64BE(BigInt(size), 2); + info.writeUInt16BE(NBD_FLAG_HAS_FLAGS | NBD_FLAG_READ_ONLY, 10); + await write(optReply(opt, NBD_REP_INFO, info)); + await write(optReply(opt, NBD_REP_ACK, Buffer.alloc(0))); + if (opt === NBD_OPT_GO) break; + } else if (opt === NBD_OPT_ABORT) { + await write(optReply(opt, NBD_REP_ACK, Buffer.alloc(0))); + socket.end(); + return; + } else { + await write(optReply(opt, NBD_REP_ERR_UNSUP, Buffer.alloc(0))); + } + } + + /* Transmission: dispatch reads concurrently, reply in completion order */ + let inFlight = 0; + const pending = new Set>(); + let slotWaiter: (() => void) | null = null; + + for (;;) { + if (inFlight >= MAX_IN_FLIGHT) { + await new Promise((r) => (slotWaiter = r)); + } + let hdr: Buffer; + try { + hdr = await read(28); + } catch { + break; /* EOF/disconnect */ + } + if (hdr.readUInt32BE(0) !== REQ_MAGIC) throw new Error('bad request magic'); + const type = hdr.readUInt16BE(6); + const handle = Buffer.from(hdr.subarray(8, 16)); /* copy: detach from reader buffer */ + const offset = hdr.readBigUInt64BE(16); + const length = hdr.readUInt32BE(24); + + if (type === NBD_CMD_DISC) break; + + if (type === NBD_CMD_READ) { + inFlight++; + const job = (async () => { + try { + if (offset + BigInt(length) > BigInt(size)) { + await write(simpleReply(EINVAL, handle, null)); + return; + } + const data = await source.read(Number(offset), length); + opts.onRead?.(Number(offset), length); + await write(simpleReply(0, handle, data)); + } catch { + await write(simpleReply(EIO, handle, null)); + } finally { + inFlight--; + if (slotWaiter) { + const w = slotWaiter; + slotWaiter = null; + w(); + } + } + })(); + pending.add(job); + job.finally(() => pending.delete(job)); + } else if (type === NBD_CMD_FLUSH) { + await write(simpleReply(0, handle, null)); + } else { + await write(simpleReply(EINVAL, handle, null)); + } + } + + await Promise.allSettled([...pending]); +} +``` + +- [ ] **Step 2: Add an out-of-order concurrency unit test** + +Append to `ts/packages/nbd-proxy/test/unit.mjs` (before the final `console.log`): + +```javascript +/* NbdServer: out-of-order completion must reply keyed by handle. + * Drive serveNbd over a socketpair-like net pipe with a mock DataSource + * whose reads resolve in REVERSE order. */ +await test('NbdServer replies keyed by handle under out-of-order completion', async () => { + const net = await import('node:net'); + const { serveNbd } = await import(dist); + + /* Mock source: read(offset) resolves after a delay inversely proportional + * to offset, so later requests complete first. Returns offset-tagged bytes. */ + const SIZE = 1 << 20; + const source = { + async size() { + return SIZE; + }, + async read(offset, length) { + const delay = offset === 0 ? 60 : 5; /* offset 0 finishes LAST */ + await new Promise((r) => setTimeout(r, delay)); + const b = Buffer.alloc(length); + b.writeUInt32BE(offset >>> 0, 0); + return b; + }, + async close() {}, + }; + + /* In-process TCP pair so we can drive a real client handshake. */ + const server = net.createServer((sock) => { + serveNbd(sock, source, { size: SIZE }).catch(() => {}); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const port = server.address().port; + + const replies = await new Promise((resolve, reject) => { + const c = net.connect(port, '127.0.0.1'); + const got = []; + let phase = 'hello'; + let buf = Buffer.alloc(0); + c.on('data', (d) => { + buf = Buffer.concat([buf, d]); + if (phase === 'hello' && buf.length >= 18) { + buf = buf.subarray(18); + c.write(Buffer.alloc(4)); /* client flags */ + /* NBD_OPT_GO with empty data */ + const o = Buffer.alloc(16); + o.writeBigUInt64BE(0x49484156454f5054n, 0); + o.writeUInt32BE(7, 8); + o.writeUInt32BE(0, 12); + c.write(o); + phase = 'opt'; + } + if (phase === 'opt') { + /* consume INFO reply (20+12) + ACK reply (20), then send two READs */ + if (buf.length >= 20 + 12 + 20) { + buf = buf.subarray(20 + 12 + 20); + phase = 'xmit'; + for (const off of [0, 65536]) { + const req = Buffer.alloc(28); + req.writeUInt32BE(0x25609513, 0); + req.writeUInt16BE(0, 4); + req.writeUInt16BE(0, 6); /* CMD_READ */ + req.writeUInt32BE(off, 12); /* handle = offset for identification */ + req.writeBigUInt64BE(BigInt(off), 16); + req.writeUInt32BE(8, 24); /* length */ + c.write(req); + } + } + } + if (phase === 'xmit') { + /* each reply: 16-byte header + 8 bytes data = 24 bytes */ + while (buf.length >= 24) { + const handle = buf.readUInt32BE(12); /* low 4 bytes of handle */ + const dataOff = buf.readUInt32BE(16); /* our tagged offset */ + got.push({ handle, dataOff }); + buf = buf.subarray(24); + if (got.length === 2) { + c.end(); + resolve(got); + } + } + } + }); + c.on('error', reject); + setTimeout(() => reject(new Error('timeout')), 3000); + }); + server.close(); + + /* The offset-65536 read completes first (5ms) before offset-0 (60ms), + * so replies arrive reversed — but each reply's handle must match its + * own data. */ + for (const r of replies) { + assert.strictEqual(r.handle, r.dataOff, 'handle must pair with its data'); + } + assert.strictEqual(replies[0].handle, 65536, 'faster read replied first'); +}); +``` + +- [ ] **Step 3: Build and run** + +Run: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -8` +Then: `cd ~/anyfs-reader/ts/packages/nbd-proxy && node test/unit.mjs` +Expected: both tests pass; the out-of-order test prints `ok - NbdServer replies keyed by handle +under out-of-order completion` and confirms the faster (offset 65536) read replied first while +each handle still paired with its own data. + +- [ ] **Step 4: Update src/index.ts to export serveNbd, and commit** + +Add to `ts/packages/nbd-proxy/src/index.ts`: +```ts +export { serveNbd } from './nbd-server.js'; +``` + +```bash +cd ~/anyfs-reader && cd ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -3 && cd .. +git add ts/packages/nbd-proxy/src/nbd-server.ts ts/packages/nbd-proxy/src/index.ts ts/packages/nbd-proxy/test/unit.mjs +git commit -m "feat(nbd-proxy): async multi-in-flight NbdServer + +Ports the PoC wire format to TS; transmission loop dispatches reads +without awaiting (cap 16 in-flight), replies drained through a serialized +FIFO writer, paired by handle. Unit test proves out-of-order completion +still pairs each reply with its own handle. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 4: endpoint.ts + qemu-img integration test (FileSource end-to-end) + +**Files:** +- Create: `ts/packages/nbd-proxy/src/endpoint.ts` +- Create: `ts/packages/nbd-proxy/test/integration.mjs` + +- [ ] **Step 1: Write endpoint.ts** + +Create `ts/packages/nbd-proxy/src/endpoint.ts`: + +```ts +import net from 'node:net'; +import type { DataSource } from './data-source.js'; +import { serveNbd } from './nbd-server.js'; + +/** Serve NBD on an already-open socket fd (inherited from the parent). */ +export async function serveOnFd(fd: number, source: DataSource): Promise { + const size = await source.size(); + const socket = new net.Socket({ fd }); + return serveNbd(socket, source, { size }); +} + +/** + * Serve NBD on a 127.0.0.1 loopback listener. Resolves with the bound port + * and a stop() that closes the listener. Each connection is served from the + * same source. + */ +export async function serveOnLoopback( + source: DataSource, + port = 0, +): Promise<{ port: number; stop: () => Promise }> { + const size = await source.size(); + const server = net.createServer((sock) => { + serveNbd(sock, source, { size }).catch(() => {}); + }); + await new Promise((r) => server.listen(port, '127.0.0.1', r)); + const addr = server.address() as net.AddressInfo; + return { + port: addr.port, + stop: () => new Promise((r) => server.close(() => r())), + }; +} +``` + +Also add to `src/index.ts`: +```ts +export { serveOnFd, serveOnLoopback } from './endpoint.js'; +``` + +- [ ] **Step 2: Write the qemu-img/qemu-io integration test (FileSource over loopback)** + +Create `ts/packages/nbd-proxy/test/integration.mjs`: + +```javascript +/* Integration: open the PoC fixture qcow2 THROUGH the proxy with the real + * QEMU client (qemu-img + qemu-io over a 127.0.0.1 loopback NBD endpoint). + * Proves format detection + byte-accurate delivery for FileSource. + * + * Reuses the PoC fixture: scripts/poc-nbd/fixtures/test.qcow2 + meta.json. + * MUST use async execFile (the in-process NBD server shares this event loop; + * a sync execFileSync would deadlock it). */ +import assert from 'node:assert'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileP = promisify(execFile); +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '../../../..'); +const dist = path.join(here, '..', 'dist', 'index.js'); +const metaPath = path.join(repoRoot, 'scripts/poc-nbd/fixtures/meta.json'); + +if (!fs.existsSync(metaPath)) { + console.error('fixture missing; run: node scripts/poc-nbd/make-test-image.mjs'); + process.exit(1); +} +const meta = JSON.parse(fs.readFileSync(metaPath, 'utf8')); +const { createDataSource, serveOnLoopback } = await import(dist); + +const source = await createDataSource({ kind: 'file', target: meta.qcow2 }); +const { port, stop } = await serveOnLoopback(source); +let fail = false; +try { + const uri = `nbd://127.0.0.1:${port}`; + /* (a) qemu-img detects qcow2 over NBD */ + const { stdout: info } = await execFileP('qemu-img', ['info', uri], { + encoding: 'utf8', + }); + console.log(info); + if (!/file format:\s*qcow2/.test(info)) { + console.error('FAIL: qemu-img did not detect qcow2 over NBD'); + fail = true; + } + + /* (b) qemu-io reads the marker back byte-for-byte through qcow2-over-NBD */ + const len = Buffer.from(meta.verifyBytesHex, 'hex').length; + const qio = `json:{"driver":"qcow2","file":{"driver":"nbd","server":{"type":"inet","host":"127.0.0.1","port":"${port}"}}}`; + const { stdout: dump } = await execFileP( + 'qemu-io', + ['-r', '-c', `read -v ${meta.verifyOffset} ${len}`, qio], + { encoding: 'utf8' }, + ); + const hex = [...dump.matchAll(/^[0-9a-f]{8}:\s+((?:[0-9a-f]{2}\s?)+)/gim)] + .map((m) => m[1].replace(/\s+/g, '')) + .join('') + .slice(0, len * 2); + console.log(dump); + if (hex !== meta.verifyBytesHex) { + console.error(`FAIL: marker mismatch: got ${hex} want ${meta.verifyBytesHex}`); + fail = true; + } +} finally { + await stop(); + await source.close(); +} + +if (fail) process.exit(1); +console.log(`\nINTEGRATION PASS: qcow2 detected + marker "${meta.verifyAscii}" verified through proxy (FileSource)`); +``` + +- [ ] **Step 3: Ensure the fixture exists, build, run integration** + +Run: `cd ~/anyfs-reader && [ -f scripts/poc-nbd/fixtures/meta.json ] || node scripts/poc-nbd/make-test-image.mjs` +Then: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -5` +Then: `cd ~/anyfs-reader/ts/packages/nbd-proxy && node test/integration.mjs 2>&1 | tail -25` +Expected: `qemu-img info` shows `file format: qcow2` (8 MiB), a qemu-io hexdump of the marker, and +`INTEGRATION PASS`. + +- [ ] **Step 4: Commit** + +```bash +cd ~/anyfs-reader && cd ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -3 && cd .. +git add ts/packages/nbd-proxy/src/endpoint.ts ts/packages/nbd-proxy/src/index.ts ts/packages/nbd-proxy/test/integration.mjs +git commit -m "feat(nbd-proxy): endpoint (fd + loopback) + qemu-img integration test + +serveOnFd / serveOnLoopback bind NbdServer to an inherited fd or a +127.0.0.1 listener. Integration test opens the PoC fixture qcow2 through +the proxy with real qemu-img/qemu-io, verifying format detection + the +marker byte-for-byte (FileSource end-to-end). + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 5: HttpSource (global fetch + keep-alive Agent) + +**Files:** +- Create: `ts/packages/nbd-proxy/src/sources/http.ts` +- Test: append an HttpSource case to `test/integration.mjs` (local Range server) + +- [ ] **Step 1: Write HttpSource** + +Create `ts/packages/nbd-proxy/src/sources/http.ts`: + +```ts +import { Buffer } from 'node:buffer'; +import { Agent as HttpAgent } from 'node:http'; +import { Agent as HttpsAgent } from 'node:https'; +import type { DataSource } from '../data-source.js'; + +/** + * HTTP(S) Range-backed source using Node's global fetch with a keep-alive + * agent so range requests reuse upstream connections (no per-request + * reconnect). H2 multiplexing is deferred; v1 relies on HTTP/1.1 keep-alive. + */ +export class HttpSource implements DataSource { + private agent: HttpAgent | HttpsAgent; + + private constructor( + private url: string, + private bytes: number, + secure: boolean, + ) { + this.agent = secure + ? new HttpsAgent({ keepAlive: true, maxSockets: 16 }) + : new HttpAgent({ keepAlive: true, maxSockets: 16 }); + } + + static async open(url: string): Promise { + const head = await fetch(url, { method: 'HEAD', redirect: 'follow' }); + if (!head.ok) throw new Error(`HttpSource: HEAD ${url} -> ${head.status}`); + const len = Number(head.headers.get('content-length') ?? '0'); + if (!len) throw new Error(`HttpSource: missing content-length for ${url}`); + if ((head.headers.get('accept-ranges') ?? '').toLowerCase() !== 'bytes') { + throw new Error(`HttpSource: server lacks Accept-Ranges: bytes for ${url}`); + } + return new HttpSource(head.url || url, len, new URL(url).protocol === 'https:'); + } + + async size(): Promise { + return this.bytes; + } + + async read(offset: number, length: number): Promise { + const end = Math.min(offset + length, this.bytes) - 1; + const res = await fetch(this.url, { + headers: { Range: `bytes=${offset}-${end}` }, + redirect: 'follow', + // @ts-expect-error Node-specific dispatcher/agent option + agent: this.agent, + }); + if (res.status !== 206 && res.status !== 200) { + throw new Error(`HttpSource: range ${offset}-${end} -> ${res.status}`); + } + let buf = Buffer.from(await res.arrayBuffer()); + /* Defensive: server ignored Range and returned 200 full body. */ + if (res.status === 200 && buf.length > end - offset + 1) { + buf = buf.subarray(offset, end + 1); + } + return buf; + } + + async close(): Promise { + this.agent.destroy(); + } +} +``` + +> NOTE on the `agent` option: Node's global `fetch` (undici) does NOT honor the legacy +> `http.Agent` via an `agent` field — it uses a `dispatcher`. If the `@ts-expect-error agent` +> approach does not actually reuse connections at runtime, switch to a `keepAlive`-by-default +> behavior: undici's global dispatcher already pools connections per origin, so simply calling +> `fetch(url, {headers:{Range}})` without an agent reuses connections within the process. In Step 3, +> VERIFY connection reuse empirically (count TCP connections on the local server); if the explicit +> agent doesn't help, remove the agent field and rely on undici's default global pool, and update +> the comment. Either way the REQUIREMENT is: N range reads must NOT open N fresh TCP connections. + +- [ ] **Step 2: Append the HttpSource integration case** + +Append to `ts/packages/nbd-proxy/test/integration.mjs` (before the final pass log; reuse `dist`, +`meta`, `execFileP`). Stand up a local HTTP Range server backed by the fixture qcow2, point +HttpSource at it through the proxy, and verify qemu-img detects qcow2 — AND assert connection +reuse: + +```javascript +/* HttpSource: local Range server backed by the fixture, served through the + * proxy; verify qcow2 detection + that range reads REUSE connections. */ +{ + const http = await import('node:http'); + const { createDataSource, serveOnLoopback } = await import(dist); + const raw = fs.readFileSync(meta.qcow2); + let connections = 0; + const upstream = http.createServer((req, res) => { + const m = /bytes=(\d+)-(\d+)/.exec(req.headers.range || ''); + if (req.method === 'HEAD') { + res.writeHead(200, { + 'content-length': String(raw.length), + 'accept-ranges': 'bytes', + }); + return res.end(); + } + if (!m) { + res.writeHead(200, { 'content-length': String(raw.length) }); + return res.end(raw); + } + const start = +m[1]; + const end = +m[2]; + res.writeHead(206, { + 'content-range': `bytes ${start}-${end}/${raw.length}`, + 'content-length': String(end - start + 1), + 'accept-ranges': 'bytes', + }); + res.end(raw.subarray(start, end + 1)); + }); + upstream.on('connection', () => connections++); + await new Promise((r) => upstream.listen(0, '127.0.0.1', r)); + const upPort = upstream.address().port; + + const src = await createDataSource({ + kind: 'url', + target: `http://127.0.0.1:${upPort}/disk.qcow2`, + }); + const { port, stop } = await serveOnLoopback(src); + try { + const { stdout } = await execFileP('qemu-img', ['info', `nbd://127.0.0.1:${port}`], { + encoding: 'utf8', + }); + if (!/file format:\s*qcow2/.test(stdout)) { + console.error('FAIL: HttpSource — qcow2 not detected through proxy'); + fail = true; + } + console.log(`HttpSource: upstream TCP connections opened = ${connections}`); + /* qemu-img issues many range reads; with keep-alive these must share a + * small number of connections, NOT one per read. Allow a small ceiling. */ + if (connections > 4) { + console.error(`FAIL: HttpSource opened ${connections} connections (no keep-alive reuse)`); + fail = true; + } + } finally { + await stop(); + await src.close(); + await new Promise((r) => upstream.close(r)); + } + console.log('HttpSource case done'); +} +``` + +- [ ] **Step 3: Build, run, verify qcow2 detection + connection reuse** + +Run: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -5` +Then: `cd ~/anyfs-reader/ts/packages/nbd-proxy && node test/integration.mjs 2>&1 | tail -30` +Expected: FileSource pass (from Task 4) + HttpSource detects qcow2 + `upstream TCP connections +opened = ` + overall `INTEGRATION PASS`. If connections > 4, fix per the NOTE in +Step 1 (use undici's default pool / dispatcher) until reuse is demonstrated, then re-run. + +- [ ] **Step 4: Commit** + +```bash +cd ~/anyfs-reader && cd ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -3 && cd .. +git add ts/packages/nbd-proxy/src/sources/http.ts ts/packages/nbd-proxy/test/integration.mjs +git commit -m "feat(nbd-proxy): HttpSource (global fetch + keep-alive) + +Range-backed source over Node's global fetch with connection reuse (no +new undici dep). Integration test serves the fixture qcow2 over a local +Range server through the proxy and asserts qcow2 detection + that range +reads share connections (keep-alive), not one-per-read. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 6: BlockDeviceSource (Linux v1) + +**Files:** +- Create: `ts/packages/nbd-proxy/src/sources/blockdev.ts` +- Test: append a BlockDeviceSource case to `test/unit.mjs` (loop device, or skip without privilege) + +- [ ] **Step 1: Write BlockDeviceSource** + +Create `ts/packages/nbd-proxy/src/sources/blockdev.ts`: + +```ts +import { Buffer } from 'node:buffer'; +import { open, readFile, type FileHandle } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import type { DataSource } from '../data-source.js'; + +/** + * Raw block device source. v1 is Linux-only: the kernel block layer serves + * non-aligned preads, so no app-level alignment is needed. Size comes from + * `drivelist` (optional dependency) or, as a fallback, /sys/block//size. + * macOS/Windows raw-device specifics (sector alignment, IOCTL size) are + * deferred — open() throws a clear error there. + */ +export class BlockDeviceSource implements DataSource { + private constructor( + private fh: FileHandle, + private bytes: number, + ) {} + + static async open(devPath: string): Promise { + if (os.platform() !== 'linux') { + throw new Error( + `BlockDeviceSource: v1 is Linux-only (got ${os.platform()}); ` + + `macOS/Windows raw-device support is not yet implemented`, + ); + } + const bytes = await BlockDeviceSource.sizeOf(devPath); + const fh = await open(devPath, 'r'); + return new BlockDeviceSource(fh, bytes); + } + + private static async sizeOf(devPath: string): Promise { + /* Prefer drivelist (optional dep): enumerate and match the device. */ + try { + const drivelist: { list: () => Promise> } = + await import('drivelist'); + const drives = await drivelist.list(); + const hit = drives.find((d) => d.device === devPath); + if (hit && hit.size > 0) return hit.size; + } catch { + /* drivelist absent or failed — fall through to /sys/block. */ + } + /* Fallback: /sys/block//size is in 512-byte sectors. */ + const name = path.basename(devPath); + const sysSize = path.join('/sys/block', name, 'size'); + const sectors = Number((await readFile(sysSize, 'utf8')).trim()); + if (!Number.isFinite(sectors) || sectors <= 0) { + throw new Error(`BlockDeviceSource: could not determine size of ${devPath}`); + } + return sectors * 512; + } + + async size(): Promise { + return this.bytes; + } + + async read(offset: number, length: number): Promise { + const buf = Buffer.alloc(length); + let got = 0; + while (got < length) { + const { bytesRead } = await this.fh.read(buf, got, length - got, offset + got); + if (bytesRead === 0) break; + got += bytesRead; + } + return got === length ? buf : buf.subarray(0, got); + } + + async close(): Promise { + await this.fh.close(); + } +} +``` + +Add to `src/index.ts`: +```ts +export { FileSource } from './sources/file.js'; +export { HttpSource } from './sources/http.js'; +export { BlockDeviceSource } from './sources/blockdev.js'; +``` + +- [ ] **Step 2: Add a BlockDeviceSource unit test (loop device, privilege-gated skip)** + +Append to `ts/packages/nbd-proxy/test/unit.mjs` (before the final log). Use a loop device if we +can create one (needs root); otherwise SKIP with a clear message (do not fail): + +```javascript +/* BlockDeviceSource: back a loop device with the fixture image if we have + * privilege; otherwise skip. Verifies size + a known-offset read. */ +await test('BlockDeviceSource reads via loop device (or skips)', async () => { + const { execFileSync } = await import('node:child_process'); + const osMod = await import('node:os'); + if (osMod.platform() !== 'linux') { + console.log(' (skip: blockdev v1 is Linux-only)'); + return; + } + let loopDev = null; + const img = path.join(os.tmpdir(), 'nbdproxy-blockdev-test.img'); + try { + /* 1 MiB image with a marker at offset 4096. */ + const data = Buffer.alloc(1 << 20); + data.write('BLOCKDEV-MARK', 4096, 'ascii'); + fs.writeFileSync(img, data); + try { + loopDev = execFileSync('losetup', ['--find', '--show', img], { + encoding: 'utf8', + }).trim(); + } catch { + console.log(' (skip: cannot create loop device — needs root/losetup)'); + return; + } + const src = await createDataSource({ kind: 'blockdev', target: loopDev }); + assert.ok((await src.size()) >= 1 << 20, 'size detected'); + const slice = await src.read(4096, 13); + assert.strictEqual(slice.toString('ascii'), 'BLOCKDEV-MARK', 'marker read'); + await src.close(); + } finally { + if (loopDev) { + try { + (await import('node:child_process')).execFileSync('losetup', ['-d', loopDev]); + } catch {} + } + try { + fs.unlinkSync(img); + } catch {} + } +}); +``` + +- [ ] **Step 3: Build and run unit tests** + +Run: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -5` +Then: `cd ~/anyfs-reader/ts/packages/nbd-proxy && node test/unit.mjs` +Expected: FileSource + NbdServer tests pass; BlockDeviceSource either passes (if losetup works) or +prints a skip line. Either way `N test(s) passed` with no failure. + +- [ ] **Step 4: Commit** + +```bash +cd ~/anyfs-reader && cd ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -3 && cd .. +git add ts/packages/nbd-proxy/src/sources/blockdev.ts ts/packages/nbd-proxy/src/index.ts ts/packages/nbd-proxy/test/unit.mjs +git commit -m "feat(nbd-proxy): BlockDeviceSource (Linux v1) + +Raw block device source; size via optional drivelist with /sys/block +fallback, reads via plain pread (Linux kernel handles non-aligned). +Non-Linux throws a clear 'v1 Linux-only' error. Unit test uses a loop +device when privileged, skips otherwise. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 7: CLI (bin/anyfs-nbd-proxy.ts) + CLI smoke test + +**Files:** +- Create: `ts/packages/nbd-proxy/bin/anyfs-nbd-proxy.ts` +- Test: append a CLI smoke case to `test/integration.mjs` + +- [ ] **Step 1: Write the CLI** + +Create `ts/packages/nbd-proxy/bin/anyfs-nbd-proxy.ts`: + +```ts +#!/usr/bin/env node +import { createDataSource, type DataSourceSpec } from '../src/data-source.js'; +import { serveOnFd, serveOnLoopback } from '../src/endpoint.js'; + +interface Args { + source?: 'file' | 'blockdev' | 'url'; + target?: string; + fd?: number; + port?: number; +} + +function parse(argv: string[]): Args { + const a: Args = {}; + for (let i = 0; i < argv.length; i++) { + const k = argv[i]; + const v = argv[i + 1]; + switch (k) { + case '--source': + a.source = v as Args['source']; + i++; + break; + case '--target': + a.target = v; + i++; + break; + case '--fd': + a.fd = Number(v); + i++; + break; + case '--port': + a.port = Number(v); + i++; + break; + case '--help': + case '-h': + usage(); + process.exit(0); + } + } + return a; +} + +function usage(): void { + process.stderr.write( + 'Usage: anyfs-nbd-proxy --source file|blockdev|url --target ' + + '(--fd | --port

)\n', + ); +} + +async function main(): Promise { + const a = parse(process.argv.slice(2)); + if (!a.source || !a.target || (a.fd === undefined && a.port === undefined)) { + usage(); + process.exit(2); + } + const source = await createDataSource({ + kind: a.source, + target: a.target, + } as DataSourceSpec); + + if (a.fd !== undefined) { + await serveOnFd(a.fd, source); + await source.close(); + return; + } + const { port, stop } = await serveOnLoopback(source, a.port); + process.stdout.write(`${port}\n`); /* report the bound port on stdout */ + const shutdown = async () => { + await stop(); + await source.close(); + process.exit(0); + }; + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); + /* Lifecycle binding: exit when stdin closes (parent gone). */ + process.stdin.on('end', shutdown); + process.stdin.resume(); +} + +main().catch((e) => { + process.stderr.write(`anyfs-nbd-proxy: ${e instanceof Error ? e.message : String(e)}\n`); + process.exit(1); +}); +``` + +- [ ] **Step 2: Append a CLI smoke test (spawn the CLI, qemu-img against its port)** + +Append to `ts/packages/nbd-proxy/test/integration.mjs`: + +```javascript +/* CLI smoke: spawn the built CLI in loopback mode, read its port from stdout, + * confirm qemu-img opens the fixture qcow2 through it. */ +{ + const { spawn } = await import('node:child_process'); + const cliPath = path.join(here, '..', 'dist', 'anyfs-nbd-proxy.js'); + const child = spawn( + 'node', + [cliPath, '--source', 'file', '--target', meta.qcow2, '--port', '0'], + { stdio: ['pipe', 'pipe', 'inherit'] }, + ); + const port = await new Promise((resolve, reject) => { + let out = ''; + child.stdout.on('data', (d) => { + out += d; + const m = /^(\d+)\s*$/m.exec(out); + if (m) resolve(Number(m[1])); + }); + child.on('exit', (c) => reject(new Error(`CLI exited early: ${c}`))); + setTimeout(() => reject(new Error('CLI did not report a port')), 5000); + }); + try { + const { stdout } = await execFileP('qemu-img', ['info', `nbd://127.0.0.1:${port}`], { + encoding: 'utf8', + }); + if (!/file format:\s*qcow2/.test(stdout)) { + console.error('FAIL: CLI — qcow2 not detected'); + fail = true; + } else { + console.log('CLI smoke: qcow2 detected through spawned proxy'); + } + } finally { + child.stdin.end(); /* triggers lifecycle-bound shutdown */ + child.kill('SIGTERM'); + } +} +``` + +- [ ] **Step 3: Build and run full integration** + +Run: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -5` +Then: `cd ~/anyfs-reader/ts/packages/nbd-proxy && node test/integration.mjs 2>&1 | tail -35` +Expected: FileSource + HttpSource + CLI smoke all pass; final `INTEGRATION PASS`. + +- [ ] **Step 4: Commit** + +```bash +cd ~/anyfs-reader && cd ts && pnpm --filter @anyfs/nbd-proxy build 2>&1 | tail -3 && cd .. +git add ts/packages/nbd-proxy/bin/anyfs-nbd-proxy.ts ts/packages/nbd-proxy/test/integration.mjs +git commit -m "feat(nbd-proxy): CLI + spawned-proxy integration test + +Thin CLI wires a DataSource to an fd/loopback endpoint, reports the bound +port on stdout, and shuts down on stdin EOF (lifecycle binding). Smoke +test spawns the CLI and confirms qemu-img opens the fixture through it. + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Task 8: Wire test script + web-regression confirmation + +**Files:** +- Modify: `ts/packages/nbd-proxy/package.json` (test script runs both unit + integration) + +- [ ] **Step 1: Make `pnpm test` run both suites** + +Edit `ts/packages/nbd-proxy/package.json` test script from: +```json + "test": "node test/unit.mjs" +``` +to: +```json + "test": "node test/unit.mjs && node test/integration.mjs" +``` + +- [ ] **Step 2: Run the package's full test via the workspace** + +Run: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/nbd-proxy test 2>&1 | tail -30` +Expected: unit tests pass, integration tests pass, no failures. + +- [ ] **Step 3: Confirm no web regression (the must-not-break constraint)** + +This package added a new workspace package and touched NOTHING in `@anyfs/core`, URLFS, WORKERFS, +or the http proxies. Confirm the core package still builds and its smoke tests are unaffected: + +Run: `cd ~/anyfs-reader/ts && pnpm --filter @anyfs/core build 2>&1 | tail -5` +Expected: `@anyfs/core` builds clean (the new package did not perturb it). +Then confirm git shows ONLY new files under `ts/packages/nbd-proxy/` plus the lockfile — no edits to +core/URLFS/WORKERFS/http-proxy files: +Run: `cd ~/anyfs-reader && git diff --stat main -- ts/ | grep -vE 'ts/packages/nbd-proxy/|pnpm-lock' || echo "(no other ts/ files changed — web paths intact)"` +Expected: prints `(no other ts/ files changed — web paths intact)`. + +- [ ] **Step 4: Commit** + +```bash +cd ~/anyfs-reader +git add ts/packages/nbd-proxy/package.json +git commit -m "test(nbd-proxy): run unit+integration in pnpm test; confirm web paths intact + +Package test runs both suites. Verified @anyfs/core still builds and no +files outside ts/packages/nbd-proxy were touched (web blob/URLFS/WORKERFS/ +curl-proxy paths untouched per the non-goals constraint). + +Co-Authored-By: Claude Opus 4.8 (1M context) " +``` + +--- + +## Done criteria + +The production proxy v1 is complete when: +- `pnpm --filter @anyfs/nbd-proxy test` passes (unit: FileSource exact bytes, NbdServer + out-of-order-by-handle; integration: FileSource + HttpSource + CLI all open the fixture qcow2 + through real qemu-img/qemu-io). +- HttpSource demonstrably reuses upstream connections (≤4 TCP connections for many range reads). +- BlockDeviceSource passes on Linux with a loop device, or skips cleanly without privilege; throws + a clear error on non-Linux. +- No files outside `ts/packages/nbd-proxy/` (plus the lockfile) are changed — the web + blob/URLFS/WORKERFS/curl-proxy paths are untouched (§8 of the spec). +- The PoC `scripts/poc-nbd/` and the existing `http-disk-server.ts`/`http-proxy-worker.ts` remain + untouched. diff --git a/docs/superpowers/plans/2026-06-10-build-and-test-hardening.md b/docs/superpowers/plans/2026-06-10-build-and-test-hardening.md new file mode 100644 index 0000000..d489eb7 --- /dev/null +++ b/docs/superpowers/plans/2026-06-10-build-and-test-hardening.md @@ -0,0 +1,2074 @@ +# Build & Test Hardening Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Finish the P1 build-system refactor (all build scripts read `build.config.toml`, shellcheck-clean), make the wasm export table drift-proof and single-sourced, give the patched wasm-ld and the wasm-sysroot reproducible release pipelines (mingw-style prebuilt provisioning), extract the shared daemon skeleton out of ksmbd/nfsd mains, add real C and TS unit-test suites wired into CI plus a triage of the legacy `tests/*.mjs` suites, and hook the linux + mingw64 CI jobs up to the sccache-dist compile farm. + +**Architecture:** Seven phases, each shippable on its own, in the order the user specified: (1) config.sh migration for the 7 remaining hand-rolled scripts, with a shellcheck gate; (2) wasm export list generated from `ts/native/anyfs_ts.c` (single source of truth) + delete the bit-rotted `build_anyfs_browser_wasm.sh`; (2A) wasm-ld release pipeline on the `xdqi/llvm-wasm` fork (built on a sccache-dist farm) + fetch-script consumption in anyfs-reader; (2B) wasm-sysroot brought to the mingw provisioning model — in-repo recipe script + CI-published tarball + fetch + doctor manifest check; (3) new `src/server_common/` linked into both servers; (4) meson-registered C unit tests + node:test/vitest TS unit tests + CI wiring + triage of the legacy `tests/*.mjs` zoo (CDP↔Playwright parity, keep-don't-delete diagnostics); (5) `xdqi/sccache-dist-action` coordinator/worker jobs in `linux.yml` **and** (experimental) `mingw64.yml`, with graceful fallback when the farm or secret is absent. + +**Tech Stack:** bash + shellcheck, meson/ninja, emscripten, node:test (Node ≥22), vitest + @testing-library/react + jsdom, GitHub Actions, sccache-dist, GitHub releases as artifact channel. + +**Branch:** Start from `main` (NOT the current `zig-oldglibc-aligned-new` checkout): `git checkout main && git checkout -b build-and-test-hardening`. Use a worktree if executing with superpowers:using-git-worktrees. Phase 2A additionally works in a clone of `xdqi/llvm-wasm`. + +**Out of scope (decided):** repo-root junk cleanup / committing `patches/` (separate hygiene pass), `fuse_main.c` decomposition and fuse adoption of server_common (libfuse owns its own signal handling), @anyfs/trees component tests (covered by Playwright E2E), packaging wasm sysroot libs into the msys-cross pacman repo (possible future unification, msys2-cross project's call). + +**Shellcheck rule (applies to every task that touches a `scripts/*.sh` file):** the verification step must also run `shellcheck -x scripts/` and fix (or explicitly `# shellcheck disable=`-annotate with a reason) every finding of severity warning or higher. shellcheck is installed locally. + +**Key facts the executor must know (verified 2026-06-10):** +- `scripts/lib/config.sh` loads `build.config.toml` + `build.user.toml` into `ANYFS_*` vars (e.g. `ANYFS_PATHS_LINUX_SRC`, `ANYFS_TOOLCHAINS_MSYS2_CROSS`, `ANYFS_TOOLCHAINS_EMSDK`) and materializes `""` defaults to `/deps/...`. +- `scripts/lint-no-hardcoded-paths.sh` greps migrated scripts for `\$HOME|/opt/msys2|/home/[a-z]+/` — **including comments**, so doc comments mentioning `$HOME` must be reworded during migration. +- `scripts/build_anyfs_browser_wasm.sh` is **dead**: it compiles `src/core/anyfs.c`, `src/core/kindprobe.c`, `src/core/qemu_blk_backend.c` — none exist (renamed to `anyfs_kernel.c`/`anyfs_probe.c`/`qemu_backend.c`), and its `EXPORTED_FUNCS` uses pre-refactor `anyfs_ts_disk_*` names. It is untracked (never committed). The live script is `scripts/build_anyfs_wasm.sh` (QEMU always on, outputs `ts/packages/core/wasm/anyfs.mjs`). +- `build_anyfs.sh` must keep `LKL_SRC`/`QEMU_ROOT`/`KSMBD_ROOT` **relative** when inside the source tree (meson rejects absolute paths into the source root — see the NOTE at scripts/build_anyfs.sh:77). +- `anyfs_path_parse` rejects `p0` (`parse_component` requires `v != 0`), accepts `P1`, strips leading/trailing `/`, rejects bare `disk0`, max 8 components. +- The native bridge global is `globalThis.anyfsNative` (ts/packages/core/src/native-session.ts:32). +- Existing meson test executables (`test_raw_mount` etc.) are built but **never registered with `test()`** — there is no meson test suite yet. +- Local machine deps live at `~/linux`, `~/qemu`, `~/util-linux`, `~/ksmbd-tools`, `~/emsdk`, `~/wasm-sysroot`, `~/linux-wasm` (repo `deps/` is not synced locally); CI uses `deps/*` via peru. +- `xdqi/llvm-wasm` exists (default branch `wasm-18.1.2`; peru pins `wasm-18.1.2-anyfs`) but has **no releases** and only upstream LLVM workflows; there is no local clone. The expected wasm-ld path convention is `/workspace/install/llvm/bin/wasm-ld` and doctor's hint is `./linux-wasm.sh build-llvm` — inspect the `wasm-18.1.2-anyfs` branch layout before writing the release workflow (Task 9A Step 1). +- wasm-sysroot provenance: only libblkid has an in-repo recipe (`build_libblkid_wasm.sh`); glib/gio/gobject/gmodule/gthread, pcre2 (×4), libffi, zlib, libzstd, libbz2, libuuid, libresolv were hand-built and are undocumented. Authoritative version pins live in `~/wasm-sysroot/lib/pkgconfig/*.pc`. +- Dependency provisioning today: linux-amd64 = apt system packages; mingw32/64 = prebuilt pacman packages fetched from `msys.kosaka.moe/repo` (bootstrap.tar.xz + `pacman -S`); wasm = the undocumented hand-built sysroot. Phase 2B moves wasm to the mingw model (prebuilt artifact + fetch). +- Playwright E2E (`ts/tests/e2e`) already runs 3 projects — `web`, `electron-native`, `electron-wasm` — over shared flows (open/browse/download, url-load incl. local Range server, formats matrix, errors) + `electron-only/backend-switch`. The legacy CDP suite (`tests/test-cdp.mjs` + `run-all.mjs`, 6 target×source combos) therefore overlaps heavily; parity must be checked combo-by-combo before demoting it (Task 18A). + +--- + +## Phase 1 — finish config.sh migration + +### Task 1: New config keys + machine-local override file + +**Files:** +- Modify: `build.config.toml` +- Modify: `scripts/lib/config.sh` +- Modify: `build.user.toml.example` +- Create: `build.user.toml` (gitignored — local machine only) + +- [ ] **Step 1: Add `wasm_sysroot` to `[paths]` in `build.config.toml`** + +After the `ksmbd_tools` line add: + +```toml +# Sysroot holding wasm static libs (libblkid/libz/libbz2/libzstd/glib...) +# produced by build_libblkid_wasm.sh and friends. "" => /wasm-sysroot +wasm_sysroot = "" +``` + +- [ ] **Step 2: Materialize the new defaults in `scripts/lib/config.sh`** + +In `anyfs_load_config()`, after the existing `: "${ANYFS_PATHS_KSMBD_TOOLS:=...}"` line, add: + +```bash + : "${ANYFS_PATHS_WASM_SYSROOT:=$root/wasm-sysroot}" + : "${ANYFS_TOOLCHAINS_WASM_LD:=$pfx$deps/llvm-wasm/workspace/install/llvm/bin/wasm-ld}" +``` + +and extend the final `export` line to include `ANYFS_PATHS_WASM_SYSROOT ANYFS_TOOLCHAINS_WASM_LD`. (The `wasm_ld` key already exists in the toml; only the `""`→default materialization is missing. The default path matches what `scripts/doctor.sh:36` already probes.) + +- [ ] **Step 3: Document the new key in `build.user.toml.example`** + +Add a commented sample: + +```toml +# [paths] +# wasm_sysroot = "/path/to/wasm-sysroot" +``` + +- [ ] **Step 4: Write the local `build.user.toml`** (preserves current local builds once script defaults move from `$HOME` to `deps/`): + +```toml +# Machine-local overrides (gitignored). This host keeps source trees in $HOME +# instead of peru-synced deps/. +[paths] +linux_src = "/home/kosaka/linux" +qemu_src = "/home/kosaka/qemu" +util_linux = "/home/kosaka/util-linux" +ksmbd_tools = "/home/kosaka/ksmbd-tools" +wasm_sysroot = "/home/kosaka/wasm-sysroot" + +[toolchains] +emsdk = "/home/kosaka/emsdk" +wasm_ld = "/home/kosaka/linux-wasm/workspace/install/llvm/bin/wasm-ld" +``` + +- [ ] **Step 5: Verify** + +Run: `bash -c 'source scripts/lib/config.sh && echo "$ANYFS_PATHS_WASM_SYSROOT"; echo "$ANYFS_TOOLCHAINS_WASM_LD"; echo "$ANYFS_PATHS_QEMU_SRC"'` +Expected: `/home/kosaka/wasm-sysroot`, `/home/kosaka/linux-wasm/.../wasm-ld`, `/home/kosaka/qemu` (user.toml wins). +Also run with the override file moved away (`mv build.user.toml /tmp && bash -c '...' && mv /tmp/build.user.toml .`) — expected: `/wasm-sysroot`, `/deps/llvm-wasm/...`, `/deps/qemu`. + +- [ ] **Step 6: Commit** + +```bash +git add build.config.toml scripts/lib/config.sh build.user.toml.example +git commit -m "feat(build): add wasm_sysroot config key + materialize wasm_ld default" +``` + +### Task 2: Migrate `gen_lkl_config_wasm.sh` + `build_lkl_wasm.sh` + +**Files:** +- Modify: `scripts/gen_lkl_config_wasm.sh:27-29` +- Modify: `scripts/build_lkl_wasm.sh:24-26,92-96` +- Modify: `scripts/lint-no-hardcoded-paths.sh:6` + +- [ ] **Step 1 (red): Add both scripts to the lint allowlist** + +In `scripts/lint-no-hardcoded-paths.sh` change line 6 to: + +```bash +migrated='scripts/gen_lkl_config.sh scripts/build_lkl.sh scripts/gen_lkl_config_wasm.sh scripts/build_lkl_wasm.sh' +``` + +Run: `./scripts/lint-no-hardcoded-paths.sh` — Expected: FAIL listing the `$HOME` lines in both scripts. + +- [ ] **Step 2: Migrate `gen_lkl_config_wasm.sh`** + +Replace lines 27–29: + +```bash +LINUX_DIR="$HOME/linux" +OUT_PARENT="$HOME/anyfs-reader" +EMSDK_DIR="$HOME/emsdk" +``` + +with (mirroring the already-migrated `gen_lkl_config.sh`): + +```bash +# shellcheck source=lib/config.sh +source "$(dirname "$0")/lib/config.sh" + +# CLI --linux=/--out=/--emsdk= win; config.sh provides the defaults. +LINUX_DIR="${LINUX_DIR:-$ANYFS_PATHS_LINUX_SRC}" +OUT_PARENT="${OUT_PARENT:-$(cd "$(dirname "$0")/.." && pwd)}" +EMSDK_DIR="${EMSDK_DIR:-$ANYFS_TOOLCHAINS_EMSDK}" +``` + +Also update the header comment's option defaults (`default: ~/linux` → `default: from build.config.toml; linux_src or deps/linux`, etc.) so no `$HOME` text remains. + +- [ ] **Step 3: Migrate `build_lkl_wasm.sh`** + +Same replacement for its lines 24–26. Then replace line 92: + +```bash +JOEL_WASM_LD="$HOME/linux-wasm/workspace/install/llvm/bin/wasm-ld" +``` + +with: + +```bash +JOEL_WASM_LD="${JOEL_WASM_LD:-$ANYFS_TOOLCHAINS_WASM_LD}" +``` + +and update the not-found error message to say `set toolchains.wasm_ld in build.user.toml or build it: (cd deps/llvm-wasm && ./linux-wasm.sh build-llvm)`. Reword any remaining `$HOME` mentions in the header comment. + +- [ ] **Step 4 (green): Verify** + +Run: `./scripts/lint-no-hardcoded-paths.sh` — Expected: PASS. +Run: `bash -n scripts/gen_lkl_config_wasm.sh scripts/build_lkl_wasm.sh` — Expected: no output. +Run: `./scripts/build_lkl_wasm.sh --help` — Expected: usage text, exit 0. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/gen_lkl_config_wasm.sh scripts/build_lkl_wasm.sh scripts/lint-no-hardcoded-paths.sh +git commit -m "refactor(build): wire gen_lkl_config_wasm + build_lkl_wasm to build.config.toml" +``` + +### Task 3: Migrate `build_boot_wasm.sh` + `build_libblkid_wasm.sh` + +**Files:** +- Modify: `scripts/build_boot_wasm.sh:27-29` +- Modify: `scripts/build_libblkid_wasm.sh:31-32,36-41` +- Modify: `scripts/lint-no-hardcoded-paths.sh:6` (append both) + +- [ ] **Step 1 (red): Append both scripts to the lint allowlist; run lint; expect FAIL.** + +- [ ] **Step 2: Migrate `build_boot_wasm.sh`** — replace lines 27–29 with: + +```bash +# shellcheck source=lib/config.sh +source "$(dirname "$0")/lib/config.sh" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +LINUX_DIR="${LINUX_DIR:-$ANYFS_PATHS_LINUX_SRC}" +OUT="${OUT:-$REPO_ROOT/lkl-wasm}" +EMSDK_DIR="${EMSDK_DIR:-$ANYFS_TOOLCHAINS_EMSDK}" +``` + +- [ ] **Step 3: Migrate `build_libblkid_wasm.sh`** — replace lines 31–32 with: + +```bash +# shellcheck source=lib/config.sh +source "$SCRIPT_DIR/lib/config.sh" + +UL_SRC="${UL_SRC:-$ANYFS_PATHS_UTIL_LINUX}" +SYSROOT="${SYSROOT:-$ANYFS_PATHS_WASM_SYSROOT}" +``` + +(`SCRIPT_DIR`/`REPO_ROOT` already exist in this script.) Then make the emsdk activation fall back to config when `EMSDK_ENV` is unset — replace the `if [[ -n "${EMSDK_ENV:-}" ...` block's precondition with: + +```bash +EMSDK_ENV="${EMSDK_ENV:-${ANYFS_TOOLCHAINS_EMSDK:+$ANYFS_TOOLCHAINS_EMSDK/emsdk_env.sh}}" +if [[ -n "${EMSDK_ENV:-}" && -f "$EMSDK_ENV" ]]; then +``` + +Reword the header comments and the `emconfigure not on PATH` error so they reference `toolchains.emsdk` in build.config.toml instead of `$HOME/emsdk` (the lint matches `\$HOME` even in strings). + +- [ ] **Step 4 (green):** lint PASS; `bash -n` both; `./scripts/build_libblkid_wasm.sh --help 2>/dev/null || true` (script has no --help; instead run with `UL_SRC=/nonexistent` and expect its own clear error, not a `$HOME` path). + +- [ ] **Step 5: Commit** + +```bash +git add scripts/build_boot_wasm.sh scripts/build_libblkid_wasm.sh scripts/lint-no-hardcoded-paths.sh +git commit -m "refactor(build): wire build_boot_wasm + build_libblkid_wasm to build.config.toml" +``` + +### Task 4: Migrate `build_libblkid_mingw.sh` + +**Files:** +- Modify: `scripts/build_libblkid_mingw.sh:34-40,48` +- Modify: `scripts/lint-no-hardcoded-paths.sh:6` (append) + +- [ ] **Step 1 (red):** append to allowlist, lint FAIL. +- [ ] **Step 2:** The target-selection `case` (lines 31–43) consumes `CROSS_PREFIX` defaults *before* `SCRIPT_DIR` is defined, so add the config source right after `set -euo pipefail` using the dirname form: `source "$(dirname "$0")/lib/config.sh"`. Then change the two defaults: + +```bash + mingw64) + CROSS_PREFIX="${CROSS_PREFIX:-$ANYFS_TOOLCHAINS_MSYS2_CROSS/bin/x86_64-w64-mingw32}" + ;; + mingw32) + CROSS_PREFIX="${CROSS_PREFIX:-$ANYFS_TOOLCHAINS_MSYS2_CROSS/bin/i686-w64-mingw32}" + ;; +``` + +and line 48: `UL_SRC="${UL_SRC:-$ANYFS_PATHS_UTIL_LINUX}"`. Reword the header comment's `$HOME/util-linux` mention. + +- [ ] **Step 3 (green):** lint PASS; `bash -n`; run `./scripts/build_libblkid_mingw.sh badtarget` → expected usage error exit 1. +- [ ] **Step 4: Commit** — `git commit -m "refactor(build): wire build_libblkid_mingw to build.config.toml"` + +### Task 5: Migrate `build_qemu.sh` + +**Files:** +- Modify: `scripts/build_qemu.sh:33,118-129` (+ header comments) +- Modify: `scripts/lint-no-hardcoded-paths.sh:6` (append) + +- [ ] **Step 1 (red):** append to allowlist, lint FAIL. +- [ ] **Step 2:** Below `SCRIPT_DIR=` add `source "$SCRIPT_DIR/lib/config.sh"`, change line 33 to `QEMU_SRC="${QEMU_SRC:-$ANYFS_PATHS_QEMU_SRC}"`, and in `configure_for()` replace the four `/opt/msys2-cross/...` literals: + +```bash + "--extra-cflags=-I$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw32/include" \ + "--extra-ldflags=-L$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw32/lib" +``` + +(and the mingw64 pair likewise). Reword header comments mentioning `~/qemu` and `/opt/msys2-cross`. +- [ ] **Step 3 (green):** lint PASS; `bash -n`; `./scripts/build_qemu.sh --help` → usage, exit 0. +- [ ] **Step 4: Commit** — `git commit -m "refactor(build): wire build_qemu to build.config.toml"` + +### Task 6: Migrate `build_anyfs.sh` + +**Files:** +- Modify: `scripts/build_anyfs.sh:44-46,374-377` (+ header comments) +- Modify: `scripts/lint-no-hardcoded-paths.sh:6` (append) + +- [ ] **Step 1 (red):** append to allowlist, lint FAIL. +- [ ] **Step 2:** Below `SRC_DIR=` add: + +```bash +# shellcheck source=lib/config.sh +source "$SCRIPT_DIR/lib/config.sh" + +# Meson refuses absolute paths that point inside the source tree (see NOTE +# below), so config defaults are re-relativized against SRC_DIR when inside it. +rel_to_src() { + case "$1" in + "$SRC_DIR"/*) printf '%s\n' "${1#"$SRC_DIR"/}" ;; + *) printf '%s\n' "$1" ;; + esac +} +``` + +then replace lines 44–46: + +```bash +QEMU_ROOT="$(rel_to_src "$ANYFS_PATHS_QEMU_SRC")" +KSMBD_ROOT="$(rel_to_src "$ANYFS_PATHS_KSMBD_TOOLS")" +LKL_SRC="$(rel_to_src "$ANYFS_PATHS_LINUX_SRC")" +``` + +(CLI `--qemu-root/--ksmbd-root/--lkl-src` still override afterwards — they're parsed later, unchanged.) Replace `mingw_sysroot_for()`'s two literals with `echo "$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw32"` / `.../mingw64`. Reword header-comment defaults (`default: ~/qemu` → `default: from build.config.toml`). +- [ ] **Step 3 (green):** lint PASS; `bash -n`; `./scripts/build_anyfs.sh --help` → usage, exit 0. +- [ ] **Step 4: Full local verification of Phase 1** — rebuild one cheap target end-to-end to prove config resolution works: + +Run: `./scripts/build_anyfs.sh --targets=linux-amd64 --components=core,server,fuse -j"$(nproc)"` +Expected: completes, `build-anyfs-linux-amd64/bin/` populated (this uses the local `build.user.toml` paths). +- [ ] **Step 5: Commit** — `git commit -m "refactor(build): wire build_anyfs to build.config.toml; lint covers all native scripts"` + +### Task 6A: shellcheck gate + +**Files:** +- Create: `scripts/lint-shellcheck.sh` +- Modify: `.github/workflows/linux.yml` (lint step) + +- [ ] **Step 1:** Write `scripts/lint-shellcheck.sh` — same allowlist philosophy as the hardcoded-path lint (scripts get added as they're cleaned, starting with everything migrated in this plan): + +```bash +#!/usr/bin/env bash +# shellcheck gate for build scripts (severity >= warning). Extend the list +# as scripts are cleaned; new scripts must be added here. +set -uo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +checked=' +scripts/lib/config.sh +scripts/lib/wasm_exports.sh +scripts/gen_lkl_config.sh +scripts/build_lkl.sh +scripts/gen_lkl_config_wasm.sh +scripts/build_lkl_wasm.sh +scripts/build_boot_wasm.sh +scripts/build_libblkid_wasm.sh +scripts/build_libblkid_mingw.sh +scripts/build_qemu.sh +scripts/build_anyfs.sh +scripts/build_anyfs_wasm.sh +scripts/lint-no-hardcoded-paths.sh +scripts/lint-shellcheck.sh +' +# shellcheck disable=SC2086 +shellcheck -x -S warning $(printf "$root/%s " $checked) +``` + +`chmod +x scripts/lint-shellcheck.sh`. Fix every finding it reports in the listed scripts (or annotate with `# shellcheck disable=SCxxxx` + a reason where the pattern is intentional). +- [ ] **Step 2:** In `.github/workflows/linux.yml`, extend the lint step: + +```yaml + - name: Lint — no hardcoded paths in migrated build scripts + run: ./scripts/lint-no-hardcoded-paths.sh + + - name: Lint — shellcheck + run: | + sudo apt-get install -y --no-install-recommends shellcheck + ./scripts/lint-shellcheck.sh +``` + +- [ ] **Step 3: Verify** — `./scripts/lint-shellcheck.sh` exits 0 locally. +- [ ] **Step 4: Commit** — `git commit -m "ci(lint): shellcheck gate for build scripts"` + +(`scripts/lib/wasm_exports.sh` and `scripts/build_anyfs_wasm.sh` enter this list in Phase 2 — if executing strictly in order, add them to `checked` during Task 8 instead of here.) + +--- + +## Phase 2 — single-source wasm exports + unify on build_anyfs_wasm.sh + +### Task 7: Export-list generator + drift gate (TDD) + +**Files:** +- Create: `scripts/lib/wasm_exports.sh` +- Create: `tests/test_wasm_exports.sh` + +- [ ] **Step 1: Write the failing test** + +`tests/test_wasm_exports.sh`: + +```bash +#!/usr/bin/env bash +# Gate for the generated wasm export list: +# 1. the generator emits every known-core symbol, +# 2. every ccall'd anyfs_ts_* name in the TS worker layer is exported +# (catches TS<->C drift that previously bit build_anyfs_browser_wasm.sh). +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=../scripts/lib/wasm_exports.sh +source "$root/scripts/lib/wasm_exports.sh" + +list="$(anyfs_wasm_exports "$root/ts/native/anyfs_ts.c")" + +for must in _main _malloc _free _anyfs_ts_kernel_init _anyfs_ts_init_async \ + _anyfs_ts_session_open _anyfs_ts_session_enter_async \ + _anyfs_ts_session_enter_result_p _anyfs_ts_pread_p _anyfs_ts_close_p; do + [[ ",$list," == *",$must,"* ]] || { echo "FAIL: $must missing from generated exports"; exit 1; } +done + +n="$(tr ',' '\n' <<<"$list" | grep -c '^_anyfs_ts_')" +[[ "$n" -ge 30 ]] || { echo "FAIL: only $n anyfs_ts_* exports (expected >= 30)"; exit 1; } + +# TS drift gate: every ccall('anyfs_ts_...') in the worker layer must be exported. +missing=0 +while IFS= read -r sym; do + [[ ",$list," == *",_$sym,"* ]] || { echo "FAIL: worker ccalls $sym but it is not exported"; missing=1; } +done < <(grep -rhoE "ccall\(\s*'(anyfs_ts_[a-z0-9_]+)'" \ + "$root/ts/packages/core/src" | grep -oE 'anyfs_ts_[a-z0-9_]+' | sort -u) +[[ "$missing" -eq 0 ]] + +echo "OK: $n anyfs_ts_* exports, worker ccalls all covered" +``` + +`chmod +x tests/test_wasm_exports.sh` + +- [ ] **Step 2: Run to verify it fails** + +Run: `./tests/test_wasm_exports.sh` +Expected: FAIL — `scripts/lib/wasm_exports.sh: No such file or directory`. + +- [ ] **Step 3: Write the generator** + +`scripts/lib/wasm_exports.sh`: + +```bash +# scripts/lib/wasm_exports.sh — derive -sEXPORTED_FUNCTIONS for the wasm +# bundle from the TS glue source. ts/native/anyfs_ts.c is the single source +# of truth: every non-static anyfs_ts_* function defined at column 0 is +# exported (renaming a glue function can therefore never silently drop it +# from the bundle — the failure surfaces in the node smoke test instead). +anyfs_wasm_exports() { + local glue="$1" syms s out + syms="$(grep -hE '^[A-Za-z_][A-Za-z0-9_* ]*[ *]anyfs_ts_[A-Za-z0-9_]+\(' "$glue" \ + | grep -v '^static' \ + | grep -oE 'anyfs_ts_[A-Za-z0-9_]+' \ + | sort -u)" + if [[ -z "$syms" ]]; then + echo "wasm_exports: no anyfs_ts_* definitions found in $glue" >&2 + return 1 + fi + out="_main,_malloc,_free" + for s in $syms; do out+=",_$s"; done + printf '%s\n' "$out" +} +``` + +- [ ] **Step 4: Run the test — expect PASS.** If the symbol count differs from the hand-written list, diff them before proceeding: + +```bash +bash -c 'source scripts/lib/wasm_exports.sh; anyfs_wasm_exports ts/native/anyfs_ts.c | tr "," "\n" | sort' > /tmp/gen.txt +grep -A14 "^EXPORTED_FUNCS=" scripts/build_anyfs_wasm.sh | tr -d "'\\\\" | tr ',' '\n' | grep '^_' | sort > /tmp/hand.txt +diff /tmp/hand.txt /tmp/gen.txt +``` + +Expected: empty diff (the hand list has 38 entries: `_main,_malloc,_free` + 35 glue symbols). Any extra symbol in `gen.txt` means anyfs_ts.c gained a function the hand list missed — that is the bug class this fixes; keep the generated version. + +- [ ] **Step 5: Commit** + +```bash +git add scripts/lib/wasm_exports.sh tests/test_wasm_exports.sh +git commit -m "feat(build): generate wasm export list from anyfs_ts.c + drift gate" +``` + +### Task 8: build_anyfs_wasm.sh consumes the generator + config.sh; delete the stale browser script + +**Files:** +- Modify: `scripts/build_anyfs_wasm.sh:28-39,63-68,164-165,188-205` +- Delete: `scripts/build_anyfs_browser_wasm.sh` (untracked — plain `rm`) +- Modify: `scripts/lint-no-hardcoded-paths.sh:6` (append `scripts/build_anyfs_wasm.sh`) + +- [ ] **Step 1 (red):** append `scripts/build_anyfs_wasm.sh` to the lint allowlist; lint FAIL. +- [ ] **Step 2: Migrate the header block.** Replace lines 28–39 with: + +```bash +# shellcheck source=lib/config.sh +source "$(dirname "$0")/lib/config.sh" +# shellcheck source=lib/wasm_exports.sh +source "$(dirname "$0")/lib/wasm_exports.sh" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +LINUX_DIR="${LINUX_DIR:-$ANYFS_PATHS_LINUX_SRC}" +OUT="${OUT:-$REPO_ROOT/lkl-wasm}" +EMSDK_DIR="${EMSDK_DIR:-$ANYFS_TOOLCHAINS_EMSDK}" +BLD="${BLD:-$REPO_ROOT/build-anyfs-wasm}" +TS="${TS:-$REPO_ROOT/ts}" +QEMU_ROOT="${QEMU_ROOT:-$ANYFS_PATHS_QEMU_SRC}" +QBLD="${QBLD:-$QEMU_ROOT/build-anyfs-wasm}" +SYS="${WASM_SYSROOT:-$ANYFS_PATHS_WASM_SYSROOT}" +SRC_CORE="${SRC_CORE:-$REPO_ROOT/src/core}" +GLUE="$TS/native/anyfs_ts.c" +LIBLKL="$OUT/tools/lkl/liblkl.a" +TARGET="${ANYFS_TARGET:-browser}" +``` + +- [ ] **Step 3:** Replace the two remaining `-I "$HOME/anyfs-reader/include"` occurrences (in the `INC` array ~line 66 and the qemu_backend.c compile ~line 164) with `-I "$REPO_ROOT/include"`. Reword header comments mentioning `$HOME`. +- [ ] **Step 4:** Replace the 14-line `EXPORTED_FUNCS='...'` literal (lines 192–205) with: + +```bash +EXPORTED_FUNCS="$(anyfs_wasm_exports "$GLUE")" +``` + +- [ ] **Step 5:** `rm scripts/build_anyfs_browser_wasm.sh`, then confirm nothing live references it: + +Run: `grep -rn "build_anyfs_browser_wasm" --include="*.sh" --include="*.yml" --include="*.json" --include="*.ts" --include="*.mjs" . | grep -v node_modules | grep -v docs/` +Expected: no output (the only mention is the historical design doc, which stays). + +- [ ] **Step 6 (green):** lint PASS; `bash -n scripts/build_anyfs_wasm.sh`; `./tests/test_wasm_exports.sh` PASS. +- [ ] **Step 7: Commit** + +```bash +git add scripts/build_anyfs_wasm.sh scripts/lint-no-hardcoded-paths.sh +git commit -m "refactor(build): build_anyfs_wasm uses config.sh + generated exports; drop stale browser_wasm script" +``` + +### Task 9: Prove the bundle still works (full wasm rebuild + smoke) + +**Files:** none (build + test only) + +- [ ] **Step 1:** Rebuild the node-target bundle: `ANYFS_TARGET=node ./scripts/build_anyfs_wasm.sh` +Expected: links `ts/packages/core/wasm/anyfs.node.mjs` without `undefined symbol` errors. +- [ ] **Step 2:** Rebuild the browser bundle: `./scripts/build_anyfs_wasm.sh` +Expected: `ts/packages/core/wasm/anyfs.mjs` regenerated. +- [ ] **Step 3:** Run the existing core smoke suite (exercises ccall→export coverage end-to-end): + +```bash +cd ts && pnpm --filter @anyfs/core build && pnpm --filter @anyfs/core test +``` + +Expected: `smoke.node.mjs single|multi|big` all PASS. +- [ ] **Step 4: Commit** (wasm artifacts if they are tracked; check `git status -- ts/packages/core/wasm` first — if untracked, no commit needed): + +```bash +git add -A ts/packages/core/wasm 2>/dev/null || true +git commit -m "build(wasm): regenerate bundles from unified export list" || echo "nothing tracked to commit" +``` + +--- + +## Phase 2A — wasm-ld release pipeline (xdqi/llvm-wasm fork) + +**Why:** the patched wasm-ld (GNU-ld SECTIONS{} support for the LKL vmlinux link) currently requires every machine to build LLVM from `~/linux-wasm` sources. The fork gets a CI release pipeline (accelerated by the sccache-dist farm — LLVM is a huge C++ build, the ideal customer); anyfs-reader consumes the release binary. + +### Task 9A: Release workflow in the llvm-wasm fork + +**Files (in a fresh clone, NOT in anyfs-reader):** +- Clone: `gh repo clone xdqi/llvm-wasm ~/llvm-wasm -- --branch wasm-18.1.2-anyfs --depth 1` +- Create: `~/llvm-wasm/.github/workflows/wasm-ld-release.yml` + +- [ ] **Step 1: Inspect the branch layout** — `ls ~/llvm-wasm` and check whether it is (a) a plain llvm-project fork (has `llvm/`, `lld/` at root) or (b) carries Joel's `linux-wasm.sh` harness (doctor's hint suggests the consumed artifact lives at `workspace/install/llvm/bin/wasm-ld`). Record which; the build step below assumes (a) — if (b), replace the cmake/ninja block with `CC="sccache clang" CXX="sccache clang++" ./linux-wasm.sh build-llvm` and adjust the artifact path to `workspace/install/llvm/bin/wasm-ld`. +- [ ] **Step 2: Write `.github/workflows/wasm-ld-release.yml`** + +```yaml +name: wasm-ld-release + +# Builds the patched wasm-ld (GNU-ld linker-script support for LKL's +# vmlinux link) and publishes it as a GitHub release asset consumed by +# anyfs-reader's scripts/fetch_wasm_ld.sh. Compilation is distributed +# over an ephemeral sccache-dist farm. + +on: + workflow_dispatch: + inputs: + tag: + description: 'release tag (e.g. wasm-ld-18.1.2-anyfs-r1)' + required: true + +jobs: + sccache-workers: + name: sccache worker ${{ matrix.idx }} + runs-on: ubuntu-24.04 + timeout-minutes: 120 + strategy: + matrix: + idx: [1, 2, 3] + steps: + - uses: xdqi/sccache-dist-action@v0.0.4 + with: + mode: worker + worker-index: '${{ matrix.idx }}' + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' + + build: + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends cmake ninja-build clang lld python3 + - uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-wasm-ld-${{ github.sha }} + restore-keys: | + sccache-wasm-ld- + - name: Bring up sccache-dist farm + uses: xdqi/sccache-dist-action@v0.0.4 + with: + mode: coordinator + expected-workers: 3 + min-workers: 1 + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' + - name: Build wasm-ld + run: | + cmake -S llvm -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ + -DLLVM_ENABLE_PROJECTS=lld \ + -DLLVM_TARGETS_TO_BUILD=WebAssembly \ + -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_BENCHMARKS=OFF + ninja -C build -j"${SCCACHE_J:-$(nproc)}" lld + sccache --show-stats + - name: Package + run: | + mkdir -p out + cp build/bin/wasm-ld out/wasm-ld # lld symlink target; verify with `build/bin/wasm-ld --version` + strip out/wasm-ld + out/wasm-ld --version | grep -q 'LLD 18' + tar -C out -cJf wasm-ld-linux-amd64.tar.xz wasm-ld + - name: Release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${{ inputs.tag }}" wasm-ld-linux-amd64.tar.xz \ + --target "$GITHUB_SHA" \ + --title "${{ inputs.tag }}" \ + --notes "Patched wasm-ld (GNU-ld SECTIONS{} support) for anyfs-reader's LKL wasm link." +``` + +**Note:** `bin/wasm-ld` is a symlink to `lld` in LLVM builds — `cp` dereferences it, which is what we want. If the patch set lives in `lld/wasm/` of this fork, the plain cmake build picks it up automatically. +- [ ] **Step 3:** Verify the fork has the `TS_OAUTH_SECRET` secret (`gh secret list -R xdqi/llvm-wasm`); set it if absent (same Tailscale OAuth client as anyfs-reader's). +- [ ] **Step 4:** Commit to a branch of the fork, push, and trigger: `gh workflow run wasm-ld-release.yml -R xdqi/llvm-wasm --ref -f tag=wasm-ld-18.1.2-anyfs-r1`. Watch with `gh run watch -R xdqi/llvm-wasm`. Expected: release `wasm-ld-18.1.2-anyfs-r1` exists with the tarball; sccache stats show distributed compiles. +- [ ] **Step 5:** Smoke the artifact locally: download, untar, `./wasm-ld --version` → `LLD 18...`; then run one real LKL wasm link with `JOEL_WASM_LD= ./scripts/build_lkl_wasm.sh` → links clean. + +### Task 9B: Consume the release in anyfs-reader + +**Files:** +- Create: `scripts/fetch_wasm_ld.sh` +- Modify: `scripts/lib/config.sh` (default chain) +- Modify: `scripts/doctor.sh` (accept the fetched location + suggest the fetch script) +- Modify: `scripts/lint-shellcheck.sh` + `scripts/lint-no-hardcoded-paths.sh` (add fetch_wasm_ld.sh) + +- [ ] **Step 1:** Write `scripts/fetch_wasm_ld.sh`: + +```bash +#!/usr/bin/env bash +# Download the patched wasm-ld release (built by xdqi/llvm-wasm's +# wasm-ld-release workflow) into /.toolchain/wasm-ld/. +# Pin: bump WASM_LD_TAG when a new release is cut. +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +WASM_LD_TAG="${WASM_LD_TAG:-wasm-ld-18.1.2-anyfs-r1}" +dest="$root/.toolchain/wasm-ld" +if [[ -x "$dest/wasm-ld" ]] && "$dest/wasm-ld" --version | grep -q 'LLD 18'; then + echo "wasm-ld already present: $dest/wasm-ld"; exit 0 +fi +mkdir -p "$dest" +url="https://github.com/xdqi/llvm-wasm/releases/download/$WASM_LD_TAG/wasm-ld-linux-amd64.tar.xz" +echo "fetching $url" +curl -fsSL --retry 3 "$url" | tar -xJ -C "$dest" +"$dest/wasm-ld" --version | grep -q 'LLD 18' +echo "OK: $dest/wasm-ld" +``` + +`chmod +x`, add `.toolchain/` to `.gitignore`. +- [ ] **Step 2:** In `scripts/lib/config.sh`, change the wasm_ld default chain to prefer the fetched binary when present: + +```bash + if [[ -z "${ANYFS_TOOLCHAINS_WASM_LD:-}" && -x "$root/.toolchain/wasm-ld/wasm-ld" ]]; then + ANYFS_TOOLCHAINS_WASM_LD="$root/.toolchain/wasm-ld/wasm-ld" + fi + : "${ANYFS_TOOLCHAINS_WASM_LD:=$pfx$deps/llvm-wasm/workspace/install/llvm/bin/wasm-ld}" +``` + +(Explicit `build.user.toml` still wins because it arrives non-empty.) +- [ ] **Step 3:** Update `scripts/doctor.sh`'s wasm-ld failure hint to: `run scripts/fetch_wasm_ld.sh (or build from deps/llvm-wasm)`. +- [ ] **Step 4: Verify** — `rm -rf .toolchain && ./scripts/fetch_wasm_ld.sh` downloads + validates; `bash -c 'source scripts/lib/config.sh && echo $ANYFS_TOOLCHAINS_WASM_LD'` → the `.toolchain` path when `build.user.toml`'s `wasm_ld` is commented out; shellcheck + both lint gates PASS. +- [ ] **Step 5: Commit** — `git commit -m "feat(build): fetch patched wasm-ld from llvm-wasm fork releases"` + +--- + +## Phase 2B — wasm-sysroot: recipe + prebuilt tarball (mingw provisioning model) + +**Why:** today only libblkid has an in-repo recipe; the other ~19 static libs in `~/wasm-sysroot` were hand-built and are unreproducible. Target model = mingw's: an in-repo recipe (the PKGBUILD role), a CI-published tarball (the bootstrap.tar.xz role), a fetch script + doctor manifest check (the pacman role). + +### Task 9C: Sysroot manifest + doctor check (do this first — it locks the contract) + +**Files:** +- Create: `scripts/lib/wasm_sysroot.manifest` +- Modify: `scripts/doctor.sh` + +- [ ] **Step 1:** Generate the manifest from the known-good local sysroot — record both the lib list and the versions that pkgconfig knows: + +```bash +{ ls ~/wasm-sysroot/lib/*.a | xargs -n1 basename; + for p in ~/wasm-sysroot/lib/pkgconfig/*.pc; do + printf '# %s %s\n' "$(basename "$p" .pc)" "$(grep -m1 '^Version:' "$p" | cut -d' ' -f2)"; + done; } | sort > scripts/lib/wasm_sysroot.manifest +``` + +Review the output (expect ~20 `.a` entries: libblkid, libbz2, libffi, libgio-2.0, libglib-2.0, libgmodule-2.0, libgobject-2.0, libgthread-2.0, libgirepository-2.0, libpcre2-{8,16,32,posix}, libresolv, libuuid, libz, libzstd + version comment lines). +- [ ] **Step 2:** Add a doctor section that checks every `.a` in the manifest exists under `$ANYFS_PATHS_WASM_SYSROOT/lib` and reports missing ones with the hint `run scripts/fetch_wasm_sysroot.sh (or scripts/build_wasm_sysroot.sh)`. +- [ ] **Step 3: Verify** — `./scripts/doctor.sh` reports the sysroot section OK locally (and FAIL when pointed at an empty dir via `ANYFS_PATHS_WASM_SYSROOT=/tmp/empty ./scripts/doctor.sh`). +- [ ] **Step 4: Commit** — `git commit -m "feat(doctor): wasm-sysroot manifest check"` + +### Task 9D: `build_wasm_sysroot.sh` recipe (exploratory — acceptance is manifest parity) + +**Files:** +- Create: `scripts/build_wasm_sysroot.sh` +- Create: `scripts/lib/emscripten-cross.meson` (glib needs a meson cross file) +- Modify: lint allowlists (both) + +This task is the archaeology: the recipe must rebuild, from pinned upstream tarballs, a sysroot whose `.a` set matches `scripts/lib/wasm_sysroot.manifest`. Versions come from the manifest's `#` comment lines (zlib/zstd/bz2 have no `.pc` — pin to the versions currently linkable; start with zlib 1.3.x, bzip2 1.0.8, zstd 1.5.x and verify the existing `.a`s' object names match). + +- [ ] **Step 1:** Script skeleton — config-sourced, per-lib functions, `SYSROOT` output dir argument (default `$ANYFS_PATHS_WASM_SYSROOT`), `--only=` for iteration: + +```bash +#!/usr/bin/env bash +# Build the complete wasm sysroot from pinned sources (the PKGBUILD role +# in the mingw-style provisioning model; see docs/wasm-sysroot.md). +# Order matters: zlib -> bz2 -> zstd -> libffi -> pcre2 -> glib -> util-linux(blkid+uuid). +set -euo pipefail +source "$(dirname "$0")/lib/config.sh" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SYSROOT="${SYSROOT:-$ANYFS_PATHS_WASM_SYSROOT}" +WORK="${WORK:-$REPO_ROOT/build-wasm-sysroot}" +# ... per-lib build_zlib / build_bz2 / build_zstd / build_ffi / build_pcre2 / +# build_glib / build_blkid functions; each: fetch pinned tarball into +# $WORK, emconfigure/emcmake/meson-cross build, install into $SYSROOT. +``` + +Known-good per-lib approaches (verify/adjust while iterating): + - zlib: `emconfigure ./configure --static --prefix=$SYSROOT && emmake make install` + - bzip2: no configure — `emmake make libbz2.a CC=emcc AR=emar RANLIB=emranlib` + manual install + - zstd: `emmake make -C lib libzstd.a` + manual install (or `emcmake cmake build/cmake -DZSTD_BUILD_PROGRAMS=OFF -DZSTD_BUILD_SHARED=OFF`) + - libffi: needs the emscripten port patches — use upstream libffi + `emconfigure ./configure --host=wasm32-unknown-linux --disable-shared`; if that fails, the known-working route is the `libffi-emscripten` fork. Record which in the script comment. + - pcre2: `emcmake cmake -DPCRE2_BUILD_PCRE2_16=ON -DPCRE2_BUILD_PCRE2_32=ON -DBUILD_SHARED_LIBS=OFF` + - glib: `meson setup --cross-file scripts/lib/emscripten-cross.meson -Ddefault_library=static -Dtests=false -Dglib_assert=false`; the cross file maps c='emcc', ar='emar', pkg-config sysroot to `$SYSROOT`. glib-on-emscripten is the fiddly one — budget iteration time; the existing `~/wasm-sysroot` proves a working flag set exists. + - blkid+uuid: reuse the logic from `build_libblkid_wasm.sh` (fold it in or call it; keep `build_libblkid_wasm.sh` as a thin wrapper for compat). +- [ ] **Step 2 (acceptance):** Build into a fresh dir and diff against the manifest: + +```bash +SYSROOT=/tmp/sysroot-rebuild ./scripts/build_wasm_sysroot.sh +ls /tmp/sysroot-rebuild/lib/*.a | xargs -n1 basename | sort > /tmp/rebuilt.txt +grep -v '^#' scripts/lib/wasm_sysroot.manifest | sort > /tmp/expected.txt +diff /tmp/expected.txt /tmp/rebuilt.txt +``` + +Expected: empty diff (libgirepository may be droppable if nothing links it — check `grep -rn girepository scripts/ meson.build`; if unused, remove it from the manifest instead of building it, and note that in the commit message). +- [ ] **Step 3 (proof):** Rebuild the wasm bundle against the fresh sysroot and run the smoke suite: `WASM_SYSROOT=/tmp/sysroot-rebuild ANYFS_TARGET=node ./scripts/build_anyfs_wasm.sh && (cd ts && pnpm --filter @anyfs/core test)` → PASS. +- [ ] **Step 4:** shellcheck + lint gates; commit — `git commit -m "feat(build): reproducible wasm-sysroot recipe (manifest-parity with the legacy hand-built sysroot)"` + +### Task 9E: CI tarball release + fetch script + +**Files:** +- Create: `.github/workflows/wasm-sysroot.yml` +- Create: `scripts/fetch_wasm_sysroot.sh` +- Create: `docs/wasm-sysroot.md` + +- [ ] **Step 1:** `.github/workflows/wasm-sysroot.yml` — workflow_dispatch with a `tag` input (e.g. `wasm-sysroot-r1`): install emsdk (`mymindstorm/setup-emsdk@v14`, version matching local — check `emcc --version` locally and pin it), meson/ninja/cmake, run `SYSROOT="$PWD/out-sysroot" ./scripts/build_wasm_sysroot.sh`, run the manifest diff from Task 9D Step 2 as the gate, `tar -cJf wasm-sysroot-linux.tar.xz -C out-sysroot .`, `gh release create "$tag" ...` (same shape as Task 9A's release step, `permissions: contents: write`). +- [ ] **Step 2:** `scripts/fetch_wasm_sysroot.sh` — mirror of `fetch_wasm_ld.sh`: pin `WASM_SYSROOT_TAG`, download from `https://github.com/xdqi/anyfs/releases/download/$WASM_SYSROOT_TAG/wasm-sysroot-linux.tar.xz` into `/.toolchain/wasm-sysroot/`, presence check = manifest check, and the same config.sh preference hook: + +```bash + if [[ -z "${ANYFS_PATHS_WASM_SYSROOT:-}" && -d "$root/.toolchain/wasm-sysroot/lib" ]]; then + ANYFS_PATHS_WASM_SYSROOT="$root/.toolchain/wasm-sysroot" + fi +``` + +(insert before the existing `: "${ANYFS_PATHS_WASM_SYSROOT:=$root/wasm-sysroot}"`). +- [ ] **Step 3:** `docs/wasm-sysroot.md` — document the three-role model (recipe=PKGBUILD, release tarball=bootstrap.tar.xz, fetch+manifest=pacman), version-bump procedure, and the per-target provisioning table (apt / msys.kosaka.moe pacman / this tarball). +- [ ] **Step 4:** Trigger the workflow, verify the release exists, then on a clean checkout simulate a fresh machine: `mv ~/wasm-sysroot ~/wasm-sysroot.bak; ./scripts/fetch_wasm_sysroot.sh && ./scripts/doctor.sh` → sysroot section OK; restore `~/wasm-sysroot`. +- [ ] **Step 5: Commit** — `git commit -m "feat(build): wasm-sysroot CI tarball release + fetch (mingw-style provisioning)"` + +--- + +## Phase 3 — extract server_common from ksmbd/nfsd + +### Task 10: Create `src/server_common/` + meson wiring + +**Files:** +- Create: `src/server_common/server_common.h` +- Create: `src/server_common/server_common.c` +- Modify: `meson.build` (the `anyfs-ksmbd` and `anyfs-nfsd` `executable()` blocks — grep `lkl_ksmbd = executable`) + +- [ ] **Step 1: Write `src/server_common/server_common.h`** + +```c +/* + * server_common.h — Shared daemon skeleton for the LKL server surfaces + * (anyfs-ksmbd, anyfs-nfsd): stop flag + signal install, kernel boot with + * loopback up, the --share resolution loop, and shutdown. Arg parsing and + * the serving loops stay in the respective mains. + */ +#ifndef ANYFS_SERVER_COMMON_H +#define ANYFS_SERVER_COMMON_H + +#include "anyfs.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* 1 while serving; cleared by SIGINT/SIGTERM. Serving loops poll it. */ +extern volatile sig_atomic_t anyfs_server_running; + +/* Unbuffer stdout and route SIGINT/SIGTERM to clearing + * anyfs_server_running. */ +void anyfs_server_install_signals(void); + +/* Boot the LKL kernel and bring up loopback (ifindex 1, idempotent — + * the in-kernel listeners bind to it). Returns anyfs_kernel_init()'s + * result: 0 on success. */ +int anyfs_server_boot(const AnyfsKernelOpts* opts); + +typedef struct AnyfsShareEntry { + char name[64]; + char lkl_path[ANYFS_LKL_PATH_MAX]; +} AnyfsShareEntry; + +/* Resolve every --share spec to a (name, lkl_path) pair via + * anyfs_share_resolve(), which prints its own diagnostics. Returns the + * number of entries filled, or -1 on the first failure. */ +int anyfs_server_resolve_shares(char* const* specs, int n_specs, + AnyfsSession** disks, int n_disks, + uint32_t enter_flags, AnyfsShareEntry* out, + int max_out); + +/* Close every non-NULL session and halt the kernel. */ +void anyfs_server_shutdown(AnyfsSession** disks, int n_disks); + +#ifdef __cplusplus +} +#endif + +#endif +``` + +- [ ] **Step 2: Write `src/server_common/server_common.c`** + +```c +// SPDX-License-Identifier: GPL-2.0-or-later +#include "server_common.h" + +#include +#include + +volatile sig_atomic_t anyfs_server_running = 1; + +static void handle_stop(int sig) +{ + (void)sig; + anyfs_server_running = 0; +} + +void anyfs_server_install_signals(void) +{ + setbuf(stdout, NULL); + signal(SIGINT, handle_stop); + signal(SIGTERM, handle_stop); +} + +int anyfs_server_boot(const AnyfsKernelOpts* opts) +{ + int ret = anyfs_kernel_init(opts); + if (ret) + return ret; + /* lo is auto-up after boot, but the call is idempotent and + * documents intent. */ + lkl_if_up(1); + return 0; +} + +int anyfs_server_resolve_shares(char* const* specs, int n_specs, + AnyfsSession** disks, int n_disks, + uint32_t enter_flags, AnyfsShareEntry* out, + int max_out) +{ + int n = 0; + + for (int si = 0; si < n_specs; si++) { + if (n >= max_out) { + fprintf(stderr, "error: too many shares (max %d)\n", + max_out); + return -1; + } + AnyfsShareEntry* e = &out[n]; + if (anyfs_share_resolve(specs[si], disks, n_disks, enter_flags, + e->name, sizeof(e->name), e->lkl_path, + sizeof(e->lkl_path)) < 0) + return -1; + n++; + } + return n; +} + +void anyfs_server_shutdown(AnyfsSession** disks, int n_disks) +{ + for (int i = 0; i < n_disks; i++) { + if (disks[i]) + anyfs_session_close(disks[i]); + } + anyfs_kernel_halt(); +} +``` + +(Verify `anyfs.h` transitively provides `AnyfsKernelOpts`, `ANYFS_LKL_PATH_MAX`, `anyfs_share_resolve`, `anyfs_session_close`, `anyfs_kernel_halt`, `anyfs_kernel_init` — both mains already get all of these from `"anyfs.h"` + ``. If `lkl_if_up` needs another header, mirror whatever `ksmbd_main.c` includes.) + +- [ ] **Step 3: Wire into meson.** In `meson.build`, add the source file and include dir to **both** server executables: + - `lkl_ksmbd = executable('anyfs-ksmbd', ...)`: add `'src/server_common/server_common.c'` to the source list and `include_directories('src/server_common')` to its `include_directories:` array. + - `lkl_nfsd = executable('anyfs-nfsd', ...)`: likewise. + +- [ ] **Step 4: Compile check** (server_common is not yet referenced, it must just build): + +Run: `./scripts/build_anyfs.sh --targets=linux-amd64 --components=core,server -j"$(nproc)"` +Expected: clean build. + +- [ ] **Step 5: Commit** + +```bash +git add src/server_common meson.build +git commit -m "feat(server): add server_common daemon skeleton (signals/boot/shares/shutdown)" +``` + +### Task 11: Port ksmbd_main.c to server_common + +**Files:** +- Modify: `src/ksmbd/ksmbd_main.c` + +- [ ] **Step 1:** Add `#include "server_common.h"` next to the other local includes. +- [ ] **Step 2:** Delete the local `static volatile int running = 1;` and `sigint_handler` (lines ~71–77); replace every remaining `running` reference (the `while (running)` IPC loop) with `anyfs_server_running`. +- [ ] **Step 3:** Replace the ShareInfo typedef with the shared entry — delete the `typedef struct { char name[64]; char lkl_path[ANYFS_LKL_PATH_MAX]; } ShareInfo;` block and add `typedef AnyfsShareEntry ShareInfo;` (keeps `setup_ksmbd_config(const ShareInfo*, ...)` and all `sh->name`/`sh->lkl_path` uses unchanged). +- [ ] **Step 4:** In `main()`: + - replace the `setbuf(stdout, NULL); signal(SIGINT, sigint_handler); signal(SIGTERM, sigint_handler);` triple with `anyfs_server_install_signals();` + - replace the kernel-boot block + + ```c + int ret = anyfs_kernel_init(&kern_opts); + if (ret) { pr_err("Failed to start kernel\n"); return 1; } + pr_info("LKL kernel started (ksmbd built-in)\n"); + lkl_if_up(1); + ``` + + with + + ```c + int ret = anyfs_server_boot(&kern_opts); + if (ret) { pr_err("Failed to start kernel\n"); return 1; } + pr_info("LKL kernel started (ksmbd built-in)\n"); + ``` + + (delete the now-redundant standalone `lkl_if_up(1);` and its comment block — the comment content moved to server_common.c). + - replace the share-resolve loop + + ```c + ShareInfo shares[ANYFS_MAX_SHARES]; + int n_shares = 0; + for (int si = 0; si < n_share_specs; si++) { ... } + ``` + + with + + ```c + ShareInfo shares[ANYFS_MAX_SHARES]; + int n_shares = anyfs_server_resolve_shares(share_specs, n_share_specs, + disks, n_images, 0, shares, + ANYFS_MAX_SHARES); + if (n_shares < 0) + goto halt; + for (int i = 0; i < n_shares; i++) + pr_info("Share [%s] -> %s\n", shares[i].name, shares[i].lkl_path); + ``` + + - replace the `halt:` epilogue's close-loop + `anyfs_kernel_halt();` with `anyfs_server_shutdown(disks, n_images);`. +- [ ] **Step 5: Build + verify** — `./scripts/build_anyfs.sh --targets=linux-amd64 --components=core,server -j"$(nproc)"` → clean. +- [ ] **Step 6: Commit** — `git commit -m "refactor(ksmbd): use server_common skeleton"` + +### Task 12: Port nfsd_main.c to server_common + smoke both servers + +**Files:** +- Modify: `src/nfsd/nfsd_main.c` + +- [ ] **Step 1:** Add `#include "server_common.h"`; delete local `running`/`sigint_handler`; replace `while (running)` with `while (anyfs_server_running)`. +- [ ] **Step 2:** In `main()`: replace the signal/setbuf triple with `anyfs_server_install_signals();`; replace the kernel init + `lkl_if_up(1)` block with `anyfs_server_boot(&kern_opts)` (same pattern as Task 11, keeping nfsd's own `printf("LKL kernel started (nfsd built-in)\n");`). +- [ ] **Step 3:** Replace the export-resolve loop. `ExportInfo` keeps its `bind_path` member, so resolve into shared entries first, then copy: + +```c + /* ── 5. Resolve --share specs to LKL paths ───────────────────────── */ + { + AnyfsShareEntry ents[ANYFS_MAX_SHARES]; + uint32_t eflags = read_only ? ANYFS_SESSION_READONLY : 0; + int n = anyfs_server_resolve_shares(share_specs, n_share_specs, + disks, n_images, eflags, + ents, ANYFS_MAX_SHARES); + if (n < 0) + goto halt; + for (int i = 0; i < n; i++) { + ExportInfo* ex = &g_exports[g_n_exports]; + memcpy(ex->name, ents[i].name, sizeof(ex->name)); + memcpy(ex->lkl_path, ents[i].lkl_path, + sizeof(ex->lkl_path)); + printf("Export [%d] /%s -> %s\n", g_n_exports, ex->name, + ex->lkl_path); + g_n_exports++; + } + } +``` + +(`ExportInfo.name` is `char[64]` and `lkl_path` is `char[ANYFS_LKL_PATH_MAX]` — identical sizes, so `memcpy` of the full member is safe.) +- [ ] **Step 4:** Replace the `halt:` close-loop + `anyfs_kernel_halt();` with `anyfs_server_shutdown(disks, n_images);` (keep the final `printf("Done\n"); return 0;`). +- [ ] **Step 5: Build** — same build_anyfs.sh invocation → clean. +- [ ] **Step 6: Behavior verification — run the integration smoke test against the local Debian image:** + +Run: `./tests/smoke-debian-qcow2.sh --build-dir="$PWD/build-anyfs-linux-amd64" "$PWD/debian.qcow2"` +Expected: lspart + smbclient + NFS checks PASS (no `--nfs-mount` locally — that needs sudo). Ctrl+C handling is covered by the script's own teardown. +- [ ] **Step 7: Commit** — `git commit -m "refactor(nfsd): use server_common skeleton"` + +--- + +## Phase 4 — C and TS unit tests + +### Task 13: C unit suite — path DSL (TDD scaffolding for `meson test`) + +**Files:** +- Create: `tests/unit/test_path_dsl.c` +- Modify: `meson.build` (after the existing `# --- Tests ---` section) + +- [ ] **Step 1: Write the test** + +`tests/unit/test_path_dsl.c`: + +```c +// SPDX-License-Identifier: GPL-2.0-or-later +/* Unit tests for the path DSL parser (src/core/anyfs_path.c). + * Pure userspace — no LKL boot, runs in milliseconds under `meson test`. */ +#include "anyfs_path.h" + +#include +#include + +static int failures; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, \ + __LINE__, #cond); \ + failures++; \ + } \ + } while (0) + +static void test_simple(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("p1", &ap) == 0); + CHECK(ap.n_comp == 1); + CHECK(ap.comp[0].p == 1); + CHECK(ap.comp[0].query == NULL); + CHECK(ap.disk_idx_set == 0); + anyfs_path_free(&ap); +} + +static void test_disk_prefix(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("disk0/p1", &ap) == 0); + CHECK(ap.disk_idx_set == 1); + CHECK(ap.disk_idx == 0); + CHECK(ap.n_comp == 1 && ap.comp[0].p == 1); + anyfs_path_free(&ap); + + CHECK(anyfs_path_parse("disk12/p2/p1", &ap) == 0); + CHECK(ap.disk_idx == 12); + CHECK(ap.n_comp == 2); + CHECK(ap.comp[0].p == 2 && ap.comp[1].p == 1); + anyfs_path_free(&ap); +} + +static void test_slashes_and_case(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("/p1/", &ap) == 0); /* leading+trailing ok */ + CHECK(ap.n_comp == 1 && ap.comp[0].p == 1); + anyfs_path_free(&ap); + + CHECK(anyfs_path_parse("P3", &ap) == 0); /* uppercase P accepted */ + CHECK(ap.comp[0].p == 3); + anyfs_path_free(&ap); +} + +static void test_query(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("p1?keyref=LUKS_KEY", &ap) == 0); + CHECK(ap.comp[0].query != NULL); + CHECK(strcmp(ap.comp[0].query, "keyref=LUKS_KEY") == 0); + anyfs_path_free(&ap); + + /* Query is percent-decoded in place; %2F must not re-split. */ + CHECK(anyfs_path_parse("p1?keyfile=%2Ftmp%2Fk", &ap) == 0); + CHECK(strcmp(ap.comp[0].query, "keyfile=/tmp/k") == 0); + anyfs_path_free(&ap); + + /* Bad escape in query is a parse error. */ + CHECK(anyfs_path_parse("p1?key=%zz", &ap) == -1); +} + +static void test_errors(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("", &ap) == -1); + CHECK(anyfs_path_parse("/", &ap) == -1); + CHECK(anyfs_path_parse("p0", &ap) == -1); /* index must be > 0 */ + CHECK(anyfs_path_parse("p", &ap) == -1); + CHECK(anyfs_path_parse("x1", &ap) == -1); + CHECK(anyfs_path_parse("disk0", &ap) == -1); /* disk alone invalid */ + CHECK(anyfs_path_parse("disk/p1", &ap) == -1); + CHECK(anyfs_path_parse("diskX/p1", &ap) == -1); + CHECK(anyfs_path_parse("p1x", &ap) == -1); + /* 9 components > ANYFS_PATH_MAX_COMPONENTS (8) */ + CHECK(anyfs_path_parse("p1/p1/p1/p1/p1/p1/p1/p1/p1", &ap) == -1); + CHECK(anyfs_path_parse(NULL, &ap) == -1); +} + +static void test_pct_decode(void) +{ + char a[] = "a%2Fb"; + CHECK(anyfs_path_pct_decode(a) == 0); + CHECK(strcmp(a, "a/b") == 0); + + char b[] = "plain"; + CHECK(anyfs_path_pct_decode(b) == 0); + CHECK(strcmp(b, "plain") == 0); + + char c[] = "%zz"; + CHECK(anyfs_path_pct_decode(c) == -1); + + char d[] = "%4"; /* truncated escape */ + CHECK(anyfs_path_pct_decode(d) == -1); + + CHECK(anyfs_path_pct_decode(NULL) == 0); +} + +int main(void) +{ + test_simple(); + test_disk_prefix(); + test_slashes_and_case(); + test_query(); + test_errors(); + test_pct_decode(); + if (failures) { + fprintf(stderr, "%d failure(s)\n", failures); + return 1; + } + printf("test_path_dsl: all OK\n"); + return 0; +} +``` + +- [ ] **Step 2: Register in meson.** In `meson.build`, after the existing tests block (`test_nfsd_pseudo_root = ...`), add: + +```meson +# --- Unit tests (pure userspace, no LKL boot; run via `meson test --suite unit`) --- +test_path_dsl = executable('test_path_dsl', + 'tests/unit/test_path_dsl.c', + 'src/core/anyfs_path.c', + include_directories: [anyfs_inc], + install: false, +) +test('path_dsl', test_path_dsl, suite: 'unit') +``` + +(`anyfs_path.c` is freestanding — no LKL/glib deps — so the test links in milliseconds.) + +- [ ] **Step 3: Run to verify it fails… then passes.** First deliberately check the harness catches failures: temporarily flip one assertion (e.g. `CHECK(ap.comp[0].p == 2)` in `test_simple`), run, see FAIL; revert. Then: + +Run: `ninja -C build-anyfs-linux-amd64 && meson test -C build-anyfs-linux-amd64 --suite unit --print-errorlogs` +Expected: `1/1 path_dsl OK`. + +- [ ] **Step 4: Commit** — `git add tests/unit meson.build && git commit -m "test(core): meson unit suite + path DSL parser tests"` + +### Task 14: C unit suite — share helpers + +**Files:** +- Create: `tests/unit/test_share_helpers.c` +- Modify: `meson.build` (append below the path_dsl test) + +- [ ] **Step 1: Write the test** + +```c +// SPDX-License-Identifier: GPL-2.0-or-later +/* Unit tests for the pure --share helpers (src/core/anyfs_share.c). */ +#include "anyfs_share.h" + +#include +#include + +static int failures; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, \ + __LINE__, #cond); \ + failures++; \ + } \ + } while (0) + +static void test_auto_name(void) +{ + char out[32]; + anyfs_share_auto_name("disk0/p1", out, sizeof(out)); + CHECK(strcmp(out, "disk0_p1") == 0); + + anyfs_share_auto_name("p2", out, sizeof(out)); + CHECK(strcmp(out, "p2") == 0); + + /* Truncation: output is always NUL-terminated. */ + char tiny[5]; + anyfs_share_auto_name("disk0/p1", tiny, sizeof(tiny)); + CHECK(strcmp(tiny, "disk") == 0); + + /* out_sz == 0 must not write anything (no crash). */ + anyfs_share_auto_name("x", tiny, 0); +} + +static void test_split(void) +{ + const char *name, *path; + + char a[] = "data=disk0/p1"; + CHECK(anyfs_share_split(a, &name, &path) == 0); + CHECK(name && strcmp(name, "data") == 0); + CHECK(strcmp(path, "disk0/p1") == 0); + + char b[] = "disk0/p1"; + CHECK(anyfs_share_split(b, &name, &path) == 0); + CHECK(name == NULL); + CHECK(strcmp(path, "disk0/p1") == 0); + + /* Only the FIRST '=' splits; the rest stays in path. */ + char c[] = "n=p1?key=v"; + CHECK(anyfs_share_split(c, &name, &path) == 0); + CHECK(strcmp(name, "n") == 0); + CHECK(strcmp(path, "p1?key=v") == 0); +} + +int main(void) +{ + test_auto_name(); + test_split(); + if (failures) { + fprintf(stderr, "%d failure(s)\n", failures); + return 1; + } + printf("test_share_helpers: all OK\n"); + return 0; +} +``` + +- [ ] **Step 2: Register in meson** (anyfs_share.c calls into sessions, so link the full dep like `test_raw_mount` does): + +```meson +test_share_helpers = executable('test_share_helpers', + 'tests/unit/test_share_helpers.c', + include_directories: [anyfs_inc], + dependencies: [anyfs_dep], + c_args: gio_args + qemu_args, + install: false, +) +test('share_helpers', test_share_helpers, suite: 'unit') +``` + +- [ ] **Step 3: Run** — `meson test -C build-anyfs-linux-amd64 --suite unit --print-errorlogs` → `2/2 OK`. +- [ ] **Step 4: Commit** — `git commit -m "test(core): share helper unit tests"` + +### Task 15: CI — run the C unit suite + wasm-exports gate in linux.yml + +**Files:** +- Modify: `.github/workflows/linux.yml` + +- [ ] **Step 1:** After the `Build anyfs-reader (core + server + fuse)` step, add: + +```yaml + - name: Run C unit tests + run: meson test -C build-anyfs-linux-amd64 --suite unit --print-errorlogs + + - name: Wasm export drift gate + run: ./tests/test_wasm_exports.sh +``` + +- [ ] **Step 2: Validate YAML** — `python3 -c "import yaml,sys; yaml.safe_load(open('.github/workflows/linux.yml'))"` +- [ ] **Step 3: Commit** — `git commit -m "ci(linux): run C unit suite + wasm export gate"` + +### Task 16: TS unit tests — @anyfs/core (node:test, no wasm needed) + +**Files:** +- Modify: `ts/packages/core/src/index.ts` (export `AnyfsSessionBase` if not already exported — check with `grep -n "session-base" ts/packages/core/src/index.ts`) +- Create: `ts/packages/core/test/format.test.mjs` +- Create: `ts/packages/core/test/session-base.test.mjs` +- Create: `ts/packages/core/test/dispatch.test.mjs` (supersedes `dispatch.test.ts`, which is currently not executed by any script) +- Delete: `ts/packages/core/test/dispatch.test.ts` +- Modify: `ts/packages/core/package.json` (scripts) + +- [ ] **Step 1:** Ensure exports. In `ts/packages/core/src/index.ts` add (if missing): + +```ts +export { AnyfsSessionBase } from './session-base.js'; +``` + +Verify `createSession` and the `format.ts` helpers (`fmtBytes`, `fmtMode`, `splitExt`, `formatSize`) are exported from the index; add `export * from './format.js';` if absent. + +- [ ] **Step 2:** Write `ts/packages/core/test/format.test.mjs` (runs against built `dist/`): + +```js +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fmtBytes, fmtMode, splitExt, formatSize } from '../dist/index.js'; + +test('fmtBytes tiers', () => { + assert.equal(fmtBytes(512), '512 B'); + assert.equal(fmtBytes(2048), '2048 B (2.0 KiB)'); + assert.match(fmtBytes(5 * 1024 * 1024), /5\.0 MiB/); + assert.match(fmtBytes(3 * 1024 ** 3), /3\.00 GiB/); +}); + +test('fmtMode common cases', () => { + assert.equal(fmtMode(0o100644), '-rw-r--r-- (0644)'); + assert.equal(fmtMode(0o040755), 'drwxr-xr-x (0755)'); + assert.equal(fmtMode(0o120777), 'lrwxrwxrwx (0777)'); + assert.equal(fmtMode(0o104755), '-rwsr-xr-x (4755)'); // setuid + assert.equal(fmtMode(0o041777), 'drwxrwxrwt (1777)'); // sticky /tmp +}); + +test('splitExt — Chonky-bug-safe rules', () => { + assert.equal(splitExt('a.txt'), '.txt'); + assert.equal(splitExt('noext'), ''); // no dot -> '' + assert.equal(splitExt('.bashrc'), ''); // dotfile -> '' + assert.equal(splitExt('.pwd.lock'), '.lock'); // later dot splits + assert.equal(splitExt('trailing.'), ''); // trailing dot -> '' +}); + +test('formatSize adaptive units', () => { + assert.equal(formatSize(undefined), ''); + assert.equal(formatSize(0), '0 B'); + assert.equal(formatSize(1536), '1.5 KiB'); + assert.equal(formatSize(10 * 1024 * 1024), '10 MiB'); +}); +``` + +- [ ] **Step 3:** Write `ts/packages/core/test/session-base.test.mjs` — a fake in-memory session exercising the shared base-class logic: + +```js +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { AnyfsSessionBase } from '../dist/index.js'; + +/** In-memory tree: dirs map path -> entries; files map path -> bytes. */ +class FakeSession extends AnyfsSessionBase { + constructor({ dirs = {}, files = {}, symlinkSizes = {} } = {}) { + super(); + this.dirs = dirs; + this.files = files; + this.symlinkSizes = symlinkSizes; // lstat-size != follow-size repro + this.openCount = 0; + this.closedFds = []; + this.readdirCalls = 0; + } + async attachBlob() {} async attachUrl() {} async attachPath() {} + async enter() { return '/mnt'; } + async listParts() { return []; } + async meta() { return {}; } + async readdir(path) { + this.readdirCalls++; + const e = this.dirs[path]; + if (!e) throw new Error(`ENOENT ${path}`); + return e; + } + async stat(path) { + if (path in this.symlinkSizes) return { size: this.symlinkSizes[path], mode: 0o120777 }; + return { size: this.files[path]?.length ?? 0, mode: 0o100644 }; + } + async statFollow(path) { return { size: this.files[path]?.length ?? 0, mode: 0o100644 }; } + async readlink() { return ''; } + async realpath(p) { return p; } + async readKernelFile() { return ''; } + onProgress() { return () => {}; } + async _openFdRaw() { this.openCount++; return this.openCount; } + async _readFdRaw(fd, offset, length) { + const bytes = this.currentBytes; + return bytes.subarray(offset, offset + length); + } + async _closeFdRaw(fd) { this.closedFds.push(fd); } + async _dispose() { this.disposedCalled = true; } +} + +async function drain(stream) { + const chunks = []; + for await (const c of stream) chunks.push(c); + return chunks; +} + +test('openReadable sizes from statFollow, not lstat (symlink truncation bug)', async () => { + const data = new Uint8Array(100).fill(7); + const s = new FakeSession({ + files: { '/etc/os-release': data }, + symlinkSizes: { '/etc/os-release': 21 }, // lstat sees link-target length + }); + s.currentBytes = data; + const { stream, size } = await s.openReadable('/etc/os-release'); + assert.equal(size, 100); // would be 21 if base used stat() + const chunks = await drain(stream); + assert.equal(chunks.reduce((n, c) => n + c.length, 0), 100); +}); + +test('openReadable chunks and closes the fd exactly once', async () => { + const data = new Uint8Array(2500).fill(1); + const s = new FakeSession({ files: { '/f': data } }); + s.currentBytes = data; + const { stream } = await s.openReadable('/f', { chunkSize: 1000 }); + const chunks = await drain(stream); + assert.deepEqual(chunks.map((c) => c.length), [1000, 1000, 500]); + assert.deepEqual(s.closedFds, [1]); +}); + +test('openReadable cancel closes the fd', async () => { + const data = new Uint8Array(5000).fill(1); + const s = new FakeSession({ files: { '/f': data } }); + s.currentBytes = data; + const { stream } = await s.openReadable('/f', { chunkSize: 1000 }); + const reader = stream.getReader(); + await reader.read(); + await reader.cancel(); + assert.deepEqual(s.closedFds, [1]); +}); + +test('walk is BFS, skips unreadable dirs, honors chunkSize', async () => { + const s = new FakeSession({ + dirs: { + '/': [ + { name: 'a', kind: 'dir' }, + { name: 'f1', kind: 'file' }, + ], + '/a': [{ name: 'f2', kind: 'file' }, { name: 'bad', kind: 'dir' }], + // '/a/bad' missing -> readdir throws -> silently skipped + }, + }); + const seen = []; + for await (const chunk of s.walk('/', 2)) seen.push(...chunk); + assert.deepEqual(seen, ['/a', '/f1', '/a/f2', '/a/bad']); +}); + +test('close() closes tracked fds, disposes once, and poisons the session', async () => { + const data = new Uint8Array(10); + const s = new FakeSession({ files: { '/f': data } }); + s.currentBytes = data; + await s.openFd('/f'); + await s.close(); + assert.deepEqual(s.closedFds, [1]); + assert.equal(s.disposedCalled, true); + await assert.rejects(() => s.openFd('/f'), /already disposed/); + await s.close(); // idempotent +}); +``` + +- [ ] **Step 4:** Write `ts/packages/core/test/dispatch.test.mjs` — full matrix, replacing the never-run `.ts` file: + +```js +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { createSession } from '../dist/index.js'; + +afterEach(() => { delete globalThis.anyfsNative; }); + +const fakeBridge = { sessionOpen() {}, kernelInit() {} }; + +test('web → wasm, blob+url only, no bridge', () => { + const r = createSession('web'); + assert.equal(r.backend, 'wasm'); + assert.deepEqual(r.allowedKinds, new Set(['blob', 'url'])); + assert.equal(r.nativeBridge, undefined); + assert.equal(typeof r.wasmCaps, 'object'); +}); + +test('node → node-wasm, path only', () => { + const r = createSession('node'); + assert.equal(r.backend, 'node-wasm'); + assert.deepEqual(r.allowedKinds, new Set(['path'])); +}); + +test('electron + bridge → native, path+url', () => { + globalThis.anyfsNative = fakeBridge; + const r = createSession('electron'); + assert.equal(r.backend, 'native'); + assert.equal(r.nativeBridge, fakeBridge); + assert.deepEqual(r.allowedKinds, new Set(['path', 'url'])); +}); + +test('electron + bridge + disableNative → wasm fallback', () => { + globalThis.anyfsNative = fakeBridge; + const r = createSession('electron', { disableNative: true }); + assert.equal(r.backend, 'wasm'); + assert.deepEqual(r.allowedKinds, new Set(['blob', 'url'])); +}); + +test('electron without bridge → wasm fallback', () => { + const r = createSession('electron'); + assert.equal(r.backend, 'wasm'); + assert.deepEqual(r.allowedKinds, new Set(['blob', 'url'])); +}); + +test('electron wasm + pathLoopbackUrl cap → path allowed', () => { + const r = createSession('electron', { + disableNative: true, + electronWasmCaps: { pathLoopbackUrl: 'http://127.0.0.1:9999/d0' }, + }); + assert.equal(r.backend, 'wasm'); + assert.ok(r.allowedKinds.has('path')); + assert.equal(r.wasmCaps.pathLoopbackUrl, 'http://127.0.0.1:9999/d0'); +}); + +test('electron wasm caps forwarded verbatim (urlProxyPrefix)', () => { + const r = createSession('electron', { + disableNative: true, + electronWasmCaps: { urlProxyPrefix: 'anyfs-url://proxy/?u=' }, + }); + assert.equal(r.wasmCaps.urlProxyPrefix, 'anyfs-url://proxy/?u='); + assert.ok(!r.allowedKinds.has('path')); // no loopback cap -> no path +}); +``` + +`rm ts/packages/core/test/dispatch.test.ts` + +**Caveat for the executor:** if `getAnyfsNative()` caches or checks `window` instead of `globalThis`, read `ts/packages/core/src/native-session.ts:30-40` and adapt the global stubbing accordingly (verified 2026-06-10: it reads `globalThis.anyfsNative`). + +- [ ] **Step 5:** Wire scripts. In `ts/packages/core/package.json`: + +```json +"scripts": { + "build": "tsup", + "test:unit": "node --test test/*.test.mjs", + "test": "npm run test:unit && node test/smoke.node.mjs single && node test/smoke.node.mjs multi && node test/smoke.node.mjs big" +} +``` + +- [ ] **Step 6: Run to verify they fail first** (before `pnpm build`, dist is stale and `AnyfsSessionBase` export missing): `cd ts && pnpm --filter @anyfs/core run test:unit` → expect import error. Then `pnpm --filter @anyfs/core build && pnpm --filter @anyfs/core run test:unit` → expect all PASS. +- [ ] **Step 7: Commit** — `git add ts/packages/core && git commit -m "test(core): node:test unit suite — format, session-base, dispatch matrix"` + +### Task 17: TS unit tests — @anyfs/react (vitest + testing-library) + +**Files:** +- Modify: `ts/packages/react/package.json` +- Create: `ts/packages/react/vitest.config.ts` +- Create: `ts/packages/react/test/provider.test.tsx` +- Create: `ts/packages/react/test/hooks.test.tsx` + +- [ ] **Step 1:** Add dev deps + scripts in `ts/packages/react/package.json`: + +```json +"scripts": { + "build": "tsup src/index.ts --format esm --dts --clean --external react --external @anyfs/core", + "test": "vitest run" +}, +"devDependencies": { + "typescript": "^5.5.4", + "tsup": "^8.3.0", + "@types/react": "^19.0.0", + "react": "^19.0.0", + "react-dom": "^19.0.0", + "vitest": "^3.0.0", + "jsdom": "^25.0.0", + "@testing-library/react": "^16.1.0" +} +``` + +Run: `cd ts && pnpm install` + +- [ ] **Step 2:** `ts/packages/react/vitest.config.ts`: + +```ts +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'jsdom', + include: ['test/**/*.test.tsx'], + globals: false, + }, +}); +``` + +- [ ] **Step 3:** Write `ts/packages/react/test/provider.test.tsx`. Mock `@anyfs/core` so the provider's state machine is tested without wasm/IPC: + +```tsx +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import React from 'react'; + +const fakeSession = () => ({ + attachBlob: vi.fn(async () => {}), + attachUrl: vi.fn(async () => {}), + attachPath: vi.fn(async () => {}), + onProgress: vi.fn(() => () => {}), + close: vi.fn(async () => {}), + readdir: vi.fn(async () => []), + stat: vi.fn(async () => ({ size: 0, mode: 0o100644 })), + openFd: vi.fn(async () => 3), + readFd: vi.fn(async () => new Uint8Array(0)), + closeFd: vi.fn(async () => {}), +}); + +const prewarmMock = vi.fn(); + +vi.mock('@anyfs/core', () => ({ + createSession: (env: string) => ({ + backend: 'wasm', + allowedKinds: new Set(['blob', 'url']), + wasmCaps: {}, + }), + prewarm: (...args: unknown[]) => prewarmMock(...args), + prewarmNative: vi.fn(), + NativeSession: class NativeSessionMock {}, +})); + +import { AnyfsProvider, useAnyfsDisk } from '../src/index.js'; + +function Status() { + const { status, error } = useAnyfsDisk(); + return

{status}{error ? `:${error.message}` : ''}
; +} + +beforeEach(() => { + prewarmMock.mockReset(); +}); + +describe('AnyfsProvider', () => { + it('idle without source or prewarm', () => { + render( + + + , + ); + expect(screen.getByTestId('status').textContent).toBe('idle'); + }); + + it('prewarm → booting → booted', async () => { + let resolve!: (s: unknown) => void; + prewarmMock.mockReturnValue(new Promise((r) => (resolve = r))); + render( + + + , + ); + expect(screen.getByTestId('status').textContent).toBe('booting'); + resolve(fakeSession()); + await waitFor(() => expect(screen.getByTestId('status').textContent).toBe('booted')); + }); + + it('blob source attaches and reaches ready', async () => { + const s = fakeSession(); + prewarmMock.mockResolvedValue(s); + const blob = new Blob([new Uint8Array(16)]); + render( + + + , + ); + await waitFor(() => expect(screen.getByTestId('status').textContent).toBe('ready')); + expect(s.attachBlob).toHaveBeenCalledWith(blob); + }); + + it('disallowed source kind → error state', async () => { + prewarmMock.mockResolvedValue(fakeSession()); + render( + + + , + ); + await waitFor(() => + expect(screen.getByTestId('status').textContent).toMatch(/^error:.*not supported/), + ); + }); + + it('prewarm failure → error state', async () => { + prewarmMock.mockRejectedValue(new Error('boot failed')); + render( + + + , + ); + await waitFor(() => + expect(screen.getByTestId('status').textContent).toBe('error:boot failed'), + ); + }); +}); +``` + +- [ ] **Step 4:** Write `ts/packages/react/test/hooks.test.tsx`: + +```tsx +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { render, screen, waitFor } from '@testing-library/react'; +import React from 'react'; + +const prewarmMock = vi.fn(); +vi.mock('@anyfs/core', () => ({ + createSession: () => ({ backend: 'wasm', allowedKinds: new Set(['blob', 'url']), wasmCaps: {} }), + prewarm: (...args: unknown[]) => prewarmMock(...args), + prewarmNative: vi.fn(), + NativeSession: class NativeSessionMock {}, +})); + +import { AnyfsProvider, useAnyfsDir, useAnyfsFile } from '../src/index.js'; + +const entriesRoot = [ + { name: 'etc', kind: 'dir' }, + { name: 'README', kind: 'file' }, +]; + +function makeSession() { + return { + attachBlob: vi.fn(async () => {}), + onProgress: vi.fn(() => () => {}), + close: vi.fn(async () => {}), + readdir: vi.fn(async (p: string) => { + if (p === '/') return entriesRoot; + throw new Error('ENOENT'); + }), + stat: vi.fn(async () => ({ size: 4, mode: 0o100644 })), + openFd: vi.fn(async () => 7), + readFd: vi.fn(async () => new Uint8Array([1, 2, 3, 4])), + closeFd: vi.fn(async () => {}), + }; +} + +function Tree({ path }: { path: string }) { + const { entries, loading, error } = useAnyfsDir(path); + if (error) return
error:{error.message}
; + if (loading || !entries) return
loading
; + return
{entries.map((e) => e.name).join(',')}
; +} + +function FileBytes({ path }: { path: string }) { + const { data, error } = useAnyfsFile(path); + if (error) return
error
; + return
{data ? Array.from(data).join(',') : 'loading'}
; +} + +const blob = new Blob([new Uint8Array(8)]); + +beforeEach(() => prewarmMock.mockReset()); + +describe('useAnyfsDir', () => { + it('lists entries once ready and caches per (session,path)', async () => { + const s = makeSession(); + prewarmMock.mockResolvedValue(s); + const { rerender } = render( + + + , + ); + await waitFor(() => expect(screen.getByTestId('dir').textContent).toBe('etc,README')); + // Remount the consumer — cache must serve it without a second readdir. + rerender( + + + , + ); + await waitFor(() => expect(screen.getByTestId('dir').textContent).toBe('etc,README')); + expect(s.readdir).toHaveBeenCalledTimes(1); + }); + + it('surfaces readdir errors', async () => { + prewarmMock.mockResolvedValue(makeSession()); + render( + + + , + ); + await waitFor(() => + expect(screen.getByTestId('dir').textContent).toBe('error:ENOENT'), + ); + }); +}); + +describe('useAnyfsFile', () => { + it('stat + openFd + readFd + closeFd round trip', async () => { + const s = makeSession(); + prewarmMock.mockResolvedValue(s); + render( + + + , + ); + await waitFor(() => expect(screen.getByTestId('file').textContent).toBe('1,2,3,4')); + expect(s.openFd).toHaveBeenCalledWith('/README'); + expect(s.readFd).toHaveBeenCalledWith(7, 0, 4); + expect(s.closeFd).toHaveBeenCalledWith(7); + }); +}); +``` + +- [ ] **Step 5: Run to verify red → green.** `cd ts && pnpm --filter @anyfs/react test` — first run may fail on mock-shape mismatches with the real provider; fix the **tests** (the provider is the spec here), not the provider, unless the failure exposes a real provider bug — in that case stop and report it before changing provider code. +- [ ] **Step 6:** Confirm the workspace-level `pnpm test` (`pnpm -r --filter './packages/*' test`) now runs core unit + smoke, react vitest, and the existing nbd-proxy suite. Run from `ts/`: `pnpm test` → all green. +- [ ] **Step 7: Commit** — `git add ts/packages/react ts/pnpm-lock.yaml && git commit -m "test(react): vitest provider + hooks unit suite"` + +### Task 18: CI — TS unit workflow + +**Files:** +- Create: `.github/workflows/ts.yml` + +- [ ] **Step 1:** + +```yaml +name: ts + +# Pure TS unit tests (no wasm bundle, no LKL): @anyfs/core node:test suite + +# @anyfs/react vitest suite + @anyfs/nbd-proxy. The wasm smoke tests +# (test/smoke.node.mjs) need a built bundle and stay local/linux-job-only. + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: + +jobs: + unit: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + package_json_file: ts/package.json + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: pnpm + cache-dependency-path: ts/pnpm-lock.yaml + - name: Install + working-directory: ts + run: pnpm install --frozen-lockfile + - name: Build packages + working-directory: ts + run: pnpm -r --filter './packages/*' build + - name: Core unit tests + working-directory: ts + run: pnpm --filter @anyfs/core run test:unit + - name: React unit tests + working-directory: ts + run: pnpm --filter @anyfs/react test + - name: nbd-proxy tests + working-directory: ts + run: pnpm --filter @anyfs/nbd-proxy test +``` + +**Caveat:** `@anyfs/anyfs-native` build (node-gyp) may fail or be skipped on CI — if `pnpm -r build` chokes on it, exclude it: `pnpm -r --filter './packages/*' --filter '!@anyfs/anyfs-native' build`. Check `ts/packages/anyfs-native/package.json`'s build script first. + +- [ ] **Step 2:** Validate: `python3 -c "import yaml; yaml.safe_load(open('.github/workflows/ts.yml'))"` +- [ ] **Step 3: Commit** — `git commit -m "ci: ts unit-test workflow"` + +### Task 18A: CDP↔Playwright parity audit, then demote the CDP suite to diagnostics + +**Files:** +- Create: `tests/diagnostics/README.md` +- Move: `tests/{test-cdp,run-all,common-cdp}.mjs` → `tests/diagnostics/` +- Possibly create: new specs under `ts/tests/e2e/flows/` (only where the audit finds gaps) + +Principle (user decision): temporary tests are regression guards — nothing with assertion value gets deleted; the CDP suite is demoted, not removed, once Playwright parity is confirmed. + +- [ ] **Step 1: Build the parity matrix.** Read `tests/test-cdp.mjs` and list every assertion per combo (6 combos: web/electron-wasm/electron-native × local/http). Map each to the Playwright coverage: projects `web`/`electron-wasm`/`electron-native` × specs `open-browse-download` (local open + browse + download), `url-load` (URL incl. local Range server), `formats`, `errors`, `electron-only/backend-switch`. Write the matrix as a table into `tests/diagnostics/README.md`. +- [ ] **Step 2:** For every CDP assertion with NO Playwright equivalent (expected candidates: any DOM detail test-cdp checks that the flows skip — e.g. specific tree-row interactions), add it to the matching existing spec in `ts/tests/e2e/flows/` rather than creating new files. Run the touched specs: `cd ts/tests/e2e && pnpm exec playwright test --project=web ` → PASS. +- [ ] **Step 3:** `git mv tests/test-cdp.mjs tests/run-all.mjs tests/common-cdp.mjs tests/diagnostics/` and finish the README: state that Playwright (`ts/tests/e2e`) is the primary regression suite, these are manual diagnostics, and include the parity matrix + how to run (`node tests/diagnostics/run-all.mjs --image ...`). Fix the `TEST_SCRIPT` relative path inside `run-all.mjs` (it resolves `test-cdp.mjs` via `__dirname` — moving the files together keeps it valid; verify by running `node tests/diagnostics/run-all.mjs --help || true` and checking it locates the script). +- [ ] **Step 4: Commit** — `git commit -m "test: CDP suite demoted to diagnostics after Playwright parity audit"` + +### Task 18B: Relocate test-core.mjs + corral the debug scripts + +**Files:** +- Move: `tests/test-core.mjs`, `tests/common.mjs` → `ts/tests/integration/` +- Move: `tests/{test-atomics,test-async-boot,test-prewarm-direct,test-prewarm-e2e,test-direct-module,test-worker-debug}.mjs` → `tests/diagnostics/` +- Modify: `ts/package.json`, `tests/diagnostics/README.md` + +- [ ] **Step 1:** `git mv tests/test-core.mjs tests/common.mjs ts/tests/integration/`. Fix any relative imports/paths inside them (`grep -n "\.\./" ts/tests/integration/*.mjs` — common.mjs path references to wasm bundles / fixtures must be updated for the new depth). Add to `ts/package.json` scripts: + +```json +"test:integration": "node tests/integration/test-core.mjs" +``` + +Run it: `cd ts && pnpm run test:integration` → the 4 combos (native/wasm × file/url) PASS locally (requires built wasm bundle + native addon; this script is local-only, not wired into CI). +- [ ] **Step 2:** `git mv` the six debug scripts into `tests/diagnostics/`; extend the README with one line per script stating which regression it guards (atomics = Atomics.wait-on-main-thread regressions; async-boot = workeronly async boot path; prewarm-direct/e2e = landing prewarm; direct-module = main-page-load failure mode, expected to fail by design; worker-debug = worker boot logging). Verify each still parses: `for f in tests/diagnostics/*.mjs; do node --check "$f"; done`. +- [ ] **Step 3:** Confirm `tests/` now contains only: C test sources, `images/`, `setup.sh`, `run_tests.sh`, `smoke-debian-qcow2.sh`, `test_wasm_exports.sh`, `unit/`, `diagnostics/`. +- [ ] **Step 4: Commit** — `git commit -m "test: relocate test-core to ts integration; corral debug scripts under tests/diagnostics"` + +--- + +## Phase 5 — sccache-dist-action in linux CI + +**Prerequisite (manual, one-time):** repo secret `TS_OAUTH_SECRET` must exist on `xdqi/anyfs` — a Tailscale OAuth client secret with `auth_keys` write scope tagged `tag:ci-sccache` (already provisioned for the distcc/sccache projects; verify with `gh secret list -R xdqi/anyfs`). If absent, the farm steps are skipped and CI builds locally — no breakage. + +### Task 19: `--cc` plumb-through for build_lkl.sh and build_qemu.sh + +**Files:** +- Modify: `scripts/build_lkl.sh` +- Modify: `scripts/build_qemu.sh` + +- [ ] **Step 1:** `build_lkl.sh`: add to the options help text `# --cc=CMD C compiler override passed to make as CC= (e.g. "sccache gcc")`, add `CC_OVERRIDE=""` next to the other defaults, add the case arm: + +```bash + --cc=*) CC_OVERRIDE="${1#--cc=}"; shift ;; + --cc) CC_OVERRIDE="$2"; shift 2 ;; +``` + +and in `build_one()` extend the make invocations: + +```bash + local cc_arg=() + [[ -n "$CC_OVERRIDE" ]] && cc_arg=(CC="$CC_OVERRIDE") +``` + +then append `"${cc_arg[@]}"` to both `make` calls (clean + build), after `"${cross_arg[@]}"`. + +- [ ] **Step 2:** `build_qemu.sh`: same `--cc=` option parsing into `CC_OVERRIDE=""`. In the configure invocation (`"$QEMU_SRC/configure" ...` around line 225), prepend: + +```bash + local cc_cfg=() + [[ -n "$CC_OVERRIDE" && "$target" == "linux-amd64" ]] && cc_cfg=("--cc=$CC_OVERRIDE") +``` + +and pass `"${cc_cfg[@]}"` to configure. (mingw targets keep their cross_prefix configure untouched — out of scope.) Note: `--cc` only takes effect on a fresh configure; document in the help text that `--reconfigure` is needed to switch compilers. + +- [ ] **Step 3: Verify** — `bash -n` both; `./scripts/build_lkl.sh --help | grep -- --cc` shows the new option. Quick functional check: `./scripts/build_lkl.sh --targets=linux-amd64 --cc=gcc -j"$(nproc)"` → builds (no-op rebuild, CC=gcc is the default compiler anyway). +- [ ] **Step 4: Commit** — `git commit -m "feat(build): --cc compiler override for LKL and QEMU builds"` + +### Task 20: Wire the farm into linux.yml + +**Files:** +- Modify: `.github/workflows/linux.yml` + +- [ ] **Step 1:** Add the worker fleet as a sibling job (top-level under `jobs:`): + +```yaml + # Ephemeral sccache-dist compile farm. Workers serve the coordinator + # (the build job) and exit when it goes offline. Skipped on PRs — + # fork PRs have no TS_OAUTH_SECRET and same-repo PRs don't need the + # farm for cache-warm builds. + sccache-workers: + name: sccache worker ${{ matrix.idx }} + if: ${{ github.event_name != 'pull_request' }} + runs-on: ubuntu-24.04 + timeout-minutes: 75 + strategy: + matrix: + idx: [1, 2] + steps: + - uses: xdqi/sccache-dist-action@v0.0.4 + with: + mode: worker + worker-index: '${{ matrix.idx }}' + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' +``` + +- [ ] **Step 2:** In the `build` job, right after `Lint — no hardcoded paths...`, add: + +```yaml + # ── sccache-dist farm (best-effort; build proceeds locally if absent) ── + - name: Cache sccache objects + if: ${{ github.event_name != 'pull_request' }} + uses: actions/cache@v4 + with: + path: ~/.cache/sccache + key: sccache-linux-amd64-${{ github.sha }} + restore-keys: | + sccache-linux-amd64- + + - name: Bring up sccache-dist farm + if: ${{ github.event_name != 'pull_request' }} + continue-on-error: true + uses: xdqi/sccache-dist-action@v0.0.4 + with: + mode: coordinator + expected-workers: 2 + min-workers: 1 + wait-timeout: 180s + oauth-secret: '${{ secrets.TS_OAUTH_SECRET }}' +``` + +- [ ] **Step 3:** Make the two heavy build steps farm-aware. Replace the `Build LKL kernel` run block's make invocation with: + +```yaml + - name: Build LKL kernel + run: | + CC_ARGS=() + if command -v sccache >/dev/null 2>&1; then + CC_ARGS=(--cc="sccache gcc") + fi + ./scripts/build_lkl.sh \ + --linux=deps/linux \ + --out="$GITHUB_WORKSPACE" \ + --targets=linux-amd64 \ + "${CC_ARGS[@]}" \ + -j"${SCCACHE_J:-$(nproc)}" + mkdir -p deps/linux/tools/lkl/lib + ln -sf "$GITHUB_WORKSPACE/lkl-linux-amd64/tools/lkl/lib/liblkl.so" \ + deps/linux/tools/lkl/lib/liblkl.so +``` + +and the `Build QEMU block layer` step likewise: + +```yaml + - name: Build QEMU block layer + run: | + CC_ARGS=() + if command -v sccache >/dev/null 2>&1; then + CC_ARGS=(--cc="sccache gcc") + fi + ./scripts/build_qemu.sh \ + --qemu-src=deps/qemu \ + --targets=linux-amd64 \ + "${CC_ARGS[@]}" \ + -j"${SCCACHE_J:-$(nproc)}" +``` + +(`command -v sccache` is the farm-up probe: the coordinator action puts `sccache` on PATH and exports `SCCACHE_J`; on PRs or farm failure neither exists and the build runs exactly as today. `SCCACHE_DIST_FALLBACK=true` — the action default — keeps non-distributable jobs local.) + +- [ ] **Step 4:** After the build steps, add stats for observability: + +```yaml + - name: sccache stats + if: ${{ github.event_name != 'pull_request' }} + continue-on-error: true + run: sccache --show-stats || true +``` + +- [ ] **Step 5:** Validate YAML (`python3 -c "import yaml; yaml.safe_load(open('.github/workflows/linux.yml'))"`); commit — `git commit -m "ci(linux): sccache-dist compile farm (2 workers, best-effort)"` + +### Task 20A: Wire the farm into mingw64.yml (experimental) + +**Files:** +- Modify: `.github/workflows/mingw64.yml` + +This is the unproven leg: sccache-dist must package the msys2-cross `x86_64-w64-mingw32-gcc` toolchain (under `/opt/msys2-cross`) and ship it to workers. The engine supports arbitrary-toolchain packaging, and the same fork engine already distributes zig cross-compiles for the msys2-cross triples — but plain cross-gcc with its sysroot has not been exercised. Everything is best-effort: on any farm failure the build falls back to local compilation exactly as today. + +- [ ] **Step 1:** Add an `sccache-workers` job to `mingw64.yml` — identical shape to linux.yml's (matrix `idx: [1, 2]`, `if: github.event_name != 'pull_request'`). +- [ ] **Step 2:** In the build job, after the msys2-cross install step, add the same `Cache sccache objects` (key `sccache-mingw64-...`) + `Bring up sccache-dist farm` (coordinator, `continue-on-error: true`) steps as Task 20 Step 2. +- [ ] **Step 3:** Make the LKL kernel step farm-aware — same probe pattern, cross compiler spelled out: + +```yaml + CC_ARGS=() + if command -v sccache >/dev/null 2>&1; then + CC_ARGS=(--cc="sccache /opt/msys2-cross/bin/x86_64-w64-mingw32-gcc") + fi + ./scripts/build_lkl.sh ... --targets=mingw64 "${CC_ARGS[@]}" -j"${SCCACHE_J:-$(nproc)}" +``` + +(Absolute compiler path so sccache hashes/ships the right toolchain. QEMU's mingw configure is left alone — cross_prefix plumbing there is out of scope.) +- [ ] **Step 4:** Add the `sccache stats` step. Validate YAML, commit — `git commit -m "ci(mingw64): experimental sccache-dist farm for the LKL cross build"` +- [ ] **Step 5 (assessment gate):** After Task 21's run, check this job's stats: if cross compiles show `dist errors` / all-local fallback, file the finding in `docs/ci-sccache.md` ("mingw64 distribution blocked on X") and leave the wiring in place — it costs nothing when it falls back. + +### Task 21: Validate on real Actions + document + +**Files:** +- Create: `docs/ci-sccache.md` + +- [ ] **Step 1:** Push the branch and trigger: `git push -u origin build-and-test-hardening && gh workflow run linux.yml --ref build-and-test-hardening` (workflow_dispatch is enabled; if `gh workflow run` rejects a non-default-branch workflow, open a draft PR instead — the `pull_request` event skips the farm but still validates the C unit tests, wasm gate, and that the farm-less fallback path builds green). +- [ ] **Step 2:** Watch: `gh run watch` — for **both** linux.yml and mingw64.yml assert (a) workers register, (b) the LKL build log shows the sccache-wrapped compiler, (c) `sccache --show-stats` shows non-zero distributed/cached compiles (mingw64 may legitimately show all-local fallback — record it per Task 20A Step 5), (d) artifacts still upload. Known wrinkle from the sccache-dist deployments: the v0.0.4 engine has **no S3 feature** — irrelevant here, the cache is local disk + actions/cache. +- [ ] **Step 3:** Write `docs/ci-sccache.md` documenting: required secret (`TS_OAUTH_SECRET`, Tailscale OAuth client with `auth_keys` write + `tag:ci-sccache`, needed on both `xdqi/anyfs` and `xdqi/llvm-wasm`), farm topology (2 workers + coordinator-in-build-job per workflow; 3 workers for llvm-wasm releases), the `command -v sccache` fallback contract, why PRs skip the farm, and the mingw64 cross-distribution status from Task 20A's assessment. +- [ ] **Step 4: Commit** — `git add docs/ci-sccache.md && git commit -m "docs: sccache-dist CI farm setup + fallback contract"` + +--- + +## Final verification (whole plan) + +- [ ] `./scripts/lint-no-hardcoded-paths.sh` + `./scripts/lint-shellcheck.sh` — PASS with every build script in the allowlists (`build_anyfs_browser_wasm.sh` is deleted). +- [ ] `./tests/test_wasm_exports.sh` — PASS. +- [ ] `./scripts/fetch_wasm_ld.sh && ./scripts/fetch_wasm_sysroot.sh && ./scripts/doctor.sh` — wasm-ld + sysroot sections OK from fetched artifacts alone. +- [ ] `meson test -C build-anyfs-linux-amd64 --suite unit --print-errorlogs` — PASS. +- [ ] `./tests/smoke-debian-qcow2.sh --build-dir="$PWD/build-anyfs-linux-amd64" "$PWD/debian.qcow2"` — PASS. +- [ ] `cd ts && pnpm test` — PASS (core unit+smoke, react, nbd-proxy); `pnpm run test:integration` PASS locally. +- [ ] `cd ts/tests/e2e && pnpm exec playwright test` — PASS (incl. any specs added by the parity audit). +- [ ] linux.yml + mingw64.yml + ts.yml green on GitHub Actions; `wasm-ld-18.1.2-anyfs-r1` and `wasm-sysroot-r1` releases exist and are consumed. +- [ ] Use superpowers:requesting-code-review, then superpowers:finishing-a-development-branch. diff --git a/docs/superpowers/specs/2026-05-31-nbd-fd-transport-poc-design.md b/docs/superpowers/specs/2026-05-31-nbd-fd-transport-poc-design.md new file mode 100644 index 0000000..a87f948 --- /dev/null +++ b/docs/superpowers/specs/2026-05-31-nbd-fd-transport-poc-design.md @@ -0,0 +1,329 @@ +# NBD-over-inherited-fd Transport — PoC Design + +**Date:** 2026-05-31 +**Status:** Design approved, pre-implementation +**Scope:** Proof-of-concept only (native, Linux-first) + +## 1. Motivation + +The long-term goal is to run the QEMU block engine in a **separate, high-privilege +process** so anyfs can read sources that need elevated access (privileged files, +raw physical disks) and so the engine's QEMU `AioContext` stops tangling with the +Electron/Chromium main loop (the deferred hang documented in the +`native-needs-utilityProcess` project memory). + +The transport between the parent (Node/Electron) and the QEMU child is deliberately +**not** a listening TCP port. Instead: + +- The parent creates a connected socket pair and hands **one end's fd** to the QEMU + child via process inheritance + a command-line argument. +- QEMU's NBD client driver adopts that fd as its transport channel. +- The Node parent speaks the NBD protocol (read-only subset) over its end, serving + bytes from whatever privileged data source it has. + +This buys two properties that justify the "twisted" transport over plain TCP: + +1. **Parent/child lifecycle binding** (primary): when either side dies, the socket + pair hits EOF and the other side tears down cleanly — no dangling connections, + no leaked ports, no third party able to connect. This is the strong motivation. +2. **No port occupied/exposed** (secondary): nothing is bound to a listening port + on the machine; the channel is private to the process pair. + +### NBD wraps an image *file*, not a block device + +QEMU's `block/nbd.c` registers NBD as a **protocol driver** +(`.protocol_name = "nbd"`), not a format driver. QEMU's automatic format probe +therefore layers a format driver *on top of* the NBD protocol channel: + +``` +qcow2/vmdk/vhdx/... (format, auto-probed from the first bytes) + | + nbd: (protocol, our inherited-fd channel) +``` + +So `blk_new_open` over an NBD channel that serves a qcow2 byte stream yields a +fully-parsed qcow2 disk. The NBD layer only needs to serve **reads** (read-only + +future `BDRV_O_SNAPSHOT` overlay for writes), so the server implements a tiny +command subset. + +## 2. Key verified facts (foundation of the approach) + +These were confirmed by reading the QEMU tree (`~/qemu`) and anyfs source: + +- **`block/nbd.c` talks only through `s->ioc` (a `QIOChannel *`)** — it never touches + a raw socket fd directly. Any `QIOChannel` subclass can back it. +- **QEMU already ships a `SOCKET_ADDRESS_TYPE_FD`** SocketAddress type, and + `util/qemu-sockets.c:socket_get_fd()` accepts a **bare numeric fd** when there is + **no monitor** (`monitor_cur() == NULL`, which is exactly our library/`blk_new_open` + case): it does `qemu_strtoi(fdstr)` then `fd_is_socket(fd)`. +- **`qio_channel_socket_new_fd(fd)`** (`io/channel-socket.c:150`) adopts an existing + fd as a channel, skipping listen/connect. It calls `getpeername`/`getsockname`, so + **the fd must be a socket** (a pipe fails). This is why the transport is a + *socket* pair, not a pipe. + +**Consequence:** the native path needs **zero QEMU source changes**. The only C +changes are in anyfs's own glue (`lspart_main.c` + `qemu_backend.c`). + +## 3. Approach decision + +**Chosen: Approach A — reuse QEMU's `fd:N` SocketAddress, zero QEMU source changes.** + +The parent creates `socketpair(AF_UNIX, SOCK_STREAM)`, inherits one end (no +`CLOEXEC`) into the lspart child, and lspart constructs NBD server options as +`server.type=fd, server.str="N"`. QEMU's no-monitor path turns N into the channel via +`qio_channel_socket_new_fd`. + +Rejected alternatives: + +- **Approach B — custom `QIOChannel` subclass** (io_readv/writev/close over any fd + incl. pipes). More flexible and pipe-capable, but requires non-trivial QEMU source + changes and a hook in `nbd_open` to substitute `s->ioc`. **Deferred** as the future + evolution path if a true pipe/SAB transport is ever needed; out of PoC scope. +- **Approach C — `channel-command.c`** (QEMU spawns the proxy and uses its stdio). + Inverts the lifecycle direction (QEMU becomes the parent); wrong shape. Rejected. + +## 4. Architecture + +``` ++------------------+ socketpair(AF_UNIX) +-------------------+ +| Parent (Node) | fd0 <---- NBD frames (R/W) ----> fd1 | lspart child | +| hand-written | (no TCP listen) | (QEMU block layer)| +| NBD server | | | +| fs.read(local | fd1 inherited (no CLOEXEC), | blk_new_open( | +| qcow2/img) | passed as --nbd-fd N | nbd: type=fd | +| -> NBD READ | | str="N") | ++------------------+ +-------------------+ + | qcow2 driver auto-layered + v + list partitions + read a slice +``` + +### Three-stage verification (increasing risk, gated in order) + +1. **fd passing without QEMU source changes** (highest leverage) — confirm the + `fd:N` SocketAddress adopts an inherited socketpair fd under no-monitor. +2. **Full Linux chain** — Node NBD server (read-only, plain-file data source only) + → socketpair → lspart opens the qcow2 over NBD → list partitions + read & verify + a slice. +3. **Windows/wine fd-passing scouting** — separate minimal experiment; socketpair is + unreliable on Windows so this falls back to TCP loopback. + +### Explicitly out of PoC scope (YAGNI) + +Physical-disk / privileged reads · wasm `nbdfs` · custom `QIOChannel` (Approach B) · +N-API/Electron integration · write path (read-only + future `BDRV_O_SNAPSHOT`) · NBD +TLS / structured-reply / advanced features. + +## 5. Components & interfaces + +### (1) Node NBD server — new, `scripts/poc-nbd-server.mjs` + +- **Does:** speaks NBD newstyle (read-only subset) over an already-open socket fd. +- **Interface:** `startNbdServer(fd, imagePath)`. Data source abstracted as + `read(offset, len) -> Buffer`. **PoC simplification:** synchronous + `fs.readSync(imageFd, ...)`, serving one request at a time. The production server is + fully async with multiple in-flight requests — see §9. +- **Protocol subset:** newstyle fixed handshake → `NBD_OPT_GO` (advertise export size + + flags with `NBD_FLAG_READ_ONLY`) → command loop handling only `NBD_CMD_READ`, + `NBD_CMD_FLUSH` (no-op), `NBD_CMD_DISC`; everything else returns `EINVAL`. Simple + replies (no structured reply). +- **Depends on:** Node built-in `net` / `fs` only. + +### (2) fd-bridge launcher + native transport addon — new + +- `scripts/poc-nbd-launch.mjs` (the parent role) plus a small N-API addon + (`scripts/poc-native-transport/`, or reusing the anyfs-native build skeleton). +- **Addon exports:** + - `socketpair() -> [fd0, fd1]` — Linux/macOS: libc + `socketpair(AF_UNIX, SOCK_STREAM, 0)`, returns two bare fds. + - reserved Windows abstraction slot, e.g. `createLoopbackPair() -> + { serverHandle, childPort }` (implementation filled in for the Windows branch). +- **This addon is the seed of a unified "Node-layer native transport abstraction."** + Future wasm twisted transport and Windows handle passing layer here, exposing one + interface to callers (launcher today, Electron main process later). +- **Launcher:** calls the addon for fds, inherits `fd1` into the lspart child via + `child_process.spawn` (`stdio`/fd options), keeps `fd0` for the NBD server. +- **Critical detail:** the inherited fd must **not** be `CLOEXEC`; the child learns + its number via `--nbd-fd N`. + +### (3) lspart NBD entry — modify `src/lspart/lspart_main.c` + +- **Does:** parse `--nbd-fd N` (and Windows `--nbd-port P`), and feed a pseudo-path + (e.g. `nbd-fd:N`) into the existing `anyfs_session_open` → `anyfs_disk_add` → + `qemu_blk_open` path (today it opens plain image paths at `lspart_main.c:95`). +- **Interface:** reuse existing lspart output (partition table + optional hexdump of a + slice). +- This plus (4) are the **only C changes**, and per Approach A they do **not** touch + QEMU source. + +### (4) qemu_backend nbd path — modify `src/core/qemu_backend.c` + +- **Does:** when `image_path` looks like `nbd-fd:N` (resp. `nbd-port:P` on Windows), + build a QDict with `server.type=fd, server.str="N"` (resp. + `server.type=inet, host=127.0.0.1, port=P`) and pass it to `blk_new_open` instead of + treating `image_path` as a filename. +- **Interface:** add an `is_nbd_fd` branch alongside the existing `is_url` options + block (`qemu_backend.c:175-184`), reusing the existing QDict-options mechanism. + +## 6. Data flow + +### Startup & handshake (once) + +``` +launcher.mjs addon lspart (child / QEMU) NBD server (Node) + | socketpair() ----->| | | + |<-- [fd0, fd1] ------| | | + | spawn(lspart, --nbd-fd , | + | inherit fd1 non-CLOEXEC) ---------->| | + | startNbdServer(fd0, imagePath) ----------------------------------> | open imageFd + | | qemu_blk_open("nbd-fd:N") | + | | QDict server.type=fd,str=N| + | | blk_new_open --handshake->| + | |<-- NBD_OPT_GO: size, RO ---| + | | probe: read header --READ>| fs.readSync + | |<-- qcow2 magic ------------| + | | auto-layer qcow2 driver | +``` + +### Steady-state read (per partition / byte read) + +``` +lspart: list partitions / hexdump + | blk_pread(offset, len) -> qcow2 driver resolves cluster map + | -> emits NBD_CMD_READ(phys offset, len) -> socketpair fd -> NBD server + | fs.readSync(...) + |<-------------- NBD simple reply (handle + data) <----------------| + | verify bytes +``` + +### Shutdown / lifecycle binding + +- **Normal:** lspart done → `blk_unref` → NBD client sends `NBD_CMD_DISC` → socketpair + EOF → NBD server closes imageFd and exits. +- **Child crash:** lspart dies → inherited fd1 closed by OS → NBD server reads + EOF/`ECONNRESET` on fd0 → exits. **This is where lifecycle binding pays off** — no + dangling state, no leaked port. +- **Parent crash:** NBD server dies → fd0 closed → lspart's NBD client reads EOF on + the socketpair → `blk_pread` returns `-EIO` → lspart errors out. + +### Windows branch differences + +`socketpair()` is replaced by the addon's `createLoopbackPair()` (`127.0.0.1:0` bind + +accept, no outward listen); the child gets `--nbd-port P` instead of `--nbd-fd N`, and +QEMU uses `server.type=inet, host=127.0.0.1, port=P`. Protocol frames and data flow are +identical — only the transport-endpoint construction differs. Windows anonymous pipes +were ruled out: they are unidirectional and `channel-file`'s POSIX `readv`/poll is +unreliable for pipe fds under win32's HANDLE-based main loop; named pipes would require +a hand-written `QIOChannel` (ReadFile/WriteFile + overlapped + win32 main-loop +integration), which is out of PoC scope. + +## 7. Error handling + +| Failure point | Symptom | Handling | +|---|---|---| +| fd is not a socket | `socket_get_fd` → `fd_is_socket` fails → "File descriptor 'N' is not a socket" | launcher must pass a socketpair fd; this error is itself the stage-1 probe | +| fd closed by CLOEXEC | child can't see fd, `blk_new_open` → EBADF | spawn without CLOEXEC; test asserts child `fstat(N)` succeeds | +| NBD handshake mismatch | QEMU NBD client protocol error | server strictly follows newstyle fixed handshake; cross-check with `qemu-nbd`/`nbdinfo` | +| image not qcow2 / probe fails | `blk_getlength` < 0 or empty partition list | use a known qcow2 test image; first confirm the plain file path opens the same image as a control | +| read out of bounds | server sees offset+len > size | server returns `NBD_REP_ERR`/`EINVAL`, lspart reports EIO | +| Windows fd path unavailable | (expected) socketpair fails | addon takes the loopback branch directly on win; never attempts socketpair | + +## 8. Testing strategy & success criteria + +Each stage has one executable criterion and gates the next. + +### Stage 1 — fd passing, zero QEMU source changes (highest leverage) + +- **Test:** launcher creates the socketpair, spawns a minimal lspart invocation that + only does `blk_new_open("nbd-fd:N")` then `blk_getlength` and prints capacity. +- **PASS:** lspart prints the **same capacity** as opening the plain file, and QEMU is + **unmodified at the source level** (only anyfs-side C changed). +- **FAIL handling:** "not a socket" → revisit launcher fd passing. If QEMU source must + be changed → re-evaluate, possibly escalate to Approach B. + +### Stage 2 — full Linux chain + +- **Test:** with a known-content qcow2 (GPT + ext4, containing a verifiable magic byte + pattern), run full lspart: + 1. partition list matches `qemu-img info` / plain-file lspart output **item by item**; + 2. hexdump at a chosen offset is **byte-for-byte** equal to the expected magic. +- **PASS:** partition list matches + byte verification passes + the NBD server log shows + a non-trivial number of `NBD_CMD_READ`s (proving data actually traversed the + socketpair, not some other path). + +### Stage 3 — Windows/wine scouting + +- **Test:** under wine, run the mingw64 lspart through the loopback branch on the same + qcow2. +- **PASS:** opens + partition list matches. +- This is a go/no-go probe; **failure is allowed and recorded** in FINDINGS.md (per the + repo rule of documenting blocking findings inline) — Windows does not block the Linux + main-chain conclusion. + +### Cross-validation + +Use the system `qemu-nbd` + `nbdinfo`/`qemu-img` to independently validate the +hand-written Node NBD server's protocol correctness: first get a real QEMU client to +connect to the hand-written server successfully, then swap in lspart. This decouples +"protocol implementation bug" from "transport bug." + +### Regression guarantee + +All changes live on the anyfs side (lspart + qemu_backend branches); the existing +file/URL paths are untouched. Existing lspart plain-file usage must keep passing. + +## 9. Production server (post-PoC, design intent) + +The PoC server (§5.1) is deliberately synchronous and one-request-at-a-time to keep +the verification minimal. The **production** Node NBD server is **fully asynchronous** +end to end, on both the local-fs and the http/url data sources. This section records +the intent so the PoC interface (`read(offset, len) -> Buffer/Promise`) doesn't +have to be re-shaped later. + +### Concurrency model — exploit NBD's multi-in-flight + +QEMU's NBD client issues up to **16 concurrent in-flight requests** +(`MAX_NBD_REQUESTS = 16`, `block/nbd.c:50`), pairing requests to replies by `cookie` +(handle) and accepting **out-of-order** replies (`nbd_receive_replies` matches by +cookie, `block/nbd.c:421`). The production server therefore: + +- Reads NBD command headers in a loop **without waiting** for the previous command's + data — it can hold several `NBD_CMD_READ`s outstanding at once. +- Dispatches each read to an async data source, and emits each reply (handle + data) as + soon as that source resolves, in whatever order they complete. The handle guarantees + the client re-associates correctly. +- Serializes only the *write* of reply frames onto the socket (one reply on the wire at + a time), not the *servicing* of reads. + +This is what keeps a keep-alive http upstream's throughput up: multiple Range requests +can be in flight to the upstream concurrently instead of a serial request/wait/respond +cycle. + +### Async data sources + +How the server fans the (up to 16) in-flight NBD reads out to the upstream depends on +the source. **Note the distinction: HTTP/1.1 keep-alive is connection *reuse*, not +multiplexing** — a single keep-alive connection is still serial (one response must be +fully read before the next request goes out on that connection; HTTP/1.1 pipelining is +deprecated and unusable due to head-of-line blocking). Concurrency on H1 comes from the +*number of pooled connections*, not from muxing one connection. + +- **Local fs:** `fs.promises` / async `fs.read` (or a small read pool), never + `readSync`. No connection concept — the 16 reads simply run concurrently. +- **http/url over HTTP/1.1:** a keep-alive **connection pool** + (`http.Agent({ keepAlive: true, maxSockets: N })` or `undici.Pool`). This eliminates + per-request handshake/TLS churn (the problem URLFS has today) and gives **up to N + concurrent** Range requests; the rest queue. Throughput-bound by N, typically 6–16. +- **http/url over HTTP/2:** a single connection genuinely multiplexes — the 16 in-flight + Range reads become up to 16 parallel streams on one connection (bounded by the + server's `SETTINGS_MAX_CONCURRENT_STREAMS`). `undici` negotiates H2 when the upstream + supports it, so object-storage/CDN image URLs (which usually speak H2) get real mux; + H1-only upstreams fall back to the connection-pool behavior above. + +### Shared concern with URLFS + +URLFS (`ts/packages/core/src/url-fs.ts`) currently issues a fresh synchronous XHR per +chunk with **no upstream connection reuse and no multiplexing**. The production http +data source should solve connection reuse (H1 keep-alive pool) and prefer H2 mux where +available, in a way URLFS can later adopt. This is noted as a **shared future +improvement**, not part of this PoC. diff --git a/docs/superpowers/specs/2026-06-01-nbd-proxy-design.md b/docs/superpowers/specs/2026-06-01-nbd-proxy-design.md new file mode 100644 index 0000000..46f7317 --- /dev/null +++ b/docs/superpowers/specs/2026-06-01-nbd-proxy-design.md @@ -0,0 +1,239 @@ +# Production NBD Proxy — Design + +**Date:** 2026-06-01 +**Status:** Design approved, pre-planning +**Builds on:** the NBD-over-inherited-fd transport PoC (proven 2026-06-01, `scripts/poc-nbd/`, spec `2026-05-31-nbd-fd-transport-poc-design.md`). + +## 1. Purpose + +A standalone, optionally-privileged Node process that exposes a data source — +**a local file, a physical disk / privileged file, or an http(s) URL** — as a +read-only NBD endpoint. A QEMU engine (separate, low-privilege process) connects +over that NBD channel via an inherited socket fd or a `127.0.0.1` loopback port, +and QEMU's format drivers (qcow2/vmdk/…) auto-layer atop the NBD protocol. + +This is the production form of the PoC's hand-written synchronous one-request-at-a-time +server: fully asynchronous, with multiple in-flight requests, and a real data-source +abstraction. + +### Why NBD (not the existing HTTP-range proxy) + +The repo already has HTTP-range proxies (`ts/packages/anyfs-native/src/http-disk-server.ts`, +`ts/examples/electron-demo/src/http-proxy-worker.ts`) that feed QEMU's **curl** driver +over plain HTTP. We deliberately use **NBD** instead because it gives what those cannot: +a **private channel that is not a listening service** (inherited fd), **lifecycle binding** +to the process pair (channel EOF tears both sides down cleanly), no exposed port, and +NBD's native read-only + up-to-16 in-flight model. These are the original motivations for +the "twisted transport." The existing HTTP proxies remain for their existing URL scenarios. + +### Process & privilege model + +The proxy is a **separate process** that may be launched as root/Administrator solely to +read privileged sources (physical disks, protected files). The QEMU engine stays in a +**low-privilege** process and only speaks NBD to the proxy. Privilege isolation is clean: +elevation lives entirely in the proxy. + +## 2. Architecture + +``` ++------------------------------+ NBD (inherited fd / 127.0.0.1 loopback) +------------------+ +| anyfs-nbd-proxy (separate) | <--- multi in-flight, out-of-order replies -> | QEMU engine | +| may run root/Administrator | | (low privilege) | +| | | qcow2/vmdk -> | +| NbdServer (async, 16-way) | | nbd: protocol | +| | read(off,len) +------------------+ +| v +| DataSource (interface) +| |- FileSource (fs.promises) +| |- BlockDeviceSource (/dev/sdX ; size via drivelist) +| '- HttpSource (undici keep-alive pool / H2) ++------------------------------+ +``` + +### Package layout — `ts/packages/anyfs-nbd-proxy/` + +| File | Responsibility | +|---|---| +| `src/nbd-server.ts` | Async multi-in-flight NBD newstyle server (read-only subset). Out-of-order replies keyed by handle; only reply-frame writes are serialized. | +| `src/data-source.ts` | `DataSource` interface (`size()`, `read(offset,len)`, `close()`) + `createDataSource(spec)` factory. | +| `src/sources/file.ts` | `FileSource` — `fs.promises` async pread. | +| `src/sources/blockdev.ts` | `BlockDeviceSource` — open raw device, size via `drivelist`, plain pread (Linux v1). | +| `src/sources/http.ts` | `HttpSource` — undici keep-alive pool (H1) / H2 mux, Range requests. | +| `src/endpoint.ts` | Bind an NbdServer to an inherited fd (`net.Socket({fd})`) or a `127.0.0.1:0` loopback listener. | +| `bin/anyfs-nbd-proxy.ts` | Thin CLI: parse args, build a DataSource + endpoint, run. No business logic. | +| `test/*.test.ts` | Unit + qemu-img/qemu-io integration tests. | + +### Relationship to existing code + +- The PoC `scripts/poc-nbd/` stays as a runnable regression baseline — untouched. +- Existing `http-disk-server.ts` / `http-proxy-worker.ts` (QEMU curl route) stay for their + existing URL scenarios — untouched and not in conflict (this package is the NBD route). +- The `DataSource` abstraction is meant to be reusable by the later native-integration step. + +## 3. NbdServer — async multi-in-flight concurrency + +PoC was strictly serial (read header → sync read → write reply → next). Production reads +headers without blocking, holds multiple in-flight reads, and replies in completion order. + +### Read loop (producer) + +``` +loop: + hdr = await readExactly(28) // next request header; does not await in-flight reads + if DISC: break + if READ: + if inFlight >= 16: await slotFreed // backpressure, aligned to client MAX_NBD_REQUESTS + inFlight++ + handleRead(handle, offset, length) // NOT awaited — dispatch and continue reading + .then(data => enqueueReply(handle, 0, data)) + .catch(err => enqueueReply(handle, errnoOf(err), null)) + .finally(() => { inFlight--; signalSlotFreed() }) + if FLUSH: enqueueReply(handle, 0, null) // read-only, no-op +``` + +### Write side (consumer, serialized) + +A socket may carry only one reply frame at a time (header+data must not interleave). All +`enqueueReply` calls feed a FIFO drained by a single writer loop that `await socket.write()`s +each frame. Reply order = completion order (out-of-order vs request order); the 8-byte +handle lets the client re-pair — exactly NBD simple-reply semantics. + +### Why this saturates throughput + +16 READs can `await` their data sources concurrently (16 parallel Range requests over the +keep-alive pool / H2 streams, or 16 parallel fs preads). Only frame-writing is serialized, +and that is memory→socket — far faster than source IO. + +### Reader simplification + +The read loop frames request headers **serially** (one after another), so the byte reader +need only support a single outstanding `read()` — concurrency happens only *after* a request +is parsed, in data fetching. We do not parse multiple headers concurrently; we only service +them concurrently. + +## 4. DataSource implementations + +```ts +interface DataSource { + size(): Promise; // NBD export size (bytes) + read(offset: number, length: number): Promise; // exactly `length` bytes + close(): Promise; +} +``` + +NbdServer depends only on this interface. `createDataSource({kind, target})` dispatches. + +### FileSource (`sources/file.ts`) + +`fs.promises.open(path,'r')` → `FileHandle`; `size()` via `fh.stat()`; `read()` via +`fh.read(buf, 0, length, offset)`. Pure async (the PoC `readSync` upgraded). + +### BlockDeviceSource (`sources/blockdev.ts`) — Linux v1 + +- **size**: `drivelist.list()` (already integrated in the repo, enumeration-only) → match the + target device → use its `.size` field. Cross-platform metadata, no `/sys/block` parsing, no + native ioctl. Fallback to `/sys/block//size`×512 on Linux if drivelist is unavailable. +- **read**: plain `fs.promises` `fh.read(buf, 0, len, offset)`. + - **Linux**: the kernel block layer handles non-aligned reads — **no app-level alignment + needed**. This is the v1 target. + - **Windows** (deferred): `\\.\PhysicalDriveN` `ReadFile` requires sector alignment → a later + version adds an align-expand-and-crop wrapper. `/dev/rdiskN` on macOS similar; use buffered + `/dev/diskN` to avoid alignment. +- **v1 scope: Linux only.** macOS/Windows raw-device specifics + alignment are deferred; the CLI + reports a clear "blockdev v1 is Linux-only" error on other platforms rather than failing + silently. **No native code is required for v1** (drivelist for size, Linux pread for reads). + +### HttpSource (`sources/http.ts`) + +`undici`: a `Pool` (H1 keep-alive, connection reuse) or automatic H2 multiplexing. `size()` +via HEAD (follow redirects; read `content-length` + `accept-ranges`); `read()` sends +`Range: bytes=off-end`. The 16 in-flight reads map to concurrent pool requests (H1) or H2 +streams. Defensive: if a server ignores Range and returns 200 with the full body, crop +(same defense URLFS uses). + +> Note: this keep-alive/pool approach is what spec §9 of the PoC design flagged as something +> URLFS *may* later adopt. That is a **future, optional** improvement to URLFS — **not** part of +> this package and not a change to URLFS now. + +## 5. Endpoint & CLI + +### Endpoint (`src/endpoint.ts`) — reuses the two PoC-proven paths + +| Mode | Use | Implementation | +|---|---|---| +| inherited fd | Linux/macOS: parent makes a socketpair, passes one end to the proxy child | `--fd N` → `new net.Socket({fd:N})` → NbdServer serves it | +| loopback | Windows, or when fd-passing is impractical | `--port P` (or `0` to auto-pick) → `net.createServer().listen(P,'127.0.0.1')`, one NbdServer per connection | + +These are PoC Stage 1 (fd) and Stage 3 (loopback) lifted into a reusable layer; NbdServer is +agnostic to which backs it. + +### CLI (`bin/anyfs-nbd-proxy.ts`) — thin wrapper + +``` +anyfs-nbd-proxy \ + --source file|blockdev|url \ + --target \ + (--fd | --port

) \ + [--export-size ] # optional, overrides auto-detected size +``` + +Flow: parse → `createDataSource({kind,target})` → `await source.size()` → build endpoint per +`--fd`/`--port` → run NbdServer(s) → exit on endpoint EOF / SIGTERM. + +`bin/` contains no business logic — NbdServer, DataSource, and endpoint all live in importable +`src/`, so the later native-integration step can either spawn this CLI process (preferred) or +import the library. + +### Lifecycle binding (the core "twisted transport" payoff, from the PoC) + +- inherited fd: parent dies → fd EOF → NbdServer reads EOF → proxy exits. proxy dies → fd closes + → QEMU's NBD client reads EOF → EIO. No dangling state, no leaked port. +- loopback: the parent kills the proxy child on its own exit (or the proxy self-exits on stdin EOF). + +## 6. Error handling + +| Failure | Symptom | Handling | +|---|---|---| +| out-of-bounds read | offset+len > size | NBD reply `EINVAL`; server keeps running | +| DataSource IO error (fs/blockdev/http) | read rejects | that request replies `EIO` (errno-mapped); other in-flight reads continue | +| http upstream 5xx / dropped conn | undici rejects | that read → EIO; pool reconnects for later requests | +| privileged source permission denied | open fails (EACCES) | startup `size()` fails → CLI prints a clear error and exits non-zero (never enters the NBD loop) | +| client disconnect / EOF | read loop hits EOF | clean teardown: close DataSource, proxy exits | +| blockdev on non-Linux in v1 | blockdev on mac/win | CLI reports "blockdev v1 is Linux-only"; no silent failure | + +## 7. Testing strategy + +- **Unit:** a mock DataSource that completes out of order (verify handle pairing); FileSource + pread against a known byte pattern; HttpSource against a local Range server incl. pool reuse; + out-of-bounds and EIO paths. +- **Integration (reuse the PoC fixture approach):** + - `qemu-img info` / `qemu-io read` open a fixture qcow2 through this server (over a unix socket), + verifying format detection + byte correctness (as PoC Tasks 6/8, but against the new server). + - Each data source once: FileSource (qcow2 file), BlockDeviceSource (a loopback-mounted device, + or skipped if no privilege), HttpSource (local Range server serving the qcow2). + - **Out-of-order concurrency test:** make HttpSource impose differing per-request delays; assert + the server still replies correctly keyed by handle. +- **Web regression (must run):** confirm the web demo's blob open and URL open are unaffected + (this package touches none of those files; verify once at acceptance regardless). + +## 8. Non-goals & must-not-break constraints + +This package is the **NBD route**, serving only native's private fd/loopback channel for +privileged files / physical disks / (optionally) http sources. It **must not modify or replace** +the existing web paths, which continue to work exactly as before: + +- **blob**: browser File/Blob → WORKERFS (`/work/`) → wasm kernel. +- **CORS-relaxed URL**: existing URLFS (sync XHR + Range + LRU); existing `http-disk-server.ts` / + electron `http-proxy-worker.ts` (QEMU curl driver). + +This package introduces **no changes** to `ts/packages/core`, URLFS, or WORKERFS. The HttpSource +keep-alive/pool design is a *future, optional* reference for URLFS — not a change to URLFS now. +Acceptance must confirm zero regression in the web demo's blob and URL opens. + +## 9. Future direction (NOT this scope) + +Later, the Electron build may route **all** data sources through this NBD proxy (unifying local +file / URL / physical disk behind one NBD channel and moving the QEMU engine fully into the +separate proxy process). That convergence is explicitly **out of scope for this package's first +version** — it is recorded here as intent. For now, NBD is an additive parallel route and the +existing web/Electron data-source paths are left intact. diff --git a/docs/wasm-sysroot.md b/docs/wasm-sysroot.md new file mode 100644 index 0000000..76c8bf2 --- /dev/null +++ b/docs/wasm-sysroot.md @@ -0,0 +1,68 @@ +# wasm sysroot provisioning + +The wasm build links against a sysroot of 17 prebuilt static libraries +(glib/gio, pcre2, libffi, zlib, bzip2, zstd, blkid/uuid, a libresolv stub — +see `scripts/lib/wasm_sysroot.manifest`). It is provisioned the same way as +the mingw toolchain, with three roles: + +| Role | mingw analogue | wasm piece | +|---|---|---| +| Recipe (how to build it from pinned sources) | PKGBUILD | `scripts/build_wasm_sysroot.sh` | +| Prebuilt artifact (what consumers download) | `bootstrap.tar.xz` | `wasm-sysroot-linux.tar.xz` GitHub release asset, built by `.github/workflows/wasm-sysroot.yml` | +| Installer / integrity check | pacman | `scripts/fetch_wasm_sysroot.sh` + manifest-completeness check | + +`scripts/fetch_wasm_sysroot.sh` downloads the release tarball into +`/.toolchain/wasm-sysroot/` (gitignored). It short-circuits if every +manifest lib is already present, and re-validates the manifest after +extraction. `scripts/lib/config.sh` auto-prefers that location when +`paths.wasm_sysroot` is unset/empty; an explicit `build.user.toml` value +always wins. + +## Per-target dependency provisioning + +| Target | Library dependencies come from | +|---|---| +| linux-amd64 | apt packages (distro `-dev` packages) | +| mingw32 / mingw64 | msys.kosaka.moe pacman repo (msys2-cross packages) | +| wasm | this tarball (`fetch_wasm_sysroot.sh`) | + +## Version-bump procedure + +1. Edit the version pins (`*_V` / `*_URL` / `*_SHA`) in + `scripts/build_wasm_sysroot.sh`, and update + `scripts/lib/wasm_sysroot.manifest` if the lib set or pins change. +2. Dispatch `.github/workflows/wasm-sysroot.yml` with a new tag + (`wasm-sysroot-rN`). The workflow rebuilds from source — the recipe's + built-in manifest parity check is the acceptance gate — and publishes the + release. +3. Bump the `WASM_SYSROOT_TAG` default in `scripts/fetch_wasm_sysroot.sh`. + +## The two excavated glib hacks + +Both were reverse-engineered from the known-good hand-built sysroot and are +applied by the recipe (see the comments in `build_glib()` in +`scripts/build_wasm_sysroot.sh`): + +1. **`scripts/lib/glib-2.88.0-emscripten-fd-query-path.patch`** — Emscripten + port patch applied to the glib tree before `meson setup` (rationale in the + patch header). +2. **config.h post-edit** — Emscripten's libc *declares* `posix_spawn{,p}` + and `pthread_getname_np` (so meson's compile-only checks pass) but does + not *define* them, which breaks the tool executables at wasm-ld. The + recipe deletes `HAVE_POSIX_SPAWN` and `HAVE_PTHREAD_GETNAME_NP` from the + generated `_build/config.h` after setup, matching the known-good sysroot's + preserved config.h. + +## Local rebuild + +```sh +./scripts/build_wasm_sysroot.sh # into paths.wasm_sysroot +SYSROOT=/tmp/sysroot ./scripts/build_wasm_sysroot.sh +``` + +Takes ~12 minutes. Requires emsdk (the recipe was validated with emcc 5.0.7 +— the same version `.github/workflows/wasm-sysroot.yml` pins), meson, ninja, +pkg-config, and a util-linux v2.40.4 checkout with a generated `./configure` +(`./autogen.sh`); the tree is located via `paths.util_linux` or the `UL_SRC` +env override. Full runs end with the manifest parity check and fail on any +drift. diff --git a/meson.build b/meson.build index 9f7c94f..9962865 100644 --- a/meson.build +++ b/meson.build @@ -368,6 +368,24 @@ test_nfsd_pseudo_root = executable('test_nfsd_pseudo_root', c_args: gio_args + qemu_args, ) +# --- Unit tests (pure userspace, no LKL boot; run via `meson test --suite unit`) --- +test_path_dsl = executable('test_path_dsl', + 'tests/unit/test_path_dsl.c', + 'src/core/anyfs_path.c', + include_directories: [anyfs_inc], + install: false, +) +test('path_dsl', test_path_dsl, suite: 'unit') + +test_share_helpers = executable('test_share_helpers', + 'tests/unit/test_share_helpers.c', + include_directories: [anyfs_inc], + dependencies: [anyfs_dep], + c_args: gio_args + qemu_args, + install: false, +) +test('share_helpers', test_share_helpers, suite: 'unit') + bench_backends = executable('bench_backends', 'tests/bench_backends.c', include_directories: [anyfs_inc, include_directories('src/core')], @@ -478,9 +496,11 @@ if enable_ksmbd 'src/ksmbd/ksmbd_main.c', 'src/ksmbd/ksmbd_ipc.c', 'src/ksmbd/ksmbd_stubs.c', + 'src/server_common/server_common.c', fastsync_srcs, ksmbd_tools_sources, - include_directories: [anyfs_inc, ksmbd_tools_inc], + include_directories: [anyfs_inc, ksmbd_tools_inc, + include_directories('src/server_common')], dependencies: [lkl_dep, glib_sys_dep2, thread_dep, host_proxy_dep], link_with: [ksmbd_tools_lib, anyfs_core], link_args: fastsync_link_args, @@ -502,7 +522,9 @@ if enable_ksmbd # reuses same LKL build with NFSD config # on the path. lkl_nfsd = executable('anyfs-nfsd', 'src/nfsd/nfsd_main.c', - include_directories: [anyfs_inc], + 'src/server_common/server_common.c', + include_directories: [anyfs_inc, + include_directories('src/server_common')], dependencies: [lkl_dep, thread_dep, host_proxy_dep], link_with: [anyfs_core], install: true, diff --git a/scripts/build_anyfs.sh b/scripts/build_anyfs.sh index 19095bc..d9405ce 100755 --- a/scripts/build_anyfs.sh +++ b/scripts/build_anyfs.sh @@ -15,9 +15,10 @@ # --src=DIR anyfs-reader source root (default: ) # --out-prefix=PFX Build-dir prefix (default: build-anyfs) # Produces /-/. -# --qemu-root=DIR QEMU source tree (default: ~/qemu) -# --ksmbd-root=DIR ksmbd-tools source tree (default: ~/ksmbd-tools) -# --lkl-src=DIR Linux kernel source tree (default: ~/linux). Meson +# --qemu-root=DIR QEMU source tree (default: from build.config.toml) +# --ksmbd-root=DIR ksmbd-tools source tree (default: from build.config.toml) +# --lkl-src=DIR Linux kernel source tree (default: from +# build.config.toml). Meson # needs this for tools/lkl/include/{lkl.h,lkl_host.h}; # the option default in meson_options.txt is the literal # string '${LINUX_SRC}' which cannot be auto-resolved, @@ -38,12 +39,24 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" SRC_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=lib/config.sh +source "$SCRIPT_DIR/lib/config.sh" + +# Meson refuses absolute paths that point inside the source tree (see NOTE +# below), so config defaults are re-relativized against SRC_DIR when inside it. +rel_to_src() { + case "$1" in + "$SRC_DIR"/*) printf '%s\n' "${1#"$SRC_DIR"/}" ;; + *) printf '%s\n' "$1" ;; + esac +} + TARGETS_REQ="linux-amd64,mingw32,mingw64" COMPONENTS_REQ="core,server,fuse" OUT_PFX="build-anyfs" -QEMU_ROOT="$HOME/qemu" -KSMBD_ROOT="$HOME/ksmbd-tools" -LKL_SRC="$HOME/linux" +QEMU_ROOT="$(rel_to_src "$ANYFS_PATHS_QEMU_SRC")" +KSMBD_ROOT="$(rel_to_src "$ANYFS_PATHS_KSMBD_TOOLS")" +LKL_SRC="$(rel_to_src "$ANYFS_PATHS_LINUX_SRC")" RECONFIGURE=0 JOBS="$(nproc)" @@ -372,8 +385,8 @@ for t in data: # Returns empty string for linux-amd64. mingw_sysroot_for() { case "$1" in - mingw32) echo "/opt/msys2-cross/mingw32" ;; - mingw64) echo "/opt/msys2-cross/mingw64" ;; + mingw32) echo "$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw32" ;; + mingw64) echo "$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw64" ;; *) echo "" ;; esac } diff --git a/scripts/build_anyfs_wasm.sh b/scripts/build_anyfs_wasm.sh index 0db7d5a..9592b56 100755 --- a/scripts/build_anyfs_wasm.sh +++ b/scripts/build_anyfs_wasm.sh @@ -5,10 +5,10 @@ # and libblkid-based fstype probing. # # Inputs: -# - $LINUX_DIR kernel source tree (default: ~/linux) +# - $LINUX_DIR kernel source tree (default: paths.linux_src) # - $OUT/tools/lkl/liblkl.a pre-built LKL wasm (build_lkl_wasm.sh) -# - $HOME/anyfs-reader/src/core/ core C sources -# - $HOME/anyfs-reader/ts/native/anyfs_ts.c TypeScript glue +# - /src/core/ core C sources +# - /ts/native/anyfs_ts.c TypeScript glue # - $QEMU_ROOT/build-anyfs-wasm/ pre-built QEMU wasm archives # - $WASM_SYSROOT sysroot with libblkid/libz/libbz2/libzstd etc. # @@ -25,15 +25,21 @@ # the asyncify unwinding happens on the dedicated pthread. set -euo pipefail -LINUX_DIR="${LINUX_DIR:-$HOME/linux}" -OUT="${OUT:-$HOME/anyfs-reader/lkl-wasm}" -EMSDK_DIR="${EMSDK_DIR:-$HOME/emsdk}" -BLD="${BLD:-$HOME/anyfs-reader/build-anyfs-wasm}" -TS="${TS:-$HOME/anyfs-reader/ts}" -QEMU_ROOT="${QEMU_ROOT:-$HOME/qemu}" +# shellcheck source=lib/config.sh +source "$(dirname "$0")/lib/config.sh" +# shellcheck source=lib/wasm_exports.sh +source "$(dirname "$0")/lib/wasm_exports.sh" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +LINUX_DIR="${LINUX_DIR:-$ANYFS_PATHS_LINUX_SRC}" +OUT="${OUT:-$REPO_ROOT/lkl-wasm}" +EMSDK_DIR="${EMSDK_DIR:-$ANYFS_TOOLCHAINS_EMSDK}" +BLD="${BLD:-$REPO_ROOT/build-anyfs-wasm}" +TS="${TS:-$REPO_ROOT/ts}" +QEMU_ROOT="${QEMU_ROOT:-$ANYFS_PATHS_QEMU_SRC}" QBLD="${QBLD:-$QEMU_ROOT/build-anyfs-wasm}" -SYS="${WASM_SYSROOT:-$HOME/wasm-sysroot}" -SRC_CORE="${SRC_CORE:-$HOME/anyfs-reader/src/core}" +SYS="${WASM_SYSROOT:-$ANYFS_PATHS_WASM_SYSROOT}" +SRC_CORE="${SRC_CORE:-$REPO_ROOT/src/core}" GLUE="$TS/native/anyfs_ts.c" LIBLKL="$OUT/tools/lkl/liblkl.a" TARGET="${ANYFS_TARGET:-browser}" @@ -43,12 +49,14 @@ source "$EMSDK_DIR/emsdk_env.sh" >/dev/null 2>&1 case "$TARGET" in browser) + # shellcheck disable=SC2054 # comma is part of the emcc value, not an element separator ENV_FLAG=(-sENVIRONMENT=web,worker) FS_FLAG=(-lworkerfs.js) FS_RUNTIME="WORKERFS" OUT_STEM="anyfs" ;; node) + # shellcheck disable=SC2054 # comma is part of the emcc value, not an element separator ENV_FLAG=(-sENVIRONMENT=node,worker) FS_FLAG=(-lnodefs.js) FS_RUNTIME="NODEFS" @@ -63,7 +71,7 @@ esac INC=( -I "$LINUX_DIR/tools/lkl/include" -I "$OUT/tools/lkl/include" - -I "$HOME/anyfs-reader/include" + -I "$REPO_ROOT/include" -I "$SRC_CORE" ) @@ -161,7 +169,7 @@ QEMU_BLK_OBJ="$BLD/qemu_blk_backend.${TARGET}.o" echo " CC src/core/qemu_backend.c -> $QEMU_BLK_OBJ" emcc -pthread -O2 -g \ -I "$SRC_CORE" \ - -I "$HOME/anyfs-reader/include" \ + -I "$REPO_ROOT/include" \ -I "$QEMU_ROOT" -I "$QEMU_ROOT/include" \ -I "$QBLD" -I "$QBLD/qapi" \ -I "$SYS/include" -I "$SYS/include/glib-2.0" \ @@ -189,20 +197,7 @@ mkdir -p "$TS/packages/core/wasm" OUT_DIR="$TS/packages/core/wasm" OUT_JS="$OUT_DIR/${OUT_STEM}.mjs" -EXPORTED_FUNCS='_main,_malloc,_free,'\ -'_anyfs_ts_kernel_init,_anyfs_ts_init_async,_anyfs_ts_is_boot_complete,_anyfs_ts_boot_result,_anyfs_ts_kernel_halt,'\ -'_anyfs_ts_session_open,_anyfs_ts_session_close,'\ -'_anyfs_ts_session_list_json,_anyfs_ts_session_meta_json,'\ -'_anyfs_ts_session_enter,'\ -'_anyfs_ts_session_enter_async,_anyfs_ts_session_enter_is_complete,_anyfs_ts_session_enter_result_p,'\ -'_anyfs_ts_readdir_json,_anyfs_ts_lstat_json,_anyfs_ts_stat_json,'\ -'_anyfs_ts_realpath,_anyfs_ts_readlink,_anyfs_ts_read_kernel_file,'\ -'_anyfs_ts_open,_anyfs_ts_pread,_anyfs_ts_close,'\ -'_anyfs_ts_session_open_p,_anyfs_ts_session_list_json_p,_anyfs_ts_session_meta_json_p,'\ -'_anyfs_ts_session_enter_p,'\ -'_anyfs_ts_readdir_json_p,_anyfs_ts_lstat_json_p,_anyfs_ts_stat_json_p,'\ -'_anyfs_ts_realpath_p,_anyfs_ts_readlink_p,_anyfs_ts_read_kernel_file_p,'\ -'_anyfs_ts_open_p,_anyfs_ts_pread_p,_anyfs_ts_close_p' +EXPORTED_FUNCS="$(anyfs_wasm_exports "$GLUE")" EXPORTED_RUNTIME="ccall,cwrap,HEAPU8,HEAP32,HEAPU32,FS,${FS_RUNTIME},UTF8ToString,stringToUTF8,getValue,setValue" @@ -220,6 +215,7 @@ EXTRA_ARCHIVES=( "$SYS/lib/libblkid.a" "$SYS/lib/libbz2.a" "$SYS/lib/libzstd.a" ) +# shellcheck disable=SC2054 # commas are literal in -Wl,... linker flags, not element separators LDFLAGS=( -pthread -sPROXY_TO_PTHREAD=1 diff --git a/scripts/build_boot_wasm.sh b/scripts/build_boot_wasm.sh index 947f79f..4233e7c 100755 --- a/scripts/build_boot_wasm.sh +++ b/scripts/build_boot_wasm.sh @@ -24,9 +24,13 @@ # -sNODERAWFS=0 don't expose host fs (we don't need it) set -e -LINUX_DIR="${LINUX_DIR:-$HOME/linux}" -OUT="${OUT:-$HOME/anyfs-reader/lkl-wasm}" -EMSDK_DIR="${EMSDK_DIR:-$HOME/emsdk}" +# shellcheck source=lib/config.sh +source "$(dirname "$0")/lib/config.sh" +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +LINUX_DIR="${LINUX_DIR:-$ANYFS_PATHS_LINUX_SRC}" +OUT="${OUT:-$REPO_ROOT/lkl-wasm}" +EMSDK_DIR="${EMSDK_DIR:-$ANYFS_TOOLCHAINS_EMSDK}" LIBLKL="$OUT/tools/lkl/liblkl.a" if [[ ! -f "$LIBLKL" ]]; then @@ -61,6 +65,9 @@ for src in boot.c test.c; do emcc "${CFLAGS[@]}" "${INC[@]}" -c "$TESTS_DIR/$src" -o "$obj" done +# shellcheck disable=SC2054 +# SC2054: false positive — the comma inside -sENVIRONMENT=node,worker is part +# of the emcc flag value, not an array separator. LDFLAGS=( -pthread -sPROXY_TO_PTHREAD=1 diff --git a/scripts/build_libblkid_mingw.sh b/scripts/build_libblkid_mingw.sh index 3cef5d7..44c938f 100755 --- a/scripts/build_libblkid_mingw.sh +++ b/scripts/build_libblkid_mingw.sh @@ -2,8 +2,8 @@ # Cross-build libblkid.a for mingw (i686 or x86_64). # # Inputs: -# - util-linux source tree (default: $HOME/util-linux; override -# with UL_SRC=...) +# - util-linux source tree (default: util_linux from build.config.toml; +# falls back to deps/util-linux; override with UL_SRC=...) # - shim headers + TUs under /patches/libblkid/shim/ # # Output: @@ -25,16 +25,19 @@ set -euo pipefail +# shellcheck source=lib/config.sh +source "$(dirname "$0")/lib/config.sh" + # --------------------------------------------------------------------------- # Target selection # --------------------------------------------------------------------------- TARGET="${1:-mingw64}" case "$TARGET" in mingw64) - CROSS_PREFIX="${CROSS_PREFIX:-/opt/msys2-cross/bin/x86_64-w64-mingw32}" + CROSS_PREFIX="${CROSS_PREFIX:-$ANYFS_TOOLCHAINS_MSYS2_CROSS/bin/x86_64-w64-mingw32}" ;; mingw32) - CROSS_PREFIX="${CROSS_PREFIX:-/opt/msys2-cross/bin/i686-w64-mingw32}" + CROSS_PREFIX="${CROSS_PREFIX:-$ANYFS_TOOLCHAINS_MSYS2_CROSS/bin/i686-w64-mingw32}" ;; *) echo "Usage: $0 [mingw32|mingw64]" >&2 @@ -45,7 +48,7 @@ esac SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -UL_SRC="${UL_SRC:-$HOME/util-linux}" +UL_SRC="${UL_SRC:-$ANYFS_PATHS_UTIL_LINUX}" # Resolve UL_SRC to absolute: we `cd "$UL_SRC"` below and pass -I"$UL_SRC/...", # so a relative UL_SRC (e.g. deps/util-linux) would break after the cd. [[ -d "$UL_SRC" ]] && UL_SRC="$(cd "$UL_SRC" && pwd)" diff --git a/scripts/build_libblkid_wasm.sh b/scripts/build_libblkid_wasm.sh index 6609761..18f7c09 100755 --- a/scripts/build_libblkid_wasm.sh +++ b/scripts/build_libblkid_wasm.sh @@ -1,141 +1,13 @@ -#!/bin/bash -# Build libblkid.a for emscripten and install into the wasm sysroot. -# -# Inputs: -# - util-linux source tree (default: $HOME/util-linux; override via -# UL_SRC=...). Must contain a generated `configure` (run autogen.sh once -# if not). -# - emsdk on PATH (`source $HOME/emsdk/emsdk_env.sh` before running, or -# set EMSDK_ENV=/path/to/emsdk_env.sh). -# -# Output: -# $SYSROOT/lib/libblkid.a -# $SYSROOT/include/blkid/blkid.h -# (default sysroot: $HOME/wasm-sysroot) -# -# Why a separate script: build_anyfs_wasm.sh expects libblkid.a + -# blkid.h to already exist in the sysroot; this script provides the recipe -# that produces them. The mingw port lives in patches/libblkid/shim/ and -# uses a hand-compiled source list, but wasm tolerates util-linux's full -# autotools build via emconfigure, so we just drive that. -# -# Usage: -# ./scripts/build_libblkid_wasm.sh # uses defaults -# UL_SRC=/path/to/util-linux SYSROOT=/path/to/sysroot ./scripts/build_libblkid_wasm.sh - +#!/usr/bin/env bash +# Thin compatibility wrapper: the canonical libblkid wasm recipe now lives in +# build_wasm_sysroot.sh (which sets the mandatory -O3 -pthread flags and the +# util-linux 2.40.4 version gate). This wrapper just delegates. +# +# Environment variables UL_SRC and SYSROOT are honoured by the delegate +# (build_wasm_sysroot.sh reads UL_SRC and SYSROOT with the same names), so +# existing callers that set those variables continue to work unchanged. +# BLD_DIR (previously accepted by this script) has no equivalent in the +# delegate; the delegate always uses WORK (default: /build-wasm-sysroot) +# for its per-library build directories. set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" - -UL_SRC="${UL_SRC:-$HOME/util-linux}" -SYSROOT="${SYSROOT:-$HOME/wasm-sysroot}" -BLD_DIR="${BLD_DIR:-$REPO_ROOT/build-blkid-wasm}" - -if [[ -n "${EMSDK_ENV:-}" && -f "$EMSDK_ENV" ]]; then - # shellcheck source=/dev/null - source "$EMSDK_ENV" -fi -if ! command -v emconfigure >/dev/null 2>&1; then - echo "emconfigure not on PATH — \`source \$HOME/emsdk/emsdk_env.sh\` or set EMSDK_ENV first" >&2 - exit 1 -fi - -if [[ ! -f "$UL_SRC/configure" ]]; then - echo "util-linux configure not found at $UL_SRC/configure" >&2 - echo " run \`cd $UL_SRC && ./autogen.sh\` first" >&2 - exit 1 -fi - -mkdir -p "$BLD_DIR" "$SYSROOT/lib" "$SYSROOT/include/blkid" -cd "$BLD_DIR" - -# emconfigure works by setting CC=emcc / AR=emar / RANLIB=emranlib in the -# environment so the autoconf checks pick them up. Several util-linux -# autoconf probes assume Linux behaviour that emscripten doesn't quite -# match (openat, BSD-style ttyent, /proc/self/mountinfo); we work around -# them by: -# * --disable-all-programs --enable-libblkid: only build the library, -# skip every CLI tool (which would pull in much more libc surface). -# * --enable-static --disable-shared: emscripten can produce side -# modules but we want a plain .a to feed our final link. -# * --without-systemd, --disable-nls, --disable-asciidoc: -# unconditionally off; they would require host-side tooling that -# doesn't help an embedded archive. -# * --without-tinfo --without-readline --without-ncurses --without-cap-ng -# --without-audit --without-libmagic --without-libuser --without-econf -# --without-cryptsetup --without-util --without-systemdsystemunitdir: -# same idea — drop probe-only dependencies that just add link work. -# -# Notes on quirks: -# * configure tests `int openat()` at configure-time. Emscripten exposes -# it, so HAVE_OPENAT is set and libblkid compiles the sysfs/path code -# paths. They don't run at our use site (anyfs_probe.c feeds blkid a -# regular tmpfile path), so this is fine. -# * The C99 flag is set by util-linux's configure; emcc inherits it. - -CONFIGURE_ARGS=( - --host=wasm32-unknown-emscripten - --prefix=/usr - --enable-static --disable-shared - --enable-libblkid - --disable-all-programs - --disable-nls - --disable-asciidoc - --without-systemd - --without-systemdsystemunitdir - --without-tinfo - --without-readline - --without-ncurses - --without-ncursesw - --without-cap-ng - --without-audit - --without-libmagic - --without-libuser - --without-econf - --without-cryptsetup - --without-util - --without-python - --without-selinux - --without-utempter -) - -# Emscripten's musl ships openat but configure's runtime test would try to -# execute the cross binary — set ac_cv_func_openat=yes upfront so it skips -# the runtime stage. Similar overrides may need to be added if util-linux -# adds more `AC_RUN_IFELSE` probes in future releases. -export ac_cv_func_openat=yes -export ac_cv_func_fstatat=yes -export ac_cv_func_fdopendir=yes -export ac_cv_func_dirfd=yes - -echo ">>> emconfigure $UL_SRC/configure ${CONFIGURE_ARGS[*]}" -emconfigure "$UL_SRC/configure" "${CONFIGURE_ARGS[@]}" - -echo ">>> emmake make libblkid.la -j$(nproc)" -emmake make -j"$(nproc)" libblkid.la - -# libtool produces .libs/libblkid.a (the static archive) — the .la is just -# the libtool descriptor. Install both the archive and the public header. -if [[ -f .libs/libblkid.a ]]; then - cp .libs/libblkid.a "$SYSROOT/lib/libblkid.a" -else - echo "libblkid.a not found under .libs/" >&2 - exit 1 -fi - -# blkid.h is generated from blkid.h.in (autoconf substitutes the version -# string in). After `make`, it lives in libblkid/src/blkid.h relative to -# the build dir; fall back to the source tree if the in-tree build wrote -# it there (autotools VPATH quirk). -if [[ -f libblkid/src/blkid.h ]]; then - cp libblkid/src/blkid.h "$SYSROOT/include/blkid/blkid.h" -elif [[ -f "$UL_SRC/libblkid/src/blkid.h" ]]; then - cp "$UL_SRC/libblkid/src/blkid.h" "$SYSROOT/include/blkid/blkid.h" -else - echo "blkid.h not generated" >&2 - exit 1 -fi - -echo "Done." -ls -la "$SYSROOT/lib/libblkid.a" "$SYSROOT/include/blkid/blkid.h" +exec "$(dirname "$0")/build_wasm_sysroot.sh" --only=blkid "$@" diff --git a/scripts/build_lkl.sh b/scripts/build_lkl.sh index 8f2fdb1..5490a8b 100755 --- a/scripts/build_lkl.sh +++ b/scripts/build_lkl.sh @@ -11,6 +11,7 @@ # linux-amd64,linux-arm64,mingw32,mingw64 # (default: linux-amd64,mingw32,mingw64) # --clean Run `make clean` in each target before building +# --cc=CMD C compiler override passed to make as CC= (e.g. "sccache gcc") # -j N Parallelism (default: nproc) # # Expects each lkl-/ to already contain a .config and (for mingw @@ -27,6 +28,7 @@ OUT_PARENT="${OUT_PARENT:-$(cd "$(dirname "$0")/.." && pwd)}" TARGETS_REQ="" DO_CLEAN=0 JOBS="$(nproc)" +CC_OVERRIDE="" while [[ $# -gt 0 ]]; do case "$1" in @@ -37,10 +39,12 @@ while [[ $# -gt 0 ]]; do --targets=*) TARGETS_REQ="${1#--targets=}"; shift ;; --targets) TARGETS_REQ="$2"; shift 2 ;; --clean) DO_CLEAN=1; shift ;; + --cc=*) CC_OVERRIDE="${1#--cc=}"; shift ;; + --cc) CC_OVERRIDE="$2"; shift 2 ;; -j) JOBS="$2"; shift 2 ;; -j*) JOBS="${1#-j}"; shift ;; -h|--help) - sed -n '2,18p' "$0" | sed 's/^# \{0,1\}//' + awk 'NR==1{next} /^#/{sub(/^# ?/,""); print; next} {exit}' "$0" exit 0 ;; *) echo "Unknown argument: $1" >&2; exit 1 ;; @@ -83,16 +87,19 @@ build_one() { local cross_arg=() [[ -n "$CROSS" ]] && cross_arg=(CROSS_COMPILE="$CROSS") + local cc_arg=() + [[ -n "$CC_OVERRIDE" ]] && cc_arg=(CC="$CC_OVERRIDE") + # OUTPUT must go through the environment, not as a make CLI arg — the # tools/lkl Makefile rewrites OUTPUT to "$OUTPUT/tools/lkl/", and a CLI # assignment would defeat that rewrite (GNU make precedence). if [[ $DO_CLEAN -eq 1 ]]; then OUTPUT="$OUT" make -C "$LINUX_DIR/tools/lkl" \ - ARCH=lkl "${cross_arg[@]}" clean || true + ARCH=lkl "${cross_arg[@]}" "${cc_arg[@]}" clean || true fi OUTPUT="$OUT" make -C "$LINUX_DIR/tools/lkl" -j"$JOBS" \ - ARCH=lkl "${cross_arg[@]}" + ARCH=lkl "${cross_arg[@]}" "${cc_arg[@]}" echo echo "Output for lkl-$NAME:" diff --git a/scripts/build_lkl_wasm.sh b/scripts/build_lkl_wasm.sh index f856d96..07cedca 100755 --- a/scripts/build_lkl_wasm.sh +++ b/scripts/build_lkl_wasm.sh @@ -4,9 +4,9 @@ # Usage: ./build_lkl_wasm.sh [OPTIONS] # # Options: -# --linux=DIR Kernel source tree (default: ~/linux) -# --out=DIR Parent dir containing lkl-wasm/ (default: ~/anyfs-reader) -# --emsdk=DIR emsdk install root (default: ~/emsdk) +# --linux=DIR Kernel source tree (default: linux_src from build.config.toml; falls back to deps/linux) +# --out=DIR Parent dir containing lkl-wasm/ (default: repo root) +# --emsdk=DIR emsdk install root (default: toolchains.emsdk from build.config.toml) # --clean Run `make clean` before building # -j N Parallelism (default: nproc) # @@ -19,11 +19,15 @@ # - LD=emcc, since the kernel uses partial linking ($(LD) -r vmlinux); emcc # forwards -r to wasm-ld and keeps the output in wasm format # - No CROSS_COMPILE (it would prefix the tool names, defeating emcc/emar) -set -e +set -e -o pipefail -LINUX_DIR="$HOME/linux" -OUT_PARENT="$HOME/anyfs-reader" -EMSDK_DIR="$HOME/emsdk" +# shellcheck source=lib/config.sh +source "$(dirname "$0")/lib/config.sh" + +# CLI --linux=/--out=/--emsdk= win; config.sh provides the defaults. +LINUX_DIR="${LINUX_DIR:-$ANYFS_PATHS_LINUX_SRC}" +OUT_PARENT="${OUT_PARENT:-$(cd "$(dirname "$0")/.." && pwd)}" +EMSDK_DIR="${EMSDK_DIR:-$ANYFS_TOOLCHAINS_EMSDK}" DO_CLEAN=0 JOBS="$(nproc)" @@ -64,6 +68,23 @@ if [[ ! -f "$PRESEED_SYSCALL_DEFS_H" ]]; then fi export PRESEED_SYSCALL_DEFS_H +# Post-processing tools vendored in scripts/lkl-wasm-tools/. Check early so a +# missing tool fails BEFORE any kernel compile rather than after liblkl.a is +# built, which would leave a half-processed archive that crashes at boot. +# FIXER/PREFIXER env vars override the vendored copies. +REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TOOLS_DIR="$REPO_ROOT/scripts/lkl-wasm-tools" +FIXER="${FIXER:-$TOOLS_DIR/wasm_fix_absolute_brackets.py}" +PREFIXER="${PREFIXER:-$TOOLS_DIR/wasm_prefix_kernel_symbols.py}" +for tool in "$FIXER" "$PREFIXER"; do + if [[ ! -f "$tool" ]]; then + echo "Error: required post-processing tool not found: $tool" >&2 + echo "Skipping it would produce a liblkl.a that crashes at boot" >&2 + echo "(absolute SECTIONS{} brackets dereference garbage)." >&2 + exit 1 + fi +done + # Activate emsdk environment so emcc/emar resolve from $PATH. # shellcheck disable=SC1091 source "$EMSDK_DIR/emsdk_env.sh" >/dev/null 2>&1 @@ -89,10 +110,11 @@ export CLANG_TARGET_FLAGS_lkl="wasm32-unknown-emscripten" # bracket symbols (__setup_start, init_thread_union, ...) actually get # materialised into vmlinux.unstripped. Route the script link to Joel's # wasm-ld; everything else still goes through emsdk's stock wasm-ld. -JOEL_WASM_LD="$HOME/linux-wasm/workspace/install/llvm/bin/wasm-ld" +JOEL_WASM_LD="${JOEL_WASM_LD:-$ANYFS_TOOLCHAINS_WASM_LD}" if [[ ! -x "$JOEL_WASM_LD" ]]; then - echo "Error: Joel's wasm-ld not found at $JOEL_WASM_LD" >&2 - echo "Build it with: cd ~/linux-wasm && ./linux-wasm.sh build-llvm" >&2 + echo "Error: patched wasm-ld not found at $JOEL_WASM_LD" >&2 + echo "Fix: set toolchains.wasm_ld in build.user.toml, run scripts/fetch_wasm_ld.sh," >&2 + echo " or build it from deps/llvm-wasm (./linux-wasm.sh build-llvm)" >&2 exit 1 fi LD_WRAPPER="$OUT/wasm-ld-wrapper" @@ -237,16 +259,21 @@ OUTPUT="$OUT" make -C "$LINUX_DIR/tools/lkl" -j"$JOBS" ARCH=lkl "${TOOLS[@]}" \ # absolute values that don't match where it actually places the segments, # and any segment whose only references are these absolute brackets gets # DCE'd. See wasm_fix_absolute_brackets.py for details. -FIXER="${FIXER:-${LKLFTPD_SRC}/wasm_fix_absolute_brackets.py}" -PREFIXER="${PREFIXER:-${LKLFTPD_SRC}/wasm_prefix_kernel_symbols.py}" +# +# These two rewrites are NOT optional. A liblkl.a built without the bracket +# fixer boots into `RuntimeError: table index is out of bounds` the moment +# the kernel iterates a bracketed section (first hit: trace_event_init +# walking __start_ftrace_events..__stop_ftrace_events — CONFIG_TRACING is +# force-selected by `config LKL` in arch/lkl/Kconfig, so it can't be +# config'd away; __initcallN brackets are equally load-bearing). The tools +# are vendored in scripts/lkl-wasm-tools/; FIXER/PREFIXER env vars override. +# (FIXER/PREFIXER/TOOLS_DIR are set and checked in the preflight section above.) LKLO="$OUT/tools/lkl/lib/lkl.o" LIBA="$OUT/tools/lkl/liblkl.a" -if [[ -f "$FIXER" ]]; then - echo - echo " FIX $LKLO (absolute -> segment-relative brackets)" - python3 "$FIXER" "$LKLO" "$LKLO.fixed" | tail -20 - mv "$LKLO.fixed" "$LKLO" -fi +echo +echo " FIX $LKLO (absolute -> segment-relative brackets)" +python3 "$FIXER" "$LKLO" "$LKLO.fixed" | tail -20 +mv "$LKLO.fixed" "$LKLO" # Namespace kernel symbols so they stop colliding with libc at final-link. # Without this, the kernel's vsnprintf / memcpy / etc. outrank musl's weak # copies and any libc caller in user code ends up running the kernel @@ -254,12 +281,10 @@ fi # PAGE_SIZE. ELF/PE LKL solves the same problem with `objcopy # --prefix-symbols=_`; llvm-objcopy advertises that flag but actually # rejects it on wasm objects, so we do the rewrite ourselves. -if [[ -f "$PREFIXER" ]]; then - echo - echo " NS $LKLO (prefix kernel symbols)" - python3 "$PREFIXER" "$LKLO" "$LKLO.prefixed" | tail -10 - mv "$LKLO.prefixed" "$LKLO" -fi +echo +echo " NS $LKLO (prefix kernel symbols)" +python3 "$PREFIXER" "$LKLO" "$LKLO.prefixed" | tail -10 +mv "$LKLO.prefixed" "$LKLO" if [[ -f "$LKLO" ]]; then "${EMSDK_DIR}/upstream/bin/llvm-ar" rs "$LIBA" "$LKLO" >/dev/null fi diff --git a/scripts/build_qemu.sh b/scripts/build_qemu.sh index 81ffd63..91d30fd 100755 --- a/scripts/build_qemu.sh +++ b/scripts/build_qemu.sh @@ -5,13 +5,16 @@ # Usage: ./build_qemu.sh [OPTIONS] # # Options: -# --qemu-src=DIR QEMU source tree (default: ~/qemu) +# --qemu-src=DIR QEMU source tree (default: qemu_src from +# build.config.toml; falls back to deps/qemu) # --out-prefix=PFX Build-dir prefix inside qemu-src (default: build-anyfs) # Produces /-/. # --targets=LIST Comma-separated subset of: # linux-amd64,mingw32,mingw64 # (default: linux-amd64,mingw32,mingw64) # --reconfigure Wipe build dir and re-run configure +# --cc=CMD C compiler override passed to configure as --cc= for +# linux-amd64 only; switching compilers needs --reconfigure # -j N Parallelism (default: nproc) # -h, --help # @@ -21,20 +24,24 @@ # (mingw32/mingw64, supplied by msys-cross-pkgconfig) provide glib/zstd/zlib. # # Prereqs: -# - QEMU source tree at $QEMU_SRC (default ~/qemu). util/fdmon-poll.c must -# have 'static' removed from its __thread declarations so a shared object -# can resolve them — handled by the project's local patch. +# - QEMU source tree at $QEMU_SRC (qemu_src in build.config.toml). +# util/fdmon-poll.c must have 'static' removed from its __thread +# declarations so a shared object can resolve them — handled by the +# project's local patch. # - For mingw: msys2-cross with mingw-w64-{i686,x86_64}-{glib2,zlib,zstd} -# installed under /opt/msys2-cross/{mingw32,mingw64}. +# installed under /{mingw32,mingw64}. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=lib/config.sh +source "$SCRIPT_DIR/lib/config.sh" -QEMU_SRC="$HOME/qemu" +QEMU_SRC="${QEMU_SRC:-$ANYFS_PATHS_QEMU_SRC}" OUT_PFX="build-anyfs" TARGETS_REQ="linux-amd64,mingw32,mingw64" RECONFIGURE=0 JOBS="$(nproc)" +CC_OVERRIDE="" while [[ $# -gt 0 ]]; do case "$1" in @@ -45,6 +52,8 @@ while [[ $# -gt 0 ]]; do --targets=*) TARGETS_REQ="${1#--targets=}"; shift ;; --targets) TARGETS_REQ="$2"; shift 2 ;; --reconfigure) RECONFIGURE=1; shift ;; + --cc=*) CC_OVERRIDE="${1#--cc=}"; shift ;; + --cc) CC_OVERRIDE="$2"; shift 2 ;; -j) JOBS="$2"; shift 2 ;; -j*) JOBS="${1#-j}"; shift ;; -h|--help) @@ -117,16 +126,16 @@ configure_for() { '--cpu=i386' \ '--disable-pixman' \ '--disable-png' \ - "--extra-cflags=-I/opt/msys2-cross/mingw32/include" \ - "--extra-ldflags=-L/opt/msys2-cross/mingw32/lib" + "--extra-cflags=-I$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw32/include" \ + "--extra-ldflags=-L$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw32/lib" ;; mingw64) printf '%s\n' \ '--cross-prefix=x86_64-w64-mingw32-' \ '--disable-pixman' \ '--disable-png' \ - "--extra-cflags=-I/opt/msys2-cross/mingw64/include" \ - "--extra-ldflags=-L/opt/msys2-cross/mingw64/lib" + "--extra-cflags=-I$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw64/include" \ + "--extra-ldflags=-L$ANYFS_TOOLCHAINS_MSYS2_CROSS/mingw64/lib" ;; esac } @@ -219,11 +228,14 @@ build_one() { mapfile -t target_cfg < <(configure_for "$target") + local cc_cfg=() + [[ -n "$CC_OVERRIDE" && "$target" == "linux-amd64" ]] && cc_cfg=("--cc=$CC_OVERRIDE") + if [[ ! -f "$builddir/build.ninja" ]]; then rm -rf "$builddir" mkdir -p "$builddir" ( cd "$builddir" && "$QEMU_SRC/configure" \ - "${COMMON_CONFIGURE[@]}" "${target_cfg[@]}" ) + "${COMMON_CONFIGURE[@]}" "${target_cfg[@]}" "${cc_cfg[@]}" ) # b_pie=false matches the -fno-pie/-fPIC flags; needed for the shared # link on Linux and harmless on mingw. werror=false keeps the build # from tripping over glibc-vs-QEMU prototype drift (e.g. diff --git a/scripts/build_wasm_sysroot.sh b/scripts/build_wasm_sysroot.sh new file mode 100755 index 0000000..12c129a --- /dev/null +++ b/scripts/build_wasm_sysroot.sh @@ -0,0 +1,403 @@ +#!/usr/bin/env bash +# Reproducible build recipe for the wasm sysroot. +# +# Rebuilds, from pinned upstream sources, every static library listed in +# scripts/lib/wasm_sysroot.manifest and installs them (plus headers and +# pkgconfig files) into $SYSROOT. Acceptance gate: +# +# SYSROOT=/tmp/sysroot-rebuild ./scripts/build_wasm_sysroot.sh +# ls /tmp/sysroot-rebuild/lib/*.a | xargs -n1 basename | sort \ +# | diff <(grep -vE '^#|^$' scripts/lib/wasm_sysroot.manifest | sort) - +# +# Provenance of the known-good hand-built sysroot (reverse-engineered from +# the build trees/logs it preserved under /src and the glib +# checkout's meson-info; snapshot 2026-06-10): +# - zlib/zstd/libffi: pinned release tarballs, configure lines recovered +# from the preserved configure.log / config.log. +# - bzip2: 7 objects named *.c.o in the archive — hand-compiled with emcc +# (no .pc file, matching upstream bzip2 which ships none). +# - pcre2 (-8/-16/-32/-posix): NOT built standalone; produced by glib's +# meson subproject fallback (-Dforce_fallback_for=pcre2), pinned by +# glib's own subprojects/pcre2.wrap (10.46, sha256 in the wrap file). +# libpcre2-posix.a is installed but gets no .pc — matches the manifest. +# - glib 2.88.0 (incl. libgirepository-2.0.a via the default +# introspection=auto): meson cross build; exact option set recovered +# from build-wasm32/meson-info/intro-buildoptions.json. +# - blkid+uuid 2.40.4: util-linux tree via emconfigure (same approach as +# scripts/build_libblkid_wasm.sh, which remains as the thin +# blkid-only iteration helper), plus libuuid and the generated .pc +# files the hand-built sysroot carries. +# - libresolv.a: a hand-written one-function stub (res_query() returning +# HOST_NOT_FOUND) so glib/gio's resolver references link; the original +# source is preserved verbatim below (it was /src/res_query.c). +# +# All libraries are compiled with -O3 -pthread: the final anyfs link is +# pthread-enabled, so every object must carry the atomics/bulk-memory +# target features or wasm-ld rejects the mix. +# +# Usage: +# ./scripts/build_wasm_sysroot.sh # full build into config sysroot +# SYSROOT=/tmp/sysroot-rebuild ./scripts/build_wasm_sysroot.sh +# ./scripts/build_wasm_sysroot.sh --only=glib # iterate one recipe +# ./scripts/build_wasm_sysroot.sh --clean # wipe work dir first +# +# Env overrides: SYSROOT, WORK, UL_SRC, EMSDK_ENV. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +# shellcheck source=lib/config.sh +source "$SCRIPT_DIR/lib/config.sh" + +SYSROOT="${SYSROOT:-$ANYFS_PATHS_WASM_SYSROOT}" +WORK="${WORK:-$REPO_ROOT/build-wasm-sysroot}" +UL_SRC="${UL_SRC:-$ANYFS_PATHS_UTIL_LINUX}" + +ONLY="" +CLEAN=0 +for arg in "$@"; do + case "$arg" in + --only=*) ONLY="${arg#--only=}" ;; + --clean) CLEAN=1 ;; + -h|--help) + sed -n '2,50p' "$0" | sed 's/^# \{0,1\}//' + exit 0 ;; + *) echo "unknown argument: $arg (try --only=, --clean)" >&2; exit 1 ;; + esac +done + +EMSDK_ENV="${EMSDK_ENV:-${ANYFS_TOOLCHAINS_EMSDK:+$ANYFS_TOOLCHAINS_EMSDK/emsdk_env.sh}}" +if [[ -n "${EMSDK_ENV:-}" && -f "$EMSDK_ENV" ]]; then + # shellcheck source=/dev/null + source "$EMSDK_ENV" >/dev/null 2>&1 +fi +for tool in emcc emconfigure emmake emar meson ninja pkg-config curl python3; do + command -v "$tool" >/dev/null 2>&1 || { + echo "$tool not on PATH — set toolchains.emsdk in build.config.toml (or EMSDK_ENV) and install meson/ninja" >&2 + exit 1 + } +done + +[[ $CLEAN -eq 1 ]] && rm -rf "$WORK" +mkdir -p "$WORK" "$SYSROOT/lib/pkgconfig" "$SYSROOT/include" + +NPROC="$(nproc)" + +# fetch — download (with cache) and verify. +fetch() { + local url="$1" sha="$2" dest="$3" + if [[ ! -f "$dest" ]] || ! echo "$sha $dest" | sha256sum --check --quiet - 2>/dev/null; then + echo ">>> fetch $url" + curl -fL --retry 3 -o "$dest" "$url" + fi + echo "$sha $dest" | sha256sum --check --quiet - +} + +# unpack — fresh-extract into $WORK/. +unpack() { + local tarball="$1" dirname="$2" + rm -rf "${WORK:?}/$dirname" + tar -xf "$tarball" -C "$WORK" + [[ -d "$WORK/$dirname" ]] || { echo "expected $dirname after extracting $tarball" >&2; exit 1; } +} + +# --------------------------------------------------------------------------- +# zlib 1.3.1 +# --------------------------------------------------------------------------- +ZLIB_V=1.3.1 +# zlib.net 404s superseded releases; fossils/ archives every version. +ZLIB_URL="https://zlib.net/fossils/zlib-$ZLIB_V.tar.gz" +ZLIB_SHA=9a93b2b7dfdac77ceba5a558a580e74667dd6fede4585b91eefb60f03b72df23 +build_zlib() { + echo "=== zlib $ZLIB_V ===" + fetch "$ZLIB_URL" "$ZLIB_SHA" "$WORK/zlib-$ZLIB_V.tar.gz" + unpack "$WORK/zlib-$ZLIB_V.tar.gz" "zlib-$ZLIB_V" + cd "$WORK/zlib-$ZLIB_V" + # zlib's configure reads CFLAGS; -pthread keeps the objects compatible + # with the pthread-enabled final link (atomics target feature). + CFLAGS="-O3 -pthread" emconfigure ./configure --prefix="$SYSROOT" --static + emmake make -j"$NPROC" libz.a + emmake make install # installs libz.a, zlib.h/zconf.h, lib/pkgconfig/zlib.pc +} + +# --------------------------------------------------------------------------- +# bzip2 1.0.8 — upstream has no .pc and its Makefile hardcodes cc tests, so +# compile the 7 library sources directly (the hand-built archive's members +# are exactly these, named *.c.o). +# --------------------------------------------------------------------------- +BZ2_V=1.0.8 +BZ2_URL="https://sourceware.org/pub/bzip2/bzip2-$BZ2_V.tar.gz" +BZ2_SHA=ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269 +build_bzip2() { + echo "=== bzip2 $BZ2_V ===" + fetch "$BZ2_URL" "$BZ2_SHA" "$WORK/bzip2-$BZ2_V.tar.gz" + unpack "$WORK/bzip2-$BZ2_V.tar.gz" "bzip2-$BZ2_V" + cd "$WORK/bzip2-$BZ2_V" + local srcs=(blocksort bzlib compress crctable decompress huffman randtable) + local objs=() + for s in "${srcs[@]}"; do + emcc -O3 -pthread -D_FILE_OFFSET_BITS=64 -c "$s.c" -o "$s.c.o" + objs+=("$s.c.o") + done + rm -f libbz2.a + emar rcs libbz2.a "${objs[@]}" + install -m644 libbz2.a "$SYSROOT/lib/libbz2.a" + install -m644 bzlib.h "$SYSROOT/include/bzlib.h" +} + +# --------------------------------------------------------------------------- +# zstd 1.5.7 — lib/Makefile native targets (the hand-built tree shows the +# obj/conf_*/static layout that Makefile produces). +# --------------------------------------------------------------------------- +ZSTD_V=1.5.7 +ZSTD_URL="https://github.com/facebook/zstd/releases/download/v$ZSTD_V/zstd-$ZSTD_V.tar.gz" +ZSTD_SHA=eb33e51f49a15e023950cd7825ca74a4a2b43db8354825ac24fc1b7ee09e6fa3 +build_zstd() { + echo "=== zstd $ZSTD_V ===" + fetch "$ZSTD_URL" "$ZSTD_SHA" "$WORK/zstd-$ZSTD_V.tar.gz" + unpack "$WORK/zstd-$ZSTD_V.tar.gz" "zstd-$ZSTD_V" + cd "$WORK/zstd-$ZSTD_V" + CFLAGS="-O3 -pthread" emmake make -C lib -j"$NPROC" libzstd.a + # install-static/-includes/-pc are plain file copies; PREFIX routes them + # into the sysroot. (Skip install-shared: static-only sysroot.) + CFLAGS="-O3 -pthread" emmake make -C lib PREFIX="$SYSROOT" \ + install-static install-includes install-pc +} + +# --------------------------------------------------------------------------- +# libffi 3.5.2 — upstream wasm32 support via emconfigure; exact configure +# line recovered from the hand-built tree's config.log. +# --------------------------------------------------------------------------- +FFI_V=3.5.2 +FFI_URL="https://github.com/libffi/libffi/releases/download/v$FFI_V/libffi-$FFI_V.tar.gz" +FFI_SHA=f3a3082a23b37c293a4fcd1053147b371f2ff91fa7ea1b2a52e335676bac82dc +build_libffi() { + echo "=== libffi $FFI_V ===" + fetch "$FFI_URL" "$FFI_SHA" "$WORK/libffi-$FFI_V.tar.gz" + unpack "$WORK/libffi-$FFI_V.tar.gz" "libffi-$FFI_V" + cd "$WORK/libffi-$FFI_V" + CFLAGS="-O3 -pthread" emconfigure ./configure \ + --host=wasm32-unknown-linux \ + --prefix="$SYSROOT" \ + --enable-static --disable-shared \ + --disable-dependency-tracking \ + --disable-builddir \ + --disable-multi-os-directory \ + --disable-raw-api \ + --disable-docs + emmake make -j"$NPROC" + emmake make install # libffi.a(+.la), ffi.h/ffitarget.h, libffi.pc +} + +# --------------------------------------------------------------------------- +# glib 2.88.0 — meson cross build. Also produces: +# * libpcre2-{8,16,32,posix}.a + libpcre2-{8,16,32}.pc via the forced +# pcre2 10.46 subproject fallback (pin lives in glib's +# subprojects/pcre2.wrap: source_hash 15fbc5ab...299f, wrapdb 10.46-1; +# meson verifies it on download). +# * libgirepository-2.0.a (introspection=auto is satisfied cross-side in +# glib 2.88, which hosts girepository in-tree). +# Depends on zlib + libffi (pkgconfig) AND libresolv (gio hard-requires +# res_query()) already being in $SYSROOT. +# --------------------------------------------------------------------------- +GLIB_V=2.88.0 +GLIB_URL="https://download.gnome.org/sources/glib/${GLIB_V%.*}/glib-$GLIB_V.tar.xz" +GLIB_SHA=3546251ccbb3744d4bc4eb48354540e1f6200846572bab68e3a2b7b2b64dfd07 +build_glib() { + echo "=== glib $GLIB_V (+pcre2 subproject, +girepository) ===" + fetch "$GLIB_URL" "$GLIB_SHA" "$WORK/glib-$GLIB_V.tar.xz" + unpack "$WORK/glib-$GLIB_V.tar.xz" "glib-$GLIB_V" + # Emscripten port patch (see the patch header for rationale). + patch -d "$WORK/glib-$GLIB_V" -p1 \ + < "$SCRIPT_DIR/lib/glib-$GLIB_V-emscripten-fd-query-path.patch" + # Resolve the cross-file template against this sysroot. + sed "s|@SYSROOT@|$SYSROOT|g" "$SCRIPT_DIR/lib/emscripten-cross.meson" \ + > "$WORK/cross-wasm32.meson" + cd "$WORK/glib-$GLIB_V" + # PKG_CONFIG_LIBDIR (not _PATH) so ONLY the sysroot's .pc files are + # visible — host libffi/zlib must never leak into the wasm build. + PKG_CONFIG_LIBDIR="$SYSROOT/lib/pkgconfig" meson setup _build \ + --cross-file "$WORK/cross-wasm32.meson" \ + -Dprefix="$SYSROOT" \ + -Dbuildtype=release \ + -Ddefault_library=static \ + -Dforce_fallback_for=pcre2 \ + -Dselinux=disabled \ + -Dxattr=false \ + -Dlibmount=disabled \ + -Dnls=disabled \ + -Dtests=false \ + -Dglib_debug=disabled \ + -Dglib_assert=false \ + -Dglib_checks=false + # Emscripten's libc DECLARES posix_spawn{,p} and pthread_getname_np (so + # meson's compile-only checks pass) but does not DEFINE them — the .js + # tool executables then fail at wasm-ld with undefined symbols. The + # known-good sysroot was built by dropping the two defines from the + # generated config.h after setup (verified: its preserved config.h lacks + # exactly these two lines vs a fresh setup); replicate that. config.h is + # written at setup time only, so the edit survives the compile. + # + # Guard: assert the lines are present before the deletion so a future + # glib bump that renames/removes these defines is caught loudly rather + # than silently producing a broken sysroot (the wasm-ld undefined-symbol + # failure would be cryptic and far removed from this edit). + grep -q 'HAVE_POSIX_SPAWN' _build/config.h || { + echo "ERROR: HAVE_POSIX_SPAWN not found in _build/config.h" >&2 + echo " A glib version bump may have renamed or dropped this define." >&2 + echo " Review config.h and update the sed deletion below accordingly." >&2 + exit 1 + } + grep -q 'HAVE_PTHREAD_GETNAME_NP' _build/config.h || { + echo "ERROR: HAVE_PTHREAD_GETNAME_NP not found in _build/config.h" >&2 + echo " A glib version bump may have renamed or dropped this define." >&2 + echo " Review config.h and update the sed deletion below accordingly." >&2 + exit 1 + } + sed -i '/#define HAVE_POSIX_SPAWN 1/d;/#define HAVE_PTHREAD_GETNAME_NP 1/d' \ + _build/config.h + # Post-deletion guard: assert neither define survived (catches a sed + # pattern that no longer matches, e.g. if glib changes the comment style + # or the define value). + if grep -qE 'HAVE_POSIX_SPAWN|HAVE_PTHREAD_GETNAME_NP' _build/config.h; then + echo "ERROR: HAVE_POSIX_SPAWN or HAVE_PTHREAD_GETNAME_NP still present in _build/config.h after deletion" >&2 + echo " The sed pattern may no longer match the current glib config.h format." >&2 + echo " Review config.h and update the sed deletion above accordingly." >&2 + exit 1 + fi + PKG_CONFIG_LIBDIR="$SYSROOT/lib/pkgconfig" meson compile -C _build + meson install -C _build --no-rebuild +} + +# --------------------------------------------------------------------------- +# blkid + uuid 2.40.4 — built from the util-linux tree (paths.util_linux in +# build.config.toml), same emconfigure approach as +# scripts/build_libblkid_wasm.sh, extended with libuuid and the generated +# pkgconfig files. The tree must be on the v2.40.4 release (stable/v2.40) +# and have a generated ./configure (run autogen.sh once if not). +# --------------------------------------------------------------------------- +UL_V=2.40.4 +build_blkid() { + echo "=== util-linux $UL_V (libblkid + libuuid) ===" + if [[ ! -f "$UL_SRC/configure" ]]; then + echo "util-linux configure not found at $UL_SRC/configure" >&2 + echo " run \`cd $UL_SRC && ./autogen.sh\` first" >&2 + exit 1 + fi + local v + v="$(sed -n "s/^PACKAGE_VERSION='\(.*\)'/\1/p" "$UL_SRC/configure" | head -1)" + if [[ "$v" != "$UL_V" ]]; then + echo "util-linux at $UL_SRC is version '$v', manifest pins $UL_V — check out v$UL_V" >&2 + exit 1 + fi + rm -rf "$WORK/util-linux-build" + mkdir -p "$WORK/util-linux-build" + cd "$WORK/util-linux-build" + # Autoconf runtime probes can't execute cross binaries; emscripten's + # musl provides these, so pre-seed the cache (see + # scripts/build_libblkid_wasm.sh for the full rationale). + export ac_cv_func_openat=yes ac_cv_func_fstatat=yes \ + ac_cv_func_fdopendir=yes ac_cv_func_dirfd=yes + CFLAGS="-O3 -pthread" emconfigure "$UL_SRC/configure" \ + --host=wasm32-unknown-emscripten \ + --prefix="$SYSROOT" \ + --enable-static --disable-shared \ + --enable-libblkid --enable-libuuid \ + --disable-all-programs \ + --disable-nls --disable-asciidoc \ + --without-systemd --without-systemdsystemunitdir \ + --without-tinfo --without-readline \ + --without-ncurses --without-ncursesw \ + --without-cap-ng --without-audit --without-libmagic \ + --without-libuser --without-econf --without-cryptsetup \ + --without-util --without-python --without-selinux \ + --without-utempter + emmake make -j"$NPROC" libblkid.la libuuid.la \ + libblkid/blkid.pc libuuid/uuid.pc + install -m644 .libs/libblkid.a "$SYSROOT/lib/libblkid.a" + install -m644 .libs/libuuid.a "$SYSROOT/lib/libuuid.a" + mkdir -p "$SYSROOT/include/blkid" "$SYSROOT/include/uuid" + # blkid.h is generated (version substituted); uuid.h is plain source. + if [[ -f libblkid/src/blkid.h ]]; then + install -m644 libblkid/src/blkid.h "$SYSROOT/include/blkid/blkid.h" + else + install -m644 "$UL_SRC/libblkid/src/blkid.h" "$SYSROOT/include/blkid/blkid.h" + fi + install -m644 "$UL_SRC/libuuid/src/uuid.h" "$SYSROOT/include/uuid/uuid.h" + install -m644 libblkid/blkid.pc "$SYSROOT/lib/pkgconfig/blkid.pc" + install -m644 libuuid/uuid.pc "$SYSROOT/lib/pkgconfig/uuid.pc" +} + +# --------------------------------------------------------------------------- +# libresolv — single-function stub. glib/gio reference res_query(); the +# wasm bundle never resolves DNS, so it just reports HOST_NOT_FOUND. This +# is the verbatim source the hand-built sysroot preserved (src/res_query.c). +# --------------------------------------------------------------------------- +build_libresolv() { + echo "=== libresolv (res_query stub) ===" + rm -rf "$WORK/libresolv" + mkdir -p "$WORK/libresolv" + cd "$WORK/libresolv" + cat > res_query.c <<'EOF' +#include +int res_query(const char *name, int class, + int type, unsigned char *dest, int len) +{ + h_errno = HOST_NOT_FOUND; + return -1; +} +EOF + emcc -O3 -pthread -Wno-unused-parameter -c res_query.c -o libresolv.o + rm -f libresolv.a + emar rcs libresolv.a libresolv.o + install -m644 libresolv.a "$SYSROOT/lib/libresolv.a" +} + +# --------------------------------------------------------------------------- +# Driver — dependency order matters: glib needs zlib+libffi (pkgconfig from +# this sysroot) and libresolv (gio's res_query check); everything else is +# independent. +# --------------------------------------------------------------------------- +ALL_LIBS=(zlib bzip2 zstd libffi libresolv glib blkid) + +run_one() { + case "$1" in + zlib) build_zlib ;; + bzip2|bz2) build_bzip2 ;; + zstd) build_zstd ;; + libffi|ffi) build_libffi ;; + glib|pcre2|girepository) build_glib ;; + blkid|uuid|util-linux) build_blkid ;; + libresolv|resolv) build_libresolv ;; + *) echo "unknown --only target: $1 (one of: ${ALL_LIBS[*]})" >&2; exit 1 ;; + esac +} + +if [[ -n "$ONLY" ]]; then + run_one "$ONLY" +else + for lib in "${ALL_LIBS[@]}"; do + run_one "$lib" + done +fi + +list_libs() { + find "$SYSROOT/lib" -maxdepth 1 -name '*.a' -printf '%f\n' | sort +} + +echo +echo "=== sysroot summary ($SYSROOT) ===" +list_libs + +echo +echo "=== manifest parity check ===" +if diff <(grep -vE '^#|^$' "$SCRIPT_DIR/lib/wasm_sysroot.manifest" | sort) \ + <(list_libs); then + echo "OK: sysroot lib set matches scripts/lib/wasm_sysroot.manifest" +elif [[ -n "$ONLY" ]]; then + echo "(partial build via --only=$ONLY — parity mismatch expected)" +else + echo "FAIL: sysroot lib set differs from the manifest" >&2 + exit 1 +fi diff --git a/scripts/doctor.sh b/scripts/doctor.sh index c4ad5dc..0ead4da 100755 --- a/scripts/doctor.sh +++ b/scripts/doctor.sh @@ -4,9 +4,11 @@ set -uo pipefail _root="$(cd "$(dirname "$0")/.." && pwd)" # Capture env overrides before config.sh (which exports toml defaults, overwriting env vars). _pre_wasm_ld="${ANYFS_TOOLCHAINS_WASM_LD:-}" +_pre_wasm_sysroot="${ANYFS_PATHS_WASM_SYSROOT:-}" source "$(dirname "$0")/lib/config.sh" # Restore env override if caller set it explicitly before running doctor. [ -n "$_pre_wasm_ld" ] && ANYFS_TOOLCHAINS_WASM_LD="$_pre_wasm_ld" +[ -n "$_pre_wasm_sysroot" ] && ANYFS_PATHS_WASM_SYSROOT="$_pre_wasm_sysroot" fail=0 ok() { printf ' \033[32mok\033[0m %s\n' "$1"; } bad() { printf ' \033[31mFAIL\033[0m %s\n' "$1"; fail=1; } @@ -30,20 +32,26 @@ mw="$ANYFS_TOOLCHAINS_MSYS2_CROSS/bin/x86_64-w64-mingw32-ld" [ -x "$mw" ] && ok "$($mw --version | head -1)" || bad "mingw64 ld missing: $mw" echo "== wasm-ld (patched, from xdqi/llvm-wasm — only this binary is consumed) ==" -wl="${ANYFS_TOOLCHAINS_WASM_LD:-}" -if [ -z "$wl" ]; then - case "$ANYFS_PATHS_DEPS_ROOT" in - /*) wl="$ANYFS_PATHS_DEPS_ROOT/llvm-wasm/workspace/install/llvm/bin/wasm-ld" ;; - *) wl="$_root/$ANYFS_PATHS_DEPS_ROOT/llvm-wasm/workspace/install/llvm/bin/wasm-ld" ;; - esac -fi +wl="$ANYFS_TOOLCHAINS_WASM_LD" if [ -x "$wl" ]; then ver=$("$wl" --version 2>/dev/null | grep -oE 'LLD [0-9]+' | head -1) [ "$ver" = "LLD 18" ] && ok "$ver ($wl)" || bad "wasm-ld is '$ver', expected LLD 18 ($wl)" else - bad "patched wasm-ld not built — run: (cd deps/llvm-wasm && ./linux-wasm.sh build-llvm)" + bad "patched wasm-ld missing — run scripts/fetch_wasm_ld.sh (or build from source: cd deps/llvm-wasm && ./linux-wasm.sh build-llvm)" fi +echo "== wasm sysroot (static libs for the wasm bundle; see scripts/lib/wasm_sysroot.manifest) ==" +_manifest="$_root/scripts/lib/wasm_sysroot.manifest" +_missing=0 +while IFS= read -r lib; do + case "$lib" in ''|'#'*) continue ;; esac + if [ ! -f "$ANYFS_PATHS_WASM_SYSROOT/lib/$lib" ]; then + bad "missing $ANYFS_PATHS_WASM_SYSROOT/lib/$lib — run scripts/fetch_wasm_sysroot.sh (or scripts/build_wasm_sysroot.sh)" + _missing=1 + fi +done < "$_manifest" +[ "$_missing" -eq 0 ] && ok "all manifest libs present in $ANYFS_PATHS_WASM_SYSROOT/lib" + echo "== deps synced + at locked SHA ==" if [ -d "$ANYFS_PATHS_LINUX_SRC/.git" ]; then ok "linux at $(git -C "$ANYFS_PATHS_LINUX_SRC" rev-parse --short HEAD)" diff --git a/scripts/fetch_wasm_ld.sh b/scripts/fetch_wasm_ld.sh new file mode 100755 index 0000000..7a7dbc1 --- /dev/null +++ b/scripts/fetch_wasm_ld.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Download the prebuilt patched wasm-ld (published by xdqi/llvm-wasm's +# wasm-ld-release.yml workflow) into /.toolchain/wasm-ld/. Pin: bump +# WASM_LD_TAG when a new release is cut. scripts/lib/config.sh auto-prefers +# this location unless toolchains.wasm_ld is set explicitly. +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +WASM_LD_TAG="${WASM_LD_TAG:-wasm-ld-18.1.2-anyfs-r1}" +dest="$root/.toolchain/wasm-ld" +if [[ -x "$dest/wasm-ld" ]] && "$dest/wasm-ld" --version 2>/dev/null | grep -q 'LLD 18'; then + echo "wasm-ld already present: $dest/wasm-ld"; exit 0 +fi +mkdir -p "$dest" +url="https://github.com/xdqi/llvm-wasm/releases/download/$WASM_LD_TAG/wasm-ld-linux-amd64.tar.xz" +echo "fetching $url" +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT +curl -fsSL --retry 3 -o "$tmp" "$url" || { + echo "download failed (does release tag '$WASM_LD_TAG' exist?)" >&2 + exit 1 +} +tar -xJf "$tmp" -C "$dest" +[[ -f "$dest/wasm-ld" ]] || { echo "archive did not contain wasm-ld" >&2; exit 1; } +chmod +x "$dest/wasm-ld" +"$dest/wasm-ld" --version | grep -q 'LLD 18' || { + echo "fetched wasm-ld does not report 'LLD 18':" >&2 + "$dest/wasm-ld" --version >&2 || true + exit 1 +} +echo "OK: $dest/wasm-ld ($("$dest/wasm-ld" --version | head -1))" diff --git a/scripts/fetch_wasm_sysroot.sh b/scripts/fetch_wasm_sysroot.sh new file mode 100755 index 0000000..4b1352f --- /dev/null +++ b/scripts/fetch_wasm_sysroot.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Download the prebuilt wasm sysroot (published by .github/workflows/ +# wasm-sysroot.yml) into /.toolchain/wasm-sysroot/. Pin: bump +# WASM_SYSROOT_TAG when a new release is cut. scripts/lib/config.sh +# auto-prefers this location unless paths.wasm_sysroot is set explicitly. +# See docs/wasm-sysroot.md. +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +WASM_SYSROOT_TAG="${WASM_SYSROOT_TAG:-wasm-sysroot-r1}" +dest="$root/.toolchain/wasm-sysroot" +manifest="$root/scripts/lib/wasm_sysroot.manifest" +complete() { + while IFS= read -r lib; do + case "$lib" in ''|'#'*) continue ;; esac + [[ -f "$dest/lib/$lib" ]] || return 1 + done < "$manifest" +} +if complete; then + echo "wasm sysroot already present: $dest"; exit 0 +fi +mkdir -p "$dest" +url="https://github.com/xdqi/anyfs/releases/download/$WASM_SYSROOT_TAG/wasm-sysroot-linux.tar.xz" +echo "fetching $url" +tmp="$(mktemp)" +trap 'rm -f "$tmp"' EXIT +curl -fsSL --retry 3 -o "$tmp" "$url" || { + echo "download failed (does release tag '$WASM_SYSROOT_TAG' exist?)" >&2 + exit 1 +} +tar -xJf "$tmp" -C "$dest" +complete || { echo "fetched sysroot fails the manifest check" >&2; exit 1; } +echo "OK: $dest" diff --git a/scripts/gen_lkl_config.sh b/scripts/gen_lkl_config.sh index 34a7cb8..9f43732 100755 --- a/scripts/gen_lkl_config.sh +++ b/scripts/gen_lkl_config.sh @@ -395,12 +395,11 @@ EOF #define LKL_HOST_CONFIG_VIRTIO_NET_FD y EOF elif [[ "$NAME" == mingw* ]]; then - local ELFCLASS MSYS_ARCH LDFLAGS_EXTRA + local ELFCLASS LDFLAGS_EXTRA local BINUTILS_DIR; BINUTILS_DIR="$(binutils_dir_for "$NAME")" local NT64_CONF="" NT64_AUTOCONF="" if [[ "$NAME" == "mingw64" ]]; then ELFCLASS="ELFCLASS64" - MSYS_ARCH="mingw64" LDFLAGS_EXTRA="LDFLAGS += -Wl,--image-base,0x10000" # NT64 pulls in virtio_net_wintap.c so lkl_netdev_wintap_create # (referenced by tests/net-test.c) actually has a definition. @@ -408,7 +407,6 @@ EOF NT64_AUTOCONF="#define LKL_HOST_CONFIG_NT64 y" else ELFCLASS="ELFCLASS32" - MSYS_ARCH="mingw32" LDFLAGS_EXTRA="" fi cat > "$CONF" </cross-wasm32.meson there): emcc/em++ +# with -O3 -pthread, warnings demoted for glib's K&R-isms, and the sysroot +# include/lib dirs wired into the compiler/linker args so subprojects see +# the freshly installed zlib/libffi. -sASYNCIFY only affects the .js tool +# executables meson links as build by-products, not the static libs. + +[host_machine] +system = 'emscripten' +cpu_family = 'wasm32' +cpu = 'wasm32' +endian = 'little' + +[binaries] +c = 'emcc' +cpp = 'em++' +ar = 'emar' +ranlib = 'emranlib' +pkgconfig = ['pkg-config', '--static'] + +[built-in options] +c_args = ['-O3', '-pthread', '-Wno-incompatible-function-pointer-types', '-Wno-incompatible-pointer-types', '-Wno-implicit-function-declaration', '-I@SYSROOT@/include'] +cpp_args = ['-O3', '-pthread', '-Wno-incompatible-function-pointer-types', '-Wno-incompatible-pointer-types', '-I@SYSROOT@/include'] +objc_args = ['-O3', '-pthread', '-Wno-incompatible-function-pointer-types', '-Wno-incompatible-pointer-types', '-I@SYSROOT@/include'] +c_link_args = ['-O3', '-pthread', '-sASYNCIFY=1', '-L@SYSROOT@/lib'] +cpp_link_args = ['-O3', '-pthread', '-sASYNCIFY=1', '-L@SYSROOT@/lib'] diff --git a/scripts/lib/glib-2.88.0-emscripten-fd-query-path.patch b/scripts/lib/glib-2.88.0-emscripten-fd-query-path.patch new file mode 100644 index 0000000..c75f5c3 --- /dev/null +++ b/scripts/lib/glib-2.88.0-emscripten-fd-query-path.patch @@ -0,0 +1,18 @@ +# glib 2.88.0 emscripten port patch, applied by scripts/build_wasm_sysroot.sh. +# glib-unix.c's g_unix_fd_query_path() has per-platform implementations and +# a hard #error fallback; emscripten has no F_GETPATH / /proc/self/fd, so +# report G_FILE_ERROR_NOSYS like the HURD branch does. This is the same +# (uncommitted) patch the known-good hand-built sysroot's glib tree carried. +--- a/glib/glib-unix.c ++++ b/glib/glib-unix.c +@@ -1006,6 +1006,10 @@ g_unix_fd_query_path (int fd, + g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_NOSYS, + "g_unix_fd_query_path() not supported on HURD"); + return NULL; ++#elif defined (__EMSCRIPTEN__) ++ g_set_error (error, G_FILE_ERROR, G_FILE_ERROR_NOSYS, ++ "g_unix_fd_query_path() not supported on emscripten"); ++ return NULL; + #else + #error "g_unix_fd_query_path() not supported on this platform" + #endif diff --git a/scripts/lib/wasm_exports.sh b/scripts/lib/wasm_exports.sh new file mode 100644 index 0000000..402788a --- /dev/null +++ b/scripts/lib/wasm_exports.sh @@ -0,0 +1,24 @@ +# shellcheck shell=bash +# scripts/lib/wasm_exports.sh — derive -sEXPORTED_FUNCTIONS for the wasm +# bundle from the TS glue source. ts/native/anyfs_ts.c is the single source +# of truth: every non-static anyfs_ts_* function defined at column 0 is +# exported (renaming a glue function can therefore never silently drop it +# from the bundle — the failure surfaces in the node smoke test instead). +# The DEF_P_TRAMP(name, ...) out-pointer trampolines token-paste their +# names (anyfs_ts__p), so macro invocations are parsed as well. +anyfs_wasm_exports() { + local glue="$1" syms s out + syms="$({ grep -hE '^[A-Za-z_][A-Za-z0-9_* ]*[ *]anyfs_ts_[A-Za-z0-9_]+\(' "$glue" \ + | grep -v '^static' \ + | grep -oE 'anyfs_ts_[A-Za-z0-9_]+' + grep -hE '^DEF_P_TRAMP\(' "$glue" \ + | sed -E 's/^DEF_P_TRAMP\([[:space:]]*([A-Za-z0-9_]+).*/anyfs_ts_\1_p/' + } | sort -u)" + if [[ -z "$syms" ]]; then + echo "wasm_exports: no anyfs_ts_* definitions found in $glue" >&2 + return 1 + fi + out="_main,_malloc,_free" + for s in $syms; do out+=",_$s"; done + printf '%s\n' "$out" +} diff --git a/scripts/lib/wasm_sysroot.manifest b/scripts/lib/wasm_sysroot.manifest new file mode 100644 index 0000000..2c5add3 --- /dev/null +++ b/scripts/lib/wasm_sysroot.manifest @@ -0,0 +1,56 @@ +# scripts/lib/wasm_sysroot.manifest — contract for the wasm sysroot +# ($ANYFS_PATHS_WASM_SYSROOT, default /wasm-sysroot; see build.config.toml +# [paths].wasm_sysroot). Non-comment lines are the static libs (.a) expected +# under /lib — the hand-built set the wasm bundle links against. +# +# Consumed by: +# - scripts/doctor.sh: presence check of every listed lib. +# - scripts/build_wasm_sysroot.sh (future rebuild recipe): acceptance diff — +# the freshly built lib set must match this manifest exactly. +# +# '# ' comment lines record the version pins of the +# known-good sysroot's pkgconfig files; the recipe script must build these +# exact versions. Snapshot taken 2026-06-10 from the known-good local sysroot. +# +# Version pins (from /lib/pkgconfig/*.pc): +# blkid 2.40.4 +# gio-2.0 2.88.0 +# gio-unix-2.0 2.88.0 +# girepository-2.0 2.88.0 +# glib-2.0 2.88.0 +# gmodule-2.0 2.88.0 +# gmodule-export-2.0 2.88.0 +# gmodule-no-export-2.0 2.88.0 +# gobject-2.0 2.88.0 +# gthread-2.0 2.88.0 +# libffi 3.5.2 +# libpcre2-16 10.46 +# libpcre2-32 10.46 +# libpcre2-8 10.46 +# libzstd 1.5.7 +# uuid 2.40.4 +# zlib 1.3.1 +# +# Expected static libs under /lib: +libblkid.a +libbz2.a +libffi.a +libgio-2.0.a +# NOTE: nothing in scripts/, meson.build or ts/native/ links libgirepository-2.0 +# today (checked 2026-06-10) — possibly prunable, but it is part of the +# known-good sysroot; pruning is the build_wasm_sysroot.sh recipe task's call. +libgirepository-2.0.a +libglib-2.0.a +libgmodule-2.0.a +libgobject-2.0.a +libgthread-2.0.a +libpcre2-16.a +libpcre2-32.a +libpcre2-8.a +# NOTE: nothing in scripts/, meson.build or ts/native/ links libpcre2-posix +# today (checked 2026-06-10) — possibly prunable, same caveat as above. +libpcre2-posix.a +libresolv.a +libuuid.a +libz.a +libzstd.a diff --git a/scripts/lint-no-hardcoded-paths.sh b/scripts/lint-no-hardcoded-paths.sh index fb75c11..ee3ac0d 100755 --- a/scripts/lint-no-hardcoded-paths.sh +++ b/scripts/lint-no-hardcoded-paths.sh @@ -3,7 +3,7 @@ set -uo pipefail root="$(cd "$(dirname "$0")/.." && pwd)" # Scripts that have been migrated to config (extend this allowlist as P1 progresses). -migrated='scripts/gen_lkl_config.sh scripts/build_lkl.sh' +migrated='scripts/gen_lkl_config.sh scripts/build_lkl.sh scripts/gen_lkl_config_wasm.sh scripts/build_lkl_wasm.sh scripts/build_boot_wasm.sh scripts/build_libblkid_wasm.sh scripts/build_libblkid_mingw.sh scripts/build_qemu.sh scripts/build_anyfs.sh scripts/build_anyfs_wasm.sh scripts/build_wasm_sysroot.sh scripts/fetch_wasm_sysroot.sh scripts/fetch_wasm_ld.sh' rc=0 for f in $migrated; do if grep -nE '\$HOME|/opt/msys2|/home/[a-z]+/' "$root/$f"; then diff --git a/scripts/lint-shellcheck.sh b/scripts/lint-shellcheck.sh new file mode 100755 index 0000000..041f811 --- /dev/null +++ b/scripts/lint-shellcheck.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Lint gate: runs shellcheck (severity >= warning) on build scripts. Extend +# the list below as scripts are cleaned; new scripts must be added here. +set -uo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +checked=' +scripts/lib/config.sh +scripts/gen_lkl_config.sh +scripts/build_lkl.sh +scripts/gen_lkl_config_wasm.sh +scripts/build_lkl_wasm.sh +scripts/build_boot_wasm.sh +scripts/build_libblkid_wasm.sh +scripts/build_libblkid_mingw.sh +scripts/build_qemu.sh +scripts/build_anyfs.sh +scripts/build_anyfs_wasm.sh +scripts/build_wasm_sysroot.sh +scripts/fetch_wasm_sysroot.sh +scripts/fetch_wasm_ld.sh +scripts/lib/wasm_exports.sh +scripts/doctor.sh +scripts/lint-no-hardcoded-paths.sh +scripts/lint-shellcheck.sh +' +# shellcheck disable=SC2046,SC2086 +# SC2046/SC2086: word-splitting on $checked and $(printf ...) is intentional — +# each whitespace-delimited path becomes a separate argument to shellcheck. +shellcheck -x -S warning $(printf "$root/%s " $checked) diff --git a/scripts/lkl-wasm-tools/wasm_fix_absolute_brackets.py b/scripts/lkl-wasm-tools/wasm_fix_absolute_brackets.py new file mode 100644 index 0000000..0a6e2d0 --- /dev/null +++ b/scripts/lkl-wasm-tools/wasm_fix_absolute_brackets.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +# Vendored from ~/lklftpd (canonical copy for anyfs-reader builds since 2026-06-10). +"""Convert WASM_SYM_ABSOLUTE data symbols in a relocatable .o to +segment-relative symbols, when the symbol's absolute value falls inside +(or on the boundary of) one of the file's data segments. + +Why: Joel's wasm-ld emits script-defined symbols (e.g. __start_ftrace_events +inside SECTIONS{}) with WASM_SYM_ABSOLUTE and value = offset in the merged +data layout. In the final emcc link, absolute symbols' value is used as-is +for the VA, but emcc re-packs/DCE-prunes input segments — so the +absolute brackets no longer point at the corresponding segment. + +Fix: detect absolute symbols whose value falls in segment N's virtual range, +rewrite them as segment-relative (segment=N, offset=value-N.start, clear +ABSOLUTE bit). The relocation now references a defined symbol bound to a +real segment, which (a) gives the symbol the correct final VA and (b) keeps +the segment alive through DCE. +""" +import sys, struct + +# WASM constants +WASM_SYMBOL_TYPE_FUNCTION = 0 +WASM_SYMBOL_TYPE_DATA = 1 +WASM_SYMBOL_TYPE_GLOBAL = 2 +WASM_SYMBOL_TYPE_SECTION = 3 +WASM_SYMBOL_TYPE_EVENT = 4 +WASM_SYMBOL_TYPE_TABLE = 5 + +WASM_SYM_BINDING_WEAK = 0x01 +WASM_SYM_BINDING_LOCAL = 0x02 +WASM_SYM_VISIBILITY_HIDDEN = 0x04 +WASM_SYM_UNDEFINED = 0x10 +WASM_SYM_EXPORTED = 0x20 +WASM_SYM_EXPLICIT_NAME = 0x40 +WASM_SYM_NO_STRIP = 0x80 +WASM_SYM_TLS = 0x100 +WASM_SYM_ABSOLUTE = 0x200 + +WASM_SEGMENT_INFO = 5 +WASM_INIT_FUNCS = 6 +WASM_COMDAT_INFO = 7 +WASM_SYMBOL_TABLE = 8 + + +def read_uleb(buf, off): + r, shift = 0, 0 + while True: + b = buf[off] + off += 1 + r |= (b & 0x7f) << shift + if (b & 0x80) == 0: + return r, off + shift += 7 + + +def read_sleb(buf, off): + r, shift = 0, 0 + while True: + b = buf[off] + off += 1 + r |= (b & 0x7f) << shift + shift += 7 + if (b & 0x80) == 0: + if b & 0x40: + r |= -(1 << shift) + return r, off + + +def write_uleb(v): + out = bytearray() + while True: + b = v & 0x7f + v >>= 7 + if v: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def read_string(buf, off): + n, off = read_uleb(buf, off) + s = buf[off:off + n] + return s.decode('utf-8'), off + n + + +def parse_sections(buf): + """Return list of (id, header_off, body_off, body_size) for each section.""" + off = 8 + secs = [] + while off < len(buf): + sid = buf[off] + hdr = off + off += 1 + sz, off = read_uleb(buf, off) + body = off + secs.append((sid, hdr, body, sz)) + off += sz + return secs + + +def parse_data_segments(buf, body, sz): + """Parse the DATA section. Return list of dicts {init_offset, size, body_off}. + For active segments, init_offset is decoded from the constant expr. + """ + off = body + count, off = read_uleb(buf, off) + segs = [] + for _ in range(count): + flag, off = read_uleb(buf, off) + if flag == 0: + # active, memory 0, offset expr + opc = buf[off] + off += 1 + assert opc == 0x41, f"expected i32.const, got 0x{opc:02x}" + init_off, off = read_sleb(buf, off) + eopc = buf[off] + off += 1 + assert eopc == 0x0b + seg_sz, off = read_uleb(buf, off) + segs.append({'init_offset': init_off, 'size': seg_sz, 'body_off': off, 'flag': flag}) + off += seg_sz + elif flag == 1: + # passive + seg_sz, off = read_uleb(buf, off) + segs.append({'init_offset': None, 'size': seg_sz, 'body_off': off, 'flag': flag}) + off += seg_sz + elif flag == 2: + # active with explicit memidx + mem, off = read_uleb(buf, off) + opc = buf[off]; off += 1 + init_off, off = read_sleb(buf, off) + eopc = buf[off]; off += 1 + seg_sz, off = read_uleb(buf, off) + segs.append({'init_offset': init_off, 'size': seg_sz, 'body_off': off, 'flag': flag}) + off += seg_sz + else: + raise ValueError(f'unknown data flag {flag}') + return segs + + +def parse_linking_section(buf, body, sz, n_data_segs): + """Parse the `linking` custom section. Return: + version: int + subsections: list of {id, header_off, body_off, body_size} + seg_names: list of segment names by index (None if not present) + sym_table: {symtab_body_off, count, symbols: [{type, flags, name, ...}], end_off} + """ + end = body + sz + off = body + name, off = read_string(buf, off) + assert name == 'linking' + version, off = read_uleb(buf, off) + + seg_names = [None] * n_data_segs + sym_table = None + subsections = [] + while off < end: + subid = buf[off] + sub_hdr = off + off += 1 + sub_sz, off = read_uleb(buf, off) + sub_body = off + subsections.append({'id': subid, 'header_off': sub_hdr, 'body_off': sub_body, 'body_size': sub_sz}) + if subid == WASM_SEGMENT_INFO: + so = sub_body + cnt, so = read_uleb(buf, so) + for i in range(cnt): + nm, so = read_string(buf, so) + _align, so = read_uleb(buf, so) + _flags, so = read_uleb(buf, so) + if i < n_data_segs: + seg_names[i] = nm + elif subid == WASM_SYMBOL_TABLE: + so = sub_body + cnt, so = read_uleb(buf, so) + symbols = [] + for _ in range(cnt): + sym = {} + sym['entry_off'] = so + sym['type'] = buf[so] + so += 1 + sym['flags'], so = read_uleb(buf, so) + if sym['type'] in (WASM_SYMBOL_TYPE_FUNCTION, WASM_SYMBOL_TYPE_GLOBAL, + WASM_SYMBOL_TYPE_EVENT, WASM_SYMBOL_TYPE_TABLE): + sym['index'], so = read_uleb(buf, so) + if (sym['flags'] & WASM_SYM_EXPLICIT_NAME) or not (sym['flags'] & WASM_SYM_UNDEFINED): + sym['name'], so = read_string(buf, so) + else: + sym['name'] = None + elif sym['type'] == WASM_SYMBOL_TYPE_DATA: + sym['name'], so = read_string(buf, so) + if not (sym['flags'] & WASM_SYM_UNDEFINED): + sym['segment'], so = read_uleb(buf, so) + sym['offset'], so = read_uleb(buf, so) + sym['size'], so = read_uleb(buf, so) + else: + sym['segment'] = None + elif sym['type'] == WASM_SYMBOL_TYPE_SECTION: + sym['section'], so = read_uleb(buf, so) + sym['name'] = None + else: + raise ValueError(f'unknown symbol type {sym["type"]}') + sym['entry_end'] = so + symbols.append(sym) + sym_table = { + 'sub_body': sub_body, + 'sub_size': sub_sz, + 'count': cnt, + 'symbols': symbols, + 'end_off': so, + } + off += sub_sz + return version, subsections, seg_names, sym_table + + +def encode_sym(sym): + """Serialize a symbol entry back to bytes.""" + out = bytearray() + out.append(sym['type']) + out += write_uleb(sym['flags']) + if sym['type'] in (WASM_SYMBOL_TYPE_FUNCTION, WASM_SYMBOL_TYPE_GLOBAL, + WASM_SYMBOL_TYPE_EVENT, WASM_SYMBOL_TYPE_TABLE): + out += write_uleb(sym['index']) + if sym['name'] is not None: + n = sym['name'].encode('utf-8') + out += write_uleb(len(n)) + out += n + elif sym['type'] == WASM_SYMBOL_TYPE_DATA: + n = sym['name'].encode('utf-8') + out += write_uleb(len(n)) + out += n + if not (sym['flags'] & WASM_SYM_UNDEFINED): + out += write_uleb(sym['segment']) + out += write_uleb(sym['offset']) + out += write_uleb(sym['size']) + elif sym['type'] == WASM_SYMBOL_TYPE_SECTION: + out += write_uleb(sym['section']) + return bytes(out) + + +def find_candidates(segs, value): + """Return all (seg_idx, offset_within_seg) bindings that would produce this + value in the original layout. A value can match both "end of prev seg" and + "start of next seg" if they're adjacent — both are returned so the caller + can disambiguate based on pairing. + """ + out = [] + for i, s in enumerate(segs): + if s['init_offset'] is None: + continue + if s['init_offset'] <= value <= s['init_offset'] + s['size']: + out.append((i, value - s['init_offset'])) + return out + + +# Map __start_X / __stop_X style pairs by stripping the directional prefix. +def pair_key(name): + """Return (key, role) where role is 'start' or 'stop' or None. + Used to pair symbols that bracket the same range. + """ + pairs = [ + ('__start_', '__stop_'), + ('__start', '__stop'), # __start___param / __stop___param + ('__begin_', '__end_'), + ] + for s, e in pairs: + if name.startswith(s): + return name[len(s):], 'start' + if name.startswith(e): + return name[len(e):], 'stop' + # __initcallN_start / __initcall_end style is harder; skip pairing + # __setup_start / __setup_end + if name.endswith('_start'): + return name[:-len('_start')], 'start' + if name.endswith('_end'): + return name[:-len('_end')], 'stop' + if name.endswith('_begin'): + return name[:-len('_begin')], 'start' + return None, None + + +def find_segment_for_value(segs, value): + """Single-symbol fallback: prefer start-of-segment over end-of-previous.""" + # Prefer exact start-of-segment match (or strictly inside) + for i, s in enumerate(segs): + if s['init_offset'] is None: + continue + if s['init_offset'] <= value < s['init_offset'] + s['size']: + return i, value - s['init_offset'] + # Fallback: exact end of last segment with no next + for i, s in enumerate(segs): + if s['init_offset'] is None: + continue + if value == s['init_offset'] + s['size']: + return i, s['size'] + return None, None + + +def main(): + if len(sys.argv) < 2: + print("usage: wasm_fix_absolute_brackets.py [out.o]", file=sys.stderr) + sys.exit(2) + in_path = sys.argv[1] + out_path = sys.argv[2] if len(sys.argv) > 2 else in_path + + with open(in_path, 'rb') as f: + buf = bytearray(f.read()) + + secs = parse_sections(buf) + data_sec = None + linking_sec = None + for sid, hdr, body, sz in secs: + if sid == 11: # DATA + data_sec = (hdr, body, sz) + elif sid == 0: # custom + # need to peek name + n, off = read_uleb(buf, body) + name = bytes(buf[off:off + n]).decode('utf-8') + if name == 'linking': + linking_sec = (hdr, body, sz) + + assert data_sec is not None, "no DATA section" + assert linking_sec is not None, "no linking section" + + _, dbody, dsz = data_sec + segs = parse_data_segments(buf, dbody, dsz) + print(f"data segments: {len(segs)}") + + _, lbody, lsz = linking_sec + version, subsections, seg_names, sym_table = parse_linking_section(buf, lbody, lsz, len(segs)) + print(f"linking version {version}, {len(subsections)} subsections, {sym_table['count']} symbols") + + # print first few segs with their virt range and name + for i, s in enumerate(segs[:5]): + print(f" seg[{i}] '{seg_names[i]}' init_off=0x{s['init_offset']:x} size=0x{s['size']:x}") + + # Collect all absolute DATA symbols + abs_syms = [] + for sym in sym_table['symbols']: + if sym['type'] != WASM_SYMBOL_TYPE_DATA: + continue + if sym['flags'] & WASM_SYM_UNDEFINED: + continue + if not (sym['flags'] & WASM_SYM_ABSOLUTE): + continue + abs_syms.append(sym) + + # Bucket by pair key + by_pair = {} # key -> {'start': sym, 'stop': sym} + unpaired = [] + for sym in abs_syms: + key, role = pair_key(sym['name']) + if key is None or role is None: + unpaired.append(sym) + continue + by_pair.setdefault(key, {})[role] = sym + + # Some "pairs" may only have one half; treat the orphan as unpaired + for key, d in list(by_pair.items()): + if 'start' not in d or 'stop' not in d: + for sym in d.values(): + unpaired.append(sym) + del by_pair[key] + + conversions = [] + skipped = [] + paired_resolved = 0 + + # Resolve paired symbols + for key, d in by_pair.items(): + s_sym, e_sym = d['start'], d['stop'] + s_val, e_val = s_sym['offset'], e_sym['offset'] + s_cands = find_candidates(segs, s_val) + e_cands = find_candidates(segs, e_val) + if not s_cands or not e_cands: + skipped.append((s_sym, s_val, 'no seg')) + skipped.append((e_sym, e_val, 'no seg')) + continue + # Prefer pairing where start and stop are in the same segment AND + # offset_stop - offset_start == (e_val - s_val). + size = e_val - s_val + chosen = None + for sc in s_cands: + for ec in e_cands: + if sc[0] == ec[0] and ec[1] - sc[1] == size: + chosen = (sc, ec) + break + if chosen: + break + if chosen is None: + # Multi-segment span: bind start to seg @ 0 of "next" seg + # (start-of-segment candidate) and stop to "end of prev" (size). + sc = next((c for c in s_cands if c[1] == 0), s_cands[0]) + ec = next((c for c in e_cands if c[1] == segs[c[0]]['size']), e_cands[-1]) + chosen = (sc, ec) + conversions.append((s_sym, chosen[0][0], chosen[0][1])) + conversions.append((e_sym, chosen[1][0], chosen[1][1])) + paired_resolved += 1 + + # Resolve unpaired with simple rule + for sym in unpaired: + val = sym['offset'] + seg_idx, new_off = find_segment_for_value(segs, val) + if seg_idx is None: + skipped.append((sym, val, 'no seg')) + continue + conversions.append((sym, seg_idx, new_off)) + + print(f"\nResolved {paired_resolved} bracket pairs, {len(conversions) - 2 * paired_resolved} unpaired absolutes") + for sym, val, why in skipped: + print(f" SKIP {sym['name']:40s} = 0x{val:x} ({why})") + + print(f"\nConverting {len(conversions)} absolute brackets to segment-relative:") + for sym, seg_idx, new_off in conversions: + print(f" {sym['name']:40s} 0x{sym['offset']:x} -> seg#{seg_idx} ({seg_names[seg_idx]}) @ 0x{new_off:x}") + + if not conversions: + print("nothing to do") + return + + # Apply conversions in-memory + for sym, seg_idx, new_off in conversions: + sym['flags'] &= ~WASM_SYM_ABSOLUTE + sym['segment'] = seg_idx + sym['offset'] = new_off + # size: keep as-is + + # Rebuild the symbol table subsection + new_st = bytearray() + new_st += write_uleb(sym_table['count']) + for sym in sym_table['symbols']: + new_st += encode_sym(sym) + + # Find the WASM_SYMBOL_TABLE subsection in linking + st_sub = None + for ss in subsections: + if ss['id'] == WASM_SYMBOL_TABLE: + st_sub = ss + break + assert st_sub is not None + + old_st_size = st_sub['body_size'] + new_st_size = len(new_st) + print(f"\nsymbol table: {old_st_size} -> {new_st_size} bytes") + + # Reconstruct the linking section body: + # linking section body = name_string + version + subsections + # We need to rebuild: copy [body, st_sub_body_start), then new size+payload, + # then [st_sub_body_end, linking_end). + # st_sub_body_start = st_sub['body_off'] + # st_sub_size_field_start = st_sub['header_off'] + 1 (one byte sub id) + # Replace size uleb + payload. + + st_id_off = st_sub['header_off'] + st_size_field_off = st_id_off + 1 # right after id byte + st_body_end = st_sub['body_off'] + st_sub['body_size'] + + new_size_uleb = write_uleb(new_st_size) + # Build new linking section *body* by replacing the subsection + # The original linking section body spans [lbody, lbody+lsz) + new_linking_body = bytearray() + # part before subsection's size field (i.e., up to and including id byte) + new_linking_body += buf[lbody:st_size_field_off] + new_linking_body += new_size_uleb + new_linking_body += new_st + # part after subsection body + new_linking_body += buf[st_body_end:lbody + lsz] + + new_linking_size = len(new_linking_body) + new_linking_size_uleb = write_uleb(new_linking_size) + + # The linking section header was at linking_sec[0] (hdr offset) + # Format: id byte (0) + uleb size + body + lhdr, _, _ = linking_sec + old_size_uleb_len = lbody - (lhdr + 1) + + # Build final buffer: + # [0, lhdr+1): everything up to and including section id + # new linking size uleb + # new linking body + # [lbody+lsz, end): everything after linking section + + new_buf = bytearray() + new_buf += buf[:lhdr + 1] + new_buf += new_linking_size_uleb + new_buf += new_linking_body + new_buf += buf[lbody + lsz:] + + with open(out_path, 'wb') as f: + f.write(new_buf) + print(f"\nwrote {out_path} ({len(buf)} -> {len(new_buf)} bytes)") + + +if __name__ == '__main__': + main() diff --git a/scripts/lkl-wasm-tools/wasm_prefix_kernel_symbols.py b/scripts/lkl-wasm-tools/wasm_prefix_kernel_symbols.py new file mode 100644 index 0000000..ca1f005 --- /dev/null +++ b/scripts/lkl-wasm-tools/wasm_prefix_kernel_symbols.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +# Vendored from ~/lklftpd (canonical copy for anyfs-reader builds since 2026-06-10). +"""Add a prefix to non-public kernel symbols in a relocatable wasm .o, +so they stop colliding with libc at final-link time. + +Why: LKL's `lkl.o` is the partial-link of every kernel TU. wasm-ld leaves all +non-static globals visible at link time — including the kernel's own +`vsnprintf` / `memcpy` / `strlen` / etc. When the final emcc link pulls in +`liblkl.a` with --whole-archive, those kernel symbols outrank musl libc's weak +copies, and any user-space caller of vsnprintf (e.g. tools/lkl/lib/utils.c's +`lkl_printf`) ends up running the kernel implementation. That implementation +treats wasm pointers < PAGE_SIZE as bad addresses and prints "(efault)". + +The ELF/PE flow sidesteps this with `objcopy --prefix-symbols=_` plus +`-G_lkl_*` to namespace everything except the public ABI. llvm-objcopy +rejects those flags on wasm objects (despite advertising them in --help), so +we do the same job by hand: read the linking-section symbol table, prepend a +prefix to every defined non-LOCAL function/data/global whose name isn't part +of the public LKL ABI, and write the file back. + +We only touch the symbol-table *name* field. Code-section calls and DATA +relocations refer to symbols by INDEX, so renames don't require relocation +edits. COMDAT groups also reference symbols by index. The custom `name` +section (debug info) is left alone — function/data names there are indexed +by function/data index, not linkage symbol name, so backtraces still resolve +the right code; only the *displayed* name lags the rename, which is fine. + +Usage: + wasm_prefix_kernel_symbols.py [out.o] [--prefix=_] +""" +import sys, struct + +WASM_SYMBOL_TYPE_FUNCTION = 0 +WASM_SYMBOL_TYPE_DATA = 1 +WASM_SYMBOL_TYPE_GLOBAL = 2 +WASM_SYMBOL_TYPE_SECTION = 3 +WASM_SYMBOL_TYPE_EVENT = 4 +WASM_SYMBOL_TYPE_TABLE = 5 + +WASM_SYM_BINDING_WEAK = 0x01 +WASM_SYM_BINDING_LOCAL = 0x02 +WASM_SYM_VISIBILITY_HIDDEN = 0x04 +WASM_SYM_UNDEFINED = 0x10 +WASM_SYM_EXPORTED = 0x20 +WASM_SYM_EXPLICIT_NAME = 0x40 +WASM_SYM_NO_STRIP = 0x80 +WASM_SYM_TLS = 0x100 +WASM_SYM_ABSOLUTE = 0x200 + +WASM_SEGMENT_INFO = 5 +WASM_INIT_FUNCS = 6 +WASM_COMDAT_INFO = 7 +WASM_SYMBOL_TABLE = 8 + + +def read_uleb(buf, off): + r, shift = 0, 0 + while True: + b = buf[off] + off += 1 + r |= (b & 0x7f) << shift + if (b & 0x80) == 0: + return r, off + shift += 7 + + +def write_uleb(v): + out = bytearray() + while True: + b = v & 0x7f + v >>= 7 + if v: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def read_string(buf, off): + n, off = read_uleb(buf, off) + s = bytes(buf[off:off + n]).decode('utf-8') + return s, off + n + + +def parse_sections(buf): + off = 8 + secs = [] + while off < len(buf): + sid = buf[off] + hdr = off + off += 1 + sz, off = read_uleb(buf, off) + body = off + secs.append((sid, hdr, body, sz)) + off += sz + return secs + + +def parse_linking_section(buf, body, sz): + end = body + sz + off = body + name, off = read_string(buf, off) + assert name == 'linking', f"expected linking, got {name!r}" + version, off = read_uleb(buf, off) + + sym_table = None + while off < end: + subid = buf[off] + sub_hdr = off + off += 1 + sub_sz, off = read_uleb(buf, off) + sub_body = off + if subid == WASM_SYMBOL_TABLE: + so = sub_body + cnt, so = read_uleb(buf, so) + symbols = [] + for _ in range(cnt): + sym = {'entry_off': so} + sym['type'] = buf[so]; so += 1 + sym['flags'], so = read_uleb(buf, so) + t = sym['type'] + if t in (WASM_SYMBOL_TYPE_FUNCTION, WASM_SYMBOL_TYPE_GLOBAL, + WASM_SYMBOL_TYPE_EVENT, WASM_SYMBOL_TYPE_TABLE): + sym['index'], so = read_uleb(buf, so) + if (sym['flags'] & WASM_SYM_EXPLICIT_NAME) or \ + not (sym['flags'] & WASM_SYM_UNDEFINED): + sym['name'], so = read_string(buf, so) + else: + sym['name'] = None + elif t == WASM_SYMBOL_TYPE_DATA: + sym['name'], so = read_string(buf, so) + if not (sym['flags'] & WASM_SYM_UNDEFINED): + sym['segment'], so = read_uleb(buf, so) + sym['offset'], so = read_uleb(buf, so) + sym['size'], so = read_uleb(buf, so) + else: + sym['segment'] = None + elif t == WASM_SYMBOL_TYPE_SECTION: + sym['section'], so = read_uleb(buf, so) + sym['name'] = None + else: + raise ValueError(f'unknown symbol type {t}') + sym['entry_end'] = so + symbols.append(sym) + sym_table = { + 'sub_header_off': sub_hdr, + 'sub_body_off': sub_body, + 'sub_body_size': sub_sz, + 'count': cnt, + 'symbols': symbols, + } + off += sub_sz + return version, sym_table + + +def encode_sym(sym): + out = bytearray() + out.append(sym['type']) + out += write_uleb(sym['flags']) + t = sym['type'] + if t in (WASM_SYMBOL_TYPE_FUNCTION, WASM_SYMBOL_TYPE_GLOBAL, + WASM_SYMBOL_TYPE_EVENT, WASM_SYMBOL_TYPE_TABLE): + out += write_uleb(sym['index']) + if sym['name'] is not None: + n = sym['name'].encode('utf-8') + out += write_uleb(len(n)) + out += n + elif t == WASM_SYMBOL_TYPE_DATA: + n = sym['name'].encode('utf-8') + out += write_uleb(len(n)) + out += n + if not (sym['flags'] & WASM_SYM_UNDEFINED): + out += write_uleb(sym['segment']) + out += write_uleb(sym['offset']) + out += write_uleb(sym['size']) + elif t == WASM_SYMBOL_TYPE_SECTION: + out += write_uleb(sym['section']) + return bytes(out) + + +# Names that DO clash with musl libc and whose kernel-internal copy must be +# pushed out of the way at final-link time. Anything not in this set is left +# alone — touching the rest of the kernel's symbol topology breaks tracing / +# initcall iteration / module init in subtle ways even though the wasm +# linking format claims index-based references should survive a rename. +LIBC_CONFLICTS = { + # printf family — the actual reason this tool exists. The kernel's + # vsnprintf treats wasm pointers below PAGE_SIZE as "(efault)". + 'vsnprintf', 'snprintf', 'vsprintf', 'sprintf', + # mem* + 'memcpy', 'memset', 'memcmp', 'memmove', 'memchr', + # str* + 'strlen', 'strnlen', 'strcmp', 'strncmp', + 'strcpy', 'strncpy', 'strcat', 'strncat', + 'strchr', 'strrchr', 'strstr', 'strsep', + 'strspn', 'strcspn', 'strpbrk', + # misc + 'abort', 'bsearch', + # lib/zstd/common/error_private.c — zstd's only ERR_-prefixed export + # in the kernel. Listed by exact name (rather than as ERR_* prefix) + # so we don't accidentally rename unrelated kernel helpers that + # happen to start with ERR_ in the future. + 'ERR_getErrorString', + # QEMU libqemuutil also defines these — kernel internals must yield + # so QEMU's userspace RCU / buffer / crc32c run their own versions + # when QEMU code calls them, while the kernel keeps its own + # implementations under the prefixed name for internal callers. + 'synchronize_rcu', 'buffer_init', 'crc32c', + # Sysroot libz (pulled in by QEMU/glib via -lz) defines this with the + # exact same name as lib/zlib_inflate/inffast.c. Push the kernel copy + # under the prefix so its internal zlib_inflate() still calls it, and + # libz's userspace inflate_fast resolves cleanly for libz callers. + # Only the kernel zlib that SQUASHFS_ZLIB / ZISOFS / BTRFS use needs + # this symbol — they reach it through zlib_inflate() inside the + # kernel, all of which gets renamed in lock-step. + 'inflate_fast', +} + +# Like LIBC_CONFLICTS but matched as a string prefix. Used for symbol families +# that have too many members to enumerate individually. Each kernel-internal +# call to a matching name is resolved by INDEX inside lkl.o (the prefix tool +# only rewrites the name field), so the kernel's bundled implementation keeps +# working under the renamed export — and userspace callers resolve cleanly +# against the equivalently-named symbols in their own library. +LIBC_CONFLICT_PREFIXES = ( + # lib/zstd/ inside the kernel (used by SQUASHFS_ZSTD / EROFS_FS_ZIP_ZSTD / + # ZSTD crypto) duplicates every ZSTD_* export of userspace libzstd.a. + # QEMU's block/qcow2-threads.c references the userspace copy for qcow2 + # compression_type=zstd; without prefixing, wasm-ld fails on duplicate + # symbol errors. ZSTD_ itself is ~271 funcs; HUF_/FSE_/HIST_ are + # entropy-coding helpers that zstd bundles and also exports. + 'ZSTD_', 'HUF_', 'FSE_', 'HIST_', +) + + +def should_prefix(sym, prefix): + # Only rename defined function/data/global symbols. + if sym['type'] not in (WASM_SYMBOL_TYPE_FUNCTION, + WASM_SYMBOL_TYPE_DATA, + WASM_SYMBOL_TYPE_GLOBAL, + WASM_SYMBOL_TYPE_EVENT, + WASM_SYMBOL_TYPE_TABLE): + return False + if sym['flags'] & WASM_SYM_UNDEFINED: + return False + name = sym.get('name') + if not name: + return False + if name in LIBC_CONFLICTS: + return True + for p in LIBC_CONFLICT_PREFIXES: + if name.startswith(p): + return True + return False + + +def main(): + args = sys.argv[1:] + prefix = '_' + rest = [] + for a in args: + if a.startswith('--prefix='): + prefix = a[len('--prefix='):] + else: + rest.append(a) + if len(rest) < 1: + print("usage: wasm_prefix_kernel_symbols.py [out.o] [--prefix=_]", + file=sys.stderr) + sys.exit(2) + in_path = rest[0] + out_path = rest[1] if len(rest) > 1 else in_path + + with open(in_path, 'rb') as f: + buf = bytearray(f.read()) + + secs = parse_sections(buf) + linking_sec = None + for sid, hdr, body, sz in secs: + if sid == 0: + n, off = read_uleb(buf, body) + name = bytes(buf[off:off + n]).decode('utf-8') + if name == 'linking': + linking_sec = (hdr, body, sz) + break + if linking_sec is None: + print("error: no linking section", file=sys.stderr) + sys.exit(1) + + lhdr, lbody, lsz = linking_sec + version, sym_table = parse_linking_section(buf, lbody, lsz) + if sym_table is None: + print("error: no symbol table in linking section", file=sys.stderr) + sys.exit(1) + + print(f"linking version {version}, {sym_table['count']} symbols") + + renamed = 0 + kept_lkl = 0 + sample_renamed = [] + sample_kept = [] + for sym in sym_table['symbols']: + if should_prefix(sym, prefix): + old = sym['name'] + sym['name'] = prefix + old + renamed += 1 + if len(sample_renamed) < 8: + sample_renamed.append(old) + elif (sym.get('name') or '').startswith('lkl_'): + kept_lkl += 1 + if len(sample_kept) < 8: + sample_kept.append(sym['name']) + + print(f"renamed {renamed} symbols with prefix '{prefix}'") + print(f" sample: {sample_renamed}") + print(f"kept {kept_lkl} public lkl_* symbols") + print(f" sample: {sample_kept}") + + if renamed == 0: + print("nothing to do") + return + + # Rebuild WASM_SYMBOL_TABLE subsection payload + new_st = bytearray() + new_st += write_uleb(sym_table['count']) + for sym in sym_table['symbols']: + new_st += encode_sym(sym) + + st_id_off = sym_table['sub_header_off'] + st_size_field_off = st_id_off + 1 + st_body_off = sym_table['sub_body_off'] + st_body_end = st_body_off + sym_table['sub_body_size'] + new_st_size_uleb = write_uleb(len(new_st)) + + # Rebuild linking section body + new_linking_body = bytearray() + new_linking_body += buf[lbody:st_size_field_off] + new_linking_body += new_st_size_uleb + new_linking_body += new_st + new_linking_body += buf[st_body_end:lbody + lsz] + new_linking_size_uleb = write_uleb(len(new_linking_body)) + + # Rebuild file: [0, lhdr+1) | size | body | [end of old linking, end) + new_buf = bytearray() + new_buf += buf[:lhdr + 1] + new_buf += new_linking_size_uleb + new_buf += new_linking_body + new_buf += buf[lbody + lsz:] + + with open(out_path, 'wb') as f: + f.write(new_buf) + print(f"wrote {out_path} ({len(buf)} -> {len(new_buf)} bytes)") + + +if __name__ == '__main__': + main() diff --git a/scripts/poc-nbd/.gitignore b/scripts/poc-nbd/.gitignore new file mode 100644 index 0000000..67fc140 --- /dev/null +++ b/scripts/poc-nbd/.gitignore @@ -0,0 +1 @@ +fixtures/ diff --git a/scripts/poc-nbd/FINDINGS.md b/scripts/poc-nbd/FINDINGS.md new file mode 100644 index 0000000..346ba8a --- /dev/null +++ b/scripts/poc-nbd/FINDINGS.md @@ -0,0 +1,80 @@ +# NBD-over-fd PoC — Findings + +## Stage 1 (Linux, inherited-fd) — PASS +lspart opened the qcow2 over an inherited socketpair fd (`--nbd-fd 3`), exit 0, +27 reads traversed the socketpair, table parity with the plain-file open. +Required adding `module_call_init(MODULE_INIT_QOM)` to src/core/qemu_backend.c +(needed for the qio-channel-socket QOM type the NBD client adopts) — anyfs glue +only, zero QEMU source changes. + +## Stage 2 (Linux, full chain) — PASS +lspart-over-NBD matched plain-file lspart (both 0 data rows; the synthetic +8 MiB image has no partition table). 27 inherited-fd reads. The marker +"ANYFS-NBD-POC-MARKER-v1" at offset 0x100000 read back byte-for-byte THROUGH +the qcow2-over-NBD chain via qemu-io (qcow2 driver → nbd: protocol → server). + +## Protocol cross-check (Task 6) — PASS +qemu-img detected qcow2 over the hand-written NBD server (over a unix socket), +independently confirming the NBD newstyle protocol implementation is correct. +(This caught a wrong NBD_REP_MAGIC constant, since fixed.) + +## Stage 3 (Windows/wine, loopback fallback) — PASS (2026-06-01) + +Approach: Windows has no reliable socketpair and QEMU requires the NBD fd to be +a socket, so the Windows path uses `--nbd-port P` (127.0.0.1 loopback, ephemeral +port, no outward listen) instead of an inherited fd. + +**Result with a freshly-rebuilt QEMU-enabled lspart.exe:** +``` +[qemu_blk] open(nbd-port:43113) ro=1 snap=0 +[qemu_blk] bdrv_init… +[qemu_blk] main loop ready +[qemu_blk] blk_new_open name=(nbd via options) flags=0x0 +[qemu_blk] blk_new_open returned 0x... +[qemu_blk] capacity=8716288 +--- connections=1 nbd_reads=49 exit=0 --- +STAGE3 PASS (wine loopback): qcow2 opened over NBD, data traversed 127.0.0.1 +``` +Under wine-10.0, the mingw64 QEMU NBD client connected to the host Node NBD +server on `127.0.0.1`, completed the newstyle handshake, opened the qcow2 over +NBD (capacity detected), and 49 reads traversed the connection. The Windows +loopback fallback transport is validated. + +### What it took to get there (instructive) + +The first attempts FAILED, and tracing why surfaced three real gotchas: + +1. **DLL search path** — without `WINEPATH`, the dependent DLLs + (libwinpthread-1.dll, libanyfs-qemublk.dll, liblkl.dll) live in + `build-anyfs-mingw64/bin/` but Windows loads them relative to the module, not + from a separate bin/ staging dir. Fix: run with + `WINEPATH=Z:\...\build-anyfs-mingw64\bin` (or stage DLLs next to the .exe). + Matches the known win64 DLL-search pattern. + +2. **Stale binary** — the original 2026-05-29 lspart.exe predated the + `--nbd-fd`/`--nbd-port` flags and the `MODULE_INIT_QOM` fix. Rebuilt the + mingw64 lspart. + +3. **QEMU backend not compiled in (the real blocker)** — the rebuilt lspart STILL + failed (`failed to open nbd-port:N`, zero connections) because the + `build-anyfs-mingw64` meson dir was configured with **`enable_qemu=False`**, so + `qemu_backend.c` (and its nbd branch) was never compiled into the exe — it used + the raw backend, treating `nbd-port:N` as a filename. The + `scripts/build_anyfs.sh` script only emits `-Denable_qemu=true` when the build + includes the `core` component; a `--components=server`-only build silently + re-disables QEMU. Fix: rebuild with `--components=core,server` so QEMU is + enabled and `qemu_backend.c` is compiled into `libanyfs_core.a`. + +### Build recipe to reproduce +``` +# 1. ensure the mingw QEMU libs + libanyfs-qemublk.dll are built: +bash scripts/build_qemu.sh --targets=mingw64 +# 2. build lspart WITH the qemu backend (must include 'core'): +bash scripts/build_anyfs.sh --targets=mingw64 --components=core,server +# 3. run under wine with the DLL dir on WINEPATH: +WINEPATH=$(winepath -w build-anyfs-mingw64/bin) \ + wine build-anyfs-mingw64/src/lspart/anyfs-lspart.exe --nbd-port

+``` + +Both transport variants are now validated: Linux inherited-fd (Stages 1/2) and +Windows 127.0.0.1 loopback (Stage 3). diff --git a/scripts/poc-nbd/launch.mjs b/scripts/poc-nbd/launch.mjs new file mode 100644 index 0000000..72b61e5 --- /dev/null +++ b/scripts/poc-nbd/launch.mjs @@ -0,0 +1,70 @@ +/* + * Parent/launcher for the NBD-over-fd PoC. + * + * Two entry points: + * serveOnFd(imagePath): create a socketpair, run the NBD server on fd0, + * return fd1 (the inheritable end) + a stop() handle and read counter. + * serveOnUnixSocket(imagePath, sockPath): listen on a unix socket and + * serve clients — used to cross-validate against qemu-img / qemu-io. + */ +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { createRequire } from 'node:module'; +import { startNbdServer } from './nbd-server.mjs'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); +const addon = require( + path.join(here, 'native-transport/build/Release/native_transport.node'), +); + +/* The NBD export size is the RAW byte length of the file on disk. QEMU's + * NBD client reads those raw bytes and probes the qcow2 header itself, so + * the server must serve the file verbatim (not the qcow2 virtual size). */ +function imageSize(imagePath) { + return fs.statSync(imagePath).size; +} + +export function serveOnFd(imagePath) { + const [fd0, fd1] = addon.socketpair(); + const imageFd = fs.openSync(imagePath, 'r'); + const size = imageSize(imagePath); + const sock = new net.Socket({ fd: fd0 }); + let reads = 0; + const done = startNbdServer(sock, imageFd, size, () => { + reads++; + }).catch((e) => { + if (!/EOF/.test(String(e))) console.error('[nbd-server]', e); + }); + return { + fd1, + stop: () => { + try { + sock.destroy(); + } catch {} + try { + fs.closeSync(imageFd); + } catch {} + }, + readCount: () => reads, + done, + }; +} + +export function serveOnUnixSocket(imagePath, sockPath) { + const imageFd = fs.openSync(imagePath, 'r'); + const size = imageSize(imagePath); + try { + fs.unlinkSync(sockPath); + } catch {} + const server = net.createServer((sock) => { + startNbdServer(sock, imageFd, size).catch((e) => { + if (!/EOF/.test(String(e))) console.error('[nbd-server]', e); + }); + }); + return new Promise((resolve) => { + server.listen(sockPath, () => resolve({ server, imageFd })); + }); +} diff --git a/scripts/poc-nbd/make-test-image.mjs b/scripts/poc-nbd/make-test-image.mjs new file mode 100644 index 0000000..701b570 --- /dev/null +++ b/scripts/poc-nbd/make-test-image.mjs @@ -0,0 +1,46 @@ +/* + * Build a deterministic qcow2 test image for the NBD-over-fd PoC. + * + * Strategy: synthesize a raw image filled with a deterministic byte + * pattern (no filesystem tooling needed), then convert it to qcow2 with + * qemu-img. We embed a recognizable marker at a known offset and record + * it as the verification anchor. Stage 2 reads that marker back through + * the qcow2-over-NBD chain to prove byte-accurate delivery. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const outDir = path.join(here, 'fixtures'); +const rawImg = path.join(outDir, 'test.raw'); +const qcow = path.join(outDir, 'test.qcow2'); + +const SIZE = 8 * 1024 * 1024; /* 8 MiB — small, fast to convert */ +const VERIFY_OFFSET = 0x100000; /* 1 MiB in: well past the qcow2 header region */ +const MARKER = Buffer.from('ANYFS-NBD-POC-MARKER-v1', 'ascii'); + +fs.mkdirSync(outDir, { recursive: true }); + +/* Deterministic fill: byte i = i & 0xff, so every offset has a predictable + * value. Then stamp the ASCII marker at VERIFY_OFFSET. */ +const raw = Buffer.alloc(SIZE); +for (let i = 0; i < SIZE; i++) raw[i] = i & 0xff; +MARKER.copy(raw, VERIFY_OFFSET); +fs.writeFileSync(rawImg, raw); + +execFileSync('qemu-img', ['convert', '-f', 'raw', '-O', 'qcow2', rawImg, qcow], { + stdio: 'inherit', +}); + +const meta = { + qcow2: qcow, + raw: rawImg, + size: SIZE, + verifyOffset: VERIFY_OFFSET, + verifyBytesHex: MARKER.toString('hex'), + verifyAscii: MARKER.toString('ascii'), +}; +fs.writeFileSync(path.join(outDir, 'meta.json'), JSON.stringify(meta, null, 2)); +console.log(JSON.stringify(meta, null, 2)); diff --git a/scripts/poc-nbd/native-transport/.gitignore b/scripts/poc-nbd/native-transport/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/scripts/poc-nbd/native-transport/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/scripts/poc-nbd/native-transport/binding.gyp b/scripts/poc-nbd/native-transport/binding.gyp new file mode 100644 index 0000000..a5a1fe4 --- /dev/null +++ b/scripts/poc-nbd/native-transport/binding.gyp @@ -0,0 +1,9 @@ +{ + "targets": [ + { + "target_name": "native_transport", + "sources": ["transport.c"], + "cflags_c": ["-D_GNU_SOURCE"] + } + ] +} diff --git a/scripts/poc-nbd/native-transport/package.json b/scripts/poc-nbd/native-transport/package.json new file mode 100644 index 0000000..9253fa1 --- /dev/null +++ b/scripts/poc-nbd/native-transport/package.json @@ -0,0 +1,10 @@ +{ + "name": "native-transport", + "version": "0.0.0", + "private": true, + "description": "PoC N-API addon: socketpair() for NBD-over-fd transport", + "main": "index.js", + "scripts": { + "build": "node-gyp rebuild" + } +} diff --git a/scripts/poc-nbd/native-transport/transport.c b/scripts/poc-nbd/native-transport/transport.c new file mode 100644 index 0000000..9c25ad7 --- /dev/null +++ b/scripts/poc-nbd/native-transport/transport.c @@ -0,0 +1,51 @@ +/* + * native-transport — minimal N-API addon for the NBD-over-fd PoC. + * + * Exports socketpair() -> [fd0, fd1]: a connected AF_UNIX/SOCK_STREAM + * pair. The parent keeps one fd for the NBD server and lets the child + * inherit the other (non-CLOEXEC) to use as QEMU's NBD transport. + * + * This is the seed of a unified Node-layer native transport abstraction; + * a Windows loopback-pair helper will be added here later. + */ +#define NAPI_VERSION 8 +#include +#include +#include +#include + +static napi_value Socketpair(napi_env env, napi_callback_info info) +{ + int fds[2]; + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0) { + napi_throw_error(env, NULL, "socketpair() failed"); + return NULL; + } + /* Clear CLOEXEC on the child end (fds[1]) so it survives exec into the + * spawned QEMU/lspart child. The parent end (fds[0]) keeps its default + * and is never inherited — only the child end is passed via stdio. */ + { + int fl = fcntl(fds[1], F_GETFD); + if (fl >= 0) + fcntl(fds[1], F_SETFD, fl & ~FD_CLOEXEC); + } + + napi_value arr, a, b; + napi_create_array_with_length(env, 2, &arr); + napi_create_int32(env, fds[0], &a); + napi_create_int32(env, fds[1], &b); + napi_set_element(env, arr, 0, a); + napi_set_element(env, arr, 1, b); + return arr; +} + +static napi_value Init(napi_env env, napi_value exports) +{ + napi_value fn; + napi_create_function(env, "socketpair", NAPI_AUTO_LENGTH, Socketpair, + NULL, &fn); + napi_set_named_property(env, exports, "socketpair", fn); + return exports; +} + +NAPI_MODULE(NODE_GYP_MODULE_NAME, Init) diff --git a/scripts/poc-nbd/nbd-server.mjs b/scripts/poc-nbd/nbd-server.mjs new file mode 100644 index 0000000..26199b6 --- /dev/null +++ b/scripts/poc-nbd/nbd-server.mjs @@ -0,0 +1,188 @@ +/* + * Hand-written read-only NBD newstyle server for the NBD-over-fd PoC. + * + * PoC scope: synchronous fs.readSync, one request at a time. The + * production server (spec §9) is fully async with multiple in-flight + * requests and a keep-alive http upstream — out of scope here. + * + * startNbdServer(socket, imageFd, size, onRead?) drives one client on an + * already-connected socket. Returns a promise that resolves on clean + * NBD_CMD_DISC / EOF. + */ +import { Buffer } from 'node:buffer'; +import fs from 'node:fs'; + +const NBDMAGIC = 0x4e42444d41474943n; +const IHAVEOPT = 0x49484156454f5054n; +const OPT_REPLY_MAGIC = 0x0003e889045565a9n; +const REQ_MAGIC = 0x25609513; +const SIMPLE_REPLY_MAGIC = 0x67446698; + +const NBD_FLAG_FIXED_NEWSTYLE = 1; +const NBD_FLAG_NO_ZEROES = 2; + +const NBD_OPT_ABORT = 2; +const NBD_OPT_INFO = 6; +const NBD_OPT_GO = 7; + +const NBD_REP_ACK = 1; +const NBD_REP_INFO = 3; +const NBD_REP_FLAG_ERROR = 0x80000000; +const NBD_REP_ERR_UNSUP = (NBD_REP_FLAG_ERROR | 1) >>> 0; + +const NBD_INFO_EXPORT = 0; +const NBD_FLAG_HAS_FLAGS = 1; +const NBD_FLAG_READ_ONLY = 2; + +const NBD_CMD_READ = 0; +const NBD_CMD_DISC = 2; +const NBD_CMD_FLUSH = 3; + +const EINVAL = 22; + +/* Read exactly n bytes from a socket, buffering across 'data' events. */ +function makeReader(socket) { + let chunks = []; + let buffered = 0; + let want = 0; + let resolveWant = null; + let rejectWant = null; + let ended = false; + let errored = null; + + function tryResolve() { + if (resolveWant && buffered >= want) { + const all = Buffer.concat(chunks); + const out = all.subarray(0, want); + const rest = all.subarray(want); + chunks = rest.length ? [Buffer.from(rest)] : []; + buffered = rest.length; + const r = resolveWant; + resolveWant = null; + rejectWant = null; + r(out); + } else if (resolveWant && (ended || errored)) { + const rej = rejectWant; + resolveWant = null; + rejectWant = null; + rej(errored || new Error('EOF')); + } + } + + socket.on('data', (d) => { + chunks.push(d); + buffered += d.length; + tryResolve(); + }); + socket.on('end', () => { + ended = true; + tryResolve(); + }); + socket.on('error', (e) => { + errored = e; + tryResolve(); + }); + + return (n) => + new Promise((resolve, reject) => { + want = n; + resolveWant = resolve; + rejectWant = reject; + tryResolve(); + }); +} + +export async function startNbdServer(socket, imageFd, size, onRead) { + const read = makeReader(socket); + const write = (buf) => + new Promise((res, rej) => socket.write(buf, (e) => (e ? rej(e) : res()))); + + /* --- Handshake --- */ + const hello = Buffer.alloc(18); + hello.writeBigUInt64BE(NBDMAGIC, 0); + hello.writeBigUInt64BE(IHAVEOPT, 8); + hello.writeUInt16BE(NBD_FLAG_FIXED_NEWSTYLE | NBD_FLAG_NO_ZEROES, 16); + await write(hello); + + await read(4); /* client flags (ignored for PoC) */ + + /* --- Option haggling: loop until GO (enter transmission) or ABORT --- */ + for (;;) { + const optHdr = await read(16); + const optMagic = optHdr.readBigUInt64BE(0); + const opt = optHdr.readUInt32BE(8); + const optLen = optHdr.readUInt32BE(12); + if (optMagic !== IHAVEOPT) throw new Error('bad option magic'); + if (optLen) await read(optLen); /* discard option data (export name etc.) */ + + if (opt === NBD_OPT_GO || opt === NBD_OPT_INFO) { + /* NBD_REP_INFO payload: info type (u16) + size (u64) + flags (u16) */ + const infoReply = Buffer.alloc(2 + 8 + 2); + infoReply.writeUInt16BE(NBD_INFO_EXPORT, 0); + infoReply.writeBigUInt64BE(BigInt(size), 2); + infoReply.writeUInt16BE(NBD_FLAG_HAS_FLAGS | NBD_FLAG_READ_ONLY, 10); + await sendOptReply(write, opt, NBD_REP_INFO, infoReply); + await sendOptReply(write, opt, NBD_REP_ACK, Buffer.alloc(0)); + if (opt === NBD_OPT_GO) break; /* GO enters transmission phase */ + } else if (opt === NBD_OPT_ABORT) { + await sendOptReply(write, opt, NBD_REP_ACK, Buffer.alloc(0)); + socket.end(); + return; + } else { + await sendOptReply(write, opt, NBD_REP_ERR_UNSUP, Buffer.alloc(0)); + } + } + + /* --- Transmission phase --- */ + for (;;) { + let hdr; + try { + hdr = await read(28); + } catch { + return; /* EOF / disconnect */ + } + const magic = hdr.readUInt32BE(0); + if (magic !== REQ_MAGIC) throw new Error('bad request magic'); + const type = hdr.readUInt16BE(6); + const handle = hdr.subarray(8, 16); /* opaque 8 bytes */ + const offset = hdr.readBigUInt64BE(16); + const length = hdr.readUInt32BE(24); + + if (type === NBD_CMD_DISC) return; + + if (type === NBD_CMD_READ) { + if (offset + BigInt(length) > BigInt(size)) { + await sendSimpleReply(write, EINVAL, handle, null); + continue; + } + const data = Buffer.alloc(length); + fs.readSync(imageFd, data, 0, length, Number(offset)); + if (onRead) onRead(Number(offset), length); + await sendSimpleReply(write, 0, handle, data); + } else if (type === NBD_CMD_FLUSH) { + await sendSimpleReply(write, 0, handle, null); + } else { + await sendSimpleReply(write, EINVAL, handle, null); + } + } +} + +/* NBD option reply: magic(8) + opt(4) + reptype(4) + len(4) + payload */ +async function sendOptReply(write, opt, repType, payload) { + const fixed = Buffer.alloc(16); + fixed.writeBigUInt64BE(OPT_REPLY_MAGIC, 0); + fixed.writeUInt32BE(opt, 8); + fixed.writeUInt32BE(repType >>> 0, 12); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(payload.length, 0); + await write(Buffer.concat([fixed, lenBuf, payload])); +} + +/* NBD simple reply: magic(4) + error(4) + handle(8) + [data] */ +async function sendSimpleReply(write, error, handle, data) { + const hdr = Buffer.alloc(16); + hdr.writeUInt32BE(SIMPLE_REPLY_MAGIC, 0); + hdr.writeUInt32BE(error >>> 0, 4); + handle.copy(hdr, 8); + await write(data ? Buffer.concat([hdr, data]) : hdr); +} diff --git a/scripts/poc-nbd/test-stage1.mjs b/scripts/poc-nbd/test-stage1.mjs new file mode 100644 index 0000000..f0c6aa4 --- /dev/null +++ b/scripts/poc-nbd/test-stage1.mjs @@ -0,0 +1,82 @@ +/* + * Stage 1: prove the inherited-fd NBD transport works with zero QEMU + * source changes. Parent creates a socketpair, runs the NBD server on + * one end, spawns lspart with the other end inherited as fd 3, and + * checks lspart opens the qcow2 (capacity > 0, table printed) identically + * to a plain-file open. + */ +import { spawn, execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { serveOnFd } from './launch.mjs'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '../..'); +const lspart = path.join(repoRoot, 'build-anyfs-linux-amd64/src/lspart/anyfs-lspart'); +const meta = JSON.parse(fs.readFileSync(path.join(here, 'fixtures/meta.json'))); + +/* Baseline: lspart over the plain qcow2 file (proves what a successful + * open looks like, independent of NBD). Captured for comparison. + * NOTE: this runs BEFORE serveOnFd, so execFileSync is safe (no in-proc + * NBD server is live yet to be starved by a blocked event loop). */ +const plainOut = execFileSync(lspart, [meta.qcow2], { encoding: 'utf8' }); + +const { fd1, stop, readCount, done } = serveOnFd(meta.qcow2); + +/* Child inherits fd1 at descriptor index 3 (stdio[3]). lspart is told + * --nbd-fd 3. stdio: 0,1,2 are inherit; index 3 = the socketpair end. */ +const child = spawn(lspart, ['--nbd-fd', '3'], { + stdio: ['inherit', 'pipe', 'inherit', fd1], +}); + +let out = ''; +child.stdout.on('data', (d) => (out += d)); + +const code = await new Promise((resolve) => child.on('exit', resolve)); +stop(); +await done; + +console.log('--- plain-file lspart (baseline) ---'); +console.log(plainOut); +console.log('--- nbd lspart output ---'); +console.log(out); +console.log(`--- exit=${code} nbd_reads=${readCount()} ---`); + +/* The synthetic image has no partition table / filesystem (it is a + * deterministic byte pattern), so lspart lists it as a single whole-disk + * row with FSTYPE '?'. The Stage-1 signal is therefore NOT a specific + * filesystem string — it is: (a) lspart exits 0, (b) the NBD server + * actually served reads (data traversed the socketpair), and (c) the NBD + * run produced the SAME table as the plain-file open. */ +function dataRows(s) { + return s + .split('\n') + .filter((l) => l.trim() && !/^Usage|warning:|^PATH\b/.test(l)).length; +} + +if (code !== 0) { + console.error('STAGE1 FAIL: lspart exited non-zero'); + process.exit(1); +} +if (readCount() === 0) { + console.error( + 'STAGE1 FAIL: NBD server served zero reads (data did not traverse the socketpair)', + ); + process.exit(1); +} +/* NOTE: the fixture has no partition table and no filesystem, so lspart + * prints only a header row (dataRows == 0) for BOTH the plain-file and + * the NBD open. The "disk not detected" signal we care about is therefore + * the parity check below — if NBD returned a different table than the + * plain-file open, something went wrong in transport. A zero row count + * that matches the baseline is still a valid PASS. */ +if (dataRows(out) !== dataRows(plainOut)) { + console.error( + `STAGE1 FAIL: nbd row count ${dataRows(out)} != plain-file row count ${dataRows(plainOut)}`, + ); + process.exit(1); +} +console.log( + 'STAGE1 PASS: lspart opened qcow2 over inherited-fd NBD; reads traversed the socketpair; table matches plain-file open', +); diff --git a/scripts/poc-nbd/test-stage2.mjs b/scripts/poc-nbd/test-stage2.mjs new file mode 100644 index 0000000..b7d848b --- /dev/null +++ b/scripts/poc-nbd/test-stage2.mjs @@ -0,0 +1,100 @@ +/* + * Stage 2: full Linux chain. + * (a) lspart over inherited-fd NBD matches lspart over the plain file. + * (b) the deterministic marker at meta.verifyOffset reads back + * byte-for-byte THROUGH the qcow2-over-NBD chain, using qemu-io as + * a real qcow2 client over a unix-socket NBD endpoint. + */ +import { spawn, execFileSync, execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import net from 'node:net'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { serveOnFd, serveOnUnixSocket } from './launch.mjs'; + +const execFileP = promisify(execFile); + +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '../..'); +const lspart = path.join(repoRoot, 'build-anyfs-linux-amd64/src/lspart/anyfs-lspart'); +const meta = JSON.parse(fs.readFileSync(path.join(here, 'fixtures/meta.json'))); + +function dataRows(s) { + return s + .split('\n') + .filter((l) => l.trim() && !/^Usage|warning:|^PATH\b/.test(l)).length; +} + +let fail = false; + +/* === Check (a): lspart parity over inherited-fd NBD === */ +const plainOut = execFileSync(lspart, [meta.qcow2], { encoding: 'utf8' }); +const { fd1, stop, readCount, done } = serveOnFd(meta.qcow2); +const child = spawn(lspart, ['--nbd-fd', '3'], { + stdio: ['inherit', 'pipe', 'inherit', fd1], +}); +let nbdOut = ''; +child.stdout.on('data', (d) => (nbdOut += d)); +const code = await new Promise((r) => child.on('exit', r)); +stop(); +await done; + +console.log('--- plain-file lspart ---\n' + plainOut); +console.log('--- nbd lspart ---\n' + nbdOut); + +if (code !== 0) { + console.error('STAGE2 FAIL: nbd lspart exited non-zero'); + fail = true; +} +if (dataRows(plainOut) !== dataRows(nbdOut)) { + console.error( + `STAGE2 FAIL: row count differs (plain=${dataRows(plainOut)} nbd=${dataRows(nbdOut)})`, + ); + fail = true; +} +if (readCount() === 0) { + console.error('STAGE2 FAIL: zero NBD reads over inherited fd'); + fail = true; +} + +/* === Check (b): byte verify THROUGH qcow2-over-NBD with qemu-io === */ +const sockPath = '/tmp/poc-nbd-stage2.sock'; +const { server, imageFd } = await serveOnUnixSocket(meta.qcow2, sockPath); +try { + const len = Buffer.from(meta.verifyBytesHex, 'hex').length; + const nbdUri = `json:{"driver":"qcow2","file":{"driver":"nbd","server":{"type":"unix","path":"${sockPath}"}}}`; + /* MUST be async: the in-process NBD server runs on this event loop, so a + * synchronous execFileSync would block it and deadlock the qemu-io read. */ + const { stdout: out } = await execFileP( + 'qemu-io', + ['-r', '-c', `read -v ${meta.verifyOffset} ${len}`, nbdUri], + { encoding: 'utf8' }, + ); + /* qemu-io 'read -v' prints a hexdump; collect the hex byte columns. */ + const hex = [...out.matchAll(/^[0-9a-f]{8}:\s+((?:[0-9a-f]{2}\s?)+)/gim)] + .map((m) => m[1].replace(/\s+/g, '')) + .join('') + .slice(0, len * 2); + console.log('--- qemu-io marker read (through qcow2-over-NBD) ---'); + console.log(out); + if (hex !== meta.verifyBytesHex) { + console.error( + `STAGE2 FAIL: marker mismatch through chain: got ${hex} want ${meta.verifyBytesHex}`, + ); + fail = true; + } +} catch (e) { + console.error('STAGE2 FAIL: qemu-io byte verify errored:', e.message); + fail = true; +} finally { + server.close(); + try { + fs.closeSync(imageFd); + } catch {} +} + +if (fail) process.exit(1); +console.log( + `STAGE2 PASS: lspart parity (rows=${dataRows(nbdOut)}), inherited-fd reads=${readCount()}, marker verified through qcow2-over-NBD (${meta.verifyAscii})`, +); diff --git a/src/core/qemu_backend.c b/src/core/qemu_backend.c index 457ac6b..ab20c46 100644 --- a/src/core/qemu_backend.c +++ b/src/core/qemu_backend.c @@ -19,6 +19,7 @@ #include "qapi/error.h" #include "qemu/error-report.h" #include "qemu/main-loop.h" +#include "qemu/module.h" #include "qobject/qdict.h" #include "system/block-backend-global-state.h" #include "system/block-backend-io.h" @@ -137,6 +138,11 @@ int qemu_blk_open(const char* image_path, uint32_t flags, readonly, snapshot); if (!qemu_initialized) { fprintf(stderr, "[qemu_blk] bdrv_init…\n"); + /* MODULE_INIT_QOM must be called before bdrv_init so that QOM + * types like QIOChannelSocket are registered. QEMU binaries + * (qemu-img, qemu-nbd) do this explicitly; we must do the same + * when embedding the block layer in a library. */ + module_call_init(MODULE_INIT_QOM); bdrv_init(); if (qemu_init_main_loop(&errp) < 0) { fprintf(stderr, "qemu_init_main_loop failed: %s\n", @@ -173,15 +179,36 @@ int qemu_blk_open(const char* image_path, uint32_t flags, */ QDict* options = NULL; const int is_url = image_path && strstr(image_path, "://") != NULL; - if (is_url) { + + /* PoC: NBD-over-inherited-fd transport. image_path "nbd-fd:N" means the + * NBD client should adopt already-open socket fd N (passed by the parent + * via inheritance). "nbd-port:P" is the Windows fallback: connect to a + * 127.0.0.1 loopback on port P. In both cases the server address comes + * from the options QDict (server.*), so the filename handed to + * blk_new_open must be NULL. */ + const char* open_name = image_path; + if (image_path && strncmp(image_path, "nbd-fd:", 7) == 0) { + options = qdict_new(); + qdict_put_str(options, "driver", "nbd"); + qdict_put_str(options, "server.type", "fd"); + qdict_put_str(options, "server.str", image_path + 7); + open_name = NULL; + } else if (image_path && strncmp(image_path, "nbd-port:", 9) == 0) { + options = qdict_new(); + qdict_put_str(options, "driver", "nbd"); + qdict_put_str(options, "server.type", "inet"); + qdict_put_str(options, "server.host", "127.0.0.1"); + qdict_put_str(options, "server.port", image_path + 9); + open_name = NULL; + } else if (is_url) { options = qdict_new(); qdict_put_int(options, "file.timeout", 20); } - fprintf(stderr, "[qemu_blk] blk_new_open flags=0x%x timeout=%d\n", - bdrv_flags, is_url ? 20 : 0); + fprintf(stderr, "[qemu_blk] blk_new_open name=%s flags=0x%x\n", + open_name ? open_name : "(nbd via options)", bdrv_flags); BlockBackend* blk = - blk_new_open(image_path, NULL, options, bdrv_flags, &errp); + blk_new_open(open_name, NULL, options, bdrv_flags, &errp); fprintf(stderr, "[qemu_blk] blk_new_open returned %p\n", (void*)blk); if (!blk) { anyfs_set_last_error("%s", error_get_pretty(errp)); diff --git a/src/ksmbd/ksmbd_main.c b/src/ksmbd/ksmbd_main.c index 2b23d0d..def25e1 100644 --- a/src/ksmbd/ksmbd_main.c +++ b/src/ksmbd/ksmbd_main.c @@ -33,6 +33,7 @@ #include "../host_proxy/host_proxy.h" #include "anyfs.h" #include "fastsync_win.h" +#include "server_common.h" #include #include @@ -62,19 +63,7 @@ #define GUEST_USER "guest" /* ── Share descriptor ─────────────────────────────────────────────────── */ -typedef struct { - char name[64]; /* SMB share name (section header in smb.conf) */ - char lkl_path[ANYFS_LKL_PATH_MAX]; /* absolute LKL path returned by - anyfs_session_enter */ -} ShareInfo; - -static volatile int running = 1; - -static void sigint_handler(int sig) -{ - (void)sig; - running = 0; -} +typedef AnyfsShareEntry ShareInfo; /* ── Runtime-tunable knobs ───────────────────────────────────────────── */ /* @@ -699,9 +688,7 @@ int main(int argc, char** argv) /* ── Signals / stdio ──────────────────────────────────────────────── */ - setbuf(stdout, NULL); - signal(SIGINT, sigint_handler); - signal(SIGTERM, sigint_handler); + anyfs_server_install_signals(); pr_logger_init(PR_LOGGER_STDIO); set_log_level(log_level); @@ -714,22 +701,13 @@ int main(int argc, char** argv) .mem_mb = mem_mb, .loglevel = (log_level >= PR_DEBUG) ? 8 : 4, }; - int ret = anyfs_kernel_init(&kern_opts); + int ret = anyfs_server_boot(&kern_opts); if (ret) { pr_err("Failed to start kernel\n"); return 1; } pr_info("LKL kernel started (ksmbd built-in)\n"); - /* - * No virtio-net / slirp: the data path is host TCP -> host_proxy - * threads -> lkl_sys_read/write -> LKL TCP on lo. We just need lo - * up so ksmbd's netdev notifier creates the listener socket bound - * to it. (lo is auto-up after boot, but the call is idempotent and - * documents intent.) - */ - lkl_if_up(1); /* loopback is always ifindex 1 in LKL */ - if (log_level >= PR_DEBUG) { long dfd = lkl_sys_open("/sys/class/ksmbd-control/debug", LKL_O_WRONLY, 0); @@ -761,17 +739,14 @@ int main(int argc, char** argv) /* ── 5. Resolve --share specs to LKL paths ───────────────────────── */ ShareInfo shares[ANYFS_MAX_SHARES]; - int n_shares = 0; - - for (int si = 0; si < n_share_specs; si++) { - ShareInfo* sh = &shares[n_shares]; - if (anyfs_share_resolve(share_specs[si], disks, n_images, 0, - sh->name, sizeof(sh->name), - sh->lkl_path, sizeof(sh->lkl_path)) < 0) - goto halt; - pr_info("Share [%s] -> %s\n", sh->name, sh->lkl_path); - n_shares++; - } + int n_shares = anyfs_server_resolve_shares(share_specs, n_share_specs, + disks, n_images, 0, shares, + ANYFS_MAX_SHARES); + if (n_shares < 0) + goto halt; + for (int i = 0; i < n_shares; i++) + pr_info("Share [%s] -> %s\n", shares[i].name, + shares[i].lkl_path); if (n_shares == 0) { pr_err("No shares could be mounted.\n"); @@ -812,7 +787,7 @@ int main(int argc, char** argv) /* ── 7. Event loop ────────────────────────────────────────────────── */ - while (running) { + while (anyfs_server_running) { ret = ipc_process_event(); if (ret < 0) break; @@ -830,13 +805,7 @@ int main(int argc, char** argv) usm_destroy(); halt: - /* Close all disk sessions (atexit handler inside anyfs_session_close - * will unmount any LKL-pinned mounts). */ - for (int i = 0; i < n_images; i++) { - if (disks[i]) - anyfs_session_close(disks[i]); - } - anyfs_kernel_halt(); + anyfs_server_shutdown(disks, n_images); pr_info("Done\n"); return 0; } diff --git a/src/lspart/lspart_main.c b/src/lspart/lspart_main.c index 73b7cf3..2f8af81 100644 --- a/src/lspart/lspart_main.c +++ b/src/lspart/lspart_main.c @@ -16,7 +16,8 @@ static void usage(FILE* f, const char* prog) { fprintf(f, - "Usage: %s [--json] [--help] [?] [...]\n" + "Usage: %s [--json] [--help] [--nbd-fd N | --nbd-port P] " + "[?] [...]\n" "\n" "Open each image, list partitions, print a unified table.\n" "PATH column is the canonical disk/p form (drops into\n" @@ -34,6 +35,7 @@ int main(int argc, char** argv) int json = 0; const char* images[16]; int n_images = 0; + static char nbd_path[32]; /* holds a synthesized "nbd-fd:N"/"nbd-port:P" */ for (int i = 1; i < argc; i++) { const char* a = argv[i]; @@ -45,6 +47,28 @@ int main(int argc, char** argv) json = 1; continue; } + if (strcmp(a, "--nbd-fd") == 0 || strcmp(a, "--nbd-port") == 0) { + if (nbd_path[0]) { + fprintf(stderr, + "only one --nbd-fd/--nbd-port allowed\n"); + return 2; + } + if (i + 1 >= argc) { + fprintf(stderr, "%s needs an argument\n", a); + return 2; + } + /* a[6]: "--nbd-fd"[6]=='f', "--nbd-port"[6]=='p' */ + const char* kind = (a[6] == 'f') ? "nbd-fd" : "nbd-port"; + snprintf(nbd_path, sizeof(nbd_path), "%s:%s", kind, + argv[++i]); + if (n_images >= (int)(sizeof(images) / + sizeof(images[0]))) { + fprintf(stderr, "too many images\n"); + return 2; + } + images[n_images++] = nbd_path; + continue; + } if (a[0] == '-' && a[1] != '\0') { fprintf(stderr, "unknown flag: %s\n", a); usage(stderr, argv[0]); diff --git a/src/nfsd/nfsd_main.c b/src/nfsd/nfsd_main.c index 6e3e01b..0678658 100644 --- a/src/nfsd/nfsd_main.c +++ b/src/nfsd/nfsd_main.c @@ -46,6 +46,7 @@ #include "../host_proxy/host_proxy.h" #include "anyfs.h" +#include "server_common.h" #include /* ── Compile-time limits ─────────────────────────────────────────────── */ @@ -77,14 +78,6 @@ typedef struct { char bind_path[80]; } ExportInfo; -static volatile int running = 1; - -static void sigint_handler(int sig) -{ - (void)sig; - running = 0; -} - /* Write a string to a file inside LKL */ static int lkl_write_file(const char* path, const char* data) { @@ -443,7 +436,7 @@ static void* cache_handler_thread(void* arg) } } - while (running) { + while (anyfs_server_running) { struct lkl_pollfd pfds[NUM_CHANNELS]; int nfds = 0; int fd_map[NUM_CHANNELS]; @@ -715,29 +708,19 @@ int main(int argc, char** argv) /* ── Signals / stdio ──────────────────────────────────────────────── */ - setbuf(stdout, NULL); - signal(SIGINT, sigint_handler); - signal(SIGTERM, sigint_handler); + anyfs_server_install_signals(); /* ── 1. Boot LKL kernel ───────────────────────────────────────────── */ AnyfsKernelOpts kern_opts = {.mem_mb = 0 /* anyfs default (32M) */, .loglevel = 4}; - int ret = anyfs_kernel_init(&kern_opts); + int ret = anyfs_server_boot(&kern_opts); if (ret) { fprintf(stderr, "Failed to start kernel\n"); return 1; } printf("LKL kernel started (nfsd built-in)\n"); - /* - * No virtio-net / slirp: the data path is host TCP -> host_proxy - * threads -> lkl_sys_read/write -> LKL TCP on lo. We just need lo - * up so nfsd's listener binds to it. (lo is auto-up after boot, - * but the call is idempotent and documents intent.) - */ - lkl_if_up(1); /* loopback is always ifindex 1 in LKL */ - /* ── 4. Open disk images ──────────────────────────────────────────── */ AnyfsSession* disks[ANYFS_MAX_DISKS] = {NULL}; @@ -749,16 +732,23 @@ int main(int argc, char** argv) } /* ── 5. Resolve --share specs to LKL paths ───────────────────────── */ - for (int si = 0; si < n_share_specs; si++) { - ExportInfo* ex = &g_exports[g_n_exports]; + { + AnyfsShareEntry ents[ANYFS_MAX_SHARES]; uint32_t eflags = read_only ? ANYFS_SESSION_READONLY : 0; - if (anyfs_share_resolve(share_specs[si], disks, n_images, - eflags, ex->name, sizeof(ex->name), - ex->lkl_path, sizeof(ex->lkl_path)) < 0) + int n = anyfs_server_resolve_shares(share_specs, n_share_specs, + disks, n_images, eflags, + ents, ANYFS_MAX_SHARES); + if (n < 0) goto halt; - printf("Export [%d] /%s -> %s\n", g_n_exports, ex->name, - ex->lkl_path); - g_n_exports++; + for (int i = 0; i < n; i++) { + ExportInfo* ex = &g_exports[g_n_exports]; + memcpy(ex->name, ents[i].name, sizeof(ex->name)); + memcpy(ex->lkl_path, ents[i].lkl_path, + sizeof(ex->lkl_path)); + printf("Export [%d] /%s -> %s\n", g_n_exports, ex->name, + ex->lkl_path); + g_n_exports++; + } } if (g_n_exports == 0) { @@ -846,7 +836,7 @@ int main(int argc, char** argv) /* ── 8. Serve until interrupted ───────────────────────────────────── */ - while (running) { + while (anyfs_server_running) { usleep(100000); } @@ -855,11 +845,7 @@ int main(int argc, char** argv) pthread_join(cache_tid, NULL); halt: - for (int i = 0; i < n_images; i++) { - if (disks[i]) - anyfs_session_close(disks[i]); - } - anyfs_kernel_halt(); + anyfs_server_shutdown(disks, n_images); printf("Done\n"); return 0; } diff --git a/src/server_common/server_common.c b/src/server_common/server_common.c new file mode 100644 index 0000000..f43b37b --- /dev/null +++ b/src/server_common/server_common.c @@ -0,0 +1,63 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include "server_common.h" + +#include +#include + +volatile sig_atomic_t anyfs_server_running = 1; + +static void handle_stop(int sig) +{ + (void)sig; + anyfs_server_running = 0; +} + +void anyfs_server_install_signals(void) +{ + setbuf(stdout, NULL); + signal(SIGINT, handle_stop); + signal(SIGTERM, handle_stop); +} + +int anyfs_server_boot(const AnyfsKernelOpts* opts) +{ + int ret = anyfs_kernel_init(opts); + if (ret) + return ret; + /* lo is auto-up after boot, but the call is idempotent and + * documents intent. */ + lkl_if_up(1); + return 0; +} + +int anyfs_server_resolve_shares(char* const* specs, int n_specs, + AnyfsSession** disks, int n_disks, + uint32_t enter_flags, AnyfsShareEntry* out, + int max_out) +{ + int n = 0; + + for (int si = 0; si < n_specs; si++) { + if (n >= max_out) { + fprintf(stderr, "error: too many shares (max %d)\n", + max_out); + return -1; + } + AnyfsShareEntry* e = &out[n]; + if (anyfs_share_resolve(specs[si], disks, n_disks, enter_flags, + e->name, sizeof(e->name), e->lkl_path, + sizeof(e->lkl_path)) < 0) + return -1; + n++; + } + return n; +} + +void anyfs_server_shutdown(AnyfsSession** disks, int n_disks) +{ + for (int i = 0; i < n_disks; i++) { + if (disks[i]) + anyfs_session_close(disks[i]); + } + anyfs_kernel_halt(); +} diff --git a/src/server_common/server_common.h b/src/server_common/server_common.h new file mode 100644 index 0000000..810478d --- /dev/null +++ b/src/server_common/server_common.h @@ -0,0 +1,49 @@ +/* + * server_common.h — Shared daemon skeleton for the LKL server surfaces + * (anyfs-ksmbd, anyfs-nfsd): stop flag + signal install, kernel boot with + * loopback up, the --share resolution loop, and shutdown. Arg parsing and + * the serving loops stay in the respective mains. + */ +#ifndef ANYFS_SERVER_COMMON_H +#define ANYFS_SERVER_COMMON_H + +#include "anyfs.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* 1 while serving; cleared by SIGINT/SIGTERM. Serving loops poll it. */ +extern volatile sig_atomic_t anyfs_server_running; + +/* Unbuffer stdout and route SIGINT/SIGTERM to clearing + * anyfs_server_running. */ +void anyfs_server_install_signals(void); + +/* Boot the LKL kernel and bring up loopback (ifindex 1, idempotent — + * the in-kernel listeners bind to it). Returns anyfs_kernel_init()'s + * result: 0 on success. */ +int anyfs_server_boot(const AnyfsKernelOpts* opts); + +typedef struct AnyfsShareEntry { + char name[64]; + char lkl_path[ANYFS_LKL_PATH_MAX]; +} AnyfsShareEntry; + +/* Resolve every --share spec to a (name, lkl_path) pair via + * anyfs_share_resolve(), which prints its own diagnostics. Returns the + * number of entries filled, or -1 on the first failure. */ +int anyfs_server_resolve_shares(char* const* specs, int n_specs, + AnyfsSession** disks, int n_disks, + uint32_t enter_flags, AnyfsShareEntry* out, + int max_out); + +/* Close every non-NULL session and halt the kernel. */ +void anyfs_server_shutdown(AnyfsSession** disks, int n_disks); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tests/diagnostics/README.md b/tests/diagnostics/README.md new file mode 100644 index 0000000..aac267a --- /dev/null +++ b/tests/diagnostics/README.md @@ -0,0 +1,90 @@ +# tests/diagnostics — manual diagnostic scripts + +**The Playwright E2E suite at `ts/tests/e2e` is the primary regression suite.** +The scripts in this directory are kept as *manual diagnostics* — they are not part +of any CI or `pnpm test` gate, but they remain runnable and are useful when the +Playwright harness itself is in question (raw CDP needs no Playwright install) or +when debugging a backend the Playwright suite currently gates off (see the +electron-native / F9 notes below). + +## CDP UI suite (demoted 2026-06-10) + +The legacy UI automation suite drives the real apps over raw CDP (no +dependencies beyond Node + a Chromium/Electron binary): + +- `test-cdp.mjs` — one target×source combo per invocation +- `run-all.mjs` — driver for all 6 combos +- `common-cdp.mjs` — minimal dependency-free CDP client (WebSocket framing, + `Runtime.evaluate`, console capture). Also imported by the boot/prewarm + debug scripts below (`common.mjs` re-exports it alongside the + Electron/vite/HTTP-server launchers). + +### How to run + +```sh +# everything (6 combos; needs Xvfb for the electron targets) +node tests/diagnostics/run-all.mjs [--image=path/to/disk.img] + +# one combo +node tests/diagnostics/test-cdp.mjs --target web --source local [--image ...] +node tests/diagnostics/test-cdp.mjs --target web --source http [--image ...] +node tests/diagnostics/test-cdp.mjs --target electron --mode wasm --source local +node tests/diagnostics/test-cdp.mjs --target electron --mode native --source http [--url https://...] +``` + +Default image: `tests/images/ext4.img`. The scripts spawn their own vite dev +server / Electron / headless Chromium and `pkill` stale ones first — do **not** +run them next to a live dev session you care about. + +## Parity audit (why the suite was demoted) + +Every assertion the CDP suite makes was mapped to the Playwright suite +(`ts/tests/e2e`, projects `web` / `electron-wasm` / `electron-native` over +`flows/*.spec.ts` + `electron-only/backend-switch.spec.ts`). Result: full +parity — every behavioural assertion is covered (usually strictly stronger), +and the one genuine gap found (the "Open URL…" dialog *UI* flow) was ported +into `flows/url-load.spec.ts` as part of this audit. + +| # | CDP assertion | Combos | Playwright coverage | +| --- | --- | --- | --- | +| C1 | Page title contains "anyfs" | all 6 | **Not ported (justified):** branding text, not behaviour. Every driver's `start()` waits for the `__anyfsTest` bridge — a strictly stronger "app actually booted" signal than a title substring. | +| C2 | Kernel boots (`Linux version` dmesg in console, or `crossOriginIsolated` fallback) | all 6 | Covered implicitly and strictly stronger: every flow mounts a partition and lists pinned filenames (`expectKnownTree`), impossible without a booted kernel. `open-browse-download.spec.ts` `@smoke` runs on all 3 projects. | +| C3a | Local open via "Open file…" button (native IPC dialog seam `ANYFS_TEST_LOCAL_PATH`) | electron-native-local | Covered by `electron-driver.ts` `openImage()` via the `__anyfsTest.openPath` bridge — sets the *same* `{kind:'path'}` source the button's IPC produces; the button itself is deliberately bypassed (FINDING F6, recents/IndexedDB hang — documented in the driver). Native runs are `test.fixme` (F9 teardown hang); this CDP combo stays the manual way to exercise native until the utilityProcess split lands. | +| C3b | Local open on wasm targets (served over local HTTP + `openUrl` hook — a CDP workaround for File-postMessage cloning) | web-wasm-local, electron-wasm-local | Covered *better*: Playwright drives the real local-file path (`setInputFiles` on the hidden `legacy-file-input`) in `open-browse-download.spec.ts` / `formats.spec.ts`; the URLFS-over-local-HTTP aspect is covered by `url-load.spec.ts` (local Range server). | +| C3c | HTTP open via `__anyfsTest.openUrl` hook, with UI fallback ("Open URL…" button → `aria-label="Disk image URL"` input → dialog Open) | all `*-http` | Hook path: `url-load.spec.ts` `@smoke` (web + electron-wasm; native F9-gated). UI dialog path: **GAP — closed by this audit**: `url-load.spec.ts` "open via the \"Open URL…\" dialog UI" (web; the dialog is identical renderer DOM on every shell). | +| C3d | electron-wasm external URL wrapped as `anyfs-url://proxy/?u=…` | electron-wasm-http | **Not ported (justified):** the wrapper was a CDP-test-level CORS workaround for *external* URLs. Playwright's hermetic Range server sends CORS headers, so raw URLs work (`url-load.spec.ts` passes on electron-wasm); the real-remote case is the `@network`-tagged variant. | +| C4 | First partition button clickable (`#N` button, optional/non-fatal) | `*-local` | Covered strictly stronger: `listPartitionIndices()` asserts the expected indices and `formats.spec.ts` enters **every** manifest-pinned partition (GPT multi, MBR extended/logical) plus `backToPartitions()` round-trips. | +| C5 | File tree appears (>0 rows, or fuzzy filename text in body) | all 6 | Covered strictly stronger: `expectKnownTree` asserts manifest-pinned filenames per partition; downloads then prove content (13-byte `hello.txt`). | +| C6 | No JS errors (`window.__jsErrors` empty) | all 6 | **Not ported (justified):** non-fatal even in the CDP suite ("may be headless artifacts" — it only logged on failure, never failed the run). Playwright traces capture page errors for debugging; a hard zero-JS-error assertion would be flaky by the CDP suite's own admission. | + +Combo-level note: the Playwright suite runs the shared flows on `web` and +`electron-wasm` green; `electron-native` is `test.fixme` solely on FINDING F9 +(Electron `app.close()` hangs ~2 min after a native QEMU+LKL mount — a real +shutdown defect, not missing coverage; the assertions exist in the specs and +the fixme is removed once the addon moves to a utilityProcess). The CDP suite +never observed F9 because it kills the Electron process instead of closing it +— which is exactly why it stays here as a native-backend diagnostic. + +## Debug scripts + +Ad-hoc boot/prewarm repro scripts (moved here from `tests/` 2026-06-10). Each +spawns its own vite dev server and/or Electron under Xvfb and `pkill`s stale +ones first. One line per script — the regression it guards: + +- `test-atomics.mjs` — Atomics.wait-on-main-thread regressions: drives + `debug-atomics.html` in Electron and watches its log (wasm LKL must never + block the main thread; see the dedicated-Worker rule). +- `test-async-boot.mjs` — the workeronly async boot path: Electron → + `anyfs.worker.js` → `anyfs.workeronly.mjs` must complete async kernel boot. +- `test-prewarm-direct.mjs` — landing prewarm, direct harness: Electron on + `debug-worker.html`, watches the prewarm worker boot to completion. +- `test-prewarm-e2e.mjs` — landing prewarm, end-to-end: vite-demo main page + must pre-boot the wasm LKL kernel during the landing screen. +- `test-direct-module.mjs` — main-page module load failure mode: loads + `anyfs.mjs` directly on the main page (not in a Worker) and calls + `anyfs_ts_init`. **Expected to fail by design** — the wasm bundle must live + in a dedicated Web Worker (WORKERFS asserts, Atomics.wait forbidden on the + main thread); this script documents that failure signature. +- `test-worker-debug.mjs` — worker boot logging: minimal Electron run against + `debug-worker.html` capturing verbose worker console output while waiting + for the prewarm result. diff --git a/tests/common-cdp.mjs b/tests/diagnostics/common-cdp.mjs similarity index 100% rename from tests/common-cdp.mjs rename to tests/diagnostics/common-cdp.mjs diff --git a/tests/common.mjs b/tests/diagnostics/common.mjs similarity index 99% rename from tests/common.mjs rename to tests/diagnostics/common.mjs index f5fb24d..b2be955 100644 --- a/tests/common.mjs +++ b/tests/diagnostics/common.mjs @@ -13,7 +13,7 @@ import { networkInterfaces } from 'node:os'; import { CDPClient, sleep } from './common-cdp.mjs'; const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = resolve(__dirname, '..'); +const ROOT = resolve(__dirname, '../..'); const TS = resolve(ROOT, 'ts'); const ELECTRON_DIR = resolve(TS, 'examples', 'electron-demo'); diff --git a/tests/run-all.mjs b/tests/diagnostics/run-all.mjs similarity index 93% rename from tests/run-all.mjs rename to tests/diagnostics/run-all.mjs index 3a9fe4d..0f4f756 100644 --- a/tests/run-all.mjs +++ b/tests/diagnostics/run-all.mjs @@ -2,8 +2,11 @@ /* * run-all.mjs — Run all 6 CDP-based UI tests for anyfs reader. * + * DEMOTED to manual diagnostics (see README.md in this directory): the + * Playwright suite at ts/tests/e2e is the primary regression gate. + * * Usage: - * node tests/run-all.mjs [--image path/to/disk.iso] + * node tests/diagnostics/run-all.mjs [--image path/to/disk.iso] * * The 6 combinations: * 1. web + WASM + local file diff --git a/tests/test-async-boot.mjs b/tests/diagnostics/test-async-boot.mjs similarity index 96% rename from tests/test-async-boot.mjs rename to tests/diagnostics/test-async-boot.mjs index b04741d..ebaec8a 100644 --- a/tests/test-async-boot.mjs +++ b/tests/diagnostics/test-async-boot.mjs @@ -23,7 +23,7 @@ function httpGet(url) { // Clean up any leftover processes try { process.kill(parseInt(fs.readFileSync('/tmp/xvfb_96.pid','utf8'))); } catch(e) {} -const viteDir = resolve(__dirname, '../ts/examples/vite-demo'); +const viteDir = resolve(__dirname, '../../ts/examples/vite-demo'); const viteProc = spawn('npx', ['vite', '--port', '5204'], { cwd: viteDir, stdio: ['ignore', 'pipe', 'pipe'], @@ -42,7 +42,7 @@ xvfb.pid && fs.writeFileSync('/tmp/xvfb_96.pid', String(xvfb.pid)); await sleep(500); const cdpPort = 9404; -const electronDir = resolve(__dirname, '../ts/examples/electron-demo'); +const electronDir = resolve(__dirname, '../../ts/examples/electron-demo'); const electronBin = resolve(electronDir, 'node_modules/.bin/electron'); const electronProc = spawn(electronBin, [ electronDir, `--remote-debugging-port=${cdpPort}` diff --git a/tests/test-atomics.mjs b/tests/diagnostics/test-atomics.mjs similarity index 96% rename from tests/test-atomics.mjs rename to tests/diagnostics/test-atomics.mjs index 0afc150..adf797c 100644 --- a/tests/test-atomics.mjs +++ b/tests/diagnostics/test-atomics.mjs @@ -20,7 +20,7 @@ function httpGet(url) { } // ── start vite ─────────────────────────────────────────────────────────── -const viteDir = resolve(__dirname, '../ts/examples/vite-demo'); +const viteDir = resolve(__dirname, '../../ts/examples/vite-demo'); const viteProc = spawn('npx', ['vite', '--port', '5201'], { cwd: viteDir, stdio: ['ignore', 'pipe', 'pipe'], @@ -43,7 +43,7 @@ const xvfb = spawn('Xvfb', [':99', '-screen', '0', '1280x720x24'], { stdio: 'ign await sleep(500); const cdpPort = 9401; -const electronDir = resolve(__dirname, '../ts/examples/electron-demo'); +const electronDir = resolve(__dirname, '../../ts/examples/electron-demo'); const electronBin = resolve(electronDir, 'node_modules/.bin/electron'); const electronProc = spawn(electronBin, [ electronDir, `--remote-debugging-port=${cdpPort}` diff --git a/tests/test-cdp.mjs b/tests/diagnostics/test-cdp.mjs similarity index 96% rename from tests/test-cdp.mjs rename to tests/diagnostics/test-cdp.mjs index 681fa79..5586856 100644 --- a/tests/test-cdp.mjs +++ b/tests/diagnostics/test-cdp.mjs @@ -5,11 +5,14 @@ * Simulates user clicks/typing through CDP to open a disk image and verify * the file tree appears. Covers all 6 target×source combinations. * + * DEMOTED to manual diagnostics (see tests/diagnostics/README.md): the + * Playwright suite at ts/tests/e2e is the primary regression gate. + * * Usage: - * node tests/test-cdp.mjs --target electron --source local [--image ...] - * node tests/test-cdp.mjs --target electron --source http [--image ...] - * node tests/test-cdp.mjs --target web --source local [--image ...] - * node tests/test-cdp.mjs --target web --source http [--image ...] + * node tests/diagnostics/test-cdp.mjs --target electron --source local [--image ...] + * node tests/diagnostics/test-cdp.mjs --target electron --source http [--image ...] + * node tests/diagnostics/test-cdp.mjs --target web --source local [--image ...] + * node tests/diagnostics/test-cdp.mjs --target web --source http [--image ...] * * For Electron targets, --mode native (default) and --mode wasm control * whether the native addon or WASM worker is used. Web targets always use diff --git a/tests/test-direct-module.mjs b/tests/diagnostics/test-direct-module.mjs similarity index 96% rename from tests/test-direct-module.mjs rename to tests/diagnostics/test-direct-module.mjs index 4918b6c..39ea9b5 100644 --- a/tests/test-direct-module.mjs +++ b/tests/diagnostics/test-direct-module.mjs @@ -19,7 +19,7 @@ function httpGet(url) { }); } -const viteDir = resolve(__dirname, '../ts/examples/vite-demo'); +const viteDir = resolve(__dirname, '../../ts/examples/vite-demo'); const viteProc = spawn('npx', ['vite', '--port', '5203'], { cwd: viteDir, stdio: ['ignore', 'pipe', 'pipe'], @@ -37,7 +37,7 @@ const xvfb = spawn('Xvfb', [':97', '-screen', '0', '1280x720x24'], { stdio: 'ign await sleep(500); const cdpPort = 9403; -const electronDir = resolve(__dirname, '../ts/examples/electron-demo'); +const electronDir = resolve(__dirname, '../../ts/examples/electron-demo'); const electronBin = resolve(electronDir, 'node_modules/.bin/electron'); // Use a longer timeout for CDP diff --git a/tests/test-prewarm-direct.mjs b/tests/diagnostics/test-prewarm-direct.mjs similarity index 97% rename from tests/test-prewarm-direct.mjs rename to tests/diagnostics/test-prewarm-direct.mjs index fb36453..f7f9fc1 100644 --- a/tests/test-prewarm-direct.mjs +++ b/tests/diagnostics/test-prewarm-direct.mjs @@ -21,7 +21,7 @@ function httpGet(url) { } // ── start vite ─────────────────────────────────────────────────────────── -const viteDir = resolve(__dirname, '../ts/examples/vite-demo'); +const viteDir = resolve(__dirname, '../../ts/examples/vite-demo'); const viteProc = spawn('npx', ['vite', '--port', '5200'], { cwd: viteDir, stdio: ['ignore', 'pipe', 'pipe'], @@ -44,7 +44,7 @@ const xvfb = spawn('Xvfb', [':99', '-screen', '0', '1280x720x24'], { stdio: 'ign await sleep(500); const cdpPort = 9400; -const electronDir = resolve(__dirname, '../ts/examples/electron-demo'); +const electronDir = resolve(__dirname, '../../ts/examples/electron-demo'); const electronBin = resolve(electronDir, 'node_modules/.bin/electron'); const electronProc = spawn(electronBin, [ electronDir, `--remote-debugging-port=${cdpPort}` diff --git a/tests/test-prewarm-e2e.mjs b/tests/diagnostics/test-prewarm-e2e.mjs similarity index 96% rename from tests/test-prewarm-e2e.mjs rename to tests/diagnostics/test-prewarm-e2e.mjs index e39c75b..0b5c93a 100644 --- a/tests/test-prewarm-e2e.mjs +++ b/tests/diagnostics/test-prewarm-e2e.mjs @@ -19,7 +19,7 @@ function httpGet(url) { }); } -const viteDir = resolve(__dirname, '../ts/examples/vite-demo'); +const viteDir = resolve(__dirname, '../../ts/examples/vite-demo'); const viteProc = spawn('npx', ['vite', '--port', '5209'], { cwd: viteDir, stdio: ['ignore', 'pipe', 'pipe'], @@ -37,7 +37,7 @@ const xvfb = spawn('Xvfb', [':95', '-screen', '0', '1280x720x24'], { stdio: 'ign await sleep(500); const cdpPort = 9409; -const electronDir = resolve(__dirname, '../ts/examples/electron-demo'); +const electronDir = resolve(__dirname, '../../ts/examples/electron-demo'); const electronBin = resolve(electronDir, 'node_modules/.bin/electron'); const electronProc = spawn(electronBin, [ electronDir, `--remote-debugging-port=${cdpPort}` diff --git a/tests/test-worker-debug.mjs b/tests/diagnostics/test-worker-debug.mjs similarity index 100% rename from tests/test-worker-debug.mjs rename to tests/diagnostics/test-worker-debug.mjs diff --git a/tests/test_wasm_exports.sh b/tests/test_wasm_exports.sh new file mode 100755 index 0000000..3c8d41e --- /dev/null +++ b/tests/test_wasm_exports.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Gate for the generated wasm export list: +# 1. the generator emits every known-core symbol, +# 2. every anyfs_ts_* string reference in the TS worker layer is exported +# (catches TS<->C drift that previously bit build_anyfs_browser_wasm.sh). +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +# shellcheck source=../scripts/lib/wasm_exports.sh +source "$root/scripts/lib/wasm_exports.sh" + +list="$(anyfs_wasm_exports "$root/ts/native/anyfs_ts.c")" + +for must in _main _malloc _free _anyfs_ts_kernel_init _anyfs_ts_init_async \ + _anyfs_ts_session_open _anyfs_ts_session_enter_async \ + _anyfs_ts_session_enter_result_p _anyfs_ts_pread_p _anyfs_ts_close_p; do + [[ ",$list," == *",$must,"* ]] || { echo "FAIL: $must missing from generated exports"; exit 1; } +done + +n="$(tr ',' '\n' <<<"$list" | grep -c '^_anyfs_ts_')" +[[ "$n" -ge 30 ]] || { echo "FAIL: only $n anyfs_ts_* exports (expected >= 30)"; exit 1; } + +# TS drift gate: every anyfs_ts_* string reference in the worker layer must be exported. +missing=0 +while IFS= read -r sym; do + [[ ",$list," == *",_$sym,"* ]] || { echo "FAIL: TS references $sym but it is not exported"; missing=1; } +done < <(grep -rhoE "'anyfs_ts_[a-z0-9_]+'" \ + "$root/ts/packages/core/src" | tr -d "'" | sort -u) +[[ "$missing" -eq 0 ]] + +echo "OK: $n anyfs_ts_* exports, TS string references all covered" diff --git a/tests/unit/test_path_dsl.c b/tests/unit/test_path_dsl.c new file mode 100644 index 0000000..1fdbb18 --- /dev/null +++ b/tests/unit/test_path_dsl.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Unit tests for the path DSL parser (src/core/anyfs_path.c). + * Pure userspace — no LKL boot, runs in milliseconds under `meson test`. */ +#include "anyfs_path.h" + +#include +#include + +static int failures; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, \ + __LINE__, #cond); \ + failures++; \ + } \ + } while (0) + +static void test_simple(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("p1", &ap) == 0); + CHECK(ap.n_comp == 1); + CHECK(ap.comp[0].p == 1); + CHECK(ap.comp[0].query == NULL); + CHECK(ap.disk_idx_set == 0); + anyfs_path_free(&ap); +} + +static void test_disk_prefix(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("disk0/p1", &ap) == 0); + CHECK(ap.disk_idx_set == 1); + CHECK(ap.disk_idx == 0); + CHECK(ap.n_comp == 1 && ap.comp[0].p == 1); + anyfs_path_free(&ap); + + CHECK(anyfs_path_parse("disk12/p2/p1", &ap) == 0); + CHECK(ap.disk_idx == 12); + CHECK(ap.n_comp == 2); + CHECK(ap.comp[0].p == 2 && ap.comp[1].p == 1); + anyfs_path_free(&ap); +} + +static void test_slashes_and_case(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("/p1/", &ap) == 0); /* leading+trailing ok */ + CHECK(ap.n_comp == 1 && ap.comp[0].p == 1); + anyfs_path_free(&ap); + + CHECK(anyfs_path_parse("P3", &ap) == 0); /* uppercase P accepted */ + CHECK(ap.comp[0].p == 3); + anyfs_path_free(&ap); +} + +static void test_query(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("p1?keyref=LUKS_KEY", &ap) == 0); + CHECK(ap.comp[0].query != NULL); + CHECK(strcmp(ap.comp[0].query, "keyref=LUKS_KEY") == 0); + anyfs_path_free(&ap); + + /* Query is percent-decoded in place; %2F must not re-split. */ + CHECK(anyfs_path_parse("p1?keyfile=%2Ftmp%2Fk", &ap) == 0); + CHECK(strcmp(ap.comp[0].query, "keyfile=/tmp/k") == 0); + anyfs_path_free(&ap); + + /* Bad escape in query is a parse error. */ + CHECK(anyfs_path_parse("p1?key=%zz", &ap) == -1); +} + +static void test_errors(void) +{ + AnyfsPath ap; + CHECK(anyfs_path_parse("", &ap) == -1); + CHECK(anyfs_path_parse("/", &ap) == -1); + CHECK(anyfs_path_parse("p0", &ap) == -1); /* index must be > 0 */ + CHECK(anyfs_path_parse("p", &ap) == -1); + CHECK(anyfs_path_parse("x1", &ap) == -1); + CHECK(anyfs_path_parse("disk0", &ap) == -1); /* disk alone invalid */ + CHECK(anyfs_path_parse("disk/p1", &ap) == -1); + CHECK(anyfs_path_parse("diskX/p1", &ap) == -1); + CHECK(anyfs_path_parse("p1x", &ap) == -1); + /* 9 components > ANYFS_PATH_MAX_COMPONENTS (8) */ + CHECK(anyfs_path_parse("p1/p1/p1/p1/p1/p1/p1/p1/p1", &ap) == -1); + CHECK(anyfs_path_parse(NULL, &ap) == -1); +} + +static void test_pct_decode(void) +{ + char a[] = "a%2Fb"; + CHECK(anyfs_path_pct_decode(a) == 0); + CHECK(strcmp(a, "a/b") == 0); + + char b[] = "plain"; + CHECK(anyfs_path_pct_decode(b) == 0); + CHECK(strcmp(b, "plain") == 0); + + char c[] = "%zz"; + CHECK(anyfs_path_pct_decode(c) == -1); + + char d[] = "%4"; /* truncated escape */ + CHECK(anyfs_path_pct_decode(d) == -1); + + CHECK(anyfs_path_pct_decode(NULL) == 0); +} + +int main(void) +{ + test_simple(); + test_disk_prefix(); + test_slashes_and_case(); + test_query(); + test_errors(); + test_pct_decode(); + if (failures) { + fprintf(stderr, "%d failure(s)\n", failures); + return 1; + } + printf("test_path_dsl: all OK\n"); + return 0; +} diff --git a/tests/unit/test_share_helpers.c b/tests/unit/test_share_helpers.c new file mode 100644 index 0000000..75e3388 --- /dev/null +++ b/tests/unit/test_share_helpers.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* Unit tests for the pure --share helpers (src/core/anyfs_share.c). */ +#include "anyfs_share.h" + +#include +#include + +static int failures; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, \ + __LINE__, #cond); \ + failures++; \ + } \ + } while (0) + +static void test_auto_name(void) +{ + char out[32]; + anyfs_share_auto_name("disk0/p1", out, sizeof(out)); + CHECK(strcmp(out, "disk0_p1") == 0); + + anyfs_share_auto_name("p2", out, sizeof(out)); + CHECK(strcmp(out, "p2") == 0); + + /* Truncation: output is always NUL-terminated. */ + char tiny[5]; + anyfs_share_auto_name("disk0/p1", tiny, sizeof(tiny)); + CHECK(strcmp(tiny, "disk") == 0); + + /* out_sz == 0 must not write anything (no crash). */ + anyfs_share_auto_name("x", tiny, 0); +} + +static void test_split(void) +{ + const char *name, *path; + + char a[] = "data=disk0/p1"; + CHECK(anyfs_share_split(a, &name, &path) == 0); + CHECK(name && strcmp(name, "data") == 0); + CHECK(strcmp(path, "disk0/p1") == 0); + + char b[] = "disk0/p1"; + CHECK(anyfs_share_split(b, &name, &path) == 0); + CHECK(name == NULL); + CHECK(strcmp(path, "disk0/p1") == 0); + + /* Only the FIRST '=' splits; the rest stays in path. */ + char c[] = "n=p1?key=v"; + CHECK(anyfs_share_split(c, &name, &path) == 0); + CHECK(strcmp(name, "n") == 0); + CHECK(strcmp(path, "p1?key=v") == 0); +} + +int main(void) +{ + test_auto_name(); + test_split(); + if (failures) { + fprintf(stderr, "%d failure(s)\n", failures); + return 1; + } + printf("test_share_helpers: all OK\n"); + return 0; +} diff --git a/ts/package.json b/ts/package.json index 6fd52de..ea4fbf9 100644 --- a/ts/package.json +++ b/ts/package.json @@ -6,6 +6,7 @@ "scripts": { "build": "pnpm -r --filter './packages/*' build", "test": "pnpm -r --filter './packages/*' test", + "test:integration": "node tests/integration/test-core.mjs", "format": "prettier --write '{packages,examples,tests}/**/*.{ts,tsx,mjs,js,json}'", "format:check": "prettier --check '{packages,examples,tests}/**/*.{ts,tsx,mjs,js,json}'" }, diff --git a/ts/packages/anyfs-native/test/smoke-url.mjs b/ts/packages/anyfs-native/test/smoke-url.mjs index df8ca54..830a646 100644 --- a/ts/packages/anyfs-native/test/smoke-url.mjs +++ b/ts/packages/anyfs-native/test/smoke-url.mjs @@ -83,10 +83,10 @@ if (!isMainThread) { process.exit(4); } - const meta = JSON.parse(n.sessionMetaJson(h)); + const meta = JSON.parse(await n.sessionMetaJson(h)); console.log('[smoke-url] sessionMeta:', meta); - const parts = JSON.parse(n.sessionListJson(h)); + const parts = JSON.parse(await n.sessionListJson(h)); console.log( '[smoke-url] partitions:', parts.length, @@ -101,7 +101,7 @@ if (!isMainThread) { const mount = n.sessionEnter(h, pick.index, 1); console.log(' mounted at', mount); - const entries = JSON.parse(n.readdirJson(mount)); + const entries = JSON.parse(await n.readdirJson(mount)); console.log('[smoke-url] readdir:', entries.length, 'entries'); console.log(' ', entries.slice(0, 5)); diff --git a/ts/packages/anyfs-native/test/smoke.mjs b/ts/packages/anyfs-native/test/smoke.mjs index 589a712..2fc7ff9 100644 --- a/ts/packages/anyfs-native/test/smoke.mjs +++ b/ts/packages/anyfs-native/test/smoke.mjs @@ -17,7 +17,7 @@ if (h < 0) { process.exit(4); } -const parts = JSON.parse(n.sessionListJson(h)); +const parts = JSON.parse(await n.sessionListJson(h)); console.log( '[smoke] partitions:', parts.length, @@ -25,7 +25,7 @@ console.log( ); if (parts.length === 0) process.exit(5); -const meta = JSON.parse(n.sessionMetaJson(h)); +const meta = JSON.parse(await n.sessionMetaJson(h)); console.log('[smoke] sessionMeta:', meta); // Pick the first non-journaled FS so RDONLY mount doesn't need replay. @@ -34,32 +34,32 @@ console.log(`[smoke] sessionEnter(part=${pick.index} ${pick.fstype}/${pick.label const mount = n.sessionEnter(h, pick.index, 1); // ANYFS_MOUNT_RDONLY console.log(' mounted at', mount); -const entries = JSON.parse(n.readdirJson(mount)); +const entries = JSON.parse(await n.readdirJson(mount)); console.log('[smoke] readdir:', entries.length, 'entries'); console.log(' ', entries.slice(0, 5)); -const meta2 = JSON.parse(n.lstatJson(mount)); +const meta2 = JSON.parse(await n.lstatJson(mount)); console.log('[smoke] lstat(mount):', { kind: meta2.kind, mode: meta2.mode.toString(8) }); const firstFile = entries.find((e) => e.kind === 'file'); if (firstFile) { const fpath = `${mount}/${firstFile.name}`; - const st = JSON.parse(n.statJson(fpath)); + const st = JSON.parse(await n.statJson(fpath)); console.log('[smoke] stat', firstFile.name, 'size=', st.size); - const fd = n.fileOpen(fpath, 0); + const fd = await n.fileOpen(fpath, 0); if (fd < 0) { console.error('open rc=', fd); process.exit(6); } - const buf = Buffer.alloc(Math.min(64, st.size)); - const got = n.pread(fd, buf, buf.length, 0); + // pread(fd, n, off) → Promise<{ rc, data }> (buffer allocated by the addon) + const { rc: got, data } = await n.pread(fd, Math.min(64, st.size), 0); console.log( '[smoke] pread got=', got, 'first bytes:', - buf.subarray(0, Math.min(16, got)).toString('hex'), + data.subarray(0, Math.min(16, Math.max(got, 0))).toString('hex'), ); - n.fileClose(fd); + await n.fileClose(fd); } else { console.log('[smoke] no regular files in mount root (skipping pread)'); } diff --git a/ts/packages/core/package.json b/ts/packages/core/package.json index b3a9a45..0677c1d 100644 --- a/ts/packages/core/package.json +++ b/ts/packages/core/package.json @@ -31,7 +31,8 @@ ], "scripts": { "build": "tsup", - "test": "node test/smoke.node.mjs single && node test/smoke.node.mjs multi && node test/smoke.node.mjs big" + "test:unit": "node --test test/*.test.mjs", + "test": "npm run test:unit && node test/smoke.node.mjs single && node test/smoke.node.mjs multi && node test/smoke.node.mjs big" }, "devDependencies": { "typescript": "^5.5.4", diff --git a/ts/packages/core/src/index.ts b/ts/packages/core/src/index.ts index 846352b..cf9a065 100644 --- a/ts/packages/core/src/index.ts +++ b/ts/packages/core/src/index.ts @@ -22,6 +22,7 @@ export { NativeSession, getAnyfsNative } from './native-session.js'; export type { AnyfsNativeBridge } from './native-session.js'; export { NodeWasmSession } from './node-wasm-session.js'; export type { AnyfsSession } from './session.js'; +export { AnyfsSessionBase } from './session-base.js'; export { applyUrlProxy, getUrlProxyPrefix } from './electron-proxy.js'; export { createSession } from './dispatch.js'; export type { WasmCaps, SessionEnv, SessionBackend, DispatchResult } from './dispatch.js'; diff --git a/ts/packages/core/test/dispatch.test.mjs b/ts/packages/core/test/dispatch.test.mjs new file mode 100644 index 0000000..57a23f8 --- /dev/null +++ b/ts/packages/core/test/dispatch.test.mjs @@ -0,0 +1,66 @@ +import { test, afterEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { createSession } from '../dist/index.js'; + +afterEach(() => { + delete globalThis.anyfsNative; +}); + +// getAnyfsNative() (src/native-session.ts) detects the bridge by checking +// `typeof g.init === 'function'` — the fake must provide init() or the +// electron path silently falls back to wasm. +const fakeBridge = { init() {}, sessionOpen() {}, kernelInit() {} }; + +test('web → wasm, blob+url only, no bridge', () => { + const r = createSession('web'); + assert.equal(r.backend, 'wasm'); + assert.deepEqual(r.allowedKinds, new Set(['blob', 'url'])); + assert.equal(r.nativeBridge, undefined); + assert.equal(typeof r.wasmCaps, 'object'); +}); + +test('node → node-wasm, path only', () => { + const r = createSession('node'); + assert.equal(r.backend, 'node-wasm'); + assert.deepEqual(r.allowedKinds, new Set(['path'])); +}); + +test('electron + bridge → native, path+url', () => { + globalThis.anyfsNative = fakeBridge; + const r = createSession('electron'); + assert.equal(r.backend, 'native'); + assert.equal(r.nativeBridge, fakeBridge); + assert.deepEqual(r.allowedKinds, new Set(['path', 'url'])); +}); + +test('electron + bridge + disableNative → wasm fallback', () => { + globalThis.anyfsNative = fakeBridge; + const r = createSession('electron', { disableNative: true }); + assert.equal(r.backend, 'wasm'); + assert.deepEqual(r.allowedKinds, new Set(['blob', 'url'])); +}); + +test('electron without bridge → wasm fallback', () => { + const r = createSession('electron'); + assert.equal(r.backend, 'wasm'); + assert.deepEqual(r.allowedKinds, new Set(['blob', 'url'])); +}); + +test('electron wasm + pathLoopbackUrl cap → path allowed', () => { + const r = createSession('electron', { + disableNative: true, + electronWasmCaps: { pathLoopbackUrl: 'http://127.0.0.1:9999/d0' }, + }); + assert.equal(r.backend, 'wasm'); + assert.ok(r.allowedKinds.has('path')); + assert.equal(r.wasmCaps.pathLoopbackUrl, 'http://127.0.0.1:9999/d0'); +}); + +test('electron wasm caps forwarded verbatim (urlProxyPrefix)', () => { + const r = createSession('electron', { + disableNative: true, + electronWasmCaps: { urlProxyPrefix: 'anyfs-url://proxy/?u=' }, + }); + assert.equal(r.wasmCaps.urlProxyPrefix, 'anyfs-url://proxy/?u='); + assert.ok(!r.allowedKinds.has('path')); // no loopback cap -> no path +}); diff --git a/ts/packages/core/test/dispatch.test.ts b/ts/packages/core/test/dispatch.test.ts deleted file mode 100644 index 6b0fb40..0000000 --- a/ts/packages/core/test/dispatch.test.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { strictEqual, deepStrictEqual } from 'node:assert'; -import { createSession } from '../src/dispatch.js'; - -let passed = 0; -let failed = 0; - -function test(name: string, fn: () => void) { - try { - fn(); - passed++; - console.log(` ✓ ${name}`); - } catch (e) { - failed++; - console.error(` ✗ ${name}: ${(e as Error).message}`); - } -} - -// web: wasm backend, blob + url only -test('web → wasm backend', () => { - const r = createSession('web'); - strictEqual(r.backend, 'wasm'); -}); - -test('web → allowed blob+url', () => { - const r = createSession('web'); - deepStrictEqual(r.allowedKinds, new Set(['blob', 'url'])); -}); - -test('web → no nativeBridge', () => { - const r = createSession('web'); - strictEqual(r.nativeBridge, undefined); -}); - -test('web → wasmCaps is present (empty)', () => { - const r = createSession('web'); - strictEqual(typeof r.wasmCaps, 'object'); -}); - -// node: node-wasm backend, path only -test('node → node-wasm backend', () => { - const r = createSession('node'); - strictEqual(r.backend, 'node-wasm'); -}); - -test('node → allowed path only', () => { - const r = createSession('node'); - deepStrictEqual(r.allowedKinds, new Set(['path'])); -}); - -test('node → no nativeBridge', () => { - const r = createSession('node'); - strictEqual(r.nativeBridge, undefined); -}); - -// electron: no bridge → wasm fallback -test('electron (no bridge) → wasm backend', () => { - const r = createSession('electron'); - strictEqual(r.backend, 'wasm'); -}); - -test('electron (no bridge) → allowed blob+url (no pathLoopbackUrl)', () => { - const r = createSession('electron'); - deepStrictEqual(r.allowedKinds, new Set(['blob', 'url'])); -}); - -// F8 contract: wasm-under-electron can take a blob source (local files stay -// {kind:'blob'} when native is disabled, mounted via WORKERFS like the web path). -test('electron (no bridge) → allowedKinds includes blob (F8)', () => { - const r = createSession('electron'); - strictEqual(r.allowedKinds.has('blob'), true); -}); - -// electron with pathLoopbackUrl: add path to allowed -test('electron (wasm + pathLoopbackUrl) → allowed blob+url+path', () => { - const r = createSession('electron', { - electronWasmCaps: { pathLoopbackUrl: 'http://127.0.0.1:12345/token123' }, - }); - deepStrictEqual(r.allowedKinds, new Set(['blob', 'url', 'path'])); -}); - -// electron with disableNative: force wasm even if bridge present -test('electron (disableNative) → wasm backend', () => { - const r = createSession('electron', { disableNative: true }); - strictEqual(r.backend, 'wasm'); -}); - -console.log(`\n${passed} passed, ${failed} failed`); -if (failed > 0) process.exit(1); diff --git a/ts/packages/core/test/format.test.mjs b/ts/packages/core/test/format.test.mjs new file mode 100644 index 0000000..3802577 --- /dev/null +++ b/ts/packages/core/test/format.test.mjs @@ -0,0 +1,35 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { fmtBytes, fmtMode, splitExt, formatSize } from '../dist/index.js'; + +test('fmtBytes tiers', () => { + assert.equal(fmtBytes(512), '512 B'); + assert.equal(fmtBytes(2048), '2048 B (2.0 KiB)'); + assert.match(fmtBytes(5 * 1024 * 1024), /5\.0 MiB/); + assert.match(fmtBytes(3 * 1024 ** 3), /3\.00 GiB/); +}); + +test('fmtMode common cases', () => { + assert.equal(fmtMode(0o100644), '-rw-r--r-- (0644)'); + assert.equal(fmtMode(0o040755), 'drwxr-xr-x (0755)'); + assert.equal(fmtMode(0o120777), 'lrwxrwxrwx (0777)'); + // fmtMode always prefixes a literal "0" to the octal of (mode & 0o7777), + // so 4-digit special modes render as 5 chars: (04755), (01777). + assert.equal(fmtMode(0o104755), '-rwsr-xr-x (04755)'); // setuid + assert.equal(fmtMode(0o041777), 'drwxrwxrwt (01777)'); // sticky /tmp +}); + +test('splitExt — Chonky-bug-safe rules', () => { + assert.equal(splitExt('a.txt'), '.txt'); + assert.equal(splitExt('noext'), ''); // no dot -> '' + assert.equal(splitExt('.bashrc'), ''); // dotfile -> '' + assert.equal(splitExt('.pwd.lock'), '.lock'); // later dot splits + assert.equal(splitExt('trailing.'), ''); // trailing dot -> '' +}); + +test('formatSize adaptive units', () => { + assert.equal(formatSize(undefined), ''); + assert.equal(formatSize(0), '0 B'); + assert.equal(formatSize(1536), '1.5 KiB'); + assert.equal(formatSize(10 * 1024 * 1024), '10 MiB'); +}); diff --git a/ts/packages/core/test/session-base.test.mjs b/ts/packages/core/test/session-base.test.mjs new file mode 100644 index 0000000..350d4c1 --- /dev/null +++ b/ts/packages/core/test/session-base.test.mjs @@ -0,0 +1,141 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { AnyfsSessionBase } from '../dist/index.js'; + +/** In-memory tree: dirs map path -> entries; files map path -> bytes. */ +class FakeSession extends AnyfsSessionBase { + constructor({ dirs = {}, files = {}, symlinkSizes = {} } = {}) { + super(); + this.dirs = dirs; + this.files = files; + this.symlinkSizes = symlinkSizes; // lstat-size != follow-size repro + this.openCount = 0; + this.closedFds = []; + } + async attachBlob() {} + async attachUrl() {} + async attachPath() {} + async enter() { + return '/mnt'; + } + async listParts() { + return []; + } + async meta() { + return {}; + } + async readdir(path) { + const e = this.dirs[path]; + if (!e) throw new Error(`ENOENT ${path}`); + return e; + } + async stat(path) { + if (path in this.symlinkSizes) return { size: this.symlinkSizes[path], mode: 0o120777 }; + return { size: this.files[path]?.length ?? 0, mode: 0o100644 }; + } + async statFollow(path) { + return { size: this.files[path]?.length ?? 0, mode: 0o100644 }; + } + async readlink() { + return ''; + } + async realpath(p) { + return p; + } + async readKernelFile() { + return ''; + } + onProgress() { + return () => {}; + } + async _openFdRaw() { + this.openCount++; + return this.openCount; + } + async _readFdRaw(fd, offset, length) { + return this.currentBytes.subarray(offset, offset + length); + } + async _closeFdRaw(fd) { + this.closedFds.push(fd); + } + async _dispose() { + this.disposedCalled = true; + } +} + +async function drain(stream) { + const chunks = []; + for await (const c of stream) chunks.push(c); + return chunks; +} + +test('openReadable sizes from statFollow, not lstat (symlink truncation bug)', async () => { + const data = new Uint8Array(100).fill(7); + const s = new FakeSession({ + files: { '/etc/os-release': data }, + symlinkSizes: { '/etc/os-release': 21 }, // lstat sees link-target length + }); + s.currentBytes = data; + const { stream, size } = await s.openReadable('/etc/os-release'); + assert.equal(size, 100); // would be 21 if base used stat() + const chunks = await drain(stream); + assert.equal( + chunks.reduce((n, c) => n + c.length, 0), + 100, + ); +}); + +test('openReadable chunks and closes the fd exactly once', async () => { + const data = new Uint8Array(2500).fill(1); + const s = new FakeSession({ files: { '/f': data } }); + s.currentBytes = data; + const { stream } = await s.openReadable('/f', { chunkSize: 1000 }); + const chunks = await drain(stream); + assert.deepEqual( + chunks.map((c) => c.length), + [1000, 1000, 500], + ); + assert.deepEqual(s.closedFds, [1]); +}); + +test('openReadable cancel closes the fd', async () => { + const data = new Uint8Array(5000).fill(1); + const s = new FakeSession({ files: { '/f': data } }); + s.currentBytes = data; + const { stream } = await s.openReadable('/f', { chunkSize: 1000 }); + const reader = stream.getReader(); + await reader.read(); + await reader.cancel(); + assert.deepEqual(s.closedFds, [1]); +}); + +test('walk is BFS, skips unreadable dirs, honors chunkSize', async () => { + const s = new FakeSession({ + dirs: { + '/': [ + { name: 'a', kind: 'dir' }, + { name: 'f1', kind: 'file' }, + ], + '/a': [ + { name: 'f2', kind: 'file' }, + { name: 'bad', kind: 'dir' }, + ], + // '/a/bad' missing -> readdir throws -> silently skipped + }, + }); + const seen = []; + for await (const chunk of s.walk('/', 2)) seen.push(...chunk); + assert.deepEqual(seen, ['/a', '/f1', '/a/f2', '/a/bad']); +}); + +test('close() closes tracked fds, disposes once, and poisons the session', async () => { + const data = new Uint8Array(10); + const s = new FakeSession({ files: { '/f': data } }); + s.currentBytes = data; + await s.openFd('/f'); + await s.close(); + assert.deepEqual(s.closedFds, [1]); + assert.equal(s.disposedCalled, true); + await assert.rejects(() => s.openFd('/f'), /already disposed/); + await s.close(); // idempotent +}); diff --git a/ts/packages/core/test/smoke.node.mjs b/ts/packages/core/test/smoke.node.mjs index 68152b0..72055a5 100644 --- a/ts/packages/core/test/smoke.node.mjs +++ b/ts/packages/core/test/smoke.node.mjs @@ -1,10 +1,17 @@ // Minimal Node smoke test for the wasm bundle. -// Exercises: createAnyfsModule() -> NODEFS mount -> anyfs_ts_kernel_init -> -// anyfs_ts_session_open + anyfs_ts_session_list_json against disk_multi.img. -import { createRequire } from 'node:module'; +// Exercises: createAnyfsModule() -> NODEFS mount -> kernel boot -> +// anyfs_ts_session_open_p + anyfs_ts_session_list_json_p against disk images. +// +// Call pattern mirrors src/worker.ts: every entry point that can touch the +// QEMU block layer goes through the `_p` out-pointer variant with +// ccall({async: true}). The block layer runs on emscripten fibers; a fiber +// swap (or a kernel-side wait that defers to the event loop) leaves the +// synchronous ccall path mid-unwind ("running asynchronously" assertion) +// and discards the export's return value. Sync ccall is only safe for +// calls that never reach the block layer (init/poll/halt). +import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -const require = createRequire(import.meta.url); const { default: createAnyfsModule } = await import( new URL('../wasm/anyfs.node.mjs', import.meta.url).href @@ -22,11 +29,16 @@ const IMAGES = { }; const which = process.argv[2] || 'multi'; -const imgHost = IMAGES[which]; -if (!imgHost) { +const imgLink = IMAGES[which]; +if (!imgLink) { console.error('unknown image:', which, 'choose: single|multi|big'); process.exit(2); } +// Some disk images are symlinks pointing outside DISKS_DIR. NODEFS exposes +// the symlink as-is, and Emscripten's VFS resolves its target inside the +// wasm namespace (where it doesn't exist) — so mount the realpath'd +// directory and open the real file name instead. +const imgHost = fs.realpathSync(imgLink); console.log('[smoke] loading wasm module…'); const M = await createAnyfsModule({ @@ -39,22 +51,51 @@ const M = await createAnyfsModule({ }); console.log('[smoke] module loaded; main() ran automatically'); -console.log('[smoke] anyfs_ts_kernel_init(64, 0)…'); -const rc = M.ccall('anyfs_ts_kernel_init', 'number', ['number', 'number'], [64, 0]); -console.log(' rc =', rc); -if (rc !== 0) process.exit(3); +// Async ccall against a `_p` out-pointer variant (same shape as worker.ts +// callP): the wasm export's direct return value is discarded by the fiber +// rewind path, so the C side writes the result through the trailing +// int32_t* out parameter. +const outp = M._malloc(4); +async function callP(name, argTypes, args) { + M.HEAP32[outp >> 2] = -0x7fffffff; + await M.ccall(name, null, [...argTypes, 'number'], [...args, outp], { async: true }); + return M.HEAP32[outp >> 2]; +} +async function callA(name, retType, argTypes, args) { + return await M.ccall(name, retType, argTypes, args, { async: true }); +} +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Boot: prefer the dedicated-pthread async path when the bundle exports it +// (keeps this thread's event loop free to service kthread spawns). +if (M._anyfs_ts_init_async) { + console.log('[smoke] anyfs_ts_init_async(64, 0)…'); + const arc = M.ccall('anyfs_ts_init_async', 'number', ['number', 'number'], [64, 0]); + if (arc !== 0) process.exit(3); + for (let i = 0; i < 600; i++) { + if (M.ccall('anyfs_ts_is_boot_complete', 'number', [], [])) break; + await sleep(100); + } + const rc = M.ccall('anyfs_ts_boot_result', 'number', [], []); + console.log(' rc =', rc); + if (rc !== 0) process.exit(3); +} else { + console.log('[smoke] anyfs_ts_kernel_init(64, 0)…'); + const rc = await callA('anyfs_ts_kernel_init', 'number', ['number', 'number'], [64, 0]); + console.log(' rc =', rc); + if (rc !== 0) process.exit(3); +} const fsPath = '/work/' + path.basename(imgHost); -console.log('[smoke] anyfs_ts_session_open(', fsPath, ', 0)…'); -const h = M.ccall('anyfs_ts_session_open', 'number', ['string', 'number'], [fsPath, 0]); +console.log('[smoke] anyfs_ts_session_open_p(', fsPath, ', 0)…'); +const h = await callP('anyfs_ts_session_open_p', ['string', 'number'], [fsPath, 0]); console.log(' handle =', h); if (h < 0) process.exit(4); const cap = 4096; const bufPtr = M._malloc(cap); -const n = M.ccall( - 'anyfs_ts_session_list_json', - 'number', +const n = await callP( + 'anyfs_ts_session_list_json_p', ['number', 'number', 'number'], [h, bufPtr, cap], ); @@ -78,12 +119,40 @@ const exerciseEntry = let mountPath; const mountBuf = M._malloc(128); const enterPart = exerciseEntry.mountWhole ? 0 : exerciseEntry.part; -const rc2 = M.ccall( - 'anyfs_ts_session_enter', - 'number', - ['number', 'number', 'number', 'number', 'number'], - [h, enterPart, 0, mountBuf, 128], -); +// Prefer the dedicated-pthread enter (ext4's jbd2 kthread can't spawn while +// the entering thread is blocked); fall back to the _p variant. +let rc2; +if (M._anyfs_ts_session_enter_async) { + rc2 = M.ccall( + 'anyfs_ts_session_enter_async', + 'number', + ['number', 'number', 'number'], + [h, enterPart, 0], + ); + if (rc2 === 0) { + let done = 0; + for (let i = 0; i < 600; i++) { + done = M.ccall('anyfs_ts_session_enter_is_complete', 'number', [], []); + if (done) break; + await sleep(100); + } + if (!done) { + console.error('session_enter timed out'); + process.exit(6); + } + rc2 = await callP( + 'anyfs_ts_session_enter_result_p', + ['number', 'number'], + [mountBuf, 128], + ); + } +} else { + rc2 = await callP( + 'anyfs_ts_session_enter_p', + ['number', 'number', 'number', 'number', 'number'], + [h, enterPart, 0, mountBuf, 128], + ); +} console.log('[smoke] session_enter rc =', rc2); if (rc2 !== 0) process.exit(6); mountPath = M.UTF8ToString(mountBuf); @@ -91,9 +160,8 @@ M._free(mountBuf); console.log(' mounted at', mountPath); const ddBuf = M._malloc(8192); -const ddRc = M.ccall( - 'anyfs_ts_readdir_json', - 'number', +const ddRc = await callP( + 'anyfs_ts_readdir_json_p', ['string', 'number', 'number'], [mountPath, ddBuf, 8192], ); @@ -109,24 +177,24 @@ M._free(ddBuf); // If big_ext4, also pread the first 16 bytes of big.bin. if (which === 'big') { const fdPath = mountPath + '/big.bin'; - const fd = M.ccall('anyfs_ts_open', 'number', ['string', 'number'], [fdPath, 0]); + const fd = await callP('anyfs_ts_open_p', ['string', 'number'], [fdPath, 0]); console.log('[smoke] open(big.bin) fd =', fd); if (fd < 0) process.exit(8); const rbuf = M._malloc(16); - const got = M.ccall( - 'anyfs_ts_pread', - 'number', - ['number', 'number', 'number', 'bigint'], - [fd, rbuf, 16, 0n], + const got = await callP( + 'anyfs_ts_pread_p', + ['number', 'number', 'number', 'number', 'number'], + [fd, rbuf, 16, 0, 0], ); console.log('[smoke] pread(16,0) ret =', got); const bytes = new Uint8Array(M.HEAPU8.buffer, rbuf, 16).slice(); console.log(' bytes =', Buffer.from(bytes).toString('hex')); M._free(rbuf); - M.ccall('anyfs_ts_close', 'number', ['number'], [fd]); + await callP('anyfs_ts_close_p', ['number'], [fd]); } -M.ccall('anyfs_ts_session_close', 'number', ['number'], [h]); -M.ccall('anyfs_ts_kernel_halt', 'number', [], []); +await callA('anyfs_ts_session_close', 'number', ['number'], [h]); +await callA('anyfs_ts_kernel_halt', 'number', [], []); +M._free(outp); console.log('[smoke] OK'); process.exit(0); diff --git a/ts/packages/nbd-proxy/bin/anyfs-nbd-proxy.ts b/ts/packages/nbd-proxy/bin/anyfs-nbd-proxy.ts new file mode 100644 index 0000000..1a6a503 --- /dev/null +++ b/ts/packages/nbd-proxy/bin/anyfs-nbd-proxy.ts @@ -0,0 +1,87 @@ +#!/usr/bin/env node +import { createDataSource, type DataSourceSpec } from '../src/data-source.js'; +import { serveOnFd, serveOnLoopback } from '../src/endpoint.js'; + +interface Args { + source: 'file' | 'blockdev' | 'url' | undefined; + target: string | undefined; + fd: number | undefined; + port: number | undefined; +} + +function emptyArgs(): Args { + return { source: undefined, target: undefined, fd: undefined, port: undefined }; +} + +function parse(argv: string[]): Args { + const a = emptyArgs(); + for (let i = 0; i < argv.length; i++) { + const k = argv[i]; + const v = argv[i + 1]; + switch (k) { + case '--source': + a.source = v as Args['source']; + i++; + break; + case '--target': + a.target = v; + i++; + break; + case '--fd': + a.fd = Number(v); + i++; + break; + case '--port': + a.port = Number(v); + i++; + break; + case '--help': + case '-h': + usage(); + process.exit(0); + } + } + return a; +} + +function usage(): void { + process.stderr.write( + 'Usage: anyfs-nbd-proxy --source file|blockdev|url --target ' + + '(--fd | --port

)\n', + ); +} + +async function main(): Promise { + const a = parse(process.argv.slice(2)); + if (!a.source || !a.target || (a.fd === undefined && a.port === undefined)) { + usage(); + process.exit(2); + } + const source = await createDataSource({ + kind: a.source, + target: a.target, + } as DataSourceSpec); + + if (a.fd !== undefined) { + await serveOnFd(a.fd, source); + await source.close(); + return; + } + const { port, stop } = await serveOnLoopback(source, a.port); + process.stdout.write(`${port}\n`); /* report the bound port on stdout */ + const shutdown = async () => { + await stop(); + await source.close(); + process.exit(0); + }; + process.on('SIGTERM', shutdown); + process.on('SIGINT', shutdown); + /* Lifecycle binding: exit when stdin closes (parent gone). */ + process.stdin.on('end', shutdown); + process.stdin.resume(); +} + +main().catch((e) => { + process.stderr.write(`anyfs-nbd-proxy: ${e instanceof Error ? e.message : String(e)}\n`); + process.exit(1); +}); diff --git a/ts/packages/nbd-proxy/package.json b/ts/packages/nbd-proxy/package.json new file mode 100644 index 0000000..f4064e4 --- /dev/null +++ b/ts/packages/nbd-proxy/package.json @@ -0,0 +1,27 @@ +{ + "name": "@anyfs/nbd-proxy", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "./dist/src/index.js", + "types": "./dist/src/index.d.ts", + "bin": { + "anyfs-nbd-proxy": "./dist/bin/anyfs-nbd-proxy.js" + }, + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "default": "./dist/src/index.js" + } + }, + "files": ["dist"], + "scripts": { + "build": "tsup", + "test": "node test/unit.mjs && node test/integration.mjs" + }, + "devDependencies": { + "typescript": "^5.5.4", + "tsup": "^8.3.0", + "@types/node": "^22.0.0" + } +} diff --git a/ts/packages/nbd-proxy/src/data-source.ts b/ts/packages/nbd-proxy/src/data-source.ts new file mode 100644 index 0000000..19229d5 --- /dev/null +++ b/ts/packages/nbd-proxy/src/data-source.ts @@ -0,0 +1,40 @@ +import { Buffer } from 'node:buffer'; + +/** A read-only random-access byte source backing an NBD export. */ +export interface DataSource { + /** Total export size in bytes. */ + size(): Promise; + /** Read exactly `length` bytes at `offset`. Rejects on IO error. */ + read(offset: number, length: number): Promise; + /** Release any held resources (fd, http agent, ...). */ + close(): Promise; +} + +export type DataSourceSpec = + | { kind: 'file'; target: string } + | { kind: 'blockdev'; target: string } + | { kind: 'url'; target: string }; + +export async function createDataSource(spec: DataSourceSpec): Promise { + switch (spec.kind) { + case 'file': { + const { FileSource } = await import('./sources/file.js'); + return FileSource.open(spec.target); + } + case 'blockdev': { + const { BlockDeviceSource } = await import('./sources/blockdev.js'); + return BlockDeviceSource.open(spec.target); + } + case 'url': { + const { HttpSource } = await import('./sources/http.js'); + return HttpSource.open(spec.target); + } + default: { + /* Reachable only when an invalid `kind` is forced past the type system + * (e.g. the CLI's `as DataSourceSpec` cast on unvalidated input). Fail + * with a clear message instead of returning undefined. */ + const bad = spec as { kind?: unknown }; + throw new Error(`createDataSource: unknown source kind: ${String(bad.kind)}`); + } + } +} diff --git a/ts/packages/nbd-proxy/src/endpoint.ts b/ts/packages/nbd-proxy/src/endpoint.ts new file mode 100644 index 0000000..6ef57ee --- /dev/null +++ b/ts/packages/nbd-proxy/src/endpoint.ts @@ -0,0 +1,34 @@ +import net from 'node:net'; +import type { DataSource } from './data-source.js'; +import { serveNbd } from './nbd-server.js'; + +/** Serve NBD on an already-open socket fd (inherited from the parent). */ +export async function serveOnFd(fd: number, source: DataSource): Promise { + const size = await source.size(); + const socket = new net.Socket({ fd }); + return serveNbd(socket, source, { size }); +} + +/** + * Serve NBD on a 127.0.0.1 loopback listener. Resolves with the bound port + * and a stop() that closes the listener. Each connection is served from the + * same source. + */ +export async function serveOnLoopback( + source: DataSource, + port = 0, +): Promise<{ port: number; stop: () => Promise }> { + const size = await source.size(); + const server = net.createServer((sock) => { + serveNbd(sock, source, { size }).catch(() => {}); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, '127.0.0.1', resolve); + }); + const addr = server.address() as net.AddressInfo; + return { + port: addr.port, + stop: () => new Promise((r) => server.close(() => r())), + }; +} diff --git a/ts/packages/nbd-proxy/src/index.ts b/ts/packages/nbd-proxy/src/index.ts new file mode 100644 index 0000000..46f612d --- /dev/null +++ b/ts/packages/nbd-proxy/src/index.ts @@ -0,0 +1,7 @@ +export type { DataSource, DataSourceSpec } from './data-source.js'; +export { createDataSource } from './data-source.js'; +export { BlockDeviceSource } from './sources/blockdev.js'; +export { FileSource } from './sources/file.js'; +export { HttpSource } from './sources/http.js'; +export { serveNbd } from './nbd-server.js'; +export { serveOnFd, serveOnLoopback } from './endpoint.js'; diff --git a/ts/packages/nbd-proxy/src/nbd-server.ts b/ts/packages/nbd-proxy/src/nbd-server.ts new file mode 100644 index 0000000..02f677f --- /dev/null +++ b/ts/packages/nbd-proxy/src/nbd-server.ts @@ -0,0 +1,238 @@ +import { Buffer } from 'node:buffer'; +import type { Socket } from 'node:net'; +import type { DataSource } from './data-source.js'; + +const NBDMAGIC = 0x4e42444d41474943n; +const IHAVEOPT = 0x49484156454f5054n; +const OPT_REPLY_MAGIC = 0x0003e889045565a9n; +const REQ_MAGIC = 0x25609513; +const SIMPLE_REPLY_MAGIC = 0x67446698; + +const NBD_FLAG_FIXED_NEWSTYLE = 1; +const NBD_FLAG_NO_ZEROES = 2; + +const NBD_OPT_ABORT = 2; +const NBD_OPT_INFO = 6; +const NBD_OPT_GO = 7; + +const NBD_REP_ACK = 1; +const NBD_REP_INFO = 3; +const NBD_REP_ERR_UNSUP = 0x80000001; + +const NBD_INFO_EXPORT = 0; +const NBD_FLAG_HAS_FLAGS = 1; +const NBD_FLAG_READ_ONLY = 2; + +const NBD_CMD_READ = 0; +const NBD_CMD_DISC = 2; +const NBD_CMD_FLUSH = 3; + +const EINVAL = 22; +const EIO = 5; + +const MAX_IN_FLIGHT = 16; + +/** Serial byte reader over a socket (single outstanding read at a time). */ +function makeReader(socket: Socket): (n: number) => Promise { + let chunks: Buffer[] = []; + let buffered = 0; + let want = 0; + let resolveWant: ((b: Buffer) => void) | null = null; + let rejectWant: ((e: Error) => void) | null = null; + let ended = false; + let errored: Error | null = null; + + function tryResolve() { + if (resolveWant && buffered >= want) { + const all = Buffer.concat(chunks); + const out = all.subarray(0, want); + const rest = all.subarray(want); + chunks = rest.length ? [Buffer.from(rest)] : []; + buffered = rest.length; + const r = resolveWant; + resolveWant = rejectWant = null; + r(out); + } else if (resolveWant && (ended || errored)) { + const rej = rejectWant!; + resolveWant = rejectWant = null; + rej(errored ?? new Error('EOF')); + } + } + + socket.on('data', (d: Buffer) => { + chunks.push(d); + buffered += d.length; + tryResolve(); + }); + socket.on('end', () => { + ended = true; + tryResolve(); + }); + socket.on('error', (e: Error) => { + errored = e; + tryResolve(); + }); + + return (n: number) => + new Promise((resolve, reject) => { + want = n; + resolveWant = resolve; + rejectWant = reject; + tryResolve(); + }); +} + +/** Serialized reply-frame writer: one frame on the wire at a time. */ +function makeWriter(socket: Socket) { + let chain: Promise = Promise.resolve(); + return (buf: Buffer): Promise => { + chain = chain.then( + () => + new Promise((res, rej) => + socket.write(buf, (e) => (e ? rej(e) : res())), + ), + ); + return chain; + }; +} + +function simpleReply(error: number, handle: Buffer, data: Buffer | null): Buffer { + const hdr = Buffer.alloc(16); + hdr.writeUInt32BE(SIMPLE_REPLY_MAGIC, 0); + hdr.writeUInt32BE(error >>> 0, 4); + handle.copy(hdr, 8); + return data ? Buffer.concat([hdr, data]) : hdr; +} + +function optReply(opt: number, repType: number, payload: Buffer): Buffer { + const fixed = Buffer.alloc(16); + fixed.writeBigUInt64BE(OPT_REPLY_MAGIC, 0); + fixed.writeUInt32BE(opt, 8); + fixed.writeUInt32BE(repType >>> 0, 12); + const lenBuf = Buffer.alloc(4); + lenBuf.writeUInt32BE(payload.length, 0); + return Buffer.concat([fixed, lenBuf, payload]); +} + +/** + * Serve one client on `socket` from `source`. Resolves on clean DISC/EOF. + * Reads are dispatched concurrently (up to 16 in flight); replies are written + * in completion order, keyed by the 8-byte handle. + */ +export async function serveNbd( + socket: Socket, + source: DataSource, + opts: { size: number; onRead?: (offset: number, length: number) => void }, +): Promise { + const read = makeReader(socket); + const write = makeWriter(socket); + const size = opts.size; + + /* Handshake */ + const hello = Buffer.alloc(18); + hello.writeBigUInt64BE(NBDMAGIC, 0); + hello.writeBigUInt64BE(IHAVEOPT, 8); + hello.writeUInt16BE(NBD_FLAG_FIXED_NEWSTYLE | NBD_FLAG_NO_ZEROES, 16); + await write(hello); + await read(4); /* client flags */ + + /* Option haggling until GO (enter transmission) or ABORT */ + for (;;) { + const optHdr = await read(16); + if (optHdr.readBigUInt64BE(0) !== IHAVEOPT) throw new Error('bad option magic'); + const opt = optHdr.readUInt32BE(8); + const optLen = optHdr.readUInt32BE(12); + if (optLen) await read(optLen); + + if (opt === NBD_OPT_GO || opt === NBD_OPT_INFO) { + const info = Buffer.alloc(12); + info.writeUInt16BE(NBD_INFO_EXPORT, 0); + info.writeBigUInt64BE(BigInt(size), 2); + info.writeUInt16BE(NBD_FLAG_HAS_FLAGS | NBD_FLAG_READ_ONLY, 10); + await write(optReply(opt, NBD_REP_INFO, info)); + await write(optReply(opt, NBD_REP_ACK, Buffer.alloc(0))); + if (opt === NBD_OPT_GO) break; + } else if (opt === NBD_OPT_ABORT) { + await write(optReply(opt, NBD_REP_ACK, Buffer.alloc(0))); + socket.end(); + return; + } else { + await write(optReply(opt, NBD_REP_ERR_UNSUP, Buffer.alloc(0))); + } + } + + /* Transmission: dispatch reads concurrently, reply in completion order */ + let inFlight = 0; + const pending = new Set>(); + let slotWaiter: (() => void) | null = null; + + for (;;) { + if (inFlight >= MAX_IN_FLIGHT) { + await new Promise((r) => (slotWaiter = r)); + } + let hdr: Buffer; + try { + hdr = await read(28); + } catch { + break; /* EOF/disconnect */ + } + if (hdr.readUInt32BE(0) !== REQ_MAGIC) throw new Error('bad request magic'); + const type = hdr.readUInt16BE(6); + const handle = Buffer.from(hdr.subarray(8, 16)); /* copy: detach from reader buffer */ + const offset = hdr.readBigUInt64BE(16); + const length = hdr.readUInt32BE(24); + + if (type === NBD_CMD_DISC) break; + + if (type === NBD_CMD_READ) { + inFlight++; + const job = (async () => { + try { + if (offset + BigInt(length) > BigInt(size)) { + await write(simpleReply(EINVAL, handle, null)); + return; + } + const data = await source.read(Number(offset), length); + /* A short read would desync the client (reply header claims success + * but carries fewer bytes than requested). The upstream bounds check + * + a fixed export size make this unreachable in normal operation; + * treat it as an IO error rather than emit a malformed frame. */ + if (data.length !== length) throw new Error('short read'); + opts.onRead?.(Number(offset), length); + await write(simpleReply(0, handle, data)); + } catch { + /* The reply write itself can fail on a dead socket — swallow it so + * the job never becomes an unhandled rejection. */ + try { + await write(simpleReply(EIO, handle, null)); + } catch { + /* socket gone; nothing more to do for this request */ + } + } finally { + inFlight--; + if (slotWaiter) { + const w: () => void = slotWaiter; + slotWaiter = null; + w(); + } + } + })(); + pending.add(job); + job.finally(() => pending.delete(job)); + } else if (type === NBD_CMD_FLUSH) { + try { + await write(simpleReply(0, handle, null)); + } catch { + break; /* socket died; stop reading and drain in-flight reads */ + } + } else { + try { + await write(simpleReply(EINVAL, handle, null)); + } catch { + break; + } + } + } + + await Promise.allSettled([...pending]); +} diff --git a/ts/packages/nbd-proxy/src/sources/blockdev.ts b/ts/packages/nbd-proxy/src/sources/blockdev.ts new file mode 100644 index 0000000..4d1f701 --- /dev/null +++ b/ts/packages/nbd-proxy/src/sources/blockdev.ts @@ -0,0 +1,75 @@ +import { Buffer } from 'node:buffer'; +import { open, readFile, type FileHandle } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import type { DataSource } from '../data-source.js'; + +/** + * Raw block device source. v1 is Linux-only: the kernel block layer serves + * non-aligned preads, so no app-level alignment is needed. Size comes from + * `drivelist` (optional dependency) or, as a fallback, /sys/block//size. + * macOS/Windows raw-device specifics (sector alignment, IOCTL size) are + * deferred — open() throws a clear error there. + */ +export class BlockDeviceSource implements DataSource { + private constructor( + private fh: FileHandle, + private bytes: number, + ) {} + + static async open(devPath: string): Promise { + if (os.platform() !== 'linux') { + throw new Error( + `BlockDeviceSource: v1 is Linux-only (got ${os.platform()}); ` + + `macOS/Windows raw-device support is not yet implemented`, + ); + } + const bytes = await BlockDeviceSource.sizeOf(devPath); + const fh = await open(devPath, 'r'); + return new BlockDeviceSource(fh, bytes); + } + + private static async sizeOf(devPath: string): Promise { + /* Prefer drivelist (optional dep): enumerate and match the device. + * Use a variable-based import to avoid TS2307 in declaration emit + * (drivelist has no type declarations). */ + try { + const modName = 'drivelist'; + const drivelist = (await import(modName)) as { + list: () => Promise>; + }; + const drives = await drivelist.list(); + const hit = drives.find((d) => d.device === devPath); + if (hit && hit.size > 0) return hit.size; + } catch { + /* drivelist absent or failed — fall through to /sys/block. */ + } + /* Fallback: /sys/block//size is in 512-byte sectors. */ + const name = path.basename(devPath); + const sysSize = path.join('/sys/block', name, 'size'); + const sectors = Number((await readFile(sysSize, 'utf8')).trim()); + if (!Number.isFinite(sectors) || sectors <= 0) { + throw new Error(`BlockDeviceSource: could not determine size of ${devPath}`); + } + return sectors * 512; + } + + async size(): Promise { + return this.bytes; + } + + async read(offset: number, length: number): Promise { + const buf = Buffer.alloc(length); + let got = 0; + while (got < length) { + const { bytesRead } = await this.fh.read(buf, got, length - got, offset + got); + if (bytesRead === 0) break; + got += bytesRead; + } + return got === length ? buf : buf.subarray(0, got); + } + + async close(): Promise { + await this.fh.close(); + } +} diff --git a/ts/packages/nbd-proxy/src/sources/file.ts b/ts/packages/nbd-proxy/src/sources/file.ts new file mode 100644 index 0000000..cd0b867 --- /dev/null +++ b/ts/packages/nbd-proxy/src/sources/file.ts @@ -0,0 +1,35 @@ +import { Buffer } from 'node:buffer'; +import { open, type FileHandle } from 'node:fs/promises'; +import type { DataSource } from '../data-source.js'; + +export class FileSource implements DataSource { + private constructor( + private fh: FileHandle, + private bytes: number, + ) {} + + static async open(path: string): Promise { + const fh = await open(path, 'r'); + const st = await fh.stat(); + return new FileSource(fh, st.size); + } + + async size(): Promise { + return this.bytes; + } + + async read(offset: number, length: number): Promise { + const buf = Buffer.alloc(length); + let got = 0; + while (got < length) { + const { bytesRead } = await this.fh.read(buf, got, length - got, offset + got); + if (bytesRead === 0) break; // EOF (e.g. read past end) + got += bytesRead; + } + return got === length ? buf : buf.subarray(0, got); + } + + async close(): Promise { + await this.fh.close(); + } +} diff --git a/ts/packages/nbd-proxy/src/sources/http.ts b/ts/packages/nbd-proxy/src/sources/http.ts new file mode 100644 index 0000000..f395ae8 --- /dev/null +++ b/ts/packages/nbd-proxy/src/sources/http.ts @@ -0,0 +1,48 @@ +import { Buffer } from 'node:buffer'; +import type { DataSource } from '../data-source.js'; + +/** + * HTTP(S) Range-backed source using Node's global fetch. undici's global + * dispatcher pools connections per origin, so range reads reuse upstream + * connections (no per-request reconnect). H2 multiplexing is deferred; v1 + * relies on HTTP/1.1 keep-alive via undici's default pool. + */ +export class HttpSource implements DataSource { + private constructor( + private url: string, + private bytes: number, + ) {} + + static async open(url: string): Promise { + const head = await fetch(url, { method: 'HEAD', redirect: 'follow' }); + if (!head.ok) throw new Error(`HttpSource: HEAD ${url} -> ${head.status}`); + const len = Number(head.headers.get('content-length') ?? '0'); + if (!len) throw new Error(`HttpSource: missing content-length for ${url}`); + if ((head.headers.get('accept-ranges') ?? '').toLowerCase() !== 'bytes') { + throw new Error(`HttpSource: server lacks Accept-Ranges: bytes for ${url}`); + } + return new HttpSource(head.url || url, len); + } + + async size(): Promise { + return this.bytes; + } + + async read(offset: number, length: number): Promise { + const end = Math.min(offset + length, this.bytes) - 1; + const res = await fetch(this.url, { + headers: { Range: `bytes=${offset}-${end}` }, + redirect: 'follow', + }); + if (res.status !== 206 && res.status !== 200) { + throw new Error(`HttpSource: range ${offset}-${end} -> ${res.status}`); + } + let buf = Buffer.from(await res.arrayBuffer()); + if (res.status === 200 && buf.length > end - offset + 1) { + buf = buf.subarray(offset, end + 1); + } + return buf; + } + + async close(): Promise {} +} diff --git a/ts/packages/nbd-proxy/test/integration.mjs b/ts/packages/nbd-proxy/test/integration.mjs new file mode 100644 index 0000000..2ce9bab --- /dev/null +++ b/ts/packages/nbd-proxy/test/integration.mjs @@ -0,0 +1,160 @@ +/* Integration: open the PoC fixture qcow2 THROUGH the proxy with the real + * QEMU client (qemu-img + qemu-io over a 127.0.0.1 loopback NBD endpoint). + * Proves format detection + byte-accurate delivery for FileSource. + * + * MUST use async execFile (the in-process NBD server shares this event loop; + * a sync execFileSync would deadlock it). */ +import assert from 'node:assert'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileP = promisify(execFile); +const here = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(here, '../../../..'); +const dist = path.join(here, '..', 'dist', 'src', 'index.js'); +const metaPath = path.join(repoRoot, 'scripts/poc-nbd/fixtures/meta.json'); + +if (!fs.existsSync(metaPath)) { + console.error('fixture missing; run: node scripts/poc-nbd/make-test-image.mjs'); + process.exit(1); +} +const meta = JSON.parse(fs.readFileSync(metaPath, 'utf8')); +const { createDataSource, serveOnLoopback } = await import(dist); + +const source = await createDataSource({ kind: 'file', target: meta.qcow2 }); +const { port, stop } = await serveOnLoopback(source); +let fail = false; +try { + const uri = `nbd://127.0.0.1:${port}`; + /* (a) qemu-img detects qcow2 over NBD */ + const { stdout: info } = await execFileP('qemu-img', ['info', uri], { + encoding: 'utf8', + }); + console.log(info); + if (!/file format:\s*qcow2/.test(info)) { + console.error('FAIL: qemu-img did not detect qcow2 over NBD'); + fail = true; + } + + /* (b) qemu-io reads the marker back byte-for-byte through qcow2-over-NBD */ + const len = Buffer.from(meta.verifyBytesHex, 'hex').length; + const qio = `json:{"driver":"qcow2","file":{"driver":"nbd","server":{"type":"inet","host":"127.0.0.1","port":"${port}"}}}`; + const { stdout: dump } = await execFileP( + 'qemu-io', + ['-r', '-c', `read -v ${meta.verifyOffset} ${len}`, qio], + { encoding: 'utf8' }, + ); + const hex = [...dump.matchAll(/^[0-9a-f]{8}:\s+((?:[0-9a-f]{2}\s?)+)/gim)] + .map((m) => m[1].replace(/\s+/g, '')) + .join('') + .slice(0, len * 2); + console.log(dump); + if (hex !== meta.verifyBytesHex) { + console.error(`FAIL: marker mismatch: got ${hex} want ${meta.verifyBytesHex}`); + fail = true; + } +} finally { + await stop(); + await source.close(); +} + +/* HttpSource: local Range server backed by the fixture, served through the + * proxy; verify qcow2 detection + that range reads REUSE connections. */ +{ + const http = await import('node:http'); + const { createDataSource, serveOnLoopback } = await import(dist); + const raw = fs.readFileSync(meta.qcow2); + let connections = 0; + const upstream = http.createServer((req, res) => { + if (req.method === 'HEAD') { + res.writeHead(200, { + 'content-length': String(raw.length), + 'accept-ranges': 'bytes', + }); + return res.end(); + } + const m = /bytes=(\d+)-(\d+)/.exec(req.headers.range || ''); + if (!m) { + res.writeHead(200, { 'content-length': String(raw.length) }); + return res.end(raw); + } + const start = +m[1]; + const end = +m[2]; + res.writeHead(206, { + 'content-range': `bytes ${start}-${end}/${raw.length}`, + 'content-length': String(end - start + 1), + 'accept-ranges': 'bytes', + }); + res.end(raw.subarray(start, end + 1)); + }); + upstream.on('connection', () => connections++); + await new Promise((r) => upstream.listen(0, '127.0.0.1', r)); + const upPort = upstream.address().port; + + const src = await createDataSource({ + kind: 'url', + target: `http://127.0.0.1:${upPort}/disk.qcow2`, + }); + const { port, stop } = await serveOnLoopback(src); + try { + const { stdout } = await execFileP('qemu-img', ['info', `nbd://127.0.0.1:${port}`], { + encoding: 'utf8', + }); + if (!/file format:\s*qcow2/.test(stdout)) { + console.error('FAIL: HttpSource — qcow2 not detected through proxy'); + fail = true; + } + console.log(`HttpSource: upstream TCP connections opened = ${connections}`); + if (connections > 4) { + console.error(`FAIL: HttpSource opened ${connections} connections (no keep-alive reuse)`); + fail = true; + } + } finally { + await stop(); + await src.close(); + await new Promise((r) => upstream.close(r)); + } + console.log('HttpSource case done'); +} + +/* CLI smoke: spawn the built CLI in loopback mode, read its port from stdout, + * confirm qemu-img opens the fixture qcow2 through it, then shut it down. */ +{ + const { spawn } = await import('node:child_process'); + const cliPath = path.join(here, '..', 'dist', 'bin', 'anyfs-nbd-proxy.js'); + const child = spawn( + 'node', + [cliPath, '--source', 'file', '--target', meta.qcow2, '--port', '0'], + { stdio: ['pipe', 'pipe', 'inherit'] }, + ); + const port = await new Promise((resolve, reject) => { + let out = ''; + child.stdout.on('data', (d) => { + out += d; + const m = /^(\d+)\s*$/m.exec(out); + if (m) resolve(Number(m[1])); + }); + child.on('exit', (c) => reject(new Error(`CLI exited early: ${c}`))); + setTimeout(() => reject(new Error('CLI did not report a port')), 5000); + }); + try { + const { stdout } = await execFileP('qemu-img', ['info', `nbd://127.0.0.1:${port}`], { + encoding: 'utf8', + }); + if (!/file format:\s*qcow2/.test(stdout)) { + console.error('FAIL: CLI — qcow2 not detected'); + fail = true; + } else { + console.log('CLI smoke: qcow2 detected through spawned proxy'); + } + } finally { + child.stdin.end(); /* triggers lifecycle-bound shutdown */ + child.kill('SIGTERM'); + } +} + +if (fail) process.exit(1); +console.log(`\nINTEGRATION PASS: qcow2 detected + marker "${meta.verifyAscii}" verified through proxy (FileSource + HttpSource + CLI)`); diff --git a/ts/packages/nbd-proxy/test/unit.mjs b/ts/packages/nbd-proxy/test/unit.mjs new file mode 100644 index 0000000..53eddc8 --- /dev/null +++ b/ts/packages/nbd-proxy/test/unit.mjs @@ -0,0 +1,166 @@ +/* Unit tests for @anyfs/nbd-proxy DataSource backends. + * Run against the built dist/ (tsup output). */ +import assert from 'node:assert'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const dist = path.join(here, '..', 'dist', 'src', 'index.js'); + +const { createDataSource } = await import(dist); + +let passed = 0; +async function test(name, fn) { + await fn(); + console.log('ok -', name); + passed++; +} + +/* FileSource: deterministic byte pattern */ +await test('FileSource reads exact bytes', async () => { + const tmp = path.join(os.tmpdir(), 'nbdproxy-file-test.bin'); + const data = Buffer.alloc(4096); + for (let i = 0; i < data.length; i++) data[i] = i & 0xff; + fs.writeFileSync(tmp, data); + + const src = await createDataSource({ kind: 'file', target: tmp }); + assert.strictEqual(await src.size(), 4096); + + const slice = await src.read(1000, 16); + for (let i = 0; i < 16; i++) { + assert.strictEqual(slice[i], (1000 + i) & 0xff, `byte ${i}`); + } + await src.close(); + fs.unlinkSync(tmp); +}); + +/* NbdServer: out-of-order completion must reply keyed by handle. + * Drive serveNbd over an in-process TCP pair with a mock DataSource whose + * reads resolve in REVERSE order (later request finishes first). */ +await test('NbdServer replies keyed by handle under out-of-order completion', async () => { + const net = await import('node:net'); + const { serveNbd } = await import(dist); + + const SIZE = 1 << 20; + const source = { + async size() { + return SIZE; + }, + async read(offset, length) { + const delay = offset === 0 ? 60 : 5; /* offset 0 finishes LAST */ + await new Promise((r) => setTimeout(r, delay)); + const b = Buffer.alloc(length); + b.writeUInt32BE(offset >>> 0, 0); + return b; + }, + async close() {}, + }; + + const server = net.createServer((sock) => { + serveNbd(sock, source, { size: SIZE }).catch(() => {}); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const port = server.address().port; + + const replies = await new Promise((resolve, reject) => { + const c = net.connect(port, '127.0.0.1'); + const got = []; + let phase = 'hello'; + let buf = Buffer.alloc(0); + c.on('data', (d) => { + buf = Buffer.concat([buf, d]); + if (phase === 'hello' && buf.length >= 18) { + buf = buf.subarray(18); + c.write(Buffer.alloc(4)); /* client flags */ + const o = Buffer.alloc(16); + o.writeBigUInt64BE(0x49484156454f5054n, 0); + o.writeUInt32BE(7, 8); /* NBD_OPT_GO */ + o.writeUInt32BE(0, 12); + c.write(o); + phase = 'opt'; + } + if (phase === 'opt') { + /* consume INFO reply (20+12) + ACK reply (20), then send two READs */ + if (buf.length >= 20 + 12 + 20) { + buf = buf.subarray(20 + 12 + 20); + phase = 'xmit'; + for (const off of [0, 65536]) { + const req = Buffer.alloc(28); + req.writeUInt32BE(0x25609513, 0); + req.writeUInt16BE(0, 4); + req.writeUInt16BE(0, 6); /* CMD_READ */ + req.writeUInt32BE(off, 12); /* low 4 bytes of handle = offset id */ + req.writeBigUInt64BE(BigInt(off), 16); + req.writeUInt32BE(8, 24); /* length */ + c.write(req); + } + } + } + if (phase === 'xmit') { + while (buf.length >= 24) { + /* reply: 16-byte header + 8 bytes data */ + const handle = buf.readUInt32BE(12); /* low 4 bytes of handle */ + const dataOff = buf.readUInt32BE(16); /* our tagged offset */ + got.push({ handle, dataOff }); + buf = buf.subarray(24); + if (got.length === 2) { + c.end(); + resolve(got); + } + } + } + }); + c.on('error', reject); + setTimeout(() => reject(new Error('timeout')), 3000); + }); + server.close(); + + for (const r of replies) { + assert.strictEqual(r.handle, r.dataOff, 'handle must pair with its data'); + } + assert.strictEqual(replies[0].handle, 65536, 'faster read replied first'); +}); + +/* BlockDeviceSource: back a loop device with a test image if losetup is + * available (needs root); otherwise SKIP with a clear message. */ +await test('BlockDeviceSource reads via loop device (or skips)', async () => { + const { execFileSync } = await import('node:child_process'); + if (os.platform() !== 'linux') { + console.log(' (skip: blockdev v1 is Linux-only)'); + return; + } + let loopDev = null; + const img = path.join(os.tmpdir(), 'nbdproxy-blockdev-test.img'); + try { + /* 1 MiB image with a marker at offset 4096. */ + const data = Buffer.alloc(1 << 20); + data.write('BLOCKDEV-MARK', 4096, 'ascii'); + fs.writeFileSync(img, data); + try { + loopDev = execFileSync('losetup', ['--find', '--show', img], { + encoding: 'utf8', + }).trim(); + } catch { + console.log(' (skip: cannot create loop device — needs root/losetup)'); + return; + } + const src = await createDataSource({ kind: 'blockdev', target: loopDev }); + assert.ok((await src.size()) >= 1 << 20, 'size detected'); + const slice = await src.read(4096, 13); + assert.strictEqual(slice.toString('ascii'), 'BLOCKDEV-MARK', 'marker read'); + await src.close(); + } finally { + if (loopDev) { + try { + (await import('node:child_process')).execFileSync('losetup', ['-d', loopDev]); + } catch {} + } + try { + fs.unlinkSync(img); + } catch {} + } +}); + +console.log(`\n${passed} test(s) passed`); diff --git a/ts/packages/nbd-proxy/tsconfig.json b/ts/packages/nbd-proxy/tsconfig.json new file mode 100644 index 0000000..5080225 --- /dev/null +++ b/ts/packages/nbd-proxy/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "./dist", + "rootDir": "." + }, + "include": ["src/**/*", "bin/**/*"] +} diff --git a/ts/packages/nbd-proxy/tsup.config.ts b/ts/packages/nbd-proxy/tsup.config.ts new file mode 100644 index 0000000..9bc826f --- /dev/null +++ b/ts/packages/nbd-proxy/tsup.config.ts @@ -0,0 +1,10 @@ +import { defineConfig } from 'tsup'; + +export default defineConfig({ + entry: ['src/index.ts', 'bin/anyfs-nbd-proxy.ts'], + format: ['esm'], + target: 'node20', + dts: true, + clean: true, + sourcemap: true, +}); diff --git a/ts/packages/react/package.json b/ts/packages/react/package.json index 849f707..4a295f2 100644 --- a/ts/packages/react/package.json +++ b/ts/packages/react/package.json @@ -14,7 +14,8 @@ "dist" ], "scripts": { - "build": "tsup src/index.ts --format esm --dts --clean --external react --external @anyfs/core" + "build": "tsup src/index.ts --format esm --dts --clean --external react --external @anyfs/core", + "test": "vitest run" }, "peerDependencies": { "react": "^18 || ^19" @@ -26,6 +27,10 @@ "typescript": "^5.5.4", "tsup": "^8.3.0", "@types/react": "^19.0.0", - "react": "^19.0.0" + "react": "^19.0.0", + "react-dom": "^19.0.0", + "vitest": "^3.0.0", + "jsdom": "^25.0.0", + "@testing-library/react": "^16.1.0" } } diff --git a/ts/packages/react/test/hooks.test.tsx b/ts/packages/react/test/hooks.test.tsx new file mode 100644 index 0000000..cb0f087 --- /dev/null +++ b/ts/packages/react/test/hooks.test.tsx @@ -0,0 +1,92 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, waitFor, cleanup } from '@testing-library/react'; +import React from 'react'; + +const prewarmMock = vi.fn(); +vi.mock('@anyfs/core', () => ({ + createSession: () => ({ backend: 'wasm', allowedKinds: new Set(['blob', 'url']), wasmCaps: {} }), + prewarm: (...args: unknown[]) => prewarmMock(...args), + prewarmNative: vi.fn(), + NativeSession: class NativeSessionMock {}, +})); + +import { AnyfsProvider, useAnyfsDir, useAnyfsFile } from '../src/index.js'; + +const entriesRoot = [ + { name: 'etc', kind: 'dir' }, + { name: 'README', kind: 'file' }, +]; + +function makeSession() { + return { + attachBlob: vi.fn(async () => {}), + onProgress: vi.fn(() => () => {}), + close: vi.fn(async () => {}), + readdir: vi.fn(async (p: string) => { + if (p === '/') return entriesRoot; + throw new Error('ENOENT'); + }), + stat: vi.fn(async () => ({ size: 4, mode: 0o100644 })), + openFd: vi.fn(async () => 7), + readFd: vi.fn(async () => new Uint8Array([1, 2, 3, 4])), + closeFd: vi.fn(async () => {}), + }; +} + +function Tree({ path }: { path: string }) { + const { entries, loading, error } = useAnyfsDir(path); + if (error) return

error:{error.message}
; + if (loading || !entries) return
loading
; + return
{entries.map((e) => e.name).join(',')}
; +} + +function FileBytes({ path }: { path: string }) { + const { data, error } = useAnyfsFile(path); + if (error) return
error
; + return
{data ? Array.from(data).join(',') : 'loading'}
; +} + +const blob = new Blob([new Uint8Array(8)]); + +beforeEach(() => prewarmMock.mockReset()); +// globals:false means testing-library's auto-cleanup (which hooks the global +// afterEach) never registers — without this, renders leak across tests. +afterEach(cleanup); + +describe('useAnyfsDir', () => { + it('lists entries once ready and caches per (session,path)', async () => { + const s = makeSession(); + prewarmMock.mockResolvedValue(s); + const { rerender } = render( + , + ); + await waitFor(() => expect(screen.getByTestId('dir').textContent).toBe('etc,README')); + rerender( + , + ); + await waitFor(() => expect(screen.getByTestId('dir').textContent).toBe('etc,README')); + expect(s.readdir).toHaveBeenCalledTimes(1); + }); + + it('surfaces readdir errors', async () => { + prewarmMock.mockResolvedValue(makeSession()); + render( + , + ); + await waitFor(() => expect(screen.getByTestId('dir').textContent).toBe('error:ENOENT')); + }); +}); + +describe('useAnyfsFile', () => { + it('stat + openFd + readFd + closeFd round trip', async () => { + const s = makeSession(); + prewarmMock.mockResolvedValue(s); + render( + , + ); + await waitFor(() => expect(screen.getByTestId('file').textContent).toBe('1,2,3,4')); + expect(s.openFd).toHaveBeenCalledWith('/README'); + expect(s.readFd).toHaveBeenCalledWith(7, 0, 4); + expect(s.closeFd).toHaveBeenCalledWith(7); + }); +}); diff --git a/ts/packages/react/test/provider.test.tsx b/ts/packages/react/test/provider.test.tsx new file mode 100644 index 0000000..6437185 --- /dev/null +++ b/ts/packages/react/test/provider.test.tsx @@ -0,0 +1,78 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, waitFor, cleanup } from '@testing-library/react'; +import React from 'react'; + +const fakeSession = () => ({ + attachBlob: vi.fn(async () => {}), + attachUrl: vi.fn(async () => {}), + attachPath: vi.fn(async () => {}), + onProgress: vi.fn(() => () => {}), + close: vi.fn(async () => {}), + readdir: vi.fn(async () => []), + stat: vi.fn(async () => ({ size: 0, mode: 0o100644 })), + openFd: vi.fn(async () => 3), + readFd: vi.fn(async () => new Uint8Array(0)), + closeFd: vi.fn(async () => {}), +}); + +const prewarmMock = vi.fn(); + +vi.mock('@anyfs/core', () => ({ + createSession: () => ({ + backend: 'wasm', + allowedKinds: new Set(['blob', 'url']), + wasmCaps: {}, + }), + prewarm: (...args: unknown[]) => prewarmMock(...args), + prewarmNative: vi.fn(), + NativeSession: class NativeSessionMock {}, +})); + +import { AnyfsProvider, useAnyfsDisk } from '../src/index.js'; + +function Status() { + const { status, error } = useAnyfsDisk(); + return
{status}{error ? `:${error.message}` : ''}
; +} + +beforeEach(() => { prewarmMock.mockReset(); }); +// globals:false means testing-library's auto-cleanup (which hooks the global +// afterEach) never registers — without this, renders leak across tests. +afterEach(cleanup); + +describe('AnyfsProvider', () => { + it('idle without source or prewarm', () => { + render(); + expect(screen.getByTestId('status').textContent).toBe('idle'); + }); + + it('prewarm → booting → booted', async () => { + let resolve!: (s: unknown) => void; + prewarmMock.mockReturnValue(new Promise((r) => (resolve = r))); + render(); + expect(screen.getByTestId('status').textContent).toBe('booting'); + resolve(fakeSession()); + await waitFor(() => expect(screen.getByTestId('status').textContent).toBe('booted')); + }); + + it('blob source attaches and reaches ready', async () => { + const s = fakeSession(); + prewarmMock.mockResolvedValue(s); + const blob = new Blob([new Uint8Array(16)]); + render(); + await waitFor(() => expect(screen.getByTestId('status').textContent).toBe('ready')); + expect(s.attachBlob).toHaveBeenCalledWith(blob); + }); + + it('disallowed source kind → error state', async () => { + prewarmMock.mockResolvedValue(fakeSession()); + render(); + await waitFor(() => expect(screen.getByTestId('status').textContent).toMatch(/^error:.*not supported/)); + }); + + it('prewarm failure → error state', async () => { + prewarmMock.mockRejectedValue(new Error('boot failed')); + render(); + await waitFor(() => expect(screen.getByTestId('status').textContent).toBe('error:boot failed')); + }); +}); diff --git a/ts/packages/react/vitest.config.ts b/ts/packages/react/vitest.config.ts new file mode 100644 index 0000000..fe11f13 --- /dev/null +++ b/ts/packages/react/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + environment: 'jsdom', + include: ['test/**/*.test.tsx'], + globals: false, + }, +}); diff --git a/ts/pnpm-lock.yaml b/ts/pnpm-lock.yaml index 54ddd7b..fb33fe8 100644 --- a/ts/pnpm-lock.yaml +++ b/ts/pnpm-lock.yaml @@ -121,24 +121,48 @@ importers: specifier: ^5.5.4 version: 5.9.3 + packages/nbd-proxy: + devDependencies: + '@types/node': + specifier: ^22.0.0 + version: 22.19.19 + tsup: + specifier: ^8.3.0 + version: 8.5.1(jiti@1.21.7)(postcss@8.5.15)(typescript@5.9.3) + typescript: + specifier: ^5.5.4 + version: 5.9.3 + packages/react: dependencies: '@anyfs/core': specifier: workspace:* version: link:../core devDependencies: + '@testing-library/react': + specifier: ^16.1.0 + version: 16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6) '@types/react': specifier: ^19.0.0 version: 19.2.15 + jsdom: + specifier: ^25.0.0 + version: 25.0.1 react: specifier: ^19.0.0 version: 19.2.6 + react-dom: + specifier: ^19.0.0 + version: 19.2.6(react@19.2.6) tsup: specifier: ^8.3.0 version: 8.5.1(jiti@1.21.7)(postcss@8.5.15)(typescript@5.9.3) typescript: specifier: ^5.5.4 version: 5.9.3 + vitest: + specifier: ^3.0.0 + version: 3.2.6(@types/node@24.12.4)(jsdom@25.0.1) packages/trees: dependencies: @@ -189,6 +213,9 @@ packages: resolution: {integrity: sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw==} engines: {node: '>=10'} + '@asamuzakjp/css-color@3.2.0': + resolution: {integrity: sha512-K1A6z8tS3XsmCMM86xoWdn7Fkdn9m6RSVtocUrJYIwZnFVkng/PvkEoWtOWmP+Scc6saYWHWZYbndEEXxl24jw==} + '@babel/code-frame@7.29.0': resolution: {integrity: sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==} engines: {node: '>=6.9.0'} @@ -286,6 +313,34 @@ packages: resolution: {integrity: sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==} engines: {node: '>=6.9.0'} + '@csstools/color-helpers@5.1.0': + resolution: {integrity: sha512-S11EXWJyy0Mz5SYvRmY8nJYTFFd1LCNV+7cXyAgQtOOuzb4EsgfqDufL+9esx72/eLhsRdGZwaldu/h+E4t4BA==} + engines: {node: '>=18'} + + '@csstools/css-calc@2.1.4': + resolution: {integrity: sha512-3N8oaj+0juUw/1H3YwmDDJXCgTB1gKU6Hc/bB502u9zR0q2vd786XJH9QfrKIEgFlZmhZiq6epXl4rHqhzsIgQ==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-parser-algorithms': ^3.0.5 + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-color-parser@3.1.0': + resolution: {integrity: sha512-nbtKwh3a6xNVIp/VRuXV64yTKnb1IjTAEEh3irzS+HkKjAOYLTGNb9pmVNntZ8iVBHcWDA2Dof0QtPgFI1BaTA==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-parser-algorithms': ^3.0.5 + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-parser-algorithms@3.0.5': + resolution: {integrity: sha512-DaDeUkXZKjdGhgYaHNJTV9pV7Y9B3b644jCLs9Upc3VeNGg6LWARAT6O+Q+/COo+2gg/bM5rhpMAtf70WqfBdQ==} + engines: {node: '>=18'} + peerDependencies: + '@csstools/css-tokenizer': ^3.0.4 + + '@csstools/css-tokenizer@3.0.4': + resolution: {integrity: sha512-Vd/9EVDiu6PPJt9yAh6roZP6El1xHrdvIVGjyBsHR0RYwNHgL7FJPyIIW4fANJNG6FtyZfvlRPpFI4ZM/lubvw==} + engines: {node: '>=18'} + '@electron/asar@3.4.1': resolution: {integrity: sha512-i4/rNPRS84t0vSRa2HorerGRXWyF4vThfHesw0dmcWHp+cspK743UanA0suA5Q5y8kzY2y6YKrvbIUn69BCAiA==} engines: {node: '>=10.12.0'} @@ -1144,6 +1199,28 @@ packages: resolution: {integrity: sha512-4BAffykYOgO+5nzBWYwE3W90sBgLJoUPRWWcL8wlyiM8IB8ipJz3UMJ9KXQd1RKQXpKp8Tutn80HZtWsu2u76w==} engines: {node: '>=10'} + '@testing-library/dom@10.4.1': + resolution: {integrity: sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg==} + engines: {node: '>=18'} + + '@testing-library/react@16.3.2': + resolution: {integrity: sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g==} + engines: {node: '>=18'} + peerDependencies: + '@testing-library/dom': ^10.0.0 + '@types/react': ^19.0.0 + '@types/react-dom': ^19.0.0 + react: ^18.0.0 || ^19.0.0 + react-dom: ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@types/aria-query@5.0.4': + resolution: {integrity: sha512-rfT93uj5s0PRL7EzccGMs3brplhcrghnDoV26NqKhCAS1hVo+WdNsPvE/yb6ilfr5hi2MEk6d5EWJTKdxg8jVw==} + '@types/babel__core@7.20.5': resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==} @@ -1159,10 +1236,16 @@ packages: '@types/cacheable-request@6.0.3': resolution: {integrity: sha512-IQ3EbTzGxIigb1I3qPZc1rWJnH0BmSKv5QYTalEwweFvyBDLSAe24zP0le/hyi7ecGfZVlIVAg4BZqb8WBwKqw==} + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + '@types/classnames@2.3.4': resolution: {integrity: sha512-dwmfrMMQb9ujX1uYGvB5ERDlOzBNywnZAZBtOe107/hORWP05ESgU4QyaanZMWYYfd2BzrG78y13/Bju8IQcMQ==} deprecated: This is a stub types definition. classnames provides its own type definitions, so you do not need this installed. + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} @@ -1230,6 +1313,35 @@ packages: peerDependencies: vite: ^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 + '@vitest/expect@3.2.6': + resolution: {integrity: sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ==} + + '@vitest/mocker@3.2.6': + resolution: {integrity: sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@3.2.6': + resolution: {integrity: sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA==} + + '@vitest/runner@3.2.6': + resolution: {integrity: sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q==} + + '@vitest/snapshot@3.2.6': + resolution: {integrity: sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw==} + + '@vitest/spy@3.2.6': + resolution: {integrity: sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg==} + + '@vitest/utils@3.2.6': + resolution: {integrity: sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg==} + '@xmldom/xmldom@0.9.10': resolution: {integrity: sha512-A9gOqLdi6cV4ibazAjcQufGj0B1y/vDqYrcuP6d/6x8P27gRS8643Dj9o1dEKtB6O7fwxb2FgBmJS2mX7gpvdw==} engines: {node: '>=14.6'} @@ -1263,6 +1375,10 @@ packages: resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} engines: {node: '>=8'} + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + ansi-styles@6.2.3: resolution: {integrity: sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==} engines: {node: '>=12'} @@ -1277,6 +1393,13 @@ packages: arg@5.0.2: resolution: {integrity: sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==} + aria-query@5.3.0: + resolution: {integrity: sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==} + + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + asynckit@0.4.0: resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} @@ -1388,10 +1511,18 @@ packages: caniuse-lite@1.0.30001793: resolution: {integrity: sha512-iwSsYWaCOoh26cV8NwNRViHlrfUvYsHDfRVcbtmw0Kg6PJIZZXwMkj1442FYLBGkeUf1juAsU3DTfxW579mrPA==} + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + chokidar@3.6.0: resolution: {integrity: sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==} engines: {node: '>= 8.10.0'} @@ -1503,12 +1634,20 @@ packages: engines: {node: '>=4'} hasBin: true + cssstyle@4.6.0: + resolution: {integrity: sha512-2z+rWdzbbSZv6/rhtvzvqeZQHrBaqgogqt85sqFNbabZOuFbCVFb8kPeEtZjiKkbrm395irpNKiYeFeLiQnFPg==} + engines: {node: '>=18'} + csstype@2.6.21: resolution: {integrity: sha512-Z1PhmomIfypOpoMjRQB70jfvy/wxT50qW08YXO5lMIJkrdq4yOTR+AW7FqutScmB9NkLwxo+jU+kZLbofZZq/w==} csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + data-urls@5.0.0: + resolution: {integrity: sha512-ZYP5VBHshaDAiVZxjbRVcFJpc+4xGgT0bK3vzy1HLN8jTO975HEbuYzZJcHoQEY5K1a0z8YayJkyVETa08eNTg==} + engines: {node: '>=18'} + debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -1526,10 +1665,17 @@ packages: supports-color: optional: true + decimal.js@10.6.0: + resolution: {integrity: sha512-YpgQiITW3JXGntzdUmyUR1V812Hn8T1YVXhCu+wO3OpS4eU9l4YdD3qjyiKdV6mvV29zapkMeD390UVEf2lkUg==} + decompress-response@6.0.0: resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} engines: {node: '>=10'} + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} @@ -1554,6 +1700,10 @@ packages: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} engines: {node: '>=0.4.0'} + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} @@ -1573,6 +1723,9 @@ packages: dnd-core@11.1.3: resolution: {integrity: sha512-QugF55dNW+h+vzxVJ/LSJeTeUw9MCJ2cllhmVThVPEtF16ooBkxj0WBE5RB+AceFxMFo1rO6bJKXtqKl+JNnyA==} + dom-accessibility-api@0.5.16: + resolution: {integrity: sha512-X7BJ2yElsnOJ30pZF4uIIDfBEVgF4XEBxL9Bxhy6dnrm5hkzqmsWHGTiHqRiITNhMyFLyAiWndIJP7Z1NTteDg==} + dom-helpers@5.2.1: resolution: {integrity: sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==} @@ -1607,6 +1760,10 @@ packages: end-of-stream@1.4.5: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + entities@6.0.1: + resolution: {integrity: sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==} + engines: {node: '>=0.12'} + env-paths@2.2.1: resolution: {integrity: sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==} engines: {node: '>=6'} @@ -1629,6 +1786,9 @@ packages: resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} engines: {node: '>= 0.4'} + es-module-lexer@1.7.0: + resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-object-atoms@1.1.1: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} @@ -1667,6 +1827,9 @@ packages: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} + estree-walker@3.0.3: + resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + exact-trie@1.0.13: resolution: {integrity: sha512-2N0sx9jMlzZxRmSOpFKmcuaPcLXYLGRp69DohigW5E7R/uo9i6S1zJ/PuAckf70099am1ts7YBRMLO8Nr8AJLg==} @@ -1674,6 +1837,10 @@ packages: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} + expect-type@1.3.0: + resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} + engines: {node: '>=12.0.0'} + exponential-backoff@3.1.3: resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==} @@ -1895,6 +2062,10 @@ packages: hotkeys-js@3.13.15: resolution: {integrity: sha512-gHh8a/cPTCpanraePpjRxyIlxDFrIhYqjuh01UHWEwDpglJKCnvLW8kqSx5gQtOuSsJogNZXLhOdbSExpgUiqg==} + html-encoding-sniffer@4.0.0: + resolution: {integrity: sha512-Y22oTqIU4uuPgEemfz7NDJz6OeKf12Lsu+QC+s3BVpda64lTiMYCyGwg5ki4vFxkMwQdeZDl2adZoqUgdFuTgQ==} + engines: {node: '>=18'} + http-cache-semantics@4.2.0: resolution: {integrity: sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==} @@ -1978,6 +2149,9 @@ packages: resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} engines: {node: '>=0.12.0'} + is-potential-custom-element-name@1.0.1: + resolution: {integrity: sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==} + isbinaryfile@4.0.10: resolution: {integrity: sha512-iHrqe5shvBUcFbmZq9zOQHBoeOhZJu6RQGrDpBgenUm/Am+F3JM2MgQj+rK3Z601fzrL5gLZWtAPH2OBaSVcyw==} engines: {node: '>= 8.0.0'} @@ -2007,6 +2181,18 @@ packages: js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-tokens@9.0.1: + resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + + jsdom@25.0.1: + resolution: {integrity: sha512-8i7LzZj7BF8uplX+ZyOlIz86V6TAsSs+np6m1kpW9u0JWi4z/1t+FzcK1aek+ybTnAC4KhBL4uXCNT0wcUIeCw==} + engines: {node: '>=18'} + peerDependencies: + canvas: ^2.11.2 + peerDependenciesMeta: + canvas: + optional: true + jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} @@ -2108,6 +2294,9 @@ packages: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + lowercase-keys@2.0.0: resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==} engines: {node: '>=8'} @@ -2118,6 +2307,10 @@ packages: lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} + lz-string@1.5.0: + resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} + hasBin: true + magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -2261,6 +2454,9 @@ packages: resolution: {integrity: sha512-DlL+XwOy3NxAQ8xuC0okPgK46iuVNAK01YN7RueYBqqFeGsBjV9XmCAzAdgt+667bCl5kPh9EqKKDwnaPG1I7A==} engines: {node: '>=10'} + nwsapi@2.2.24: + resolution: {integrity: sha512-7YRhZ3jS45LwmSCT4b2sVFHt/WuovaktDU07QrtOBY2PXskss5a9jfmR9jptyumwXST+rFjrmppMY1KT/yn35A==} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -2315,6 +2511,9 @@ packages: resolution: {integrity: sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ==} engines: {node: '>=0.10.0'} + parse5@7.3.0: + resolution: {integrity: sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==} + path-exists@3.0.0: resolution: {integrity: sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ==} engines: {node: '>=4'} @@ -2341,6 +2540,10 @@ packages: pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + pe-library@1.0.1: resolution: {integrity: sha512-nh39Mo1eGWmZS7y+mK/dQIqg7S1lp38DpRxkyoHf0ZcUs/HDc+yyTjuOtTvSMZHmfSLuSQaX945u05Y2Q6UWZg==} engines: {node: '>=14', npm: '>=7'} @@ -2450,6 +2653,10 @@ packages: engines: {node: '>=14'} hasBin: true + pretty-format@27.5.1: + resolution: {integrity: sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ==} + engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0} + proc-log@5.0.0: resolution: {integrity: sha512-Azwzvl90HaF0aCz1JrDdXQykFakSSNPaPoiZ9fm5qJIMHioDZEi7OAdRwSm6rSoPtY3Qutnm3L7ogmg3dc+wbQ==} engines: {node: ^18.17.0 || >=20.5.0} @@ -2472,6 +2679,10 @@ packages: pump@3.0.4: resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} @@ -2636,6 +2847,12 @@ packages: engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + rrweb-cssom@0.7.1: + resolution: {integrity: sha512-TrEMa7JGdVm0UThDJSx7ddw5nVm3UJS9o9CCIZ72B1vSyEZoziDqBYP3XIoi/12lKrJR8rE3jeFHMok2F/Mnsg==} + + rrweb-cssom@0.8.0: + resolution: {integrity: sha512-guoltQEx+9aMf2gDZ0s62EcV8lsXR+0w8915TC3ITdn2YueuNjdAYh/levpU9nFaoChh9RUS5ZdQMrKfVEN9tw==} + run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} @@ -2648,6 +2865,10 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + saxes@6.0.0: + resolution: {integrity: sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==} + engines: {node: '>=v12.22.7'} + scheduler@0.27.0: resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} @@ -2692,6 +2913,9 @@ packages: shortid@2.2.17: resolution: {integrity: sha512-GpbM3gLF1UUXZvQw6MCyulHkWbRseNO4cyBEZresZRorwl1+SLu1ZdqgVtuwqz8mB6RpwPkm541mYSqrKyJSaA==} + siginfo@2.0.0: + resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} + signal-exit@4.1.0: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} @@ -2741,6 +2965,12 @@ packages: resolution: {integrity: sha512-S7iGNosepx9RadX82oimUkvr0Ct7IjJbEbs4mJcTxst8um95J3sDYU1RBEOvdu6oL1Wek2ODI5i4MAw+dZ6cAQ==} engines: {node: ^18.17.0 || >=20.5.0} + stackback@0.0.2: + resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} + + std-env@3.10.0: + resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -2768,6 +2998,9 @@ packages: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} + strip-literal@3.1.0: + resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + strip-outer@1.0.1: resolution: {integrity: sha512-k55yxKHwaXnpYGsOzg4Vl8+tDrWylxDEpknGjhTiZB8dFRU5rTo9CAzeycivxV3s+zlTKwrs6WxMxR95n26kwg==} engines: {node: '>=0.10.0'} @@ -2809,6 +3042,9 @@ packages: resolution: {integrity: sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ==} engines: {node: '>=0.10.0'} + symbol-tree@3.2.4: + resolution: {integrity: sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==} + tailwindcss@3.4.19: resolution: {integrity: sha512-3ofp+LL8E+pK/JuPLPggVAIaEuhvIz4qNcf3nA1Xn2o/7fb7s/TYpHhwGDv1ZU3PkBluUVaF8PyCHcm48cKLWQ==} engines: {node: '>=14.0.0'} @@ -2841,6 +3077,9 @@ packages: tiny-warning@1.0.3: resolution: {integrity: sha512-lBN9zLN/oAf68o3zNXYrdCt1kP8WsiGW8Oo2ka41b2IM5JL/S1CTyX1rW0mb/zSuJun0ZUrDxx4sqvYS2FWzPA==} + tinybench@2.9.0: + resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -2848,10 +3087,37 @@ packages: resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} engines: {node: '>=12.0.0'} + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} + engines: {node: '>=14.0.0'} + + tinyspy@4.0.4: + resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} + engines: {node: '>=14.0.0'} + + tldts-core@6.1.86: + resolution: {integrity: sha512-Je6p7pkk+KMzMv2XXKmAE3McmolOQFdxkKw0R8EYNr7sELW46JqnNeTX8ybPiQgvg1ymCoF8LXs5fzFaZvJPTA==} + + tldts@6.1.86: + resolution: {integrity: sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==} + hasBin: true + to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + tough-cookie@5.1.2: + resolution: {integrity: sha512-FVDYdxtnj0G6Qm/DhNPSb8Ju59ULcup3tuJxkFb5K8Bv2pUXILbf0xZWU8PX8Ov19OXljbUyveOFwRMwkXzO+A==} + engines: {node: '>=16'} + + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -2941,6 +3207,11 @@ packages: validate-npm-package-license@3.0.4: resolution: {integrity: sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==} + vite-node@3.2.4: + resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + vite@5.4.21: resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} engines: {node: ^18.0.0 || >=20.0.0} @@ -2972,11 +3243,60 @@ packages: terser: optional: true + vitest@3.2.6: + resolution: {integrity: sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/debug': ^4.1.12 + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@vitest/browser': 3.2.6 + '@vitest/ui': 3.2.6 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/debug': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + + w3c-xmlserializer@5.0.0: + resolution: {integrity: sha512-o8qghlI8NZHU1lLPrpi2+Uq7abh4GGPpYANlalzWxyWteJOCsr/P+oPBA49TOLu5FTZO4d3F9MnWJfiMo4BkmA==} + engines: {node: '>=18'} + wait-on@8.0.5: resolution: {integrity: sha512-J3WlS0txVHkhLRb2FsmRg3dkMTCV1+M6Xra3Ho7HzZDHpE7DCOnoSoCJsZotrmW3uRMhvIJGSKUKrh/MeF4iag==} engines: {node: '>=12.0.0'} hasBin: true + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-encoding@3.1.1: + resolution: {integrity: sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==} + engines: {node: '>=18'} + deprecated: Use @exodus/bytes instead for a more spec-conformant and faster implementation + + whatwg-mimetype@4.0.0: + resolution: {integrity: sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==} + engines: {node: '>=18'} + + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} @@ -2987,6 +3307,11 @@ packages: engines: {node: ^18.17.0 || >=20.5.0} hasBin: true + why-is-node-running@2.3.0: + resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} + engines: {node: '>=8'} + hasBin: true + wrap-ansi@7.0.0: resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} engines: {node: '>=10'} @@ -2998,10 +3323,29 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.21.0: + resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + xml-name-validator@5.0.0: + resolution: {integrity: sha512-EvGK8EJ3DhaHfbRlETOWAS5pO9MZITeauHKJyb8wyajUfQUenkIg2MvLDTZ4T/TgIcm3HU0TFBgWWboAZ30UHg==} + engines: {node: '>=18'} + xmlbuilder@15.1.1: resolution: {integrity: sha512-yMqGBqtXyeN1e3TGYvgNgDVZ3j84W4cwkOXQswghol6APgZWaff9lnbvN7MHYJOiXsvGPXtjTYJEiC9J2wv9Eg==} engines: {node: '>=8.0'} + xmlchars@2.2.0: + resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + y18n@5.0.8: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} @@ -3035,6 +3379,14 @@ snapshots: '@alloc/quick-lru@5.2.0': {} + '@asamuzakjp/css-color@3.2.0': + dependencies: + '@csstools/css-calc': 2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-color-parser': 3.1.0(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + lru-cache: 10.4.3 + '@babel/code-frame@7.29.0': dependencies: '@babel/helper-validator-identifier': 7.28.5 @@ -3158,6 +3510,26 @@ snapshots: '@babel/helper-string-parser': 7.27.1 '@babel/helper-validator-identifier': 7.28.5 + '@csstools/color-helpers@5.1.0': {} + + '@csstools/css-calc@2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-color-parser@3.1.0(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/color-helpers': 5.1.0 + '@csstools/css-calc': 2.1.4(@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4))(@csstools/css-tokenizer@3.0.4) + '@csstools/css-parser-algorithms': 3.0.5(@csstools/css-tokenizer@3.0.4) + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-parser-algorithms@3.0.5(@csstools/css-tokenizer@3.0.4)': + dependencies: + '@csstools/css-tokenizer': 3.0.4 + + '@csstools/css-tokenizer@3.0.4': {} + '@electron/asar@3.4.1': dependencies: commander: 5.1.0 @@ -3820,6 +4192,29 @@ snapshots: dependencies: defer-to-connect: 2.0.1 + '@testing-library/dom@10.4.1': + dependencies: + '@babel/code-frame': 7.29.0 + '@babel/runtime': 7.29.2 + '@types/aria-query': 5.0.4 + aria-query: 5.3.0 + dom-accessibility-api: 0.5.16 + lz-string: 1.5.0 + picocolors: 1.1.1 + pretty-format: 27.5.1 + + '@testing-library/react@16.3.2(@testing-library/dom@10.4.1)(@types/react-dom@19.2.3(@types/react@19.2.15))(@types/react@19.2.15)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)': + dependencies: + '@babel/runtime': 7.29.2 + '@testing-library/dom': 10.4.1 + react: 19.2.6 + react-dom: 19.2.6(react@19.2.6) + optionalDependencies: + '@types/react': 19.2.15 + '@types/react-dom': 19.2.3(@types/react@19.2.15) + + '@types/aria-query@5.0.4': {} + '@types/babel__core@7.20.5': dependencies: '@babel/parser': 7.29.3 @@ -3848,10 +4243,17 @@ snapshots: '@types/node': 22.19.19 '@types/responselike': 1.0.3 + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + '@types/classnames@2.3.4': dependencies: classnames: 2.5.1 + '@types/deep-eql@4.0.2': {} + '@types/estree@1.0.8': {} '@types/fuzzy-search@2.1.5': {} @@ -3932,6 +4334,48 @@ snapshots: transitivePeerDependencies: - supports-color + '@vitest/expect@3.2.6': + dependencies: + '@types/chai': 5.2.3 + '@vitest/spy': 3.2.6 + '@vitest/utils': 3.2.6 + chai: 5.3.3 + tinyrainbow: 2.0.0 + + '@vitest/mocker@3.2.6(vite@5.4.21(@types/node@24.12.4))': + dependencies: + '@vitest/spy': 3.2.6 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 5.4.21(@types/node@24.12.4) + + '@vitest/pretty-format@3.2.6': + dependencies: + tinyrainbow: 2.0.0 + + '@vitest/runner@3.2.6': + dependencies: + '@vitest/utils': 3.2.6 + pathe: 2.0.3 + strip-literal: 3.1.0 + + '@vitest/snapshot@3.2.6': + dependencies: + '@vitest/pretty-format': 3.2.6 + magic-string: 0.30.21 + pathe: 2.0.3 + + '@vitest/spy@3.2.6': + dependencies: + tinyspy: 4.0.4 + + '@vitest/utils@3.2.6': + dependencies: + '@vitest/pretty-format': 3.2.6 + loupe: 3.2.1 + tinyrainbow: 2.0.0 + '@xmldom/xmldom@0.9.10': {} abbrev@3.0.1: {} @@ -3954,6 +4398,8 @@ snapshots: dependencies: color-convert: 2.0.1 + ansi-styles@5.2.0: {} + ansi-styles@6.2.3: {} any-promise@1.3.0: {} @@ -3965,6 +4411,12 @@ snapshots: arg@5.0.2: {} + aria-query@5.3.0: + dependencies: + dequal: 2.0.3 + + assertion-error@2.0.1: {} + asynckit@0.4.0: {} at-least-node@1.0.0: {} @@ -4098,11 +4550,21 @@ snapshots: caniuse-lite@1.0.30001793: {} + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 supports-color: 7.2.0 + check-error@2.1.3: {} + chokidar@3.6.0: dependencies: anymatch: 3.1.3 @@ -4255,10 +4717,20 @@ snapshots: cssesc@3.0.0: {} + cssstyle@4.6.0: + dependencies: + '@asamuzakjp/css-color': 3.2.0 + rrweb-cssom: 0.8.0 + csstype@2.6.21: {} csstype@3.2.3: {} + data-urls@5.0.0: + dependencies: + whatwg-mimetype: 4.0.0 + whatwg-url: 14.2.0 + debug@2.6.9: dependencies: ms: 2.0.0 @@ -4269,10 +4741,14 @@ snapshots: optionalDependencies: supports-color: 5.5.0 + decimal.js@10.6.0: {} + decompress-response@6.0.0: dependencies: mimic-response: 3.1.0 + deep-eql@5.0.2: {} + deep-extend@0.6.0: {} deepmerge@4.3.1: {} @@ -4295,6 +4771,8 @@ snapshots: delayed-stream@1.0.0: {} + dequal@2.0.3: {} + detect-libc@2.1.2: {} detect-node@2.1.0: @@ -4315,6 +4793,8 @@ snapshots: '@react-dnd/invariant': 2.0.0 redux: 4.2.1 + dom-accessibility-api@0.5.16: {} + dom-helpers@5.2.1: dependencies: '@babel/runtime': 7.29.2 @@ -4360,6 +4840,8 @@ snapshots: dependencies: once: 1.4.0 + entities@6.0.1: {} + env-paths@2.2.1: {} env-paths@3.0.0: {} @@ -4374,6 +4856,8 @@ snapshots: es-errors@1.3.0: {} + es-module-lexer@1.7.0: {} + es-object-atoms@1.1.1: dependencies: es-errors: 1.3.0 @@ -4479,10 +4963,16 @@ snapshots: escape-string-regexp@4.0.0: optional: true + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.8 + exact-trie@1.0.13: {} expand-template@2.0.3: {} + expect-type@1.3.0: {} + exponential-backoff@3.1.3: {} extract-zip@2.0.1: @@ -4742,6 +5232,10 @@ snapshots: hotkeys-js@3.13.15: {} + html-encoding-sniffer@4.0.0: + dependencies: + whatwg-encoding: 3.1.1 + http-cache-semantics@4.2.0: {} http-proxy-agent@7.0.2: @@ -4775,7 +5269,6 @@ snapshots: iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 - optional: true ieee754@1.2.1: {} @@ -4823,6 +5316,8 @@ snapshots: is-number@7.0.0: {} + is-potential-custom-element-name@1.0.1: {} + isbinaryfile@4.0.10: {} isexe@2.0.0: {} @@ -4851,6 +5346,36 @@ snapshots: js-tokens@4.0.0: {} + js-tokens@9.0.1: {} + + jsdom@25.0.1: + dependencies: + cssstyle: 4.6.0 + data-urls: 5.0.0 + decimal.js: 10.6.0 + form-data: 4.0.5 + html-encoding-sniffer: 4.0.0 + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + is-potential-custom-element-name: 1.0.1 + nwsapi: 2.2.24 + parse5: 7.3.0 + rrweb-cssom: 0.7.1 + saxes: 6.0.0 + symbol-tree: 3.2.4 + tough-cookie: 5.1.2 + w3c-xmlserializer: 5.0.0 + webidl-conversions: 7.0.0 + whatwg-encoding: 3.1.1 + whatwg-mimetype: 4.0.0 + whatwg-url: 14.2.0 + ws: 8.21.0 + xml-name-validator: 5.0.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + jsesc@3.1.0: {} json-buffer@3.0.1: {} @@ -4994,6 +5519,8 @@ snapshots: dependencies: js-tokens: 4.0.0 + loupe@3.2.1: {} + lowercase-keys@2.0.0: {} lru-cache@10.4.3: {} @@ -5002,6 +5529,8 @@ snapshots: dependencies: yallist: 3.1.1 + lz-string@1.5.0: {} + magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -5155,6 +5684,8 @@ snapshots: normalize-url@6.1.0: {} + nwsapi@2.2.24: {} + object-assign@4.1.1: {} object-hash@3.0.0: {} @@ -5196,6 +5727,10 @@ snapshots: dependencies: error-ex: 1.3.4 + parse5@7.3.0: + dependencies: + entities: 6.0.1 + path-exists@3.0.0: {} path-is-absolute@1.0.1: {} @@ -5215,6 +5750,8 @@ snapshots: pathe@2.0.3: {} + pathval@2.0.1: {} + pe-library@1.0.1: {} pend@1.2.0: {} @@ -5309,6 +5846,12 @@ snapshots: prettier@3.8.3: {} + pretty-format@27.5.1: + dependencies: + ansi-regex: 5.0.1 + ansi-styles: 5.2.0 + react-is: 17.0.2 + proc-log@5.0.0: {} progress@2.0.3: {} @@ -5331,6 +5874,8 @@ snapshots: end-of-stream: 1.4.5 once: 1.4.0 + punycode@2.3.1: {} + queue-microtask@1.2.3: {} quick-lru@5.1.1: {} @@ -5543,6 +6088,10 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.4 fsevents: 2.3.3 + rrweb-cssom@0.7.1: {} + + rrweb-cssom@0.8.0: {} + run-parallel@1.2.0: dependencies: queue-microtask: 1.2.3 @@ -5553,8 +6102,11 @@ snapshots: safe-buffer@5.2.1: {} - safer-buffer@2.1.2: - optional: true + safer-buffer@2.1.2: {} + + saxes@6.0.0: + dependencies: + xmlchars: 2.2.0 scheduler@0.27.0: {} @@ -5588,6 +6140,8 @@ snapshots: dependencies: nanoid: 3.3.12 + siginfo@2.0.0: {} + signal-exit@4.1.0: {} simple-concat@1.0.1: {} @@ -5638,6 +6192,10 @@ snapshots: dependencies: minipass: 7.1.3 + stackback@0.0.2: {} + + std-env@3.10.0: {} + string-width@4.2.3: dependencies: emoji-regex: 8.0.0 @@ -5666,6 +6224,10 @@ snapshots: strip-json-comments@2.0.1: {} + strip-literal@3.1.0: + dependencies: + js-tokens: 9.0.1 + strip-outer@1.0.1: dependencies: escape-string-regexp: 1.0.5 @@ -5720,6 +6282,8 @@ snapshots: symbol-observable@1.2.0: {} + symbol-tree@3.2.4: {} + tailwindcss@3.4.19: dependencies: '@alloc/quick-lru': 5.2.0 @@ -5789,6 +6353,8 @@ snapshots: tiny-warning@1.0.3: {} + tinybench@2.9.0: {} + tinyexec@0.3.2: {} tinyglobby@0.2.16: @@ -5796,10 +6362,30 @@ snapshots: fdir: 6.5.0(picomatch@4.0.4) picomatch: 4.0.4 + tinypool@1.1.1: {} + + tinyrainbow@2.0.0: {} + + tinyspy@4.0.4: {} + + tldts-core@6.1.86: {} + + tldts@6.1.86: + dependencies: + tldts-core: 6.1.86 + to-regex-range@5.0.1: dependencies: is-number: 7.0.0 + tough-cookie@5.1.2: + dependencies: + tldts: 6.1.86 + + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + tree-kill@1.2.2: {} trim-repeated@1.0.0: @@ -5883,6 +6469,24 @@ snapshots: spdx-correct: 3.2.0 spdx-expression-parse: 3.0.1 + vite-node@3.2.4(@types/node@24.12.4): + dependencies: + cac: 6.7.14 + debug: 4.4.3(supports-color@5.5.0) + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 5.4.21(@types/node@24.12.4) + transitivePeerDependencies: + - '@types/node' + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + vite@5.4.21(@types/node@24.12.4): dependencies: esbuild: 0.21.5 @@ -5892,6 +6496,49 @@ snapshots: '@types/node': 24.12.4 fsevents: 2.3.3 + vitest@3.2.6(@types/node@24.12.4)(jsdom@25.0.1): + dependencies: + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.6 + '@vitest/mocker': 3.2.6(vite@5.4.21(@types/node@24.12.4)) + '@vitest/pretty-format': 3.2.6 + '@vitest/runner': 3.2.6 + '@vitest/snapshot': 3.2.6 + '@vitest/spy': 3.2.6 + '@vitest/utils': 3.2.6 + chai: 5.3.3 + debug: 4.4.3(supports-color@5.5.0) + expect-type: 1.3.0 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.4 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.16 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 5.4.21(@types/node@24.12.4) + vite-node: 3.2.4(@types/node@24.12.4) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/node': 24.12.4 + jsdom: 25.0.1 + transitivePeerDependencies: + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + + w3c-xmlserializer@5.0.0: + dependencies: + xml-name-validator: 5.0.0 + wait-on@8.0.5: dependencies: axios: 1.16.1 @@ -5903,6 +6550,19 @@ snapshots: - debug - supports-color + webidl-conversions@7.0.0: {} + + whatwg-encoding@3.1.1: + dependencies: + iconv-lite: 0.6.3 + + whatwg-mimetype@4.0.0: {} + + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + which@2.0.2: dependencies: isexe: 2.0.0 @@ -5911,6 +6571,11 @@ snapshots: dependencies: isexe: 3.1.5 + why-is-node-running@2.3.0: + dependencies: + siginfo: 2.0.0 + stackback: 0.0.2 + wrap-ansi@7.0.0: dependencies: ansi-styles: 4.3.0 @@ -5925,8 +6590,14 @@ snapshots: wrappy@1.0.2: {} + ws@8.21.0: {} + + xml-name-validator@5.0.0: {} + xmlbuilder@15.1.1: {} + xmlchars@2.2.0: {} + y18n@5.0.8: {} yallist@3.1.1: {} diff --git a/ts/tests/e2e/flows/url-load.spec.ts b/ts/tests/e2e/flows/url-load.spec.ts index 862ed65..b76b9b1 100644 --- a/ts/tests/e2e/flows/url-load.spec.ts +++ b/ts/tests/e2e/flows/url-load.spec.ts @@ -52,3 +52,28 @@ test('@network open the real remote URL directly', async ({ driver }) => { await driver.enterPartition(part.index); await expectKnownTree(driver, part); }); + +// The "Open URL…" DIALOG UI: the legacy CDP suite's typeUrl() fallback proved a +// user can open an image by clicking the picker's "Open URL…" button, typing +// into the aria-label="Disk image URL" input, and clicking the dialog's Open +// button — a real user journey the driver.openUrl() bridge hook bypasses. +// Ported here so the dialog UI keeps a regression guard. Scoped to web: the +// dialog is renderer DOM from the same vite-demo bundle on every shell, and +// only the web project exposes the Playwright `page` for direct DOM driving +// (electron projects get their window inside ElectronDriver). +test('open via the "Open URL…" dialog UI, browse known entry', async ({ driver, page }, testInfo) => { + test.skip( + testInfo.project.name !== 'web', + 'dialog DOM is identical across shells (same vite-demo bundle); proven once on web', + ); + void driver; // fixture already navigated the page to /?e2e=1 and awaited the bridge + await page.getByTestId('open-url-button').click(); + const input = page.getByLabel('Disk image URL'); + await input.waitFor({ state: 'visible', timeout: 30_000 }); + await input.fill(server.url); + // The Open button probes the URL (HEAD via probeUrlAhead) before submitting; + // the Range server answers HEAD with Accept-Ranges + CORS, so it enables. + await page.getByTestId('url-dialog-submit').click(); + await driver.enterPartition(part.index); + await expectKnownTree(driver, part); +}); diff --git a/tests/test-core.mjs b/ts/tests/integration/test-core.mjs similarity index 75% rename from tests/test-core.mjs rename to ts/tests/integration/test-core.mjs index 33d1b8a..935de72 100644 --- a/tests/test-core.mjs +++ b/ts/tests/integration/test-core.mjs @@ -9,11 +9,11 @@ * module loader (native addon vs WASM) and disk-open target (local path vs * http:// URL via QEMU curl) differ. * - * Usage: - * node tests/test-core.mjs # all 4 combos - * node tests/test-core.mjs --only native # native addon only - * node tests/test-core.mjs --only wasm # WASM only - * node tests/test-core.mjs --image path.img # custom disk image + * Usage (from ts/): + * pnpm run test:integration # all 4 combos + * node tests/integration/test-core.mjs --only native # native addon only + * node tests/integration/test-core.mjs --only wasm # WASM only + * node tests/integration/test-core.mjs --image path.img # custom disk image */ import { resolve, dirname } from 'node:path'; @@ -23,7 +23,7 @@ import { Worker } from 'node:worker_threads'; import { createRequire } from 'node:module'; const __dirname = dirname(fileURLToPath(import.meta.url)); -const ROOT = resolve(__dirname, '..'); +const ROOT = resolve(__dirname, '../../..'); // repo root (this file lives in ts/tests/integration/) const NATIVE_NODE = resolve(ROOT, 'ts/packages/anyfs-native/build/Release/anyfs_native.node'); const WASM_DIR = resolve(ROOT, 'ts/packages/core/wasm'); const require_ = createRequire(import.meta.url); @@ -127,49 +127,74 @@ function startHttpWorker(filePath) { // Helpers // ═══════════════════════════════════════════════════════════════════════════════ -function mountAndReaddir(listFn, enterFn, mountWholeFn, readdirFn) { - const parts = JSON.parse(listFn()); +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +// Session API: part 0 = whole disk (raw filesystem, no partition table); +// part >= 1 = top-level partition by `index`. Replaces the old +// diskEnter/mountWhole split. +async function mountAndReaddir(listFn, enterFn, readdirFn) { + const parts = JSON.parse(await listFn()); const label = parts.length > 0 - ? parts.map(p => `${p.num}:${p.fstype || '?'}`).join(', ') + ? parts.map(p => `${p.index}:${p.fstype || '?'}`).join(', ') : '(raw filesystem)'; console.log(` partitions: ${label}`); - let mp; - if (parts.length === 0) { - mp = mountWholeFn('auto', 1); - assert(mp && typeof mp === 'string', `mountWhole → ${mp}`); - } else { - mp = enterFn(parts[0].num, 1); - assert(mp && typeof mp === 'string', `diskEnter → ${mp}`); - } + const part = parts.length === 0 ? 0 : parts[0].index; + const mp = await enterFn(part, 1); + assert(mp && typeof mp === 'string', `sessionEnter(part=${part}) → ${mp}`); - const entries = JSON.parse(readdirFn(mp)); + const entries = JSON.parse(await readdirFn(mp)); assert(entries.length > 0, `readdir → ${entries.length} entries`); const names = entries.filter(e => e.name !== '.' && e.name !== '..').map(e => e.name); console.log(` files: ${names.join(', ')}`); return names; } -function callJsonString(M, fnName, pathArg) { - let cap = 4096; - for (let i = 0; i < 6; i++) { - const buf = M._malloc(cap); - try { - const n = M.ccall(fnName, 'number', ['string', 'number', 'number'], [pathArg, buf, cap]); - if (n >= 0) return M.UTF8ToString(buf, n); - if (-n <= cap) throw new Error(`${fnName}(${pathArg}) rc=${n}`); - cap = Math.max(cap * 2, -n + 256); - } finally { M._free(buf); } +// ── wasm async `_p` calling pattern (mirrors packages/core/test/smoke.node.mjs) ── +// +// Every entry point that can touch the QEMU block layer goes through the `_p` +// out-pointer variant with ccall({async: true}): the block layer runs on +// emscripten fibers, and a fiber swap discards the export's direct return +// value, so the C side writes the result through a trailing int32_t* out +// parameter. Sync ccall is only safe for calls that never reach the block +// layer (init/poll/halt). + +function makeWasmCallers(M) { + const outp = M._malloc(4); + async function callP(name, argTypes, args) { + M.HEAP32[outp >> 2] = -0x7fffffff; + await M.ccall(name, null, [...argTypes, 'number'], [...args, outp], { async: true }); + return M.HEAP32[outp >> 2]; } - throw new Error(`${fnName}: overflow loop`); + async function callA(name, retType, argTypes, args) { + return await M.ccall(name, retType, argTypes, args, { async: true }); + } + return { callP, callA, dispose: () => M._free(outp) }; } -function callJsonHandle(M, fnName, h) { +// Boot: prefer the dedicated-pthread async path when the bundle exports it +// (keeps this thread's event loop free to service kthread spawns). +async function wasmBoot(M, callA, memMb, loglevel) { + if (M._anyfs_ts_init_async) { + const arc = M.ccall('anyfs_ts_init_async', 'number', ['number', 'number'], [memMb, loglevel]); + if (arc !== 0) return arc; + for (let i = 0; i < 600; i++) { + if (M.ccall('anyfs_ts_is_boot_complete', 'number', [], [])) break; + await sleep(100); + } + return M.ccall('anyfs_ts_boot_result', 'number', [], []); + } + return await callA('anyfs_ts_kernel_init', 'number', ['number', 'number'], [memMb, loglevel]); +} + +// Buffer-grow JSON-out call through a `_p` variant. `argTypes`/`args` are the +// leading args (handle or path); buf+cap are appended. +async function wasmCallJson(M, callP, fnName, argTypes, args) { let cap = 4096; for (let i = 0; i < 6; i++) { const buf = M._malloc(cap); try { - const n = M.ccall(fnName, 'number', ['number', 'number', 'number'], [h, buf, cap]); + const n = await callP(fnName, [...argTypes, 'number', 'number'], [...args, buf, cap]); if (n >= 0) return M.UTF8ToString(buf, n); if (-n <= cap) throw new Error(`${fnName} rc=${n}`); cap = Math.max(cap * 2, -n + 256); @@ -178,22 +203,39 @@ function callJsonHandle(M, fnName, h) { throw new Error(`${fnName}: overflow loop`); } -function wasmCallMount(M, fnName, h, part, flags) { - const buf = M._malloc(128); +// Prefer the dedicated-pthread enter (ext4's jbd2 kthread can't spawn while +// the entering thread is blocked); fall back to the `_p` variant. +async function wasmSessionEnter(M, callP, h, part, flags) { + const mountBuf = M._malloc(128); try { - const rc = M.ccall(fnName, 'number', ['number', 'number', 'number', 'number', 'number'], [h, part, flags, buf, 128]); - if (rc !== 0) throw new Error(`${fnName}(h=${h}) rc=${rc}`); - return M.UTF8ToString(buf); - } finally { M._free(buf); } -} - -function wasmCallMountWhole(M, fnName, h, fstype, flags) { - const buf = M._malloc(128); - try { - const rc = M.ccall(fnName, 'number', ['number', 'string', 'number', 'number', 'number'], [h, fstype ?? '', flags, buf, 128]); - if (rc !== 0) throw new Error(`${fnName}(h=${h}) rc=${rc}`); - return M.UTF8ToString(buf); - } finally { M._free(buf); } + let rc; + if (M._anyfs_ts_session_enter_async) { + rc = M.ccall( + 'anyfs_ts_session_enter_async', + 'number', + ['number', 'number', 'number'], + [h, part, flags], + ); + if (rc === 0) { + let done = 0; + for (let i = 0; i < 600; i++) { + done = M.ccall('anyfs_ts_session_enter_is_complete', 'number', [], []); + if (done) break; + await sleep(100); + } + if (!done) throw new Error('session_enter timed out'); + rc = await callP('anyfs_ts_session_enter_result_p', ['number', 'number'], [mountBuf, 128]); + } + } else { + rc = await callP( + 'anyfs_ts_session_enter_p', + ['number', 'number', 'number', 'number', 'number'], + [h, part, flags, mountBuf, 128], + ); + } + if (rc !== 0) throw new Error(`session_enter(h=${h}, part=${part}) rc=${rc}`); + return M.UTF8ToString(mountBuf); + } finally { M._free(mountBuf); } } // ═══════════════════════════════════════════════════════════════════════════════ @@ -202,20 +244,21 @@ function wasmCallMountWhole(M, fnName, h, fstype, flags) { async function testNative(imagePath) { const m = require_(NATIVE_NODE); - const rc = m.init(64, 7); - assert(rc === 0, `init(64,7) = ${rc}`); + // kernelInit/sessionOpen/sessionEnter/sessionClose/kernelHalt are sync; + // sessionListJson/readdirJson (AsyncWorker-based) return promises. + const rc = m.kernelInit(64, 7); + assert(rc === 0, `kernelInit(64,7) = ${rc}`); // ── file ────────────────────────────────────────────────────────────── console.log(`\n ${BOLD}[native+file]${RST} ${imagePath}`); - let h = m.diskOpen(imagePath, 1); - assert(h >= 0, `diskOpen(file) = ${h}`); - mountAndReaddir( - () => m.diskListJson(h), - (p, f) => m.diskEnter(h, p, f), - (fs, f) => m.mountWhole(h, fs, f), + let h = m.sessionOpen(imagePath, 1); + assert(h >= 0, `sessionOpen(file) = ${h}`); + await mountAndReaddir( + () => m.sessionListJson(h), + (p, f) => m.sessionEnter(h, p, f), (p) => m.readdirJson(p), ); - m.diskClose(h); + m.sessionClose(h); // ── URL ──────────────────────────────────────────────────────────────── console.log(`\n ${BOLD}[native+url]${RST} ${imagePath}`); @@ -226,15 +269,14 @@ async function testNative(imagePath) { const url = `http://127.0.0.1:${r.port}/disk`; console.log(` http server: ${url}`); - h = m.diskOpen(url, 1); - assert(h >= 0, `diskOpen(url) = ${h}`); - mountAndReaddir( - () => m.diskListJson(h), - (p, f) => m.diskEnter(h, p, f), - (fs, f) => m.mountWhole(h, fs, f), + h = m.sessionOpen(url, 1); + assert(h >= 0, `sessionOpen(url) = ${h}`); + await mountAndReaddir( + () => m.sessionListJson(h), + (p, f) => m.sessionEnter(h, p, f), (p) => m.readdirJson(p), ); - m.diskClose(h); + m.sessionClose(h); } finally { if (worker) worker.postMessage('stop'); } @@ -266,22 +308,23 @@ async function testWasmFile(imagePath) { }); pass('WASM module loaded'); - const rc = M.ccall('anyfs_ts_init', 'number', ['number', 'number'], [64, 7]); - assert(rc === 0, `anyfs_ts_init = ${rc}`); + const { callP, callA, dispose } = makeWasmCallers(M); + const rc = await wasmBoot(M, callA, 64, 7); + assert(rc === 0, `kernel boot = ${rc}`); const fsPath = `/work/${imgName}`; - const dk = M.ccall('anyfs_ts_disk_open', 'number', ['string', 'number'], [fsPath, 1]); - assert(dk >= 0, `diskOpen(${fsPath}) = ${dk}`); - - mountAndReaddir( - () => callJsonHandle(M, 'anyfs_ts_disk_list_json', dk), - (p, f) => wasmCallMount(M, 'anyfs_ts_disk_enter', dk, p, f), - (fs, f) => wasmCallMountWhole(M, 'anyfs_ts_mount_whole', dk, fs, f), - (p) => callJsonString(M, 'anyfs_ts_readdir_json', p), + const dk = await callP('anyfs_ts_session_open_p', ['string', 'number'], [fsPath, 1]); + assert(dk >= 0, `sessionOpen(${fsPath}) = ${dk}`); + + await mountAndReaddir( + () => wasmCallJson(M, callP, 'anyfs_ts_session_list_json_p', ['number'], [dk]), + (p, f) => wasmSessionEnter(M, callP, dk, p, f), + (p) => wasmCallJson(M, callP, 'anyfs_ts_readdir_json_p', ['string'], [p]), ); - M.ccall('anyfs_ts_disk_close', 'number', ['number'], [dk]); - M.ccall('anyfs_ts_kernel_halt', 'number', [], []); + await callA('anyfs_ts_session_close', 'number', ['number'], [dk]); + await callA('anyfs_ts_kernel_halt', 'number', [], []); + dispose(); } // ═══════════════════════════════════════════════════════════════════════════════ @@ -531,26 +574,27 @@ async function testWasmUrl(imagePath) { const M = await factory(); pass('WASM module loaded'); - const rc = M.ccall('anyfs_ts_init', 'number', ['number', 'number'], [64, 7]); - assert(rc === 0, `anyfs_ts_init = ${rc}`); + const { callP, callA, dispose } = makeWasmCallers(M); + const rc = await wasmBoot(M, callA, 64, 7); + assert(rc === 0, `kernel boot = ${rc}`); // Mount URLFS at /work — the kernel sees /work/disk as a regular file. const URLFS = createUrlFsNode(M); M.FS.mkdir('/work'); M.FS.mount(URLFS, { url, name: 'disk' }, '/work'); - const dk = M.ccall('anyfs_ts_disk_open', 'number', ['string', 'number'], ['/work/disk', 1]); - assert(dk >= 0, `diskOpen(/work/disk) = ${dk}`); + const dk = await callP('anyfs_ts_session_open_p', ['string', 'number'], ['/work/disk', 1]); + assert(dk >= 0, `sessionOpen(/work/disk) = ${dk}`); - mountAndReaddir( - () => callJsonHandle(M, 'anyfs_ts_disk_list_json', dk), - (p, f) => wasmCallMount(M, 'anyfs_ts_disk_enter', dk, p, f), - (fs, f) => wasmCallMountWhole(M, 'anyfs_ts_mount_whole', dk, fs, f), - (p) => callJsonString(M, 'anyfs_ts_readdir_json', p), + await mountAndReaddir( + () => wasmCallJson(M, callP, 'anyfs_ts_session_list_json_p', ['number'], [dk]), + (p, f) => wasmSessionEnter(M, callP, dk, p, f), + (p) => wasmCallJson(M, callP, 'anyfs_ts_readdir_json_p', ['string'], [p]), ); - M.ccall('anyfs_ts_disk_close', 'number', ['number'], [dk]); - M.ccall('anyfs_ts_kernel_halt', 'number', [], []); + await callA('anyfs_ts_session_close', 'number', ['number'], [dk]); + await callA('anyfs_ts_kernel_halt', 'number', [], []); + dispose(); } finally { worker.postMessage('stop'); } @@ -585,7 +629,10 @@ async function main() { } console.log(`\n${BOLD}[test-core]${RST} ${GREEN}${passed} pass${RST}, ${failed > 0 ? RED : ''}${failed} fail${RST}`); - if (failed > 0) process.exit(1); + // Explicit exit: lingering LKL/QEMU host threads (and the HTTP worker) + // keep the event loop alive after kernelHalt — known teardown behavior, + // all assertions have already run by this point. + process.exit(failed > 0 ? 1 : 0); } main(); diff --git a/ts/tests/native-session.test.mjs b/ts/tests/native-session.test.mjs index 03635ab..1bae671 100644 --- a/ts/tests/native-session.test.mjs +++ b/ts/tests/native-session.test.mjs @@ -22,9 +22,9 @@ const IMAGE = process.env.IMAGE || '/home/kosaka/debian-13.5.0-amd64-netinst.iso let passed = 0; let failed = 0; -function test(name, fn) { +async function test(name, fn) { try { - fn(); + await fn(); passed++; console.log(` ✓ ${name}`); } catch (err) { @@ -45,22 +45,22 @@ console.log(`Exports: ${Object.keys(addon).sort().join(', ')}\n`); let handle = -1; -test('kernelInit(256, 7) returns 0', () => { +await test('kernelInit(256, 7) returns 0', () => { const rc = addon.kernelInit(256, 7); strictEqual(rc, 0); }); // ── 2. Session open ─────────────────────────────── -test('sessionOpen(path, 1) returns handle >= 0', () => { +await test('sessionOpen(path, 1) returns handle >= 0', () => { handle = addon.sessionOpen(IMAGE, 1); assert(handle >= 0, `handle=${handle} expected >= 0`); }); // ── 3. List parts ───────────────────────────────── -test('sessionListJson returns array', () => { - const json = addon.sessionListJson(handle); +await test('sessionListJson returns array', async () => { + const json = await addon.sessionListJson(handle); const list = JSON.parse(json); assert(Array.isArray(list), 'expected array'); console.log(` ${list.length} partition entries`); @@ -74,8 +74,8 @@ test('sessionListJson returns array', () => { // ── 4. Meta ─────────────────────────────────────── -test('sessionMetaJson returns expected shape', () => { - const json = addon.sessionMetaJson(handle); +await test('sessionMetaJson returns expected shape', async () => { + const json = await addon.sessionMetaJson(handle); const meta = JSON.parse(json); assert(typeof meta.logical_size === 'number', 'expected logical_size'); assert(typeof meta.pt_type === 'string', 'expected pt_type'); @@ -84,13 +84,13 @@ test('sessionMetaJson returns expected shape', () => { // ── 5. Enter partition ────────────────────── -test('sessionEnter(handle, 2, 0) mounts partition', () => { +await test('sessionEnter(handle, 2, 0) mounts partition', async () => { const mp = addon.sessionEnter(handle, 2, 0); assert(mp && mp.length > 0, `mount path empty: "${mp}"`); console.log(` mount path: "${mp}"`); // readdir the mount point - const json = addon.readdirJson(mp); + const json = await addon.readdirJson(mp); const entries = JSON.parse(json); assert(Array.isArray(entries), 'expected entries array'); console.log(` ${entries.length} root entries`); @@ -101,7 +101,7 @@ test('sessionEnter(handle, 2, 0) mounts partition', () => { // ── 6. Close ────────────────────────────────────── -test('sessionClose(handle) returns 0', () => { +await test('sessionClose(handle) returns 0', () => { const rc = addon.sessionClose(handle); strictEqual(rc, 0); handle = -1; @@ -109,7 +109,7 @@ test('sessionClose(handle) returns 0', () => { // ── 7. Halt ─────────────────────────────────────── -test('kernelHalt succeeds', () => { +await test('kernelHalt succeeds', () => { addon.kernelHalt(); // kernelHalt is void assert(true, 'kernelHalt completed');