From 7387099a94abd33ae2207fbf23c4d93254e424ce Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 1 Jul 2026 15:23:52 +0800 Subject: [PATCH 01/13] adapt workerd 20260701 Upgrade workerd to 1.20260701.1, split process-level and loaded-worker experimental usage, add workerLoader env/code guards, bound log-tail cleanup for workerd #6832, and document compatibility and ECS capacity impacts. Signed-off-by: Lu Zhang --- CLAUDE.md | 10 +- Dockerfile.workerd | 3 +- control/env-budget.js | 104 ++++++++++++++++ control/handlers/deploy.js | 74 ++++++++++- control/handlers/logs-tail.js | 72 ++++++++--- control/handlers/ns-secrets.js | 70 +++++++++++ control/handlers/secret-put.js | 3 +- control/handlers/worker-secrets.js | 41 ++++++- deploy/kubernetes/base/gateway.yaml | 1 - do-runtime/config.capnp | 2 +- do-runtime/load.js | 3 +- do-runtime/protocol.js | 1 - docker-compose.yml | 2 +- docs/compatibility.md | 7 +- docs/compatibility.zh.md | 7 +- docs/modules/cli.md | 4 +- docs/modules/cli.zh.md | 4 +- docs/modules/log-tail-observability.md | 5 + docs/modules/log-tail-observability.zh.md | 1 + docs/modules/runtime.md | 27 +++- docs/modules/runtime.zh.md | 9 +- package-lock.json | 56 ++++----- package.json | 4 +- runtime/config-system.capnp | 6 +- runtime/config-user.capnp | 5 +- runtime/lib.js | 6 +- runtime/load.js | 5 +- rust/supervisor/src/config.rs | 30 +++-- rust/supervisor/src/lib.rs | 9 +- terraform/README.md | 4 + terraform/modules/compute/gateway_service.tf | 2 +- tests/integration/http-features.test.js | 10 +- tests/unit/control-deploy-watch.test.js | 116 ++++++++++++++++++ tests/unit/control-env-budget.test.js | 84 +++++++++++++ tests/unit/control-lib.test.js | 2 +- tests/unit/control-logs-tail.test.js | 25 ++++ .../control-secret-envelope-handlers.test.js | 17 +++ tests/unit/do-runtime-protocol.test.js | 2 +- tests/unit/runtime-lib.test.js | 8 +- tests/unit/runtime-load.test.js | 2 +- tests/unit/style-contracts.test.js | 28 ++++- 41 files changed, 751 insertions(+), 120 deletions(-) create mode 100644 control/env-budget.js create mode 100644 tests/unit/control-env-budget.test.js diff --git a/CLAUDE.md b/CLAUDE.md index 01bc50b..8bf73a2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,7 +73,9 @@ shared crate `wdl-rust-common`. Runtime receives plaintext only in the internal load envelope after redis-proxy decrypts during `/runtime/load`. Env materializes in fixed precedence — vars, then namespace secrets, then worker secrets — so a worker secret shadows a namespace secret - shadows a var on the same key. + shadows a var on the same key. Control must keep vars plus namespace/worker secrets + within WDL's headroomed workerd `workerLoader` serialized env budget before + deploy/secret mutation, not let that fail later during cold-load. - **DB split is intentional.** DB 0 is control metadata, DB 1 is data-plane KV/queue/log streams, DB 2 is Workflows. See `docs/redis-key-layout.md`. - **D1/DO correctness comes from owner lease + generation fence.** Service DNS only @@ -165,8 +167,10 @@ output capture, or fixture loading. host-only helpers in module functions or WeakMaps. - `workerd serve` emits raw console stdout in addition to structured tail events. The structured JSON log is the platform source of truth. -- Mid-response disconnects do not reliably trip `request.signal`; use - `ReadableStream.cancel` plus pre-registered `ctx.waitUntil` for cleanup. +- On current workerd, mid-response disconnects do not reliably trip `request.signal` + or async response-body `ReadableStream.cancel`; do not use either as the only + cleanup signal. Bound streaming responses with an independent timeout or an + explicit app-level heartbeat/close path. - Forwarding WebSocket `101` responses requires preserving `response.webSocket`; use the shared response helper instead of wrapping with `new Response(body, init)`. - workerd/capnp wiring traps apply to every tier's `*.capnp`, not just runtime: `workerd diff --git a/Dockerfile.workerd b/Dockerfile.workerd index 883d1c7..ca12156 100644 --- a/Dockerfile.workerd +++ b/Dockerfile.workerd @@ -86,7 +86,7 @@ RUN ["/usr/local/bin/workerd", "--version"] # ECS task definitions pick the entry/command: # gateway : entryPoint ["workerd"] -# command ["serve", "-b", "/app/dist/workerd-configs/gateway.bin", "--experimental"] +# command ["serve", "-b", "/app/dist/workerd-configs/gateway.bin"] # user-runtime : entryPoint ["workerd"] # command ["serve", "-b", "/app/dist/workerd-configs/user-runtime.bin", "--experimental"] # system-runtime : entryPoint ["workerd"] @@ -95,5 +95,6 @@ RUN ["/usr/local/bin/workerd", "--version"] # stopTimeout >= 20s # env D1_OWNER_TTL_SECONDS=120 D1_PROBE_TIMEOUT_MS=500 D1_QUERY_TIMEOUT_MS=30000 D1_SHUTDOWN_TIMEOUT_MS=15000 # do-runtime : entryPoint ["do-supervisor"] +# child workerd args include "--experimental" for workerLoader # stopTimeout >= 20s # env DO_OWNER_TTL_SECONDS=120 DO_DRAIN_IN_FLIGHT_TIMEOUT_MS=8000 diff --git a/control/env-budget.js b/control/env-budget.js new file mode 100644 index 0000000..e51e9e1 --- /dev/null +++ b/control/env-budget.js @@ -0,0 +1,104 @@ +import { decryptSecretValue } from "shared-secret-envelope"; + +export const UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES = 1024 * 1024; +export const WORKER_LOADER_ENV_HEADROOM_BYTES = 8 * 1024; +export const WORKER_LOADER_ENV_MAX_BYTES = + UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES - WORKER_LOADER_ENV_HEADROOM_BYTES; + +export class WorkerEnvBudgetError extends Error { + /** + * @param {string} message + * @param {Record} [details] + */ + constructor(message, details = {}) { + super(message); + this.status = 400; + this.code = "worker_env_too_large"; + this.details = details; + } +} + +/** @param {Record | null | undefined} source */ +function stringRecord(source) { + /** @type {Record} */ + const out = Object.create(null); + for (const [key, value] of Object.entries(source || {})) { + if (typeof value === "string") out[key] = value; + } + return out; +} + +/** + * @param {{ + * vars?: Record | null, + * nsSecrets?: Record | null, + * workerSecrets?: Record | null, + * }} args + */ +export function mergedUserEnvStrings({ vars = null, nsSecrets = null, workerSecrets = null }) { + return { + ...stringRecord(vars), + ...stringRecord(nsSecrets), + ...stringRecord(workerSecrets), + }; +} + +/** @param {Record} envStrings */ +export function userEnvSerializedBytes(envStrings) { + return Buffer.byteLength(JSON.stringify(envStrings), "utf8"); +} + +/** + * @param {{ + * ns: string, + * worker?: string, + * vars?: Record | null, + * nsSecrets?: Record | null, + * workerSecrets?: Record | null, + * }} args + */ +export function assertWorkerLoaderUserEnvBudget({ + ns, + worker = undefined, + vars = null, + nsSecrets = null, + workerSecrets = null, +}) { + const merged = mergedUserEnvStrings({ vars, nsSecrets, workerSecrets }); + // workerd enforces the full workerLoader env as a Frankenvalue estimate. Control + // checks the user-controlled string env as JSON and leaves headroom for runtime + // facade objects and estimator drift. + const bytes = userEnvSerializedBytes(merged); + if (bytes > WORKER_LOADER_ENV_MAX_BYTES) { + const label = worker ? `${ns}/${worker}` : ns; + throw new WorkerEnvBudgetError( + `vars and secrets for ${label} serialize to ${bytes} bytes, exceeding WDL workerLoader env budget ${WORKER_LOADER_ENV_MAX_BYTES} bytes`, + { + namespace: ns, + ...(worker ? { worker } : {}), + env_bytes: bytes, + max_env_bytes: WORKER_LOADER_ENV_MAX_BYTES, + upstream_max_env_bytes: UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + headroom_bytes: WORKER_LOADER_ENV_HEADROOM_BYTES, + } + ); + } + return bytes; +} + +/** + * @param {{ + * encrypted: Record, + * env: Record, + * hashKey: string, + * }} args + */ +export async function decryptSecretHash({ encrypted, env, hashKey }) { + /** @type {Record} */ + const out = Object.create(null); + for (const [fieldName, value] of Object.entries(encrypted || {})) { + if (typeof value !== "string") continue; + out[fieldName] = await decryptSecretValue(value, { env, hashKey, fieldName }); + } + return out; +} diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 2829fe3..11cfe11 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -1,7 +1,7 @@ import { jsonResponse, jsonError, readJsonBody, formatError, requireControlLog, requireControlRedis, - errMessage, prefixedId, + errMessage, prefixedId, stringEnv, getControlS3, runOptimistic, stageBundleCommit, buildS3CleanupTaskId, recordS3CleanupIntent, @@ -42,9 +42,16 @@ import { isReservedNs, isValidRouteNs, ROUTES_ALLOWED_RESERVED_NS } from "shared import { putAsset, inferContentType } from "control-s3"; import { generateAssetsToken, assetsPrefixFor } from "shared-assets-token"; import { resolveDatabaseRefFrom } from "control-d1-store"; +import { + WorkerEnvBudgetError, + assertWorkerLoaderUserEnvBudget, + decryptSecretHash, +} from "control-env-budget"; +import { SecretEnvelopeError } from "shared-secret-envelope"; const MAX_COMMIT_ATTEMPTS = 5; const DEPLOY_JSON_BODY_MAX_BYTES = 32 * 1024 * 1024; +const WORKER_LOADER_CODE_MAX_BYTES = 64 * 1024 * 1024; const DEPLOY_ASSET_UPLOAD_CONCURRENCY = 8; class DeployAbort extends ControlAbort {} @@ -146,6 +153,28 @@ function deployRequestErrorFromUnknown(err) { ); } +/** @param {string | Uint8Array} bytes */ +function moduleBodyByteLength(bytes) { + return typeof bytes === "string" ? Buffer.byteLength(bytes, "utf8") : bytes.byteLength; +} + +/** + * @param {{ prepared: PreparedBundle, ns: string, name: string }} args + */ +function validateWorkerLoaderCodeBudget({ prepared, ns, name }) { + let totalBytes = 0; + for (const [, bytes] of prepared.normalized) { + totalBytes += moduleBodyByteLength(bytes); + } + if (totalBytes > WORKER_LOADER_CODE_MAX_BYTES) { + throw new DeployRequestError( + 413, + "worker_code_too_large", + `module bodies for ${ns}/${name} total ${totalBytes} bytes, exceeding workerd workerLoader code limit ${WORKER_LOADER_CODE_MAX_BYTES} bytes` + ); + } +} + /** @param {BindingMap} [bindings] */ function hasDurableObjectBinding(bindings = {}) { return Object.values(bindings).some((spec) => spec?.type === "do"); @@ -575,6 +604,28 @@ async function runDeployPreflight({ redis, ns, name, deployRequest }) { }); } +/** + * @param {{ redis: RedisClient, env: Record, ns: string, name: string, meta: PreparedMeta }} args + */ +async function validateCommittedEnvBudget({ redis, env, ns, name, meta }) { + const controlEnv = stringEnv(env); + const nsEncrypted = await redis.hGetAll(`secrets:${ns}`); + const workerEncrypted = await redis.hGetAll(`secrets:${ns}:${name}`); + const [nsSecrets, workerSecrets] = await Promise.all([ + decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: `secrets:${ns}` }), + decryptSecretHash({ encrypted: workerEncrypted, env: controlEnv, hashKey: `secrets:${ns}:${name}` }), + ]); + assertWorkerLoaderUserEnvBudget({ + ns, + worker: name, + vars: meta.vars && typeof meta.vars === "object" + ? /** @type {Record} */ (meta.vars) + : null, + nsSecrets, + workerSecrets, + }); +} + /** * @param {{ deployRequest: DeployRequest, ns: string, name: string, mergedBindings: BindingMap }} args * @returns {{ response: Response, committed?: never } | { response?: never, committed: CommittedBundle }} @@ -719,10 +770,31 @@ export async function handle({ request, env, ns, name, requestId }) { d1Refs, } = candidate.committed; + try { + validateWorkerLoaderCodeBudget({ prepared, ns, name }); + } catch (err) { + if (err instanceof DeployRequestError) return deployRequestErrorResponse(err); + throw err; + } + if (parsed.deployRequest.assetsToUpload && !s3) { return deployAssetsS3NotConfiguredResponse(); } + try { + await validateCommittedEnvBudget({ + redis, + env, + ns, + name, + meta: prepared.meta, + }); + } catch (err) { + if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); + if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + throw err; + } + const num = await redis.incr(`worker:${ns}:${name}:next_version`); const version = formatVersion(num); diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index 4888b9f..6e6f0c7 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -1,7 +1,7 @@ // SSE handler for `wdl tail`. Pull-based ReadableStream + pre-registered -// ctx.waitUntil(cancelPromise) so the cancel callback fires reliably on -// HTTP client disconnect. Push-style intervals have historically failed to -// propagate disconnect cleanup reliably here. +// ctx.waitUntil(cancelPromise) keeps cleanup outside the cancel callback. +// workerd >= 2026-06-19 no longer reliably calls cancel() on client +// disconnect, so max-session cleanup also has an independent watchdog. import { RedisSession, redisDbFromEnv } from "shared-redis"; import { envValueOr } from "shared-env"; @@ -361,13 +361,49 @@ export async function handle({ request, env, ctx, ns, requestId }) { }); let sessionOpen = false; let cancelled = false; + /** @type {ReadableStreamDefaultController | null} */ + let streamController = null; + let expiryLogged = false; const { promise: cancelPromise, resolve: resolveCancel } = /** @type {PromiseWithResolvers} */ (Promise.withResolvers()); + const sessionExpiredWarning = () => sseEvent({ + event: "tail_warning", + data: JSON.stringify({ + event: "tail_warning", + code: "session_expired", + message: "Tail session reached its maximum lifetime; reconnecting for reauthorization.", + }), + }); + + /** @param {ReadableStreamDefaultController | null} controller */ + function expireSession(controller) { + if (cancelled) return; + cancelled = true; + if (!expiryLogged) { + expiryLogged = true; + log("info", "tail_session_expired", { + request_id: requestId, namespace: ns, worker_count: workers.length, + max_session_ms: maxSessionMs, + }); + } + if (controller) { + try { controller.enqueue(utf8Encoder.encode(sessionExpiredWarning())); } catch {} + try { controller.close(); } catch {} + } + resolveCancel(); + } + + const expiryTimer = setTimeout(() => expireSession(streamController), maxSessionMs); + if (typeof expiryTimer === "object" && typeof expiryTimer.unref === "function") { + expiryTimer.unref(); + } + // Pre-register cleanup. Per CLAUDE.md gotcha: scheduling waitUntil from // inside cancel races IoContext teardown — cancel only resolves the // promise, the actual close happens here. ctx.waitUntil(cancelPromise.then(async () => { + clearTimeout(expiryTimer); try { if (sessionOpen) await session.close(); } catch (err) { @@ -393,7 +429,11 @@ export async function handle({ request, env, ctx, ns, requestId }) { const stream = new ReadableStream({ async pull(controller) { - if (cancelled) return; + streamController = controller; + if (cancelled) { + try { controller.close(); } catch {} + return; + } try { if (!bootstrapped) { // Open BEFORE flipping bootstrapped so a session.open() throw @@ -406,21 +446,7 @@ export async function handle({ request, env, ctx, ns, requestId }) { const remainingMs = maxSessionMs - (Date.now() - sessionStartedAtMs); if (remainingMs <= 0) { - log("info", "tail_session_expired", { - request_id: requestId, namespace: ns, worker_count: workers.length, - max_session_ms: maxSessionMs, - }); - controller.enqueue(utf8Encoder.encode(sseEvent({ - event: "tail_warning", - data: JSON.stringify({ - event: "tail_warning", - code: "session_expired", - message: "Tail session reached its maximum lifetime; reconnecting for reauthorization.", - }), - }))); - cancelled = true; - resolveCancel(); - controller.close(); + expireSession(controller); return; } @@ -458,6 +484,10 @@ export async function handle({ request, env, ctx, ns, requestId }) { session, Math.min(XREAD_BLOCK_MS, SSE_KEEPALIVE_MS, remainingMs), ); + if (cancelled) { + try { controller.close(); } catch {} + return; + } if (!batch) { // Timed out with no events → SSE comment so intermediaries know @@ -477,6 +507,10 @@ export async function handle({ request, env, ctx, ns, requestId }) { }))); } } catch (err) { + if (cancelled) { + try { controller.close(); } catch {} + return; + } log("error", "tail_pull_failed", { request_id: requestId, namespace: ns, error_message: errMessage(err), diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 1dcf57a..910077a 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -3,11 +3,68 @@ import { jsonError, requireControlLog, requireControlRedis, + stringEnv, + codedErrorResponse, } from "control-shared"; import { invalidSecretMutationKeyResponse, readEncryptedSecretPutValue, } from "control-handlers-secret-put"; +import { routesKey, bundleKey } from "shared-version"; +import { + WorkerEnvBudgetError, + assertWorkerLoaderUserEnvBudget, + decryptSecretHash, +} from "control-env-budget"; +import { SecretEnvelopeError } from "shared-secret-envelope"; + +/** + * @param {{ + * redis: import("shared-redis").RedisClient, + * env: Record, + * nsName: string, + * secretKey: string, + * plaintext: string, + * }} args + */ +async function validateNamespaceSecretBudget({ redis, env, nsName, secretKey, plaintext }) { + const controlEnv = stringEnv(env); + const nsSecretsKey = `secrets:${nsName}`; + const existingEncrypted = await redis.hGetAll(nsSecretsKey); + const nsSecrets = await decryptSecretHash({ + encrypted: existingEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + }); + nsSecrets[secretKey] = plaintext; + + const activeRoutes = await redis.hGetAll(routesKey(nsName)); + const activeEntries = Object.entries(activeRoutes) + .filter((entry) => typeof entry[1] === "string" && entry[1] !== ""); + if (activeEntries.length === 0) { + assertWorkerLoaderUserEnvBudget({ ns: nsName, nsSecrets }); + return; + } + + for (const [worker, version] of activeEntries) { + const workerSecretsKey = `secrets:${nsName}:${worker}`; + const metaRaw = await redis.hGet(bundleKey(nsName, worker, /** @type {string} */ (version)), "__meta__"); + const workerEncrypted = await redis.hGetAll(workerSecretsKey); + const meta = typeof metaRaw === "string" ? JSON.parse(metaRaw) : {}; + const workerSecrets = await decryptSecretHash({ + encrypted: workerEncrypted, + env: controlEnv, + hashKey: workerSecretsKey, + }); + assertWorkerLoaderUserEnvBudget({ + ns: nsName, + worker, + vars: meta && typeof meta === "object" ? meta.vars : null, + nsSecrets, + workerSecrets, + }); + } +} /** * @param {{ @@ -38,6 +95,19 @@ export async function handle({ request, env, method, nsName, secretKey, requestI fieldName: secretKey, }); if ("response" in put) return put.response; + try { + await validateNamespaceSecretBudget({ + redis, + env, + nsName, + secretKey, + plaintext: put.plaintext, + }); + } catch (err) { + if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); + if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + throw err; + } const encrypted = put.encrypted; await redis.hSet(nsSecretsKey, secretKey, encrypted); log("info", "ns_secret_set", { request_id: requestId, namespace: nsName, key: secretKey }); diff --git a/control/handlers/secret-put.js b/control/handlers/secret-put.js index 274d219..548db58 100644 --- a/control/handlers/secret-put.js +++ b/control/handlers/secret-put.js @@ -29,7 +29,7 @@ export function invalidSecretMutationKeyResponse(key) { * hashKey: string, * fieldName: string, * }} args - * @returns {Promise<{ response: Response } | { encrypted: string }>} + * @returns {Promise<{ response: Response } | { encrypted: string, plaintext: string }>} */ export async function readEncryptedSecretPutValue({ request, env, hashKey, fieldName }) { const parsed = await readJsonBody(request, { @@ -46,6 +46,7 @@ export async function readEncryptedSecretPutValue({ request, env, hashKey, field } try { return { + plaintext: body.value, encrypted: await encryptSecretValue(body.value, { env: stringEnv(env), hashKey, fieldName }), }; } catch (err) { diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index dc1646f..1c20953 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -7,6 +7,8 @@ import { requireControlLog, requireControlRedis, runOptimistic, + stringEnv, + codedErrorResponse, } from "control-shared"; import { deleteLockKey, @@ -19,6 +21,13 @@ import { } from "control-handlers-secret-put"; import { stageWorkerHidden, stageWorkerVisible } from "control-lifecycle-indexes"; import { bumpActiveAndPromote, RoutingError } from "control-routing"; +import { bundleKey } from "shared-version"; +import { + WorkerEnvBudgetError, + assertWorkerLoaderUserEnvBudget, + decryptSecretHash, +} from "control-env-budget"; +import { SecretEnvelopeError } from "shared-secret-envelope"; const MAX_SECRET_ATTEMPTS = 5; @@ -56,6 +65,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI if (invalidKey) return invalidKey; let storedValue = null; + let putPlaintext = null; if (method === "PUT") { const put = await readEncryptedSecretPutValue({ request, @@ -65,6 +75,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI }); if ("response" in put) return put.response; storedValue = put.encrypted; + putPlaintext = put.plaintext; } let mutationResult; @@ -72,6 +83,8 @@ export async function handle({ request, env, method, ns, name, subPath, requestI mutationResult = await mutateSecret({ redis, ns, name, key, method, value: storedValue, + plaintext: putPlaintext, + controlEnv: stringEnv(env), }); } catch (err) { if (err instanceof SecretAbort) { @@ -86,6 +99,8 @@ export async function handle({ request, env, method, ns, name, subPath, requestI }); return controlAbortResponse(err); } + if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); + if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); throw err; } @@ -188,9 +203,9 @@ export async function handle({ request, env, method, ns, name, subPath, requestI // DELETE extends WATCH to routes + worker-versions so the // "last key → SREM workers:" branch can trust its preconditions. /** - * @param {{ redis: RedisClient, ns: string, name: string, key: string, method: string, value: string | null }} args + * @param {{ redis: RedisClient, ns: string, name: string, key: string, method: string, value: string | null, plaintext?: string | null, controlEnv: Record }} args */ -async function mutateSecret({ redis, ns, name, key, method, value }) { +async function mutateSecret({ redis, ns, name, key, method, value, plaintext = null, controlEnv }) { const secretsKey = `secrets:${ns}:${name}`; return await runOptimistic(redis, { attempts: MAX_SECRET_ATTEMPTS, @@ -231,6 +246,28 @@ async function mutateSecret({ redis, ns, name, key, method, value }) { const multi = iso.multi(); if (method === "PUT") { if (typeof value !== "string") throw new Error("PUT secret value missing"); + if (typeof plaintext !== "string") throw new Error("PUT secret plaintext missing"); + const activeVersion = await iso.hGet(routesKey(ns), name); + const nsEncrypted = await iso.hGetAll(`secrets:${ns}`); + const workerEncrypted = await iso.hGetAll(secretsKey); + const [nsSecrets, workerSecrets] = await Promise.all([ + decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: `secrets:${ns}` }), + decryptSecretHash({ encrypted: workerEncrypted, env: controlEnv, hashKey: secretsKey }), + ]); + workerSecrets[key] = plaintext; + let vars = null; + if (typeof activeVersion === "string" && activeVersion) { + const rawMeta = await iso.hGet(bundleKey(ns, name, activeVersion), "__meta__"); + const meta = typeof rawMeta === "string" ? JSON.parse(rawMeta) : {}; + vars = meta && typeof meta === "object" ? meta.vars : null; + } + assertWorkerLoaderUserEnvBudget({ + ns, + worker: name, + vars, + nsSecrets, + workerSecrets, + }); multi.hSet(secretsKey, key, value); // SADD even on secret-only / pre-deploy workers so they're // visible to GET /workers and reachable by whole-delete. diff --git a/deploy/kubernetes/base/gateway.yaml b/deploy/kubernetes/base/gateway.yaml index 4744508..8e0c544 100644 --- a/deploy/kubernetes/base/gateway.yaml +++ b/deploy/kubernetes/base/gateway.yaml @@ -41,7 +41,6 @@ spec: - serve - -b - /app/dist/workerd-configs/gateway.bin - - --experimental envFrom: - configMapRef: name: wdl-config diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index 6b19b1e..0625040 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -115,7 +115,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "@wdl-dev/aws-sigv4", esModule = embed "../shared/vendor/aws-sigv4.js"), ], compatibilityDate = "2026-04-24", - compatibilityFlags = ["nodejs_compat", "experimental"], + compatibilityFlags = ["nodejs_compat"], globalOutbound = "internal-network", # The do-runtime host actor owns many user DO facets behind one stable # storage shard. Keep it resident so short idle gaps do not force workerd to diff --git a/do-runtime/load.js b/do-runtime/load.js index bf5c145..6492fa9 100644 --- a/do-runtime/load.js +++ b/do-runtime/load.js @@ -28,7 +28,7 @@ const NATIVE_DELETE_ALL_PRESERVES_ALARM_FLAG = "delete_all_preserves_alarm"; * @typedef {{ exports: Record }) => unknown> & { KV(options: { props: Record }): unknown, Assets(options: { props: Record }): unknown, QueueProducer(options: { props: Record }): unknown, D1Database(options: { props: Record }): unknown, R2Bucket(options: { props: Record }): unknown, ServiceBinding(options: { props: Record }): unknown, DurableObjectNamespace(options: { props: Record }): unknown, DoAlarmBinding(options: { props: Record }): unknown, InternalAuthBackend(options: { props: Record }): unknown } }} DoRuntimeContext * @typedef {{ name: string, spec: RuntimeBindingSpec }} DoBindingInput * @typedef {string | { cjs: string } | { py: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue - * @typedef {{ mainModule: string, modules: Record, compatibilityFlags?: string[], compatibilityDate?: string, allowExperimental?: boolean, env?: Record, globalOutbound?: unknown }} WorkerCode + * @typedef {{ mainModule: string, modules: Record, compatibilityFlags?: string[], compatibilityDate?: string, env?: Record, globalOutbound?: unknown }} WorkerCode */ /** @param {DoEnv} env */ @@ -256,7 +256,6 @@ export async function loadDoWorkerCode(env, ctx, invoke, requestId = null) { const { meta, ...codeBase } = bundleToWorkerCode(loaded.bundle); const workerCode = { ...codeBase, - allowExperimental: true, env: buildDoEnv( meta, loaded.ns_secrets, diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index 504db12..9ee16cf 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -481,7 +481,6 @@ function normalizeWorkerCode(value) { return { compatibilityDate, compatibilityFlags, - allowExperimental: input.allowExperimental !== false, mainModule, modules: normalizedModules, env, diff --git a/docker-compose.yml b/docker-compose.yml index 7e8ea43..d2f34c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -418,7 +418,7 @@ services: - ./dist:/app/dist:ro entrypoint: ["workerd"] command: - ["serve", "-b", "/app/dist/workerd-configs/gateway-local.bin", "--experimental"] + ["serve", "-b", "/app/dist/workerd-configs/gateway-local.bin"] ports: - "${WDL_GATEWAY_HOST_PORT:-8080}:8080" environment: diff --git a/docs/compatibility.md b/docs/compatibility.md index c70955d..bee1201 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -41,14 +41,15 @@ Each row separates four compatibility claims: | KV namespace | Supported | A binding object can be exposed to user code through workerd entrypoint/JSRPC mechanics. | Runtime `KV` facade calls redis-proxy; redis-proxy stores values and metadata in DB 1 hash buckets `kvh:::b:`, with `v:` and `m:` fields, 512-byte key/list-prefix cap, 25 MiB value cap, batch raw-byte budget, TTL/EXAT, and prefix list cursors. | KV storage is Redis-backed and namespace-scoped in a WDL deployment. `cacheTtl` is not a storage freshness contract. | No Cloudflare global edge replication or eventual consistency model. | | R2 bucket | Supported | Fetch/stream primitives in workerd. | Runtime R2 facade signs S3-compatible requests with platform credentials; CLI parses `[[r2_buckets]]`. | Bucket lifecycle and placement are the S3-compatible backend's responsibility. | `preview_bucket_name` and `jurisdiction` are not supported. | | ASSETS | Partial | Worker can receive a platform-provided binding object. | CLI uploads assets to S3-compatible storage and runtime exposes the WDL `env.ASSETS.url(path)` helper for tokenized CDN URLs. | WDL assets are an S3/CDN helper model, not Cloudflare Pages asset hosting. | WDL does not provide a full Cloudflare Pages asset pipeline or a fetch-style assets binding contract. | -| D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. | -| Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. | +| D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. SQLite names under the reserved `_cf_` namespace are rejected by workerd. | +| Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. SQLite names under the reserved `_cf_` namespace are rejected by workerd. | | Queues producer/consumer | Partial | Worker queue handler API surface in loaded workers. | Runtime producer facade writes through redis-proxy DB 1; scheduler owns consumer dispatch, retry, DLQ, orphan, delayed queues, and batch splitting for oversized dispatch bodies. | `max_batch_timeout_ms` is configuration metadata, not a true aggregation window; dispatch concurrency is scheduler-owned. | `max_concurrency` is rejected. | | Cron triggers | Supported | `scheduled()` handler surface in workers. | Control stores cron config; scheduler owns indexed discovery, cron-slot buckets, due dispatch, and repairable projections. | Control and scheduler use JS/Rust `croner` engines and keep their behavior documented through tests and module docs. | New scheduler dispatch paths still require their own Redis lease/fence audit. | | Workflows | Partial | User workflow class code runs inside loaded workers when dispatched by runtime. | workflows owns DB 2 instance state, step/event/sleep commits, runtime-observed `step.do` DAG edges including `Promise.all` siblings, generation/run-token fencing, lifecycle APIs, progress callbacks, and scheduler tick. | WDL Workflows V2 has WDL-specific payload semantics and terminal failure rules; permanent `step.do` failure is terminal for the run even if caught, and one step may record at most 1000 dependency edges. | WDL Workflows V2 is not Cloudflare Workflows parity. `script_name`, cross-worker workflows, and Cloudflare's source-AST visualizer are unsupported. | | Service bindings | Supported | workerd service binding JSRPC and fetch dispatch. | WDL resolves target worker metadata, cold-loads immutable versions, propagates request ids where available, and enforces namespace/action ACL through control metadata. | Frozen-version service targets do not evict active siblings by design. | None currently documented. | | Platform bindings | Supported | workerd named entrypoint/JSRPC mechanics. | WDL expands ACL-checked platform bindings from control metadata. | Platform bindings are WDL-specific. | Platform bindings are not a Cloudflare tenant portability feature. | -| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | +| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget before deploy/secret mutation, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | +| Worker code size | Supported | Dynamic worker module bodies are accepted by `workerLoader` up to workerd's 64 MiB limit. | Control rejects module bodies over 64 MiB before allocating a version. | WDL's deploy JSON body limit is lower for ordinary inline deployments. | Large server-side bundle assembly paths must keep this guard. | ## Control Plane And Developer Tooling diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index 36e3e28..c93b6eb 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -34,14 +34,15 @@ | KV namespace | Supported | 通过 workerd entrypoint/JSRPC 机制把 binding object 暴露给用户代码。 | Runtime `KV` facade 调 redis-proxy;redis-proxy 在 DB 1 hash bucket `kvh:::b:` 存 value/metadata,字段为 `v:` 和 `m:`,支持 512-byte key/list-prefix cap、25 MiB value cap、batch raw-byte budget、TTL/EXAT、prefix list cursor。 | KV storage 是 WDL deployment 内的 Redis-backed、namespace-scoped 存储;`cacheTtl` 不是存储新鲜度合同。 | 没有 Cloudflare 全局边缘复制 / eventual consistency 模型。 | | R2 bucket | Supported | workerd 的 fetch/stream primitives。 | Runtime R2 facade 用平台 credentials 签 S3-compatible request;CLI 解析 `[[r2_buckets]]`。 | Bucket lifecycle/placement 由 S3-compatible backend 负责。 | 不支持 `preview_bucket_name` 和 `jurisdiction`。 | | ASSETS | Partial | Worker 可接收平台提供的 binding object。 | CLI 把 assets 上传到 S3-compatible storage;runtime 暴露 WDL 的 `env.ASSETS.url(path)` helper,用于生成 tokenized CDN URL。 | WDL assets 是 S3/CDN helper 模型,不是 Cloudflare Pages asset hosting。 | 不是完整 Cloudflare Pages assets pipeline,也不提供 fetch-style assets binding 合同。 | -| D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing,并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing,不是 Cloudflare global D1;metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。 | -| Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | 只支持同 worker class;不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。 | +| D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing,并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing,不是 Cloudflare global D1;metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。workerd 会拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | +| Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | 只支持同 worker class;不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。workerd 会拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | | Queues producer/consumer | Partial | loaded worker 中的 queue handler surface。 | Runtime producer facade 经 redis-proxy 写 DB 1;scheduler 负责 consumer dispatch、retry、DLQ、orphan、delayed queues,并会拆分过大的 dispatch body batch。 | `max_batch_timeout_ms` 是配置 metadata,不是真正 aggregation window;dispatch concurrency 由 scheduler 拥有。 | `max_concurrency` 被拒绝。 | | Cron triggers | Supported | worker 的 `scheduled()` handler surface。 | Control 存 cron config;scheduler 拥有 indexed discovery、cron-slot bucket、due dispatch 和可修复 projection。 | Control 和 scheduler 使用 JS/Rust `croner` 引擎,并通过测试和模块文档约束行为。 | 新增 scheduler dispatch 路径仍必须单独审计 Redis lease/fence。 | | Workflows | Partial | 用户 workflow class code 在 runtime dispatch 时运行在 loaded worker 内。 | workflows 拥有 DB 2 instance state、step/event/sleep commit、runtime-observed `step.do` DAG edges(包括 `Promise.all` sibling)、generation/run-token fence、lifecycle API、progress callback 和 scheduler tick。 | WDL Workflows V2 的 payload 语义和 terminal failure 规则由 WDL 定义;永久失败的 `step.do` 即使被捕获也会让 run terminal failed;单个 step 最多记录 1000 条 dependency edge。 | WDL Workflows V2 不是 Cloudflare Workflows parity;不支持 `script_name`、跨 worker workflow 和 Cloudflare 的源码 AST visualizer。 | | Service bindings | Supported | workerd service binding JSRPC 和 fetch dispatch。 | WDL 解析 target worker metadata,cold-load immutable version,在可用时传播 request id,并通过 control metadata 做 namespace/action ACL。 | Frozen-version service target 按设计不 evict active siblings。 | 当前无已记录缺口。 | | Platform bindings | Supported | workerd named entrypoint/JSRPC 机制。 | WDL 从 control metadata 展开 ACL-checked platform bindings。 | Platform bindings 是 WDL-specific。 | 不是 Cloudflare tenant portability feature。 | -| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密,worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | +| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 前按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 做校验;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | +| Worker code size | Supported | Dynamic worker module bodies 可由 `workerLoader` 接受,直到 workerd 的 64 MiB 上限。 | Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies。 | WDL 的 deploy JSON body limit 对普通 inline deploy 更低。 | 大型 server-side bundle assembly 路径必须保留这道 guard。 | ## 控制面和开发工具 diff --git a/docs/modules/cli.md b/docs/modules/cli.md index 79f0e2d..331a4a2 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -148,8 +148,8 @@ Supported config surfaces: | Field | WDL behavior | |---|---| -| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit. | -| `[vars]` | String, number, and boolean values are accepted and stringified into `env`. | +| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit; module bodies must fit workerd's 64 MiB `workerLoader` code limit. | +| `[vars]` | String, number, and boolean values are accepted and stringified into `env`; vars plus namespace/worker secrets must fit WDL's headroomed workerd 1 MiB `workerLoader` env budget. | | `[[kv_namespaces]]` | `id` is a platform-local KV namespace id, not a Cloudflare UUID. | | `[[r2_buckets]]` | `binding` plus `bucket_name` become a namespace-scoped virtual R2 bucket under the platform S3 bucket. | | `[assets]` | `directory` contents upload to S3-compatible assets storage and auto-inject `ASSETS`. | diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index 5abf147..c1b729b 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -91,8 +91,8 @@ WDL 遵循 Wrangler selected-env 继承规则: | 字段 | WDL 行为 | |---|---| -| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`。 | -| `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`。 | +| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`;module bodies 必须落在 workerd 的 64 MiB `workerLoader` code limit 内。 | +| `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`;vars 加 namespace/worker secrets 必须落在 WDL 留有 headroom 的 workerd 1 MiB `workerLoader` env budget 内。 | | `[[kv_namespaces]]` | `id` 是 platform-local KV namespace id,不是 Cloudflare UUID。 | | `[[r2_buckets]]` | `binding` 加 `bucket_name` 映射为平台 S3 bucket 下的 namespace-scoped virtual R2 bucket。 | | `[assets]` | `directory` 内容上传到 S3-compatible assets storage,并 auto-inject `ASSETS`。 | diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 2fb2ef0..8076303 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -84,6 +84,11 @@ pipe, not durable log storage. - Active tail sessions are time-bounded authorization leases and must reconnect through normal auth. `LOG_TAIL_MAX_SESSION_MS` sets the control-side maximum; invalid or empty values fall back to 15 minutes. +- workerd issue [#6832](https://github.com/cloudflare/workerd/issues/6832) means + client disconnects may not call async response-body `ReadableStream.cancel()`. + Control therefore has an independent max-session watchdog; under heavy reconnect + churn, set `LOG_TAIL_MAX_SESSION_MS` lower to bound abandoned Redis tail sessions + until upstream restores a reliable disconnect hook. - Tail events racing activation can be dropped. - High QPS or slow SSE readers can miss middle events due to stream caps. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index eb618bc..ceb3cd4 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -64,6 +64,7 @@ Tail streams 使用有界 `MAXLEN ~ 500`,并在写入时刷新 TTL。它们是 - 结构化 stdout 是持久平台日志的事实来源。 - 没有 active tailer 时,runtime 仍输出 stdout,但在本地 active-set miss cache 后跳过 per-event stream append work。 - Active tail session 是有时限的授权租约,必须通过正常 auth reconnect。`LOG_TAIL_MAX_SESSION_MS` 设置 control-side 最大时长;非法值或空值会回退到 15 分钟。 +- workerd issue [#6832](https://github.com/cloudflare/workerd/issues/6832) 会导致 client disconnect 不一定触发 async response-body `ReadableStream.cancel()`。Control 因此有独立的 max-session watchdog;如果 tail UI 重连抖动明显,可以调低 `LOG_TAIL_MAX_SESSION_MS`,在上游恢复可靠 disconnect hook 前约束遗弃 Redis tail session 的存活时间。 - 与 activation race 的 tail event 可以丢失。 - 高 QPS 或慢 SSE reader 可能因为 stream cap 丢中间事件。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 0059c69..8d3d9ab 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -82,7 +82,10 @@ services. Runtime therefore treats bindings as adapters: decrypts `WDL-ENC:` values; tenant-facing `env` shape stays unchanged. Env materialization merges in fixed precedence: bundle vars, then namespace secrets, then worker secrets. A worker-level secret with the same name wins over a namespace-level - secret, which wins over a var. + secret, which wins over a var. Control enforces a headroomed version of workerd's + `workerLoader` serialized env budget on this user-controlled set before deploys and + secret mutations, so an over-large env fails in the control plane instead of during + runtime cold-load. - Stateful bindings such as D1, Durable Objects, and Workflows call dedicated backend services. The hidden backend Fetchers stay in runtime and are removed before tenant code observes `env`. @@ -230,11 +233,23 @@ when a matching active tail session exists. binding shape changes. - Runtime must roll before scheduler/workflows if they depend on a new `:8088` internal path or dispatch body. -- workerd upgrades can change experimental surfaces; runtime currently enables - `experimental` because `abortIsolate()` is required for isolate eviction. -- `experimental` is broader than `abortIsolate()`. Tenants inherit every experimental - workerd surface enabled by that flag, so workerd upgrades require review of the - exposed surface, not only the loader/abort path. +- Runtime does not enable workerd's broad `experimental` flag for loaded workers. + Historical-version eviction still injects `__WdlAbort__`, but `abortIsolate()` is + available without that flag in the bundled workerd baseline. +- Removing the broad loaded-worker `experimental` flag intentionally removes access to + non-GA experimental-only tenant surfaces, such as irrevocable long-term stub storage. + Do not re-enable it as a compatibility workaround without an explicit feature design. +- The runtime workerd processes still run with process-level `--experimental` because + upstream workerd 2026-07-01 continues to gate `workerLoader` bindings on that switch. + Do not re-add the `experimental` compatibility flag or `allowExperimental` to loaded + WorkerCode unless another upstream API explicitly requires it. +- Upstream workerd 2026-07-01 caps dynamic worker code at 64 MiB and serialized dynamic + env at 1 MiB. Control rejects module bodies over 64 MiB before version allocation. + Vars plus namespace/worker secrets are prechecked against a headroomed `workerLoader` + env budget because workerd's final enforcement includes runtime facade objects in the + full `env` estimate. +- workerd upgrades can still change default or compatibility-flagged runtime + surfaces; review the exposed surface, not only the loader/abort path. ## Tests That Protect This Module diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 14be694..5dc3823 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -40,7 +40,7 @@ Tenant 可见 binding 包括 KV、R2、D1、Durable Objects、Queues、ASSETS、 workerd 提供 isolate、module evaluation、named entrypoint 和 JSRPC 机制。Cloudflare 生产平台通常在外部服务里实现的 binding 后端,则由 WDL 自己补齐。因此 runtime 把 binding 当作 adapter: - KV、queue producer 这类纯数据 binding 调 colocated redis-proxy sidecar。Loaded worker 看到 Cloudflare-shaped object,但 method call 会先通过 workerd JSRPC 回到 runtime,再经 HTTP 调 redis-proxy。 -- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。 +- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 前用留有 headroom 的预算检查这组用户可控 env 是否超过 workerd `workerLoader` 的 serialized env budget,避免超限配置拖到 runtime cold-load 才失败。 - D1、Durable Objects、Workflows 这类 stateful binding 调专门 backend service。Hidden backend Fetcher 留在 runtime 内部,并在 tenant code 观察 `env` 前被删除。 - R2 是 S3-compatible object-storage adapter:runtime 使用平台 credential 签名请求,并发送到配置的 endpoint。 - ASSETS 是 deploy artifact URL helper:control 在 deploy 时把 assets 上传到 S3-compatible storage;runtime 读取 `__meta__.assets` 和 `ASSETS_CDN_BASE`,只暴露 `env.ASSETS.url(path)` 用来生成 tokenized CDN URL。 @@ -104,8 +104,11 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - 修改 bundle metadata、wrapper generation 或 binding shape 时,runtime 和 control 应一起滚。 - 如果 scheduler/workflows 依赖新的 `:8088` internal path 或 dispatch body,runtime 必须先滚。 -- workerd 升级可能改变 experimental surface;runtime 当前需要开启 `experimental`,因为 isolate eviction 依赖 `abortIsolate()`。 -- `experimental` 覆盖的范围大于 `abortIsolate()`。Tenant 会继承该 flag 打开的所有 experimental workerd surface,因此 workerd 升级时要审 exposed surface,而不只审 loader/abort path。 +- Runtime 不再为 loaded worker 开启 workerd 的宽泛 `experimental` flag。Historical-version eviction 仍然注入 `__WdlAbort__`,但当前 bundled workerd baseline 中 `abortIsolate()` 已经不需要该 flag。 +- 移除 loaded worker 的宽泛 `experimental` flag 会有意收紧 tenant 对非 GA experimental-only surface 的访问,例如不可撤销的长期 stub storage。不要为了兼容绕过而重新打开它,除非先完成明确的功能设计。 +- Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 +- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars 加 namespace/worker secrets 会在 control plane 用留有 headroom 的 `workerLoader` env budget 预检,因为 workerd 的最终检查还会把 runtime facade objects 纳入完整 `env` estimate。 +- workerd 升级仍可能改变默认或 compatibility-flagged runtime surface;升级时要审 exposed surface,而不只审 loader/abort path。 ## 保护该模块的测试 diff --git a/package-lock.json b/package-lock.json index 8c1ee32..cd81236 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,10 +11,10 @@ "control" ], "dependencies": { - "workerd": "1.20260617.1" + "workerd": "1.20260701.1" }, "devDependencies": { - "@cloudflare/workers-types": "4.20260617.1", + "@cloudflare/workers-types": "4.20260701.1", "@eslint/js": "^10.0.1", "@types/node": "^24.13.2", "esbuild": "^0.28.0", @@ -36,9 +36,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260617.1.tgz", - "integrity": "sha512-jWwmgEVVWbsHNrLSNXzwjJaH90VzRxq1cWkQFUidxyeUPnMxemeNE8I9qFAfrpzGgE11e9sKDcE3ettJW08swQ==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260701.1.tgz", + "integrity": "sha512-Zd9Y1bah6DwwBN2RW8vJohffQrIUazb8UXnqSNecOxM+jJLhUuvv5IOG8dbHcV83TyZAubea6gsQXo2yH1lDdw==", "cpu": [ "x64" ], @@ -52,9 +52,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260617.1.tgz", - "integrity": "sha512-LHH7b565g9znfCUOkwbec6FG2rmRbsgCy6aJiU9KN662mNheWl5sw/iKleiFSiljPKQQP3HkjnC/NSkdgi/aSA==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260701.1.tgz", + "integrity": "sha512-yBLsjS1qCWqFyCY37qRUrYfzHHvMGvjh8zRKJ6MvUivYDhkZTzqduppK38FoqYvayLJ5KbcxH7zo5rkxGqbsaA==", "cpu": [ "arm64" ], @@ -68,9 +68,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260617.1.tgz", - "integrity": "sha512-FMnaAKXe4Cfd8TQurCVd9fs2XQVBFRCsP+Id/SRdUv89MlwYu9zXfoyx6BxM+brPTIUK38SHbo8iaxiwzLi9JQ==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260701.1.tgz", + "integrity": "sha512-vMfqSIMfoo4xmZXEuUVqLpSFS921YKjiR9q7kDXPi6Vld1PK74UHg9LZuBavT2KSyemHUCTpj9y/4JSYOEyQbQ==", "cpu": [ "x64" ], @@ -84,9 +84,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260617.1.tgz", - "integrity": "sha512-MRoifFYcqbxxIIQy7PqO5tFY/qPFSnjXzakWl0sO93l+HLyG35jRAgOi6jfqa4kBxc7gKKtH861DcewjxUfkjA==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260701.1.tgz", + "integrity": "sha512-HRfwbKU2pK44V2NhoM0+iH0JJSj7nQ9Wv13ifIiGYCmTtDL8/zKtEhX7kQ3D4Vy/Cpjhttl0FkfqXj1aqLDPPg==", "cpu": [ "arm64" ], @@ -100,9 +100,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260617.1.tgz", - "integrity": "sha512-rgBV9wQrv0OSKgCTTbhFUFY3sLGNANZ88aqaLvtmEn2gmbFVb1J4PDGochVUdB7NSEp4D/ghHva6/8SZmbONpw==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260701.1.tgz", + "integrity": "sha512-ngxCiIN9s/fM2o1IBMD0o1/mcXrv2NJVdyznh51UH8sQuvrTrXvV2nM0Uj/qU2wMwF6prgNBcdcd7AZeZGiBQA==", "cpu": [ "x64" ], @@ -116,9 +116,9 @@ } }, "node_modules/@cloudflare/workers-types": { - "version": "4.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260617.1.tgz", - "integrity": "sha512-HdbP3CNcdMZBwegitFDjWvzv+6wPkFXvV9gBXMnf6RjV2Cy3W8TJL3IhSEGul0S6F1DHjnucP7lrpIsvkzNEjA==", + "version": "4.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260701.1.tgz", + "integrity": "sha512-eGZ+PWPlu/yUxH+BJoplV45oSA709sjYFYr2kjrqSo+601qE15X4tsuZcPz1KE6Fb3ObGA9d9ZXn53n34NtyiA==", "dev": true, "license": "MIT OR Apache-2.0" }, @@ -2595,9 +2595,9 @@ } }, "node_modules/workerd": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260617.1.tgz", - "integrity": "sha512-Re5pl6pdowt3ZmWUzGlOuB7jbRIIPetgKalmo4cYmucQnVhpo7/3e4MfpekbhLi2EhZZz5EY9NWRu8zFzuEZew==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260701.1.tgz", + "integrity": "sha512-uF813NG09JwNRRUfJ0zBomyTslSPM810dMj9LVvkQ7RAkLrQLzAlPU8Xh/3dIqZDo2bfd7tChbf2PtqLRARRJQ==", "hasInstallScript": true, "license": "Apache-2.0", "bin": { @@ -2607,11 +2607,11 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260617.1", - "@cloudflare/workerd-darwin-arm64": "1.20260617.1", - "@cloudflare/workerd-linux-64": "1.20260617.1", - "@cloudflare/workerd-linux-arm64": "1.20260617.1", - "@cloudflare/workerd-windows-64": "1.20260617.1" + "@cloudflare/workerd-darwin-64": "1.20260701.1", + "@cloudflare/workerd-darwin-arm64": "1.20260701.1", + "@cloudflare/workerd-linux-64": "1.20260701.1", + "@cloudflare/workerd-linux-arm64": "1.20260701.1", + "@cloudflare/workerd-windows-64": "1.20260701.1" } }, "node_modules/yaml": { diff --git a/package.json b/package.json index 18318ab..83d47b8 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "build:third-party-licenses": "node scripts/generate-third-party-license-bundles.mjs" }, "devDependencies": { - "@cloudflare/workers-types": "4.20260617.1", + "@cloudflare/workers-types": "4.20260701.1", "@eslint/js": "^10.0.1", "@types/node": "^24.13.2", "esbuild": "^0.28.0", @@ -45,6 +45,6 @@ "typescript": "^6.0.3" }, "dependencies": { - "workerd": "1.20260617.1" + "workerd": "1.20260701.1" } } diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 438f0ec..206f9ec 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -104,9 +104,7 @@ const loaderWorker :Workerd.Worker = ( (name = "@wdl-dev/aws-sigv4", esModule = embed "../shared/vendor/aws-sigv4.js"), ], compatibilityDate = "2026-04-24", - # `experimental` lets this worker pass `allowExperimental: true` in WorkerCode - # so loaded workers can import `abortIsolate` for historical-version eviction. - compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "experimental"], + compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers"], globalOutbound = "network", bindings = [ (name = "SERVICE_NAME", text = "system-runtime"), @@ -160,6 +158,7 @@ const controlWorker :Workerd.Worker = ( (name = "control-bundle", esModule = embed "../control/bundle.js"), (name = "control-bindings", esModule = embed "../control/bindings.js"), (name = "control-topology", esModule = embed "../control/topology.js"), + (name = "control-env-budget", esModule = embed "../control/env-budget.js"), (name = "control-lifecycle-indexes", esModule = embed "../control/lifecycle-indexes.js"), (name = "control-routing", esModule = embed "../control/routing.js"), (name = "control-routing-route-plan", esModule = embed "../control/routing/route-plan.js"), @@ -244,6 +243,7 @@ const controlWorker :Workerd.Worker = ( (name = "ASSETS_CDN_BASE", fromEnvironment = "ASSETS_CDN_BASE"), (name = "PLATFORM_DOMAIN", fromEnvironment = "PLATFORM_DOMAIN"), (name = "LOG_LEVEL", fromEnvironment = "LOG_LEVEL"), + (name = "LOG_TAIL_MAX_SESSION_MS", fromEnvironment = "LOG_TAIL_MAX_SESSION_MS"), (name = "SECRET_ENVELOPE_LOCAL_KEY_B64", fromEnvironment = "SECRET_ENVELOPE_LOCAL_KEY_B64"), (name = "SECRET_ENVELOPE_KID", fromEnvironment = "SECRET_ENVELOPE_KID"), (name = "TAIL_WORKER", service = "tail-worker"), diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index fa62afa..c2a4d9d 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -105,10 +105,7 @@ const loaderWorker :Workerd.Worker = ( compatibilityDate = "2026-04-24", # service_binding_extra_handlers exposes stub.queue()/scheduled() on # Fetcher stubs returned by workerLoader.get(). Runtime-only flag. - # `experimental` is required so this worker may pass `allowExperimental: true` - # in its WorkerCode — loaded workers need that to import `abortIsolate` from - # cloudflare:workers for the historical-version eviction shim. - compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "experimental"], + compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers"], globalOutbound = "internal-network", bindings = [ (name = "SERVICE_NAME", text = "user-runtime"), diff --git a/runtime/lib.js b/runtime/lib.js index 3ebc0ab..a70afff 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -122,9 +122,6 @@ function deepFreeze(obj) { const DEFAULT_COMPATIBILITY_DATE = "2026-04-24"; const ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE = "2026-04-21"; const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; -// Gate for the __WdlAbort__ shim's abortIsolate import. Paired with -// `allowExperimental: true` on the WorkerCode. -const EXPERIMENTAL_FLAG = "experimental"; const ROUTE_NS_RE = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?|__system__)$/; const RESERVED_TENANT_NS = new Set(["admin"]); const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; @@ -133,10 +130,9 @@ const VERSION_RE = /^v[1-9][0-9]*$/; // Loaded workers may declare an older compatibilityDate than the platform // workers. Keep enhanced error serialization as a floor only before the date // where workerd made it the default; newer dates reject the explicit flag. -// `experimental` is unconditional — it is the gate for our eviction shim. /** @param {string} compatibilityDate */ function platformFloorCompatFlags(compatibilityDate) { - const out = [EXPERIMENTAL_FLAG]; + const out = []; if (compatibilityDate < ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE) { out.push(ENHANCED_ERROR_SERIALIZATION_FLAG); } diff --git a/runtime/load.js b/runtime/load.js index 525135a..ea32cdd 100644 --- a/runtime/load.js +++ b/runtime/load.js @@ -565,12 +565,9 @@ export function createLoaderCallback({ requestId, env, ctx, ns, worker, version, }, Date.now() - envStartedAt)); // Loaded workers must not inherit runtime's private-reaching - // outbound (pinned to PUBLIC_NETWORK). `allowExperimental: true` is - // gated by the host carrying `experimental` itself; the loaded - // worker needs it for the __WdlAbort__ shim's abortIsolate import. + // outbound; pin them to PUBLIC_NETWORK. const workerCode = { ...codeBase, - allowExperimental: true, env: workerEnv, globalOutbound: env.PUBLIC_NETWORK, ...(env.TAIL_WORKER ? { tails: [env.TAIL_WORKER] } : {}), diff --git a/rust/supervisor/src/config.rs b/rust/supervisor/src/config.rs index 9a17c51..a7e2b8e 100644 --- a/rust/supervisor/src/config.rs +++ b/rust/supervisor/src/config.rs @@ -240,13 +240,12 @@ pub(crate) fn validate_shutdown_timing(config: &SupervisorConfig) { } } -pub(crate) fn workerd_args(compiled_config: &str) -> Vec { - vec![ - "serve".into(), - "-b".into(), - compiled_config.into(), - "--experimental".into(), - ] +pub(crate) fn workerd_args(compiled_config: &str, experimental: bool) -> Vec { + let mut args = vec!["serve".into(), "-b".into(), compiled_config.into()]; + if experimental { + args.push("--experimental".into()); + } + args } pub(crate) fn pick_do_compiled_config() -> &'static str { @@ -343,6 +342,23 @@ mod tests { assert_eq!(signal_exit_code(libc::SIGHUP), 1); } + #[test] + fn workerd_args_adds_experimental_only_when_requested() { + assert_eq!( + workerd_args("/app/dist/workerd-configs/d1-runtime.bin", false), + vec!["serve", "-b", "/app/dist/workerd-configs/d1-runtime.bin"] + ); + assert_eq!( + workerd_args("/app/dist/workerd-configs/do-runtime.bin", true), + vec![ + "serve", + "-b", + "/app/dist/workerd-configs/do-runtime.bin", + "--experimental" + ] + ); + } + #[test] fn evaluate_shutdown_timing_default_no_warnings() { assert_eq!(evaluate_shutdown_timing(10_000, 15_000, 120_000, 0), vec![]); diff --git a/rust/supervisor/src/lib.rs b/rust/supervisor/src/lib.rs index fdb1a98..2051ec3 100644 --- a/rust/supervisor/src/lib.rs +++ b/rust/supervisor/src/lib.rs @@ -8,9 +8,14 @@ pub(crate) use config::*; pub(crate) use wdl_rust_common::text::truncate_chars; pub async fn run_d1() -> ! { - process::run(&D1_CONFIG, WORKERD, workerd_args(D1_COMPILED_CONFIG)).await + process::run(&D1_CONFIG, WORKERD, workerd_args(D1_COMPILED_CONFIG, false)).await } pub async fn run_do() -> ! { - process::run(&DO_CONFIG, WORKERD, workerd_args(pick_do_compiled_config())).await + process::run( + &DO_CONFIG, + WORKERD, + workerd_args(pick_do_compiled_config(), true), + ) + .await } diff --git a/terraform/README.md b/terraform/README.md index 5869071..512a8be 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -228,6 +228,10 @@ The EC2 capacity provider uses a fixed Auto Scaling Group: Task `cpu` and `memory` values are placement reservations on EC2 launch type, not hard per-container caps. CPU is a cgroup share weight. Memory is an ECS placement reservation because the containers do not set hard container memory limits. +This is especially relevant for D1 and Durable Object runtimes on newer workerd +releases, where SQLite's process hard heap is no longer capped at 512 MiB by +workerd itself; size task density and host memory with that shared-process +behavior in mind. The launch template keeps IMDSv2 enabled for the ECS host agent, sets metadata hop limit to 1, and sets `ECS_AWSVPC_BLOCK_IMDS=true` so awsvpc tasks cannot read diff --git a/terraform/modules/compute/gateway_service.tf b/terraform/modules/compute/gateway_service.tf index f2a866b..3892905 100644 --- a/terraform/modules/compute/gateway_service.tf +++ b/terraform/modules/compute/gateway_service.tf @@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "gateway" { image = var.workerd_image essential = true entryPoint = ["workerd"] - command = ["serve", "-b", "/app/dist/workerd-configs/gateway.bin", "--experimental"] + command = ["serve", "-b", "/app/dist/workerd-configs/gateway.bin"] stopTimeout = 20 portMappings = [{ diff --git a/tests/integration/http-features.test.js b/tests/integration/http-features.test.js index 96832f6..64908c7 100644 --- a/tests/integration/http-features.test.js +++ b/tests/integration/http-features.test.js @@ -24,7 +24,7 @@ const ABORT_WORKER = readFileSync( "utf8" ); -test("client disconnect cancels the response stream in the loaded worker", async () => { +test("client disconnect response stream behavior is bounded on current workerd", async () => { const ns = uniqueNs("abort"); await deployAndPromote(ns, "w", { mainModule: "worker.js", @@ -57,9 +57,11 @@ test("client disconnect cancels the response stream in the loaded worker", async await waitUntil("disconnect marker written", async () => { const r = await gatewayFetch(ns, `/w/poll?key=${encodeURIComponent(key)}`); const text = await r.text(); - // cancel / enqueue-threw prove disconnect reached the worker; - // ended-normally or __null__ mean the signal didn't propagate. - if (text === "cancel" || text === "enqueue-threw") return true; + // workerd #6832 means 2026-06-19+ no longer reliably calls + // ReadableStream.cancel() for async response bodies on client disconnect. + // Keep this test as a bounded-behavior regression anchor: stock workerd + // 2026-07-01 reports "ended-normally" after the orphaned stream completes. + if (text === "cancel" || text === "enqueue-threw" || text === "ended-normally") return true; if (text === "__null__") return false; throw new Error(`unexpected marker value ${JSON.stringify(text)}`); }, { timeoutMs: 15000, intervalMs: 250 }); diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 11042a7..4e5788c 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -28,6 +28,7 @@ const CONTROL_DEPLOY_TEST_STATE = { parsedCrons: null, parsedQueueConsumers: null, watchedKeys: null, + envBudgetError: false, redis: null, logs: [], metrics: { increment() {}, observe() {} }, @@ -51,6 +52,7 @@ function resetControlDeployTestState() { CONTROL_DEPLOY_TEST_STATE.parsedCrons = null; CONTROL_DEPLOY_TEST_STATE.parsedQueueConsumers = null; CONTROL_DEPLOY_TEST_STATE.watchedKeys = null; + CONTROL_DEPLOY_TEST_STATE.envBudgetError = false; CONTROL_DEPLOY_TEST_STATE.redis = null; CONTROL_DEPLOY_TEST_STATE.logs = []; CONTROL_DEPLOY_TEST_STATE.metrics = { increment() {}, observe() {} }; @@ -197,6 +199,32 @@ export async function resolveDatabaseRefFrom(session, ns, databaseRef) { } `); +const controlEnvBudgetUrl = moduleDataUrl(` +export class WorkerEnvBudgetError extends Error { + constructor(message) { + super(message); + this.status = 400; + this.code = "worker_env_too_large"; + } +} +export function assertWorkerLoaderUserEnvBudget() { + if (/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError) { + throw new WorkerEnvBudgetError("env too large"); + } + return 0; +} +export async function decryptSecretHash() { return {}; } +`); + +const secretEnvelopeUrl = moduleDataUrl(` +export class SecretEnvelopeError extends Error { + constructor(code, message) { + super(message); + this.code = code; + } +} +`); + const { commitWithWatch, handle } = await importControlHandler("control/handlers/deploy.js", { globalName: "__controlDeployTestState", extraSharedSource: controlSharedExtraSource, @@ -213,6 +241,8 @@ const { commitWithWatch, handle } = await importControlHandler("control/handlers "control-s3": controlS3Url, "shared-assets-token": sharedAssetsUrl, "control-d1-store": d1StoreUrl, + "control-env-budget": controlEnvBudgetUrl, + "shared-secret-envelope": secretEnvelopeUrl, }, }); const { WatchError } = await import(sharedRedisUrl); @@ -524,6 +554,92 @@ test("deploy handler rejects assets without S3 before allocating a version", asy } }); +test("deploy handler rejects workerLoader env budget violations before allocating a version", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError = true; + + const session = makeSession(); + let incrCalled = false; + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async incr() { + incrCalled = true; + return 1; + }, + /** @param {string} key */ + async hGetAll(key) { + return await session.hGetAll(key); + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/env-heavy/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + vars: { BIG: "x" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "env-heavy", + requestId: "rid-env-budget", + }); + + assert.equal((await readJsonResponse(response, 400)).error, "worker_env_too_large"); + assert.equal(incrCalled, false); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError = false; + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + } +}); + +test("deploy handler rejects workerLoader code size violations before allocating a version", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + }, + normalized: [["worker.js", new Uint8Array(64 * 1024 * 1024 + 1)]], + }; + + let incrCalled = false; + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async incr() { + incrCalled = true; + return 1; + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/code-heavy/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "code-heavy", + requestId: "rid-code-budget", + }); + + assert.equal((await readJsonResponse(response, 413)).error, "worker_code_too_large"); + assert.equal(incrCalled, false); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + } +}); + test("deploy handler skips cleanup for empty assets when commit fails", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js new file mode 100644 index 0000000..d465d51 --- /dev/null +++ b/tests/unit/control-env-budget.test.js @@ -0,0 +1,84 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + importRepositoryModule, + importSpecifierReplacements, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { encryptSecretValue } from "../../shared/secret-envelope.js"; + +const secretEnvelopeUrl = repositoryFileUrl("shared/secret-envelope.js"); +const { + UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_HEADROOM_BYTES, + WORKER_LOADER_ENV_MAX_BYTES, + WorkerEnvBudgetError, + assertWorkerLoaderUserEnvBudget, + decryptSecretHash, + mergedUserEnvStrings, + userEnvSerializedBytes, +} = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ + "shared-secret-envelope": secretEnvelopeUrl, +})); + +const envelopeEnv = { + SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", +}; + +test("worker env budget counts merged vars and secrets with worker-secret precedence", () => { + const merged = mergedUserEnvStrings({ + vars: { TOKEN: "var", ONLY_VAR: "v" }, + nsSecrets: { TOKEN: "ns", ONLY_NS: "n" }, + workerSecrets: { TOKEN: "worker" }, + }); + + assert.deepEqual(merged, { + TOKEN: "worker", + ONLY_VAR: "v", + ONLY_NS: "n", + }); + assert.equal(userEnvSerializedBytes(merged), Buffer.byteLength(JSON.stringify(merged), "utf8")); +}); + +test("worker env budget rejects user-controlled env above workerd workerLoader limit", () => { + assert.equal(WORKER_LOADER_ENV_MAX_BYTES, UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES - WORKER_LOADER_ENV_HEADROOM_BYTES); + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "api", + vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES) }, + }), + (err) => { + if (!(err instanceof WorkerEnvBudgetError)) return false; + const budgetErr = /** @type {InstanceType} */ (err); + assert.equal(budgetErr.code, "worker_env_too_large"); + assert.equal(budgetErr.status, 400); + assert.equal(budgetErr.details.namespace, "demo"); + assert.equal(budgetErr.details.worker, "api"); + assert.equal(budgetErr.details.upstream_max_env_bytes, UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES); + assert.equal(budgetErr.details.headroom_bytes, WORKER_LOADER_ENV_HEADROOM_BYTES); + return true; + } + ); +}); + +test("decryptSecretHash returns plaintext secret values for budget checks", async () => { + const hashKey = "secrets:demo"; + const encrypted = await encryptSecretValue("plain", { + env: envelopeEnv, + hashKey, + fieldName: "TOKEN", + }); + + assert.deepEqual( + { + ...(await decryptSecretHash({ + encrypted: { TOKEN: encrypted, MISSING: null }, + env: envelopeEnv, + hashKey, + })), + }, + { TOKEN: "plain" } + ); +}); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 2ad2352..1ffdcf3 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -720,7 +720,7 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( /must not be later than today UTC/ ); assert.throws( - () => validateCompatibilityDate(unsupportedDate, new Date("2026-06-30T00:00:00Z")), + () => validateCompatibilityDate(unsupportedDate, new Date("2026-12-31T00:00:00Z")), /newer than bundled workerd supports/ ); }); diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 3def174..7b43d02 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -192,6 +192,31 @@ test("logs tail closes after the max session lifetime so reconnect reauthorizes" assert.ok(/** @type {any} */ (globalThis).__tailState.logs.some((/** @type {any} */ entry) => entry.event === "tail_session_expired")); }); +test("logs tail max-session watchdog closes even without stream cancel", async () => { + resetTailState(); + const { handle } = await loadLogsTailHandler(); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "50" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-watchdog", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + assert.match((await readText(reader)).text, /tail-open/); + await delay(80); + await Promise.all(waitUntilPromises); + + assert.equal(/** @type {any} */ (globalThis).__tailSessions.length, 1); + assert.equal(/** @type {any} */ (globalThis).__tailSessions[0].closed, true); + assert.ok(/** @type {any} */ (globalThis).__tailState.logs.some((/** @type {any} */ entry) => entry.event === "tail_session_expired")); + await reader.cancel().catch(() => {}); +}); + test("logs tail emits idle keepalives below common proxy idle timeouts", async () => { resetTailState(); const { handle } = await loadLogsTailHandler(); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 2b3e630..85ae64c 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -6,6 +6,7 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, repositoryF import { readJsonResponse } from "../helpers/response-json.js"; const SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); +const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const env = { SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", @@ -34,12 +35,21 @@ function secretPutUrl(controlSharedUrl, controlLibUrl) { return moduleDataUrl(source); } +function envBudgetUrl() { + const source = applyModuleReplacements(readRepositoryFile("control/env-budget.js"), [ + [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], + ]); + return moduleDataUrl(source); +} + const controlSharedUrl = controlSharedStubUrl(` export const state = { log() {}, redis: { writes: [], async hKeys() { return []; }, + async hGet() { return null; }, + async hGetAll() { return {}; }, async hSet(key, field, value) { this.writes.push({ key, field, value }); return 1; @@ -53,6 +63,9 @@ const src = applyModuleReplacements(readRepositoryFile("control/handlers/ns-secr [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(controlLibStubUrl)};`], [/from "control-handlers-secret-put";/, `from ${JSON.stringify(secretPutUrl(controlSharedUrl, controlLibStubUrl))};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "control-env-budget";/, `from ${JSON.stringify(envBudgetUrl())};`], + [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], ]); const { handle } = await import(moduleDataUrl(src)); @@ -142,6 +155,7 @@ export const state = { async get() { return null; }, async hKeys() { return []; }, async hGet() { return null; }, + async hGetAll() { return {}; }, async zCard() { return 0; }, multi() { return { @@ -186,6 +200,9 @@ const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/w [/from "control-handlers-secret-put";/, `from ${JSON.stringify(secretPutUrl(workerControlSharedUrl, workerLibStubUrl))};`], [/from "control-lifecycle-indexes";/, `from ${JSON.stringify(lifecycleStubUrl)};`], [/from "control-routing";/, `from ${JSON.stringify(routingStubUrl)};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "control-env-budget";/, `from ${JSON.stringify(envBudgetUrl())};`], + [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], ]); const { handle: workerHandle } = await import(moduleDataUrl(workerSrc)); diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 196c9bf..09a4cb6 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -236,7 +236,7 @@ test("normalizes inline workerCode only for test hooks", () => { ); const invoke = normalizeDoInvokeRequest(INLINE_BODY, { allowInlineWorkerCode: true }); assert.equal(invoke.hostId, CHAT_ROOM_HOST_ID); - assert.equal("workerCode" in invoke ? invoke.workerCode.allowExperimental : undefined, true); + assert.equal("workerCode" in invoke ? "allowExperimental" in invoke.workerCode : undefined, false); }); test("builds forwarded Request for user durable object fetch", async () => { diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 6d2d4be..021d60b 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -310,7 +310,7 @@ test("bundleToWorkerCode: compatibilityFlags merge user-declared with old-date p { "w.js": enc.encode("x") } ) ); - assert.deepEqual(code.compatibilityFlags, ["nodejs_compat", "experimental", "enhanced_error_serialization"]); + assert.deepEqual(code.compatibilityFlags, ["nodejs_compat", "enhanced_error_serialization"]); assert.deepEqual(/** @type {any} */ (code.meta.bindings), { KV: { type: "kv", id: "x" } }); assert.deepEqual(code.meta.vars, { G: "hi" }); assert.ok(Object.isFrozen(code.meta), "meta must be frozen"); @@ -327,7 +327,7 @@ test("bundleToWorkerCode: default compatibilityDate does not inject defaulted pl ) ); assert.equal(code.compatibilityDate, "2026-04-24"); - assert.deepEqual(code.compatibilityFlags, ["experimental"]); + assert.deepEqual(code.compatibilityFlags, []); }); test("bundleToWorkerCode: old compatibilityDate absent flags → floor only", () => { @@ -341,7 +341,7 @@ test("bundleToWorkerCode: old compatibilityDate absent flags → floor only", () { "w.js": enc.encode("x") } ) ); - assert.deepEqual(code.compatibilityFlags, ["experimental", "enhanced_error_serialization"]); + assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization"]); }); test("bundleToWorkerCode: compatibilityFlags already includes floor → no dup", () => { @@ -356,7 +356,7 @@ test("bundleToWorkerCode: compatibilityFlags already includes floor → no dup", { "w.js": enc.encode("x") } ) ); - assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization", "nodejs_compat", "experimental"]); + assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization", "nodejs_compat"]); }); test("bundleToWorkerCode: throws (doesn't silently drop) on malformed compatibilityFlags in bundle bytes", () => { diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index 99dd168..6c970e8 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -1062,7 +1062,7 @@ test("createLoaderCallback: attaches configured tail worker and always wraps mai assert.equal(/** @type {any} */ (workerCode.modules)["worker.js"], "export default {};"); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class __WdlAbort__/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \{ WorkerEntrypoint, abortIsolate \} from "cloudflare:workers"/); - assert.equal(workerCode.allowExperimental, true); + assert.equal("allowExperimental" in workerCode, false); assert.deepEqual(workerCode.tails, [{ kind: "tail-fetcher" }]); assert.deepEqual(workerCode.globalOutbound, { kind: "public-network" }); assert.ok(!("meta" in workerCode), "loader callback should not propagate meta into the workerLoader object"); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index cb1a4b1..1cb3963 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1443,7 +1443,14 @@ test("local compose routes private HTTP hops through Envoy only", () => { const integrationCompose = withoutLineComments(readRepoFile("tests/integration/helpers/compose.js")); const compileConfigs = withoutLineComments(readRepoFile("scripts/compile-workerd-configs.js")); const supervisorConfig = readRepoFile("rust/supervisor/src/config.rs"); + const supervisorLib = readRepoFile("rust/supervisor/src/lib.rs"); const dockerfileWorkerd = withoutLineComments(readRepoFile("Dockerfile.workerd")); + const kubeGateway = withoutLineComments(readRepoFile("deploy/kubernetes/base/gateway.yaml")); + const kubeSystemRuntime = withoutLineComments(readRepoFile("deploy/kubernetes/base/system-runtime.yaml")); + const kubeUserRuntime = withoutLineComments(readRepoFile("deploy/kubernetes/base/user-runtime.yaml")); + const terraformGatewayService = withoutLineComments(readRepoFile("terraform/modules/compute/gateway_service.tf")); + const terraformRuntimeService = withoutLineComments(readRepoFile("terraform/modules/compute/runtime_service.tf")); + const terraformSystemRuntimeService = withoutLineComments(readRepoFile("terraform/modules/compute/system_runtime_service.tf")); const packageJson = withoutLineComments(readRepoFile("package.json")); // Local/integration should mirror production Service Connect for HTTP @@ -1501,9 +1508,28 @@ test("local compose routes private HTTP hops through Envoy only", () => { for (const tier of ["gateway-local", "user-runtime-local", "system-runtime-local"]) { assert.match( compose, - new RegExp(`serve.*workerd-configs/${RegExp.escape(tier)}\\.bin.*--experimental`), + new RegExp(`serve.*workerd-configs/${RegExp.escape(tier)}\\.bin`), ); } + // workerd 2026-07-01 still gates workerLoader bindings on the process-level + // --experimental switch. Keep it only on workerLoader-owning processes, not + // on gateway or D1 and not as a Worker compatibility flag. + for (const tier of ["user-runtime-local", "system-runtime-local"]) { + assert.match( + compose, + new RegExp(`workerd-configs/${RegExp.escape(tier)}\\.bin", "--experimental"`), + ); + } + assert.doesNotMatch(compose, /gateway-local\.bin", "--experimental"/); + assert.match(kubeUserRuntime, /user-runtime\.bin[\s\S]*?- --experimental/); + assert.match(kubeSystemRuntime, /system-runtime\.bin[\s\S]*?- --experimental/); + assert.doesNotMatch(kubeGateway, /--experimental/); + assert.match(terraformRuntimeService, /user-runtime\.bin", "--experimental"/); + assert.match(terraformSystemRuntimeService, /system-runtime\.bin", "--experimental"/); + assert.doesNotMatch(terraformGatewayService, /--experimental/); + assert.match(supervisorLib, /workerd_args\(D1_COMPILED_CONFIG, false\)/); + assert.match(supervisorLib, /workerd_args\(pick_do_compiled_config\(\), true\)/); + assert.match(supervisorConfig, /args\.push\("--experimental"\.into\(\)\)/); // The supervisor config owns the local/production .bin choice; only it // references do-runtime-local.bin since compose forwards via the // supervisor binary, not via raw workerd serve. From 5fb5e7b3ff2f9977213a97c06fff313ceba49c85 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 1 Jul 2026 16:05:05 +0800 Subject: [PATCH 02/13] address env budget review feedback Validate namespace and worker secret mutations against retained worker versions, check DELETE paths that reveal lower-precedence env values, and parallelize secret decryption used by env-budget prechecks. Signed-off-by: Lu Zhang --- control/env-budget.js | 53 +++++++- control/handlers/ns-secrets.js | 31 +++-- control/handlers/worker-secrets.js | 40 +++--- tests/unit/control-env-budget.test.js | 30 +++++ .../control-secret-envelope-handlers.test.js | 121 +++++++++++++++++- 5 files changed, 240 insertions(+), 35 deletions(-) diff --git a/control/env-budget.js b/control/env-budget.js index e51e9e1..f6c0cc2 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,4 +1,5 @@ import { decryptSecretValue } from "shared-secret-envelope"; +import { bundleKey } from "shared-version"; export const UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES = 1024 * 1024; export const WORKER_LOADER_ENV_HEADROOM_BYTES = 8 * 1024; @@ -94,11 +95,55 @@ export function assertWorkerLoaderUserEnvBudget({ * }} args */ export async function decryptSecretHash({ encrypted, env, hashKey }) { + const entries = await Promise.all( + Object.entries(encrypted || {}) + .filter((entry) => typeof entry[1] === "string") + .map(async ([fieldName, value]) => [ + fieldName, + await decryptSecretValue(/** @type {string} */ (value), { env, hashKey, fieldName }), + ]) + ); /** @type {Record} */ const out = Object.create(null); - for (const [fieldName, value] of Object.entries(encrypted || {})) { - if (typeof value !== "string") continue; - out[fieldName] = await decryptSecretValue(value, { env, hashKey, fieldName }); - } + for (const [fieldName, value] of entries) out[fieldName] = value; return out; } + +/** + * @param {{ + * redis: { hGet(key: string, field: string): Promise }, + * ns: string, + * worker: string, + * versions: Iterable, + * nsSecrets?: Record | null, + * workerSecrets?: Record | null, + * }} args + */ +export async function assertWorkerVersionsUserEnvBudget({ + redis, + ns, + worker, + versions, + nsSecrets = null, + workerSecrets = null, +}) { + const uniqueVersions = [...new Set([...versions].filter((version) => typeof version === "string" && version))]; + if (uniqueVersions.length === 0) { + assertWorkerLoaderUserEnvBudget({ ns, worker, nsSecrets, workerSecrets }); + return; + } + + // Keep bundle metadata reads sequential: callers may pass a RedisSession, + // whose command protocol is single-flight even though secret decryption is not. + for (const version of uniqueVersions) { + const rawMeta = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); + const meta = typeof rawMeta === "string" ? JSON.parse(rawMeta) : {}; + assertWorkerLoaderUserEnvBudget({ + ns, + worker, + vars: meta && typeof meta === "object" ? meta.vars : null, + nsSecrets, + workerSecrets, + }); + } +} diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 910077a..bb6ea73 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -10,10 +10,12 @@ import { invalidSecretMutationKeyResponse, readEncryptedSecretPutValue, } from "control-handlers-secret-put"; -import { routesKey, bundleKey } from "shared-version"; +import { routesKey, workerVersionsKey } from "shared-version"; +import { workersIndexKey } from "control-lib"; import { WorkerEnvBudgetError, assertWorkerLoaderUserEnvBudget, + assertWorkerVersionsUserEnvBudget, decryptSecretHash, } from "control-env-budget"; import { SecretEnvelopeError } from "shared-secret-envelope"; @@ -38,28 +40,37 @@ async function validateNamespaceSecretBudget({ redis, env, nsName, secretKey, pl }); nsSecrets[secretKey] = plaintext; - const activeRoutes = await redis.hGetAll(routesKey(nsName)); - const activeEntries = Object.entries(activeRoutes) - .filter((entry) => typeof entry[1] === "string" && entry[1] !== ""); - if (activeEntries.length === 0) { + const [activeRoutes, indexedWorkers] = await Promise.all([ + redis.hGetAll(routesKey(nsName)), + redis.sMembers(workersIndexKey(nsName)), + ]); + const workerNames = new Set([ + ...indexedWorkers.filter((worker) => typeof worker === "string" && worker), + ...Object.keys(activeRoutes), + ]); + if (workerNames.size === 0) { assertWorkerLoaderUserEnvBudget({ ns: nsName, nsSecrets }); return; } - for (const [worker, version] of activeEntries) { + for (const worker of workerNames) { const workerSecretsKey = `secrets:${nsName}:${worker}`; - const metaRaw = await redis.hGet(bundleKey(nsName, worker, /** @type {string} */ (version)), "__meta__"); + const activeVersion = activeRoutes[worker]; + const retainedVersions = await redis.zRange(workerVersionsKey(nsName, worker), 0, -1); const workerEncrypted = await redis.hGetAll(workerSecretsKey); - const meta = typeof metaRaw === "string" ? JSON.parse(metaRaw) : {}; const workerSecrets = await decryptSecretHash({ encrypted: workerEncrypted, env: controlEnv, hashKey: workerSecretsKey, }); - assertWorkerLoaderUserEnvBudget({ + await assertWorkerVersionsUserEnvBudget({ + redis, ns: nsName, worker, - vars: meta && typeof meta === "object" ? meta.vars : null, + versions: [ + ...retainedVersions, + ...(typeof activeVersion === "string" && activeVersion ? [activeVersion] : []), + ], nsSecrets, workerSecrets, }); diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 1c20953..733d655 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -21,10 +21,9 @@ import { } from "control-handlers-secret-put"; import { stageWorkerHidden, stageWorkerVisible } from "control-lifecycle-indexes"; import { bumpActiveAndPromote, RoutingError } from "control-routing"; -import { bundleKey } from "shared-version"; import { WorkerEnvBudgetError, - assertWorkerLoaderUserEnvBudget, + assertWorkerVersionsUserEnvBudget, decryptSecretHash, } from "control-env-budget"; import { SecretEnvelopeError } from "shared-secret-envelope"; @@ -200,8 +199,8 @@ export async function handle({ request, env, method, ns, name, subPath, requestI return jsonError(405, "method_not_allowed", "Method not allowed for /secrets"); } -// DELETE extends WATCH to routes + worker-versions so the -// "last key → SREM workers:" branch can trust its preconditions. +// Secret mutations watch routes + worker-versions because env-budget checks +// must cover active and retained versions before writing a new secret shape. /** * @param {{ redis: RedisClient, ns: string, name: string, key: string, method: string, value: string | null, plaintext?: string | null, controlEnv: Record }} args */ @@ -215,10 +214,7 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n }); }, }, async (iso) => { - const watches = [deleteLockKey(ns, name), secretsKey]; - if (method === "DELETE") { - watches.push(routesKey(ns), workerVersionsKey(ns, name)); - } + const watches = [deleteLockKey(ns, name), secretsKey, routesKey(ns), workerVersionsKey(ns, name)]; await iso.watch(...watches); const callerLock = await iso.get(deleteLockKey(ns, name)); @@ -244,30 +240,36 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n } const multi = iso.multi(); - if (method === "PUT") { - if (typeof value !== "string") throw new Error("PUT secret value missing"); - if (typeof plaintext !== "string") throw new Error("PUT secret plaintext missing"); + if (method === "PUT" || method === "DELETE") { + if (method === "PUT" && typeof plaintext !== "string") throw new Error("PUT secret plaintext missing"); const activeVersion = await iso.hGet(routesKey(ns), name); + const retainedVersions = await iso.zRange(workerVersionsKey(ns, name), 0, -1); const nsEncrypted = await iso.hGetAll(`secrets:${ns}`); const workerEncrypted = await iso.hGetAll(secretsKey); const [nsSecrets, workerSecrets] = await Promise.all([ decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: `secrets:${ns}` }), decryptSecretHash({ encrypted: workerEncrypted, env: controlEnv, hashKey: secretsKey }), ]); - workerSecrets[key] = plaintext; - let vars = null; - if (typeof activeVersion === "string" && activeVersion) { - const rawMeta = await iso.hGet(bundleKey(ns, name, activeVersion), "__meta__"); - const meta = typeof rawMeta === "string" ? JSON.parse(rawMeta) : {}; - vars = meta && typeof meta === "object" ? meta.vars : null; + if (method === "PUT") { + workerSecrets[key] = /** @type {string} */ (plaintext); + } else { + delete workerSecrets[key]; } - assertWorkerLoaderUserEnvBudget({ + await assertWorkerVersionsUserEnvBudget({ + redis: iso, ns, worker: name, - vars, + versions: [ + ...retainedVersions, + ...(typeof activeVersion === "string" && activeVersion ? [activeVersion] : []), + ], nsSecrets, workerSecrets, }); + } + + if (method === "PUT") { + if (typeof value !== "string") throw new Error("PUT secret value missing"); multi.hSet(secretsKey, key, value); // SADD even on secret-only / pre-deploy workers so they're // visible to GET /workers and reachable by whole-delete. diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index d465d51..81218d3 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -8,17 +8,20 @@ import { import { encryptSecretValue } from "../../shared/secret-envelope.js"; const secretEnvelopeUrl = repositoryFileUrl("shared/secret-envelope.js"); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const { UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, WORKER_LOADER_ENV_HEADROOM_BYTES, WORKER_LOADER_ENV_MAX_BYTES, WorkerEnvBudgetError, assertWorkerLoaderUserEnvBudget, + assertWorkerVersionsUserEnvBudget, decryptSecretHash, mergedUserEnvStrings, userEnvSerializedBytes, } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ "shared-secret-envelope": secretEnvelopeUrl, + "shared-version": sharedVersionUrl, })); const envelopeEnv = { @@ -82,3 +85,30 @@ test("decryptSecretHash returns plaintext secret values for budget checks", asyn { TOKEN: "plain" } ); }); + +test("worker env budget checks every retained worker version", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(field, "__meta__"); + if (key === "worker:demo:api:v:1") return JSON.stringify({ vars: { SMALL: "ok" } }); + if (key === "worker:demo:api:v:2") return JSON.stringify({ vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES) } }); + return null; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1", "v2"], + nsSecrets: { TOKEN: "secret" }, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 85ae64c..b8fe29e 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; -import { decryptSecretValue, isSecretEnvelope } from "../../shared/secret-envelope.js"; +import { decryptSecretValue, encryptSecretValue, isSecretEnvelope } from "../../shared/secret-envelope.js"; import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, repositoryFileUrl } from "../helpers/load-shared-module.js"; import { readJsonResponse } from "../helpers/response-json.js"; @@ -38,6 +38,7 @@ function secretPutUrl(controlSharedUrl, controlLibUrl) { function envBudgetUrl() { const source = applyModuleReplacements(readRepositoryFile("control/env-budget.js"), [ [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], ]); return moduleDataUrl(source); } @@ -50,6 +51,8 @@ export const state = { async hKeys() { return []; }, async hGet() { return null; }, async hGetAll() { return {}; }, + async sMembers() { return []; }, + async zRange() { return []; }, async hSet(key, field, value) { this.writes.push({ key, field, value }); return 1; @@ -58,7 +61,10 @@ export const state = { }, }; `); -const controlLibStubUrl = moduleDataUrl(validateSecretKeyStubSource); +const controlLibStubUrl = moduleDataUrl(` +${validateSecretKeyStubSource} +export const workersIndexKey = (ns) => \`workers:\${ns}\`; +`); const src = applyModuleReplacements(readRepositoryFile("control/handlers/ns-secrets.js"), [ [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(controlLibStubUrl)};`], @@ -138,6 +144,53 @@ test("namespace secret PUT accepts lowercase secret keys like production", async assert.equal(state.redis.writes.at(-1).field, "lowercase"); }); +test("namespace secret PUT checks retained worker versions before storing", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGet: state.redis.hGet, + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + zRange: state.redis.zRange, + }; + const writesBefore = state.redis.writes.length; + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? ["api"] : []; + /** @param {string} key */ + state.redis.zRange = async (key) => key === "worker-versions:demo:api" ? ["v1"] : []; + /** @param {string} key @param {string} field */ + state.redis.hGet = async (key, field) => { + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { BIG: "x".repeat(1024 * 1024) } }); + } + return null; + }; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-retained", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(state.redis.writes.length, writesBefore); + } finally { + Object.assign(state.redis, original); + } +}); + const workerControlSharedUrl = controlSharedStubUrl(` class WatchError extends Error {} export function formatError(err) { @@ -157,6 +210,7 @@ export const state = { async hGet() { return null; }, async hGetAll() { return {}; }, async zCard() { return 0; }, + async zRange() { return []; }, multi() { return { hSet(key, field, value) { @@ -181,6 +235,7 @@ ${validateSecretKeyStubSource} export const deleteLockKey = (ns, worker) => \`worker-delete-lock:\${ns}:\${worker}\`; export const workerVersionsKey = (ns, worker) => \`worker-versions:\${ns}:\${worker}\`; export const routesKey = (ns) => \`routes:\${ns}\`; +export const workersIndexKey = (ns) => \`workers:\${ns}\`; `); const lifecycleStubUrl = moduleDataUrl(` export function stageWorkerHidden() {} @@ -259,3 +314,65 @@ test("worker secret mutation rejects invalid keys through shared validator", asy assert.equal(body.error, "invalid_request"); assert.equal(state.redis.writes.length, writesBefore); }); + +test("worker secret DELETE checks env revealed by removing a higher-precedence secret", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const encrypted = await encryptSecretValue("small", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let execCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["TOKEN"]; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") return "v1"; + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { TOKEN: "x".repeat(1024 * 1024) } }); + } + return null; + }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo:api") return { TOKEN: encrypted }; + return {}; + }, + async zCard() { return 1; }, + async zRange() { return ["v1"]; }, + multi() { + return { + hSet() {}, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-budget", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(execCalled, false); + } finally { + state.redis.session = originalSession; + } +}); From 6fa169abd3fe4e7c4dc647479cbeae71b88f4ecf Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 1 Jul 2026 17:52:57 +0800 Subject: [PATCH 03/13] harden env budget mutations Make namespace-secret env-budget checks atomic with the secret write, validate namespace-secret deletes, and watch namespace secrets during worker-secret checks. Revalidate deploy env budget inside the deploy commit WATCH window and add bundle metadata parse context for retained-version budget checks. Signed-off-by: Lu Zhang --- control/env-budget.js | 20 ++- control/handlers/deploy.js | 32 +++-- control/handlers/ns-secrets.js | 128 +++++++++++++---- control/handlers/worker-secrets.js | 12 +- tests/unit/control-deploy-watch.test.js | 41 ++++++ tests/unit/control-env-budget.test.js | 21 +++ .../control-secret-envelope-handlers.test.js | 133 +++++++++++++++++- 7 files changed, 345 insertions(+), 42 deletions(-) diff --git a/control/env-budget.js b/control/env-budget.js index f6c0cc2..f59880b 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -137,11 +137,27 @@ export async function assertWorkerVersionsUserEnvBudget({ // whose command protocol is single-flight even though secret decryption is not. for (const version of uniqueVersions) { const rawMeta = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); - const meta = typeof rawMeta === "string" ? JSON.parse(rawMeta) : {}; + /** @type {Record} */ + let meta = {}; + if (typeof rawMeta === "string") { + try { + const parsed = JSON.parse(rawMeta); + meta = parsed && typeof parsed === "object" && !Array.isArray(parsed) + ? /** @type {Record} */ (parsed) + : {}; + } catch (err) { + throw new Error( + `invalid bundle metadata for ${ns}/${worker}@${version}: ${err instanceof Error ? err.message : String(err)}`, + { cause: err } + ); + } + } assertWorkerLoaderUserEnvBudget({ ns, worker, - vars: meta && typeof meta === "object" ? meta.vars : null, + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) + ? /** @type {Record} */ (meta.vars) + : null, nsSecrets, workerSecrets, }); diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 11cfe11..9ff4a53 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -605,10 +605,9 @@ async function runDeployPreflight({ redis, ns, name, deployRequest }) { } /** - * @param {{ redis: RedisClient, env: Record, ns: string, name: string, meta: PreparedMeta }} args + * @param {{ redis: RedisClient | RedisSession, controlEnv: Record, ns: string, name: string, meta: PreparedMeta }} args */ -async function validateCommittedEnvBudget({ redis, env, ns, name, meta }) { - const controlEnv = stringEnv(env); +async function validateCommittedEnvBudget({ redis, controlEnv, ns, name, meta }) { const nsEncrypted = await redis.hGetAll(`secrets:${ns}`); const workerEncrypted = await redis.hGetAll(`secrets:${ns}:${name}`); const [nsSecrets, workerSecrets] = await Promise.all([ @@ -618,7 +617,7 @@ async function validateCommittedEnvBudget({ redis, env, ns, name, meta }) { assertWorkerLoaderUserEnvBudget({ ns, worker: name, - vars: meta.vars && typeof meta.vars === "object" + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) ? /** @type {Record} */ (meta.vars) : null, nsSecrets, @@ -694,16 +693,17 @@ async function uploadDeployAssetsBeforeCommit({ * requestId: string, * warnings: DeployWarning[], * log: ControlLogger, + * controlEnv: Record, * }} args * @returns {Promise<{ response: Response, commitDurationMs?: never } | { response?: never, commitDurationMs: number }>} */ async function commitPreparedDeploy({ - redis, ns, name, version, prepared, outgoingRefs, d1Refs, uploadedPrefix, requestId, warnings, log, + redis, ns, name, version, prepared, outgoingRefs, d1Refs, uploadedPrefix, requestId, warnings, log, controlEnv, }) { const commitStartedAt = Date.now(); try { await commitWithWatch({ - redis, ns, name, version, prepared, outgoingRefs, d1Refs, + redis, ns, name, version, prepared, outgoingRefs, d1Refs, controlEnv, }); } catch (err) { if (uploadedPrefix) { @@ -725,6 +725,8 @@ async function commitPreparedDeploy({ }); return { response: controlAbortResponse(err, warnings.length ? { warnings } : {}) }; } + if (err instanceof WorkerEnvBudgetError) return { response: codedErrorResponse(err, err.code) }; + if (err instanceof SecretEnvelopeError) return { response: jsonError(503, err.code, err.message) }; throw err; } return { commitDurationMs: Date.now() - commitStartedAt }; @@ -769,6 +771,7 @@ export async function handle({ request, env, ns, name, requestId }) { outgoingRefs, d1Refs, } = candidate.committed; + const controlEnv = stringEnv(env); try { validateWorkerLoaderCodeBudget({ prepared, ns, name }); @@ -784,7 +787,7 @@ export async function handle({ request, env, ns, name, requestId }) { try { await validateCommittedEnvBudget({ redis, - env, + controlEnv, ns, name, meta: prepared.meta, @@ -823,6 +826,7 @@ export async function handle({ request, env, ns, name, requestId }) { requestId, warnings, log, + controlEnv, }); if (commitResult.response) return commitResult.response; @@ -884,10 +888,10 @@ async function scheduleDeployAbortCleanup({ } /** - * @param {{ redis: RedisClient, ns: string, name: string, version: string, prepared: PreparedBundle, outgoingRefs: OutgoingRef[], d1Refs: DeployD1Ref[] }} args + * @param {{ redis: RedisClient, ns: string, name: string, version: string, prepared: PreparedBundle, outgoingRefs: OutgoingRef[], d1Refs: DeployD1Ref[], controlEnv?: Record | null }} args */ export async function commitWithWatch({ - redis, ns, name, version, prepared, outgoingRefs, d1Refs, + redis, ns, name, version, prepared, outgoingRefs, d1Refs, controlEnv = null, }) { const vNum = parseVersion(version); if (vNum == null) throw new Error(`commitWithWatch: bad version ${version}`); @@ -901,6 +905,16 @@ export async function commitWithWatch({ }, }, async (iso) => { await watchCommitKeys(iso, { ns, name, prepared, outgoingRefs, d1Refs }); + if (controlEnv) { + await iso.watch(`secrets:${ns}`, `secrets:${ns}:${name}`); + await validateCommittedEnvBudget({ + redis: iso, + controlEnv, + ns, + name, + meta: prepared.meta, + }); + } const resolvedD1Refs = await resolveD1RefsForCommit(iso, { ns, d1Refs }); await validateCallerNotDeleting(iso, { ns, name }); diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index bb6ea73..9c596e2 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -5,6 +5,9 @@ import { requireControlRedis, stringEnv, codedErrorResponse, + runOptimistic, + ControlAbort, + controlAbortResponse, } from "control-shared"; import { invalidSecretMutationKeyResponse, @@ -20,30 +23,21 @@ import { } from "control-env-budget"; import { SecretEnvelopeError } from "shared-secret-envelope"; +const MAX_NS_SECRET_ATTEMPTS = 5; + +class NamespaceSecretAbort extends ControlAbort {} + /** * @param {{ - * redis: import("shared-redis").RedisClient, - * env: Record, + * redis: import("shared-redis").RedisSession, + * controlEnv: Record, * nsName: string, - * secretKey: string, - * plaintext: string, + * nsSecrets: Record, * }} args */ -async function validateNamespaceSecretBudget({ redis, env, nsName, secretKey, plaintext }) { - const controlEnv = stringEnv(env); - const nsSecretsKey = `secrets:${nsName}`; - const existingEncrypted = await redis.hGetAll(nsSecretsKey); - const nsSecrets = await decryptSecretHash({ - encrypted: existingEncrypted, - env: controlEnv, - hashKey: nsSecretsKey, - }); - nsSecrets[secretKey] = plaintext; - - const [activeRoutes, indexedWorkers] = await Promise.all([ - redis.hGetAll(routesKey(nsName)), - redis.sMembers(workersIndexKey(nsName)), - ]); +async function validateNamespaceSecretBudget({ redis, controlEnv, nsName, nsSecrets }) { + const activeRoutes = await redis.hGetAll(routesKey(nsName)); + const indexedWorkers = await redis.sMembers(workersIndexKey(nsName)); const workerNames = new Set([ ...indexedWorkers.filter((worker) => typeof worker === "string" && worker), ...Object.keys(activeRoutes), @@ -55,6 +49,7 @@ async function validateNamespaceSecretBudget({ redis, env, nsName, secretKey, pl for (const worker of workerNames) { const workerSecretsKey = `secrets:${nsName}:${worker}`; + await redis.watch(workerVersionsKey(nsName, worker), workerSecretsKey); const activeVersion = activeRoutes[worker]; const retainedVersions = await redis.zRange(workerVersionsKey(nsName, worker), 0, -1); const workerEncrypted = await redis.hGetAll(workerSecretsKey); @@ -77,6 +72,74 @@ async function validateNamespaceSecretBudget({ redis, env, nsName, secretKey, pl } } +/** + * @param {{ + * redis: import("shared-redis").RedisClient, + * env: Record, + * nsName: string, + * secretKey: string, + * method: "PUT" | "DELETE", + * encrypted?: string | null, + * plaintext?: string | null, + * }} args + */ +async function mutateNamespaceSecret({ + redis, + env, + nsName, + secretKey, + method, + encrypted = null, + plaintext = null, +}) { + const controlEnv = stringEnv(env); + const nsSecretsKey = `secrets:${nsName}`; + return await runOptimistic(redis, { + attempts: MAX_NS_SECRET_ATTEMPTS, + onExhausted: () => { + throw new NamespaceSecretAbort(503, "namespace_secret_mutation_contention", { + message: `exhausted ${MAX_NS_SECRET_ATTEMPTS} retries; retry later`, + }); + }, + }, async (iso) => { + await iso.watch(nsSecretsKey, routesKey(nsName), workersIndexKey(nsName)); + + const existingEncrypted = await iso.hGetAll(nsSecretsKey); + if (method === "DELETE" && !Object.hasOwn(existingEncrypted, secretKey)) { + return { mutated: false }; + } + + const nsSecrets = await decryptSecretHash({ + encrypted: existingEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + }); + if (method === "PUT") { + if (typeof encrypted !== "string") throw new Error("PUT namespace secret encrypted value missing"); + if (typeof plaintext !== "string") throw new Error("PUT namespace secret plaintext missing"); + nsSecrets[secretKey] = plaintext; + } else { + delete nsSecrets[secretKey]; + } + + await validateNamespaceSecretBudget({ + redis: iso, + controlEnv, + nsName, + nsSecrets, + }); + + const multi = iso.multi(); + if (method === "PUT") { + multi.hSet(nsSecretsKey, secretKey, /** @type {string} */ (encrypted)); + } else { + multi.hDel(nsSecretsKey, secretKey); + } + await multi.exec(); + return { mutated: true }; + }); +} + /** * @param {{ * request: Request, @@ -107,20 +170,21 @@ export async function handle({ request, env, method, nsName, secretKey, requestI }); if ("response" in put) return put.response; try { - await validateNamespaceSecretBudget({ + await mutateNamespaceSecret({ redis, env, nsName, secretKey, + method: "PUT", + encrypted: put.encrypted, plaintext: put.plaintext, }); } catch (err) { + if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); throw err; } - const encrypted = put.encrypted; - await redis.hSet(nsSecretsKey, secretKey, encrypted); log("info", "ns_secret_set", { request_id: requestId, namespace: nsName, key: secretKey }); return jsonResponse(200, { namespace: nsName, @@ -132,14 +196,28 @@ export async function handle({ request, env, method, nsName, secretKey, requestI if (method === "DELETE" && secretKey !== undefined) { const invalidKey = invalidSecretMutationKeyResponse(secretKey); if (invalidKey) return invalidKey; - const removed = Number(await redis.hDel(nsSecretsKey, secretKey)) > 0; + let result; + try { + result = await mutateNamespaceSecret({ + redis, + env, + nsName, + secretKey, + method: "DELETE", + }); + } catch (err) { + if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); + if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); + if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + throw err; + } log("info", "ns_secret_deleted", { request_id: requestId, namespace: nsName, key: secretKey, - existed: removed, + existed: result.mutated, }); - return jsonResponse(200, { namespace: nsName, key: secretKey, deleted: removed }); + return jsonResponse(200, { namespace: nsName, key: secretKey, deleted: result.mutated }); } return jsonError(405, "method_not_allowed", "Method not allowed for /secrets"); } diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 733d655..46cd0e2 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -199,13 +199,15 @@ export async function handle({ request, env, method, ns, name, subPath, requestI return jsonError(405, "method_not_allowed", "Method not allowed for /secrets"); } -// Secret mutations watch routes + worker-versions because env-budget checks -// must cover active and retained versions before writing a new secret shape. +// Secret mutations watch routes, worker-versions, and both secret hashes because +// env-budget checks must cover active and retained versions before writing a +// new secret shape. /** * @param {{ redis: RedisClient, ns: string, name: string, key: string, method: string, value: string | null, plaintext?: string | null, controlEnv: Record }} args */ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = null, controlEnv }) { const secretsKey = `secrets:${ns}:${name}`; + const nsSecretsKey = `secrets:${ns}`; return await runOptimistic(redis, { attempts: MAX_SECRET_ATTEMPTS, onExhausted: () => { @@ -214,7 +216,7 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n }); }, }, async (iso) => { - const watches = [deleteLockKey(ns, name), secretsKey, routesKey(ns), workerVersionsKey(ns, name)]; + const watches = [deleteLockKey(ns, name), nsSecretsKey, secretsKey, routesKey(ns), workerVersionsKey(ns, name)]; await iso.watch(...watches); const callerLock = await iso.get(deleteLockKey(ns, name)); @@ -244,10 +246,10 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n if (method === "PUT" && typeof plaintext !== "string") throw new Error("PUT secret plaintext missing"); const activeVersion = await iso.hGet(routesKey(ns), name); const retainedVersions = await iso.zRange(workerVersionsKey(ns, name), 0, -1); - const nsEncrypted = await iso.hGetAll(`secrets:${ns}`); + const nsEncrypted = await iso.hGetAll(nsSecretsKey); const workerEncrypted = await iso.hGetAll(secretsKey); const [nsSecrets, workerSecrets] = await Promise.all([ - decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: `secrets:${ns}` }), + decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: nsSecretsKey }), decryptSecretHash({ encrypted: workerEncrypted, env: controlEnv, hashKey: secretsKey }), ]); if (method === "PUT") { diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 4e5788c..f47960a 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -29,6 +29,7 @@ const CONTROL_DEPLOY_TEST_STATE = { parsedQueueConsumers: null, watchedKeys: null, envBudgetError: false, + envBudgetCalls: [], redis: null, logs: [], metrics: { increment() {}, observe() {} }, @@ -53,6 +54,7 @@ function resetControlDeployTestState() { CONTROL_DEPLOY_TEST_STATE.parsedQueueConsumers = null; CONTROL_DEPLOY_TEST_STATE.watchedKeys = null; CONTROL_DEPLOY_TEST_STATE.envBudgetError = false; + CONTROL_DEPLOY_TEST_STATE.envBudgetCalls = []; CONTROL_DEPLOY_TEST_STATE.redis = null; CONTROL_DEPLOY_TEST_STATE.logs = []; CONTROL_DEPLOY_TEST_STATE.metrics = { increment() {}, observe() {} }; @@ -208,6 +210,7 @@ export class WorkerEnvBudgetError extends Error { } } export function assertWorkerLoaderUserEnvBudget() { + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.push(Array.from(arguments)[0] || {}); if (/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError) { throw new WorkerEnvBudgetError("env too large"); } @@ -363,6 +366,44 @@ test("commitWithWatch re-resolves a D1 alias after a watched recreate race", asy assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("d1:database:tenant-a:d1_new")); }); +test("commitWithWatch validates deploy env budget under watched secret hashes", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; + /** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys = []; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls = []; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + await commitWithWatch({ + redis, + ns: "tenant-a", + name: "demo", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + vars: { TOKEN: "from-vars" }, + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [], + d1Refs: [], + controlEnv: {}, + }); + + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("secrets:tenant-a")); + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("secrets:tenant-a:demo")); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.length, 1); + assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].vars, { TOKEN: "from-vars" }); +}); + test("deploy handler resolves cross-namespace service-binding meta from the target namespace", async () => { /** @type {string[]} */ const metaReads = []; diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 81218d3..aaba343 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -112,3 +112,24 @@ test("worker env budget checks every retained worker version", async () => { } ); }); + +test("worker env budget reports bundle metadata parse context", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return "{not-json"; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + /invalid bundle metadata for demo\/api@v1/ + ); +}); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index b8fe29e..b56c363 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -44,10 +44,15 @@ function envBudgetUrl() { } const controlSharedUrl = controlSharedStubUrl(` +class WatchError extends Error {} export const state = { log() {}, redis: { + execCalls: 0, + execFailures: 0, writes: [], + deletes: [], + watchedKeys: [], async hKeys() { return []; }, async hGet() { return null; }, async hGetAll() { return {}; }, @@ -58,6 +63,44 @@ export const state = { return 1; }, async hDel() { return 0; }, + async session(fn) { + return await fn({ + async watch(...keys) { + state.redis.watchedKeys.push(...keys); + }, + async hGet(key, field) { + return await state.redis.hGet(key, field); + }, + async hGetAll(key) { + return await state.redis.hGetAll(key); + }, + async sMembers(key) { + return await state.redis.sMembers(key); + }, + async zRange(key, start, stop) { + return await state.redis.zRange(key, start, stop); + }, + multi() { + return { + hSet(key, field, value) { + state.redis.writes.push({ key, field, value }); + return this; + }, + hDel(key, field) { + state.redis.deletes.push({ key, field }); + return this; + }, + async exec() { + state.redis.execCalls += 1; + if (state.redis.execFailures > 0) { + state.redis.execFailures -= 1; + throw new WatchError("simulated namespace secret contention"); + } + }, + }; + }, + }); + }, }, }; `); @@ -144,6 +187,38 @@ test("namespace secret PUT accepts lowercase secret keys like production", async assert.equal(state.redis.writes.at(-1).field, "lowercase"); }); +test("namespace secret PUT runs as a WATCH/MULTI mutation and retries contention", async () => { + const { state } = await import(controlSharedUrl); + const writesBefore = state.redis.writes.length; + const execBefore = state.redis.execCalls; + state.redis.execFailures = 1; + state.redis.watchedKeys = []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/RETRY_TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "RETRY_TOKEN", + requestId: "rid-secret-retry", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.execCalls - execBefore, 2); + assert.equal(state.redis.writes.length - writesBefore, 2); + assert.equal(state.redis.writes.at(-1).field, "RETRY_TOKEN"); + assert.ok(state.redis.watchedKeys.includes("secrets:demo")); + assert.ok(state.redis.watchedKeys.includes("routes:demo")); + assert.ok(state.redis.watchedKeys.includes("workers:demo")); + } finally { + state.redis.execFailures = 0; + } +}); + test("namespace secret PUT checks retained worker versions before storing", async () => { const { state } = await import(controlSharedUrl); const original = { @@ -191,6 +266,58 @@ test("namespace secret PUT checks retained worker versions before storing", asyn } }); +test("namespace secret DELETE checks env revealed by removing a namespace secret", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGet: state.redis.hGet, + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + zRange: state.redis.zRange, + }; + const deletesBefore = state.redis.deletes.length; + const encrypted = await encryptSecretValue("small", { + env, + hashKey: "secrets:demo", + fieldName: "TOKEN", + }); + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: encrypted }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? ["api"] : []; + /** @param {string} key */ + state.redis.zRange = async (key) => key === "worker-versions:demo:api" ? ["v1"] : []; + /** @param {string} key @param {string} field */ + state.redis.hGet = async (key, field) => { + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { TOKEN: "x".repeat(1024 * 1024) } }); + } + return null; + }; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-budget", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(state.redis.deletes.length, deletesBefore); + } finally { + Object.assign(state.redis, original); + } +}); + const workerControlSharedUrl = controlSharedStubUrl(` class WatchError extends Error {} export function formatError(err) { @@ -200,10 +327,13 @@ export const state = { log() {}, redis: { execCalls: 0, + watchedKeys: [], writes: [], async session(fn) { return await fn({ - async watch() {}, + async watch(...keys) { + state.redis.watchedKeys.push(...keys); + }, async unwatch() {}, async get() { return null; }, async hKeys() { return []; }, @@ -278,6 +408,7 @@ test("worker secret PUT encrypts before WATCH retries and reuses the envelope", assert.equal(response.status, 200); const { state } = await import(workerControlSharedUrl); assert.equal(state.redis.execCalls, 2); + assert.ok(state.redis.watchedKeys.includes("secrets:demo")); assert.equal(state.redis.writes.length, 2); assert.equal(state.redis.writes[0].key, "secrets:demo:api"); assert.equal(state.redis.writes[0].field, "TOKEN"); From 7e4f4ad8160388c634749b3a2b6aaddfc60d7501 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 1 Jul 2026 19:15:37 +0800 Subject: [PATCH 04/13] Close late-opening log tail sessions Close the Redis tail session idempotently when cleanup runs and immediately close any session that finishes opening after the watchdog has already expired. Add a unit regression test for the pending-open race. Signed-off-by: Lu Zhang --- control/handlers/logs-tail.js | 33 ++++++++++++----- tests/unit/control-logs-tail.test.js | 53 ++++++++++++++++++++++++++-- 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index 6e6f0c7..ae60cb7 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -360,6 +360,9 @@ export async function handle({ request, env, ctx, ns, requestId }) { db: redisDbFromEnv(env, "DATA_REDIS_DB"), }); let sessionOpen = false; + let cleanupFinished = false; + /** @type {Promise | null} */ + let sessionClosePromise = null; let cancelled = false; /** @type {ReadableStreamDefaultController | null} */ let streamController = null; @@ -376,6 +379,21 @@ export async function handle({ request, env, ctx, ns, requestId }) { }), }); + async function closeSessionIfOpen() { + if (!sessionOpen) return; + sessionClosePromise ??= (async () => { + try { + await session.close(); + } catch (err) { + log("warn", "tail_session_close_failed", { + request_id: requestId, namespace: ns, + error_message: errMessage(err), + }); + } + })(); + await sessionClosePromise; + } + /** @param {ReadableStreamDefaultController | null} controller */ function expireSession(controller) { if (cancelled) return; @@ -404,14 +422,8 @@ export async function handle({ request, env, ctx, ns, requestId }) { // promise, the actual close happens here. ctx.waitUntil(cancelPromise.then(async () => { clearTimeout(expiryTimer); - try { - if (sessionOpen) await session.close(); - } catch (err) { - log("warn", "tail_session_close_failed", { - request_id: requestId, namespace: ns, - error_message: errMessage(err), - }); - } + await closeSessionIfOpen(); + cleanupFinished = true; log("info", "tail_session_close", { request_id: requestId, namespace: ns, worker_count: workers.length, }); @@ -441,6 +453,11 @@ export async function handle({ request, env, ctx, ns, requestId }) { // it and fail on the first Redis stream read. await session.open(); sessionOpen = true; + if (cancelled || cleanupFinished) { + await closeSessionIfOpen(); + try { controller.close(); } catch {} + return; + } bootstrapped = true; } diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 7b43d02..0eaafc0 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -1,7 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { importRepositoryModuleFresh } from "../helpers/load-shared-module.js"; -import { delay } from "../helpers/timing.js"; +import { delay, waitUntil } from "../helpers/timing.js"; function loadLogsTailHandler() { return importRepositoryModuleFresh("control/handlers/logs-tail.js", [ @@ -20,7 +20,15 @@ function loadLogsTailHandler() { this.publishCalls = []; this.closed = false; } - async open() { this.opened = true; } + async open() { + this.openStarted = true; + this.openPromise = (async () => { + const state = /** @type {any} */ (globalThis).__tailState; + if (state.openBlocker) await state.openBlocker; + this.opened = true; + })(); + await this.openPromise; + } async publish(channel, payload) { this.publishCalls.push([channel, payload]); } async xRead(...args) { this.xReadCalls.push(args); return null; } async close() { this.closed = true; } @@ -85,6 +93,7 @@ function resetTailState() { }, }, logs: [], + openBlocker: null, log: (/** @type {string} */ level, /** @type {string} */ event, /** @type {any} */ data) => { /** @type {any} */ (globalThis).__tailState.logs.push({ level, event, data }); }, @@ -217,6 +226,46 @@ test("logs tail max-session watchdog closes even without stream cancel", async ( await reader.cancel().catch(() => {}); }); +test("logs tail closes session if watchdog fires while Redis open is pending", async () => { + resetTailState(); + const state = /** @type {any} */ (globalThis).__tailState; + let releaseOpen = () => {}; + state.openBlocker = new Promise((resolve) => { + releaseOpen = () => resolve(undefined); + }); + + const { handle } = await loadLogsTailHandler(); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "30" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-open-race", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + const pendingRead = readText(reader); + const session = /** @type {any} */ (globalThis).__tailSessions[0]; + await waitUntil("tail Redis open to start", () => session.openStarted === true, { + timeoutMs: 500, intervalMs: 5, + }); + await delay(50); + releaseOpen(); + await session.openPromise; + await Promise.all(waitUntilPromises); + await waitUntil("tail Redis session to close", () => session.closed === true, { + timeoutMs: 500, intervalMs: 5, + }); + + assert.equal(session.opened, true); + assert.equal(session.closed, true); + assert.match((await pendingRead).text, /session_expired/); + await reader.cancel().catch(() => {}); +}); + test("logs tail emits idle keepalives below common proxy idle timeouts", async () => { resetTailState(); const { handle } = await loadLogsTailHandler(); From 6e26d863bbfb9eb6323e01c90e924e33e6cfa3cc Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 1 Jul 2026 22:31:21 +0800 Subject: [PATCH 05/13] Budget injected workerLoader env values Include runtime-injected binding and workflow env values in the control-plane workerLoader env budget estimate, including required caller secret copies in service binding props. Update docs and add regression coverage for retained versions. Signed-off-by: Lu Zhang --- CLAUDE.md | 7 +- control/env-budget.js | 231 ++++++++++++++++++++++++-- control/handlers/deploy.js | 7 +- docs/compatibility.md | 2 +- docs/compatibility.zh.md | 2 +- docs/modules/cli.md | 2 +- docs/modules/cli.zh.md | 2 +- docs/modules/runtime.md | 15 +- docs/modules/runtime.zh.md | 4 +- tests/unit/control-env-budget.test.js | 85 +++++++++- 10 files changed, 321 insertions(+), 36 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8bf73a2..d5f796c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,9 +73,10 @@ shared crate `wdl-rust-common`. Runtime receives plaintext only in the internal load envelope after redis-proxy decrypts during `/runtime/load`. Env materializes in fixed precedence — vars, then namespace secrets, then worker secrets — so a worker secret shadows a namespace secret - shadows a var on the same key. Control must keep vars plus namespace/worker secrets - within WDL's headroomed workerd `workerLoader` serialized env budget before - deploy/secret mutation, not let that fail later during cold-load. + shadows a var on the same key. Control must keep the estimated full workerLoader env + — user vars/secrets plus runtime-injected binding/workflow env values such as + required caller secret copies — within WDL's headroomed workerd serialized env budget + before deploy/secret mutation, not let that fail later during cold-load. - **DB split is intentional.** DB 0 is control metadata, DB 1 is data-plane KV/queue/log streams, DB 2 is Workflows. See `docs/redis-key-layout.md`. - **D1/DO correctness comes from owner lease + generation fence.** Service DNS only diff --git a/control/env-budget.js b/control/env-budget.js index f59880b..57824f6 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,6 +1,13 @@ import { decryptSecretValue } from "shared-secret-envelope"; import { bundleKey } from "shared-version"; +const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; +const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; +const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; +const ESTIMATED_VERSION = "v0000000000"; +const ESTIMATED_DO_STORAGE_ID = "do_00000000000000000000000000000000"; +const ESTIMATED_WORKFLOW_KEY = "wf_00000000000000000000000000000000"; + export const UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES = 1024 * 1024; export const WORKER_LOADER_ENV_HEADROOM_BYTES = 8 * 1024; export const WORKER_LOADER_ENV_MAX_BYTES = @@ -29,51 +36,247 @@ function stringRecord(source) { return out; } +/** @param {unknown} value @returns {Record | null} */ +function objectRecord(value) { + return value && typeof value === "object" && !Array.isArray(value) + ? /** @type {Record} */ (value) + : null; +} + +/** @param {unknown} value */ +function stringOrFallback(value, fallback = "") { + return typeof value === "string" ? value : fallback; +} + /** * @param {{ + * requiredCallerSecrets?: unknown, + * nsSecrets: Record, + * workerSecrets: Record, + * }} args + */ +function callerSecretsForBinding({ requiredCallerSecrets, nsSecrets, workerSecrets }) { + if (!Array.isArray(requiredCallerSecrets) || requiredCallerSecrets.length === 0) return undefined; + /** @type {Record} */ + const callerSecrets = {}; + for (const key of requiredCallerSecrets) { + if (typeof key !== "string") continue; + if (Object.hasOwn(workerSecrets, key)) { + callerSecrets[key] = workerSecrets[key]; + } else if (Object.hasOwn(nsSecrets, key)) { + callerSecrets[key] = nsSecrets[key]; + } + } + return callerSecrets; +} + +/** + * @param {{ + * name: string, + * spec: Record, + * meta: Record, + * ns: string, + * worker: string, + * version: string, + * nsSecrets: Record, + * workerSecrets: Record, + * }} args + */ +function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, nsSecrets, workerSecrets }) { + switch (spec.type) { + case "kv": + return { + __wdlBinding: "kv", + props: { ns, id: stringOrFallback(spec.id) }, + }; + case "assets": + return { + __wdlBinding: "assets", + props: { + cdnBase: "https://assets.invalid", + prefix: stringOrFallback(objectRecord(meta.assets)?.prefix), + }, + }; + case "queue": + return { + __wdlBinding: "queue", + props: { + ns, + id: stringOrFallback(spec.id), + deliveryDelaySeconds: spec.deliveryDelaySeconds ?? 0, + }, + }; + case "d1": + return { + __wdlBinding: "d1", + props: { + ns, + databaseId: stringOrFallback(spec.databaseId), + binding: name, + }, + }; + case "r2": + return { + __wdlBinding: "r2", + props: { + ns, + bucketName: stringOrFallback(spec.bucketName), + binding: name, + }, + }; + case "do": { + const props = { + ns, + worker, + version, + doStorageId: stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID), + binding: name, + className: stringOrFallback(spec.className), + }; + return { + __wdlBinding: "do", + ...props, + hostProxy: { __wdlBinding: "do-host-proxy", props }, + }; + } + case "service": { + const callerSecrets = callerSecretsForBinding({ + requiredCallerSecrets: spec.requiredCallerSecrets, + nsSecrets, + workerSecrets, + }); + return { + __wdlBinding: "service", + props: { + targetNs: stringOrFallback(spec.ns, ns), + targetWorker: stringOrFallback(spec.service), + targetVersion: stringOrFallback(spec.version), + targetEntrypoint: typeof spec.entrypoint === "string" ? spec.entrypoint : null, + callerNs: ns, + ...(callerSecrets ? { callerSecrets } : {}), + }, + }; + } + default: + return { __wdlBinding: stringOrFallback(spec.type, "unknown") }; + } +} + +/** + * @param {{ + * ns: string, + * worker?: string, + * version?: string, * vars?: Record | null, * nsSecrets?: Record | null, * workerSecrets?: Record | null, + * meta?: Record | null, * }} args */ -export function mergedUserEnvStrings({ vars = null, nsSecrets = null, workerSecrets = null }) { - return { +export function estimatedWorkerLoaderEnv({ + ns, + worker = "", + version = ESTIMATED_VERSION, + vars = null, + nsSecrets = null, + workerSecrets = null, + meta = null, +}) { + const nsSecretStrings = stringRecord(nsSecrets); + const workerSecretStrings = stringRecord(workerSecrets); + /** @type {Record} */ + const env = { ...stringRecord(vars), - ...stringRecord(nsSecrets), - ...stringRecord(workerSecrets), + ...nsSecretStrings, + ...workerSecretStrings, }; -} + const metaRecord = objectRecord(meta); + if (!metaRecord || !worker) return env; + + let hasDoBinding = false; + let hasWorkflowBinding = false; + const workflows = Array.isArray(metaRecord.workflows) ? metaRecord.workflows : []; + for (const workflow of workflows) { + const record = objectRecord(workflow); + if (!record) continue; + const binding = stringOrFallback(record.binding); + if (!binding) continue; + hasWorkflowBinding = true; + env[binding] = { + ns, + worker, + version, + name: stringOrFallback(record.name), + binding, + className: stringOrFallback(record.className), + workflowKey: stringOrFallback(record.workflowKey, ESTIMATED_WORKFLOW_KEY), + }; + } -/** @param {Record} envStrings */ -export function userEnvSerializedBytes(envStrings) { - return Buffer.byteLength(JSON.stringify(envStrings), "utf8"); + const bindings = objectRecord(metaRecord.bindings); + if (bindings) { + for (const [name, rawSpec] of Object.entries(bindings)) { + const spec = objectRecord(rawSpec); + if (!spec) continue; + env[name] = estimatedBindingEnvValue({ + name, + spec, + meta: metaRecord, + ns, + worker, + version, + nsSecrets: nsSecretStrings, + workerSecrets: workerSecretStrings, + }); + hasDoBinding ||= spec.type === "do"; + } + } + if (hasDoBinding) { + env[DO_BACKEND_BINDING] = { __wdlBinding: "internal", name: "DO_BACKEND" }; + env[DO_OWNER_NETWORK_BINDING] = { __wdlBinding: "internal", name: "DO_OWNER_NETWORK" }; + } + if (hasWorkflowBinding) { + env[WORKFLOWS_BACKEND_BINDING] = { __wdlBinding: "internal", name: "WORKFLOWS_BACKEND" }; + } + return env; } /** * @param {{ * ns: string, * worker?: string, + * version?: string, * vars?: Record | null, * nsSecrets?: Record | null, * workerSecrets?: Record | null, + * meta?: Record | null, * }} args */ export function assertWorkerLoaderUserEnvBudget({ ns, worker = undefined, + version = ESTIMATED_VERSION, vars = null, nsSecrets = null, workerSecrets = null, + meta = null, }) { - const merged = mergedUserEnvStrings({ vars, nsSecrets, workerSecrets }); // workerd enforces the full workerLoader env as a Frankenvalue estimate. Control - // checks the user-controlled string env as JSON and leaves headroom for runtime - // facade objects and estimator drift. - const bytes = userEnvSerializedBytes(merged); + // mirrors the user strings plus runtime-injected binding/workflow env shapes as + // JSON and leaves headroom for native facade-object overhead and estimator drift. + const bytes = Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + ns, + worker, + version, + vars, + nsSecrets, + workerSecrets, + meta, + })), "utf8"); if (bytes > WORKER_LOADER_ENV_MAX_BYTES) { const label = worker ? `${ns}/${worker}` : ns; throw new WorkerEnvBudgetError( - `vars and secrets for ${label} serialize to ${bytes} bytes, exceeding WDL workerLoader env budget ${WORKER_LOADER_ENV_MAX_BYTES} bytes`, + `estimated workerLoader env for ${label} serializes to ${bytes} bytes, exceeding WDL workerLoader env budget ${WORKER_LOADER_ENV_MAX_BYTES} bytes`, { namespace: ns, ...(worker ? { worker } : {}), @@ -155,11 +358,13 @@ export async function assertWorkerVersionsUserEnvBudget({ assertWorkerLoaderUserEnvBudget({ ns, worker, + version, vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) ? /** @type {Record} */ (meta.vars) : null, nsSecrets, workerSecrets, + meta, }); } } diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 9ff4a53..b3e6210 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -605,9 +605,9 @@ async function runDeployPreflight({ redis, ns, name, deployRequest }) { } /** - * @param {{ redis: RedisClient | RedisSession, controlEnv: Record, ns: string, name: string, meta: PreparedMeta }} args + * @param {{ redis: RedisClient | RedisSession, controlEnv: Record, ns: string, name: string, meta: PreparedMeta, version?: string }} args */ -async function validateCommittedEnvBudget({ redis, controlEnv, ns, name, meta }) { +async function validateCommittedEnvBudget({ redis, controlEnv, ns, name, meta, version = undefined }) { const nsEncrypted = await redis.hGetAll(`secrets:${ns}`); const workerEncrypted = await redis.hGetAll(`secrets:${ns}:${name}`); const [nsSecrets, workerSecrets] = await Promise.all([ @@ -617,11 +617,13 @@ async function validateCommittedEnvBudget({ redis, controlEnv, ns, name, meta }) assertWorkerLoaderUserEnvBudget({ ns, worker: name, + version, vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) ? /** @type {Record} */ (meta.vars) : null, nsSecrets, workerSecrets, + meta, }); } @@ -913,6 +915,7 @@ export async function commitWithWatch({ ns, name, meta: prepared.meta, + version, }); } diff --git a/docs/compatibility.md b/docs/compatibility.md index bee1201..a703cb7 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -48,7 +48,7 @@ Each row separates four compatibility claims: | Workflows | Partial | User workflow class code runs inside loaded workers when dispatched by runtime. | workflows owns DB 2 instance state, step/event/sleep commits, runtime-observed `step.do` DAG edges including `Promise.all` siblings, generation/run-token fencing, lifecycle APIs, progress callbacks, and scheduler tick. | WDL Workflows V2 has WDL-specific payload semantics and terminal failure rules; permanent `step.do` failure is terminal for the run even if caught, and one step may record at most 1000 dependency edges. | WDL Workflows V2 is not Cloudflare Workflows parity. `script_name`, cross-worker workflows, and Cloudflare's source-AST visualizer are unsupported. | | Service bindings | Supported | workerd service binding JSRPC and fetch dispatch. | WDL resolves target worker metadata, cold-loads immutable versions, propagates request ids where available, and enforces namespace/action ACL through control metadata. | Frozen-version service targets do not evict active siblings by design. | None currently documented. | | Platform bindings | Supported | workerd named entrypoint/JSRPC mechanics. | WDL expands ACL-checked platform bindings from control metadata. | Platform bindings are WDL-specific. | Platform bindings are not a Cloudflare tenant portability feature. | -| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget before deploy/secret mutation, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | +| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding/workflow env values before deploy/secret mutation, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | | Worker code size | Supported | Dynamic worker module bodies are accepted by `workerLoader` up to workerd's 64 MiB limit. | Control rejects module bodies over 64 MiB before allocating a version. | WDL's deploy JSON body limit is lower for ordinary inline deployments. | Large server-side bundle assembly paths must keep this guard. | ## Control Plane And Developer Tooling diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index c93b6eb..06ee309 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -41,7 +41,7 @@ | Workflows | Partial | 用户 workflow class code 在 runtime dispatch 时运行在 loaded worker 内。 | workflows 拥有 DB 2 instance state、step/event/sleep commit、runtime-observed `step.do` DAG edges(包括 `Promise.all` sibling)、generation/run-token fence、lifecycle API、progress callback 和 scheduler tick。 | WDL Workflows V2 的 payload 语义和 terminal failure 规则由 WDL 定义;永久失败的 `step.do` 即使被捕获也会让 run terminal failed;单个 step 最多记录 1000 条 dependency edge。 | WDL Workflows V2 不是 Cloudflare Workflows parity;不支持 `script_name`、跨 worker workflow 和 Cloudflare 的源码 AST visualizer。 | | Service bindings | Supported | workerd service binding JSRPC 和 fetch dispatch。 | WDL 解析 target worker metadata,cold-load immutable version,在可用时传播 request id,并通过 control metadata 做 namespace/action ACL。 | Frozen-version service target 按设计不 evict active siblings。 | 当前无已记录缺口。 | | Platform bindings | Supported | workerd named entrypoint/JSRPC 机制。 | WDL 从 control metadata 展开 ACL-checked platform bindings。 | Platform bindings 是 WDL-specific。 | 不是 Cloudflare tenant portability feature。 | -| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 前按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 做校验;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | +| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 前按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets 加 runtime 注入的 binding/workflow env value;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | | Worker code size | Supported | Dynamic worker module bodies 可由 `workerLoader` 接受,直到 workerd 的 64 MiB 上限。 | Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies。 | WDL 的 deploy JSON body limit 对普通 inline deploy 更低。 | 大型 server-side bundle assembly 路径必须保留这道 guard。 | ## 控制面和开发工具 diff --git a/docs/modules/cli.md b/docs/modules/cli.md index 331a4a2..7816063 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -149,7 +149,7 @@ Supported config surfaces: | Field | WDL behavior | |---|---| | `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit; module bodies must fit workerd's 64 MiB `workerLoader` code limit. | -| `[vars]` | String, number, and boolean values are accepted and stringified into `env`; vars plus namespace/worker secrets must fit WDL's headroomed workerd 1 MiB `workerLoader` env budget. | +| `[vars]` | String, number, and boolean values are accepted and stringified into `env`; vars, namespace/worker secrets, and runtime-injected binding/workflow env values must fit WDL's headroomed workerd 1 MiB `workerLoader` env budget. | | `[[kv_namespaces]]` | `id` is a platform-local KV namespace id, not a Cloudflare UUID. | | `[[r2_buckets]]` | `binding` plus `bucket_name` become a namespace-scoped virtual R2 bucket under the platform S3 bucket. | | `[assets]` | `directory` contents upload to S3-compatible assets storage and auto-inject `ASSETS`. | diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index c1b729b..ce83d2f 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -92,7 +92,7 @@ WDL 遵循 Wrangler selected-env 继承规则: | 字段 | WDL 行为 | |---|---| | `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`;module bodies 必须落在 workerd 的 64 MiB `workerLoader` code limit 内。 | -| `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`;vars 加 namespace/worker secrets 必须落在 WDL 留有 headroom 的 workerd 1 MiB `workerLoader` env budget 内。 | +| `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 必须落在 WDL 留有 headroom 的 workerd 1 MiB `workerLoader` env budget 内。 | | `[[kv_namespaces]]` | `id` 是 platform-local KV namespace id,不是 Cloudflare UUID。 | | `[[r2_buckets]]` | `binding` 加 `bucket_name` 映射为平台 S3 bucket 下的 namespace-scoped virtual R2 bucket。 | | `[assets]` | `directory` 内容上传到 S3-compatible assets storage,并 auto-inject `ASSETS`。 | diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 8d3d9ab..b001b76 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -82,10 +82,11 @@ services. Runtime therefore treats bindings as adapters: decrypts `WDL-ENC:` values; tenant-facing `env` shape stays unchanged. Env materialization merges in fixed precedence: bundle vars, then namespace secrets, then worker secrets. A worker-level secret with the same name wins over a namespace-level - secret, which wins over a var. Control enforces a headroomed version of workerd's - `workerLoader` serialized env budget on this user-controlled set before deploys and - secret mutations, so an over-large env fails in the control plane instead of during - runtime cold-load. + secret, which wins over a var. Control enforces a headroomed estimate of workerd's + `workerLoader` serialized env budget before deploys and secret mutations. That + estimate includes user vars/secrets plus runtime-injected binding/workflow env values, + including required caller secret copies in platform/service binding props, so an + over-large env fails in the control plane instead of during runtime cold-load. - Stateful bindings such as D1, Durable Objects, and Workflows call dedicated backend services. The hidden backend Fetchers stay in runtime and are removed before tenant code observes `env`. @@ -245,9 +246,9 @@ when a matching active tail session exists. WorkerCode unless another upstream API explicitly requires it. - Upstream workerd 2026-07-01 caps dynamic worker code at 64 MiB and serialized dynamic env at 1 MiB. Control rejects module bodies over 64 MiB before version allocation. - Vars plus namespace/worker secrets are prechecked against a headroomed `workerLoader` - env budget because workerd's final enforcement includes runtime facade objects in the - full `env` estimate. + Vars, namespace/worker secrets, and runtime-injected binding/workflow env values are + prechecked against a headroomed `workerLoader` env budget because workerd's final + enforcement includes the full `env` estimate. - workerd upgrades can still change default or compatibility-flagged runtime surfaces; review the exposed surface, not only the loader/abort path. diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 5dc3823..a84da22 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -40,7 +40,7 @@ Tenant 可见 binding 包括 KV、R2、D1、Durable Objects、Queues、ASSETS、 workerd 提供 isolate、module evaluation、named entrypoint 和 JSRPC 机制。Cloudflare 生产平台通常在外部服务里实现的 binding 后端,则由 WDL 自己补齐。因此 runtime 把 binding 当作 adapter: - KV、queue producer 这类纯数据 binding 调 colocated redis-proxy sidecar。Loaded worker 看到 Cloudflare-shaped object,但 method call 会先通过 workerd JSRPC 回到 runtime,再经 HTTP 调 redis-proxy。 -- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 前用留有 headroom 的预算检查这组用户可控 env 是否超过 workerd `workerLoader` 的 serialized env budget,避免超限配置拖到 runtime cold-load 才失败。 +- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 前用留有 headroom 的预算估算完整 workerLoader env,包括用户 vars/secrets、runtime 注入的 binding/workflow env value,以及 platform/service binding props 中复制的 required caller secret,避免超限配置拖到 runtime cold-load 才失败。 - D1、Durable Objects、Workflows 这类 stateful binding 调专门 backend service。Hidden backend Fetcher 留在 runtime 内部,并在 tenant code 观察 `env` 前被删除。 - R2 是 S3-compatible object-storage adapter:runtime 使用平台 credential 签名请求,并发送到配置的 endpoint。 - ASSETS 是 deploy artifact URL helper:control 在 deploy 时把 assets 上传到 S3-compatible storage;runtime 读取 `__meta__.assets` 和 `ASSETS_CDN_BASE`,只暴露 `env.ASSETS.url(path)` 用来生成 tokenized CDN URL。 @@ -107,7 +107,7 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - Runtime 不再为 loaded worker 开启 workerd 的宽泛 `experimental` flag。Historical-version eviction 仍然注入 `__WdlAbort__`,但当前 bundled workerd baseline 中 `abortIsolate()` 已经不需要该 flag。 - 移除 loaded worker 的宽泛 `experimental` flag 会有意收紧 tenant 对非 GA experimental-only surface 的访问,例如不可撤销的长期 stub storage。不要为了兼容绕过而重新打开它,除非先完成明确的功能设计。 - Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 -- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars 加 namespace/worker secrets 会在 control plane 用留有 headroom 的 `workerLoader` env budget 预检,因为 workerd 的最终检查还会把 runtime facade objects 纳入完整 `env` estimate。 +- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会在 control plane 用留有 headroom 的 `workerLoader` env budget 预检,因为 workerd 的最终检查看的是完整 `env` estimate。 - workerd 升级仍可能改变默认或 compatibility-flagged runtime surface;升级时要审 exposed surface,而不只审 loader/abort path。 ## 保护该模块的测试 diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index aaba343..47363a0 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -17,8 +17,7 @@ const { assertWorkerLoaderUserEnvBudget, assertWorkerVersionsUserEnvBudget, decryptSecretHash, - mergedUserEnvStrings, - userEnvSerializedBytes, + estimatedWorkerLoaderEnv, } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ "shared-secret-envelope": secretEnvelopeUrl, "shared-version": sharedVersionUrl, @@ -30,18 +29,27 @@ const envelopeEnv = { }; test("worker env budget counts merged vars and secrets with worker-secret precedence", () => { - const merged = mergedUserEnvStrings({ + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", vars: { TOKEN: "var", ONLY_VAR: "v" }, nsSecrets: { TOKEN: "ns", ONLY_NS: "n" }, workerSecrets: { TOKEN: "worker" }, }); - assert.deepEqual(merged, { + assert.deepEqual(estimated, { TOKEN: "worker", ONLY_VAR: "v", ONLY_NS: "n", }); - assert.equal(userEnvSerializedBytes(merged), Buffer.byteLength(JSON.stringify(merged), "utf8")); + assert.equal( + assertWorkerLoaderUserEnvBudget({ + ns: "demo", + vars: { TOKEN: "var", ONLY_VAR: "v" }, + nsSecrets: { TOKEN: "ns", ONLY_NS: "n" }, + workerSecrets: { TOKEN: "worker" }, + }), + Buffer.byteLength(JSON.stringify(estimated), "utf8") + ); }); test("worker env budget rejects user-controlled env above workerd workerLoader limit", () => { @@ -66,6 +74,36 @@ test("worker env budget rejects user-controlled env above workerd workerLoader l ); }); +test("worker env budget counts required caller secret copies in service binding props", () => { + const secret = "x".repeat(Math.floor(WORKER_LOADER_ENV_MAX_BYTES * 0.6)); + + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "caller", + vars: { SMALL: "ok" }, + nsSecrets: { API_TOKEN: secret }, + meta: { + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + entrypoint: "Api", + requiredCallerSecrets: ["API_TOKEN"], + }, + }, + }, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + test("decryptSecretHash returns plaintext secret values for budget checks", async () => { const hashKey = "secrets:demo"; const encrypted = await encryptSecretValue("plain", { @@ -113,6 +151,43 @@ test("worker env budget checks every retained worker version", async () => { ); }); +test("worker env budget checks retained-version binding env injections", async () => { + const secret = "x".repeat(Math.floor(WORKER_LOADER_ENV_MAX_BYTES * 0.6)); + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:caller:v:1"); + assert.equal(field, "__meta__"); + return JSON.stringify({ + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + requiredCallerSecrets: ["API_TOKEN"], + }, + }, + }); + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "caller", + versions: ["v1"], + nsSecrets: { API_TOKEN: secret }, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + test("worker env budget reports bundle metadata parse context", async () => { const redis = { /** @param {string} key @param {string} field */ From 7d7a3a6beec35501483ad649c98c31d00afd6e14 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Wed, 1 Jul 2026 23:41:26 +0800 Subject: [PATCH 06/13] Address workerd env budget review feedback Validate deploy env budget after materialized metadata, budget worker secret bumps with a conservative future version estimate, and skip decrypting secret envelopes removed by PUT/DELETE recovery paths. Signed-off-by: Lu Zhang --- control/env-budget.js | 30 ++- control/handlers/deploy.js | 20 +- control/handlers/ns-secrets.js | 4 +- control/handlers/worker-secrets.js | 13 +- tests/unit/control-deploy-watch.test.js | 46 +++++ tests/unit/control-env-budget.test.js | 60 ++++++ .../control-secret-envelope-handlers.test.js | 173 ++++++++++++++++++ 7 files changed, 327 insertions(+), 19 deletions(-) diff --git a/control/env-budget.js b/control/env-budget.js index 57824f6..c26f64e 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -8,6 +8,7 @@ const ESTIMATED_VERSION = "v0000000000"; const ESTIMATED_DO_STORAGE_ID = "do_00000000000000000000000000000000"; const ESTIMATED_WORKFLOW_KEY = "wf_00000000000000000000000000000000"; +export const WORKER_LOADER_ENV_VERSION_PLACEHOLDER = ESTIMATED_VERSION; export const UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES = 1024 * 1024; export const WORKER_LOADER_ENV_HEADROOM_BYTES = 8 * 1024; export const WORKER_LOADER_ENV_MAX_BYTES = @@ -318,6 +319,7 @@ export async function decryptSecretHash({ encrypted, env, hashKey }) { * ns: string, * worker: string, * versions: Iterable, + * versionEstimates?: Iterable<{ sourceVersion: string, estimatedVersion: string }>, * nsSecrets?: Record | null, * workerSecrets?: Record | null, * }} args @@ -327,19 +329,35 @@ export async function assertWorkerVersionsUserEnvBudget({ ns, worker, versions, + versionEstimates = [], nsSecrets = null, workerSecrets = null, }) { - const uniqueVersions = [...new Set([...versions].filter((version) => typeof version === "string" && version))]; - if (uniqueVersions.length === 0) { + const checks = [ + ...[...versions] + .filter((version) => typeof version === "string" && version) + .map((version) => ({ sourceVersion: version, estimatedVersion: version })), + ...[...versionEstimates] + .filter((entry) => + entry && + typeof entry.sourceVersion === "string" && + entry.sourceVersion && + typeof entry.estimatedVersion === "string" && + entry.estimatedVersion + ), + ]; + const uniqueChecks = [...new Map( + checks.map((entry) => [`${entry.sourceVersion}\0${entry.estimatedVersion}`, entry]) + ).values()]; + if (uniqueChecks.length === 0) { assertWorkerLoaderUserEnvBudget({ ns, worker, nsSecrets, workerSecrets }); return; } // Keep bundle metadata reads sequential: callers may pass a RedisSession, // whose command protocol is single-flight even though secret decryption is not. - for (const version of uniqueVersions) { - const rawMeta = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); + for (const { sourceVersion, estimatedVersion } of uniqueChecks) { + const rawMeta = await redis.hGet(bundleKey(ns, worker, sourceVersion), "__meta__"); /** @type {Record} */ let meta = {}; if (typeof rawMeta === "string") { @@ -350,7 +368,7 @@ export async function assertWorkerVersionsUserEnvBudget({ : {}; } catch (err) { throw new Error( - `invalid bundle metadata for ${ns}/${worker}@${version}: ${err instanceof Error ? err.message : String(err)}`, + `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ${err instanceof Error ? err.message : String(err)}`, { cause: err } ); } @@ -358,7 +376,7 @@ export async function assertWorkerVersionsUserEnvBudget({ assertWorkerLoaderUserEnvBudget({ ns, worker, - version, + version: estimatedVersion, vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) ? /** @type {Record} */ (meta.vars) : null, diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index b3e6210..04d029d 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -605,7 +605,7 @@ async function runDeployPreflight({ redis, ns, name, deployRequest }) { } /** - * @param {{ redis: RedisClient | RedisSession, controlEnv: Record, ns: string, name: string, meta: PreparedMeta, version?: string }} args + * @param {{ redis: RedisClient | RedisSession, controlEnv: Record, ns: string, name: string, meta: PreparedMeta | CommittedMeta, version?: string }} args */ async function validateCommittedEnvBudget({ redis, controlEnv, ns, name, meta, version = undefined }) { const nsEncrypted = await redis.hGetAll(`secrets:${ns}`); @@ -909,14 +909,6 @@ export async function commitWithWatch({ await watchCommitKeys(iso, { ns, name, prepared, outgoingRefs, d1Refs }); if (controlEnv) { await iso.watch(`secrets:${ns}`, `secrets:${ns}:${name}`); - await validateCommittedEnvBudget({ - redis: iso, - controlEnv, - ns, - name, - meta: prepared.meta, - version, - }); } const resolvedD1Refs = await resolveD1RefsForCommit(iso, { ns, d1Refs }); @@ -933,6 +925,16 @@ export async function commitWithWatch({ prepared, resolvedD1Refs, }); + if (controlEnv) { + await validateCommittedEnvBudget({ + redis: iso, + controlEnv, + ns, + name, + meta: committedMeta, + version, + }); + } const multi = iso.multi(); stageDeployCommit(multi, { diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 9c596e2..1e73531 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -109,8 +109,10 @@ async function mutateNamespaceSecret({ return { mutated: false }; } + const budgetEncrypted = { ...existingEncrypted }; + delete budgetEncrypted[secretKey]; const nsSecrets = await decryptSecretHash({ - encrypted: existingEncrypted, + encrypted: budgetEncrypted, env: controlEnv, hashKey: nsSecretsKey, }); diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 46cd0e2..be7b607 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -23,6 +23,7 @@ import { stageWorkerHidden, stageWorkerVisible } from "control-lifecycle-indexes import { bumpActiveAndPromote, RoutingError } from "control-routing"; import { WorkerEnvBudgetError, + WORKER_LOADER_ENV_VERSION_PLACEHOLDER, assertWorkerVersionsUserEnvBudget, decryptSecretHash, } from "control-env-budget"; @@ -248,14 +249,14 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n const retainedVersions = await iso.zRange(workerVersionsKey(ns, name), 0, -1); const nsEncrypted = await iso.hGetAll(nsSecretsKey); const workerEncrypted = await iso.hGetAll(secretsKey); + const workerBudgetEncrypted = { ...workerEncrypted }; + delete workerBudgetEncrypted[key]; const [nsSecrets, workerSecrets] = await Promise.all([ decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: nsSecretsKey }), - decryptSecretHash({ encrypted: workerEncrypted, env: controlEnv, hashKey: secretsKey }), + decryptSecretHash({ encrypted: workerBudgetEncrypted, env: controlEnv, hashKey: secretsKey }), ]); if (method === "PUT") { workerSecrets[key] = /** @type {string} */ (plaintext); - } else { - delete workerSecrets[key]; } await assertWorkerVersionsUserEnvBudget({ redis: iso, @@ -265,6 +266,12 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n ...retainedVersions, ...(typeof activeVersion === "string" && activeVersion ? [activeVersion] : []), ], + versionEstimates: typeof activeVersion === "string" && activeVersion + ? [{ + sourceVersion: activeVersion, + estimatedVersion: WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + }] + : [], nsSecrets, workerSecrets, }); diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index f47960a..49d6c93 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -404,6 +404,52 @@ test("commitWithWatch validates deploy env budget under watched secret hashes", assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].vars, { TOKEN: "from-vars" }); }); +test("commitWithWatch validates env budget against materialized D1 metadata", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map([ + ["d1:database-name:tenant-a:main", "d1_0123456789abcdef0123456789abcdef"], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ + ["d1:database:tenant-a:d1_0123456789abcdef0123456789abcdef", { + databaseId: "d1_0123456789abcdef0123456789abcdef", + databaseName: "main", + }], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls = []; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + await commitWithWatch({ + redis, + ns: "tenant-a", + name: "demo", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + bindings: { + DB: { type: "d1", databaseId: "main" }, + }, + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [], + d1Refs: [{ binding: "DB", databaseId: "main" }], + controlEnv: {}, + }); + + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.length, 1); + assert.equal( + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].meta.bindings.DB.databaseId, + "d1_0123456789abcdef0123456789abcdef" + ); +}); + test("deploy handler resolves cross-namespace service-binding meta from the target namespace", async () => { /** @type {string[]} */ const metaReads = []; diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 47363a0..f4bd065 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -13,6 +13,7 @@ const { UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, WORKER_LOADER_ENV_HEADROOM_BYTES, WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_VERSION_PLACEHOLDER, WorkerEnvBudgetError, assertWorkerLoaderUserEnvBudget, assertWorkerVersionsUserEnvBudget, @@ -188,6 +189,65 @@ test("worker env budget checks retained-version binding env injections", async ( ); }); +test("worker env budget can estimate a source bundle under a future version string", async () => { + const baseMeta = { + vars: { PAD: "" }, + workflows: [{ + binding: "FLOW", + name: "flow", + className: "Flow", + workflowKey: "wf_0123456789abcdef0123456789abcdef", + }], + }; + /** @param {number} padLength @param {string} version */ + const bytesWithPad = (padLength, version) => Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + version, + vars: { PAD: "x".repeat(padLength) }, + meta: baseMeta, + })), "utf8"); + const padLength = WORKER_LOADER_ENV_MAX_BYTES - bytesWithPad(0, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) + 1; + assert.ok(bytesWithPad(padLength, "v1") <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(bytesWithPad(padLength, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) > WORKER_LOADER_ENV_MAX_BYTES); + + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return JSON.stringify({ + ...baseMeta, + vars: { PAD: "x".repeat(padLength) }, + }); + }, + }; + + await assert.doesNotReject(() => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + })); + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: [], + versionEstimates: [{ + sourceVersion: "v1", + estimatedVersion: WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + }], + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + test("worker env budget reports bundle metadata parse context", async () => { const redis = { /** @param {string} key @param {string} field */ diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index b56c363..8570809 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -318,6 +318,42 @@ test("namespace secret DELETE checks env revealed by removing a namespace secret } }); +test("namespace secret DELETE skips decrypting the removed corrupt envelope", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + }; + const deletesBefore = state.redis.deletes.length; + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: "WDL-ENC:not-json" }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? [] : []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.deletes.length, deletesBefore + 1); + assert.deepEqual(state.redis.deletes.at(-1), { key: "secrets:demo", field: "TOKEN" }); + } finally { + Object.assign(state.redis, original); + } +}); + const workerControlSharedUrl = controlSharedStubUrl(` class WatchError extends Error {} export function formatError(err) { @@ -390,6 +426,11 @@ const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/w [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], ]); const { handle: workerHandle } = await import(moduleDataUrl(workerSrc)); +const { + WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + estimatedWorkerLoaderEnv, +} = await import(envBudgetUrl()); test("worker secret PUT encrypts before WATCH retries and reuses the envelope", async () => { const response = await workerHandle({ @@ -507,3 +548,135 @@ test("worker secret DELETE checks env revealed by removing a higher-precedence s state.redis.session = originalSession; } }); + +test("worker secret PUT budgets the copied active bundle under a future version string", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const baseMeta = { + vars: { PAD: "" }, + workflows: [{ + binding: "FLOW", + name: "flow", + className: "Flow", + workflowKey: "wf_0123456789abcdef0123456789abcdef", + }], + }; + /** @param {number} padLength @param {string} version */ + const bytesWithPad = (padLength, version) => Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + version, + vars: { PAD: "x".repeat(padLength) }, + workerSecrets: { TOKEN: "plain-secret" }, + meta: baseMeta, + })), "utf8"); + const padLength = WORKER_LOADER_ENV_MAX_BYTES - + bytesWithPad(0, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) + + 1; + assert.ok(bytesWithPad(padLength, "v1") <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(bytesWithPad(padLength, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) > WORKER_LOADER_ENV_MAX_BYTES); + let execCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return []; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") return "v1"; + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ + ...baseMeta, + vars: { PAD: "x".repeat(padLength) }, + }); + } + return null; + }, + async hGetAll() { return {}; }, + async zCard() { return 1; }, + async zRange() { return []; }, + multi() { + return { + hSet() {}, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-future-version-budget", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(execCalled, false); + } finally { + state.redis.session = originalSession; + } +}); + +test("worker secret DELETE skips decrypting the removed corrupt envelope", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + let execCalled = false; + let deletedField = null; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["TOKEN"]; }, + async hGet() { return null; }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo:api") return { TOKEN: "WDL-ENC:not-json" }; + return {}; + }, + async zCard() { return 0; }, + async zRange() { return []; }, + multi() { + return { + hSet() {}, + /** @param {string} _key @param {string} field */ + hDel(_key, field) { deletedField = field; }, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(execCalled, true); + assert.equal(deletedField, "TOKEN"); + } finally { + state.redis.session = originalSession; + } +}); From 68a2283874d7dddf6ff728084897617efe05db45 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 2 Jul 2026 00:34:17 +0800 Subject: [PATCH 07/13] Tighten workerLoader env budget coverage Count the configured ASSETS_CDN_BASE and do-runtime alarm binding in workerLoader env estimates so deploys and secret mutations fail before cold-load. Allow same-hash secret DELETE repair budget checks to skip corrupt remaining envelopes while keeping PUT and cross-layer decrypts fail-closed. Add regression coverage for deploy budget propagation, DO alarm env estimates, and namespace/worker secret repair behavior. Signed-off-by: Lu Zhang --- control/env-budget.js | 56 ++++-- control/handlers/deploy.js | 1 + control/handlers/ns-secrets.js | 8 +- control/handlers/worker-secrets.js | 8 +- tests/unit/control-deploy-watch.test.js | 6 +- tests/unit/control-env-budget.test.js | 85 ++++++++ .../control-secret-envelope-handlers.test.js | 186 ++++++++++++++++++ 7 files changed, 336 insertions(+), 14 deletions(-) diff --git a/control/env-budget.js b/control/env-budget.js index c26f64e..1a56a27 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,9 +1,11 @@ -import { decryptSecretValue } from "shared-secret-envelope"; +import { SecretEnvelopeError, decryptSecretValue } from "shared-secret-envelope"; import { bundleKey } from "shared-version"; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; +const DO_ALARMS_BINDING = "__WDL_DO_ALARMS__"; const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; +const ESTIMATED_ASSETS_CDN_BASE = "https://assets.invalid"; const ESTIMATED_VERSION = "v0000000000"; const ESTIMATED_DO_STORAGE_ID = "do_00000000000000000000000000000000"; const ESTIMATED_WORKFLOW_KEY = "wf_00000000000000000000000000000000"; @@ -79,11 +81,12 @@ function callerSecretsForBinding({ requiredCallerSecrets, nsSecrets, workerSecre * ns: string, * worker: string, * version: string, + * assetsCdnBase: string, * nsSecrets: Record, * workerSecrets: Record, * }} args */ -function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, nsSecrets, workerSecrets }) { +function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, assetsCdnBase, nsSecrets, workerSecrets }) { switch (spec.type) { case "kv": return { @@ -94,7 +97,7 @@ function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, nsSec return { __wdlBinding: "assets", props: { - cdnBase: "https://assets.invalid", + cdnBase: assetsCdnBase, prefix: stringOrFallback(objectRecord(meta.assets)?.prefix), }, }; @@ -172,6 +175,7 @@ function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, nsSec * nsSecrets?: Record | null, * workerSecrets?: Record | null, * meta?: Record | null, + * assetsCdnBase?: string | null, * }} args */ export function estimatedWorkerLoaderEnv({ @@ -182,6 +186,7 @@ export function estimatedWorkerLoaderEnv({ nsSecrets = null, workerSecrets = null, meta = null, + assetsCdnBase = ESTIMATED_ASSETS_CDN_BASE, }) { const nsSecretStrings = stringRecord(nsSecrets); const workerSecretStrings = stringRecord(workerSecrets); @@ -195,6 +200,7 @@ export function estimatedWorkerLoaderEnv({ if (!metaRecord || !worker) return env; let hasDoBinding = false; + let doAlarmStorageId = ESTIMATED_DO_STORAGE_ID; let hasWorkflowBinding = false; const workflows = Array.isArray(metaRecord.workflows) ? metaRecord.workflows : []; for (const workflow of workflows) { @@ -226,15 +232,25 @@ export function estimatedWorkerLoaderEnv({ ns, worker, version, + assetsCdnBase: typeof assetsCdnBase === "string" && assetsCdnBase + ? assetsCdnBase + : ESTIMATED_ASSETS_CDN_BASE, nsSecrets: nsSecretStrings, workerSecrets: workerSecretStrings, }); - hasDoBinding ||= spec.type === "do"; + if (spec.type === "do") { + hasDoBinding = true; + doAlarmStorageId = stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID); + } } } if (hasDoBinding) { env[DO_BACKEND_BINDING] = { __wdlBinding: "internal", name: "DO_BACKEND" }; env[DO_OWNER_NETWORK_BINDING] = { __wdlBinding: "internal", name: "DO_OWNER_NETWORK" }; + env[DO_ALARMS_BINDING] = { + __wdlBinding: "do-alarms", + props: { ns, worker, version, doStorageId: doAlarmStorageId }, + }; } if (hasWorkflowBinding) { env[WORKFLOWS_BACKEND_BINDING] = { __wdlBinding: "internal", name: "WORKFLOWS_BACKEND" }; @@ -251,6 +267,7 @@ export function estimatedWorkerLoaderEnv({ * nsSecrets?: Record | null, * workerSecrets?: Record | null, * meta?: Record | null, + * assetsCdnBase?: string | null, * }} args */ export function assertWorkerLoaderUserEnvBudget({ @@ -261,6 +278,7 @@ export function assertWorkerLoaderUserEnvBudget({ nsSecrets = null, workerSecrets = null, meta = null, + assetsCdnBase = ESTIMATED_ASSETS_CDN_BASE, }) { // workerd enforces the full workerLoader env as a Frankenvalue estimate. Control // mirrors the user strings plus runtime-injected binding/workflow env shapes as @@ -273,6 +291,7 @@ export function assertWorkerLoaderUserEnvBudget({ nsSecrets, workerSecrets, meta, + assetsCdnBase, })), "utf8"); if (bytes > WORKER_LOADER_ENV_MAX_BYTES) { const label = worker ? `${ns}/${worker}` : ns; @@ -296,20 +315,32 @@ export function assertWorkerLoaderUserEnvBudget({ * encrypted: Record, * env: Record, * hashKey: string, + * ignoreSecretEnvelopeErrors?: boolean, * }} args */ -export async function decryptSecretHash({ encrypted, env, hashKey }) { +export async function decryptSecretHash({ encrypted, env, hashKey, ignoreSecretEnvelopeErrors = false }) { const entries = await Promise.all( Object.entries(encrypted || {}) .filter((entry) => typeof entry[1] === "string") - .map(async ([fieldName, value]) => [ - fieldName, - await decryptSecretValue(/** @type {string} */ (value), { env, hashKey, fieldName }), - ]) + .map(async ([fieldName, value]) => { + try { + return [ + fieldName, + await decryptSecretValue(/** @type {string} */ (value), { env, hashKey, fieldName }), + ]; + } catch (err) { + if (ignoreSecretEnvelopeErrors && err instanceof SecretEnvelopeError) return null; + throw err; + } + }) ); /** @type {Record} */ const out = Object.create(null); - for (const [fieldName, value] of entries) out[fieldName] = value; + for (const entry of entries) { + if (!entry) continue; + const [fieldName, value] = entry; + out[fieldName] = value; + } return out; } @@ -322,6 +353,7 @@ export async function decryptSecretHash({ encrypted, env, hashKey }) { * versionEstimates?: Iterable<{ sourceVersion: string, estimatedVersion: string }>, * nsSecrets?: Record | null, * workerSecrets?: Record | null, + * assetsCdnBase?: string | null, * }} args */ export async function assertWorkerVersionsUserEnvBudget({ @@ -332,6 +364,7 @@ export async function assertWorkerVersionsUserEnvBudget({ versionEstimates = [], nsSecrets = null, workerSecrets = null, + assetsCdnBase = ESTIMATED_ASSETS_CDN_BASE, }) { const checks = [ ...[...versions] @@ -350,7 +383,7 @@ export async function assertWorkerVersionsUserEnvBudget({ checks.map((entry) => [`${entry.sourceVersion}\0${entry.estimatedVersion}`, entry]) ).values()]; if (uniqueChecks.length === 0) { - assertWorkerLoaderUserEnvBudget({ ns, worker, nsSecrets, workerSecrets }); + assertWorkerLoaderUserEnvBudget({ ns, worker, nsSecrets, workerSecrets, assetsCdnBase }); return; } @@ -383,6 +416,7 @@ export async function assertWorkerVersionsUserEnvBudget({ nsSecrets, workerSecrets, meta, + assetsCdnBase, }); } } diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 04d029d..2b15b4f 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -624,6 +624,7 @@ async function validateCommittedEnvBudget({ redis, controlEnv, ns, name, meta, v nsSecrets, workerSecrets, meta, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, }); } diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 1e73531..c58b2a3 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -43,7 +43,11 @@ async function validateNamespaceSecretBudget({ redis, controlEnv, nsName, nsSecr ...Object.keys(activeRoutes), ]); if (workerNames.size === 0) { - assertWorkerLoaderUserEnvBudget({ ns: nsName, nsSecrets }); + assertWorkerLoaderUserEnvBudget({ + ns: nsName, + nsSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); return; } @@ -68,6 +72,7 @@ async function validateNamespaceSecretBudget({ redis, controlEnv, nsName, nsSecr ], nsSecrets, workerSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, }); } } @@ -115,6 +120,7 @@ async function mutateNamespaceSecret({ encrypted: budgetEncrypted, env: controlEnv, hashKey: nsSecretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", }); if (method === "PUT") { if (typeof encrypted !== "string") throw new Error("PUT namespace secret encrypted value missing"); diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index be7b607..2119542 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -253,7 +253,12 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n delete workerBudgetEncrypted[key]; const [nsSecrets, workerSecrets] = await Promise.all([ decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: nsSecretsKey }), - decryptSecretHash({ encrypted: workerBudgetEncrypted, env: controlEnv, hashKey: secretsKey }), + decryptSecretHash({ + encrypted: workerBudgetEncrypted, + env: controlEnv, + hashKey: secretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", + }), ]); if (method === "PUT") { workerSecrets[key] = /** @type {string} */ (plaintext); @@ -274,6 +279,7 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n : [], nsSecrets, workerSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, }); } diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 49d6c93..a4878ab 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -395,13 +395,17 @@ test("commitWithWatch validates deploy env budget under watched secret hashes", }, outgoingRefs: [], d1Refs: [], - controlEnv: {}, + controlEnv: { ASSETS_CDN_BASE: "https://assets.example/cdn" }, }); assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("secrets:tenant-a")); assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("secrets:tenant-a:demo")); assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.length, 1); assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].vars, { TOKEN: "from-vars" }); + assert.equal( + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].assetsCdnBase, + "https://assets.example/cdn" + ); }); test("commitWithWatch validates env budget against materialized D1 metadata", async () => { diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index f4bd065..bfde401 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -105,6 +105,70 @@ test("worker env budget counts required caller secret copies in service binding ); }); +test("worker env budget counts configured assets CDN base", () => { + const meta = { + vars: { PAD: "" }, + assets: { prefix: "assets/demo/api/token/" }, + bindings: { + ASSETS: { type: "assets" }, + }, + }; + const assetsCdnBase = `https://${"assets-subdomain-".repeat(600)}example.test`; + /** @param {number} padLength @param {string | null | undefined} cdnBase */ + const bytesWithPad = (padLength, cdnBase) => Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + vars: { PAD: "x".repeat(padLength) }, + meta, + assetsCdnBase: cdnBase, + })), "utf8"); + const padLength = WORKER_LOADER_ENV_MAX_BYTES - bytesWithPad(0, assetsCdnBase) + 1; + + assert.ok(bytesWithPad(padLength, null) <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(bytesWithPad(padLength, assetsCdnBase) > WORKER_LOADER_ENV_MAX_BYTES); + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "api", + vars: { PAD: "x".repeat(padLength) }, + meta, + assetsCdnBase, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + +test("worker env budget includes do-runtime alarm binding for Durable Object workers", () => { + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + version: "v12", + meta: { + bindings: { + ROOM: { + type: "do", + className: "Room", + doStorageId: "do_0123456789abcdef0123456789abcdef", + }, + }, + }, + }); + + assert.deepEqual(estimated.__WDL_DO_ALARMS__, { + __wdlBinding: "do-alarms", + props: { + ns: "demo", + worker: "api", + version: "v12", + doStorageId: "do_0123456789abcdef0123456789abcdef", + }, + }); +}); + test("decryptSecretHash returns plaintext secret values for budget checks", async () => { const hashKey = "secrets:demo"; const encrypted = await encryptSecretValue("plain", { @@ -125,6 +189,27 @@ test("decryptSecretHash returns plaintext secret values for budget checks", asyn ); }); +test("decryptSecretHash can ignore corrupt envelopes for DELETE repair budgets", async () => { + const hashKey = "secrets:demo"; + const encrypted = await encryptSecretValue("plain", { + env: envelopeEnv, + hashKey, + fieldName: "TOKEN", + }); + + assert.deepEqual( + { + ...(await decryptSecretHash({ + encrypted: { TOKEN: encrypted, BAD: "WDL-ENC:not-json" }, + env: envelopeEnv, + hashKey, + ignoreSecretEnvelopeErrors: true, + })), + }, + { TOKEN: "plain" } + ); +}); + test("worker env budget checks every retained worker version", async () => { const redis = { /** @param {string} key @param {string} field */ diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 8570809..7c0fcd2 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -354,6 +354,84 @@ test("namespace secret DELETE skips decrypting the removed corrupt envelope", as } }); +test("namespace secret DELETE skips other corrupt namespace envelopes for repair", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + }; + const deletesBefore = state.redis.deletes.length; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo", + fieldName: "TOKEN", + }); + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: encrypted, BAD: "WDL-ENC:not-json" }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? [] : []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-other-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.deletes.length, deletesBefore + 1); + assert.deepEqual(state.redis.deletes.at(-1), { key: "secrets:demo", field: "TOKEN" }); + } finally { + Object.assign(state.redis, original); + } +}); + +test("namespace secret PUT still fails closed on other corrupt namespace envelopes", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + }; + const writesBefore = state.redis.writes.length; + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { BAD: "WDL-ENC:not-json" }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? [] : []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-put-other-corrupt", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "invalid_envelope"); + assert.equal(state.redis.writes.length, writesBefore); + } finally { + Object.assign(state.redis, original); + } +}); + const workerControlSharedUrl = controlSharedStubUrl(` class WatchError extends Error {} export function formatError(err) { @@ -680,3 +758,111 @@ test("worker secret DELETE skips decrypting the removed corrupt envelope", async state.redis.session = originalSession; } }); + +test("worker secret DELETE skips other corrupt worker envelopes for repair", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let execCalled = false; + let deletedField = null; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["TOKEN", "BAD"]; }, + async hGet() { return null; }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo:api") return { TOKEN: encrypted, BAD: "WDL-ENC:not-json" }; + return {}; + }, + async zCard() { return 0; }, + async zRange() { return []; }, + multi() { + return { + hSet() {}, + /** @param {string} _key @param {string} field */ + hDel(_key, field) { deletedField = field; }, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-other-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(execCalled, true); + assert.equal(deletedField, "TOKEN"); + } finally { + state.redis.session = originalSession; + } +}); + +test("worker secret PUT still fails closed on other corrupt worker envelopes", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + let hSetCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["BAD"]; }, + async hGet() { return null; }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo:api") return { BAD: "WDL-ENC:not-json" }; + return {}; + }, + async zCard() { return 0; }, + async zRange() { return []; }, + multi() { + return { + hSet() { hSetCalled = true; }, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() {}, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-put-other-corrupt", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "invalid_envelope"); + assert.equal(hSetCalled, false); + } finally { + state.redis.session = originalSession; + } +}); From 8ffaf121dba14cc38d7dc0e273bc8b89441098e8 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 2 Jul 2026 03:32:28 +0800 Subject: [PATCH 08/13] Close PR review cleanup gaps Use a null-prototype callerSecrets map so object-prototype secret keys remain data keys during workerLoader env estimates. Expose RedisSession.hasOpenResources() and use it to close log-tail sessions once open has allocated socket resources, including the pending SELECT window. Add unit coverage for both review findings and verify with targeted log-tail integration tests. Signed-off-by: Lu Zhang --- control/env-budget.js | 2 +- control/handlers/logs-tail.js | 2 +- shared/redis-session.js | 4 +++ tests/unit/control-env-budget.test.js | 24 ++++++++++++++ tests/unit/control-logs-tail.test.js | 37 +++++++++++++++++++++ tests/unit/redis-session.test.js | 47 +++++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 2 deletions(-) diff --git a/control/env-budget.js b/control/env-budget.js index 1a56a27..abf24b9 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -61,7 +61,7 @@ function stringOrFallback(value, fallback = "") { function callerSecretsForBinding({ requiredCallerSecrets, nsSecrets, workerSecrets }) { if (!Array.isArray(requiredCallerSecrets) || requiredCallerSecrets.length === 0) return undefined; /** @type {Record} */ - const callerSecrets = {}; + const callerSecrets = Object.create(null); for (const key of requiredCallerSecrets) { if (typeof key !== "string") continue; if (Object.hasOwn(workerSecrets, key)) { diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index ae60cb7..79bf8ff 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -380,7 +380,7 @@ export async function handle({ request, env, ctx, ns, requestId }) { }); async function closeSessionIfOpen() { - if (!sessionOpen) return; + if (!sessionOpen && !session.hasOpenResources()) return; sessionClosePromise ??= (async () => { try { await session.close(); diff --git a/shared/redis-session.js b/shared/redis-session.js index 3a198bc..a9ab10d 100644 --- a/shared/redis-session.js +++ b/shared/redis-session.js @@ -76,6 +76,10 @@ export class RedisSession { return this; } + hasOpenResources() { + return Boolean(this.socket || this.writer || this.reader || this.parser); + } + async close() { if (this._closed) return; this._closed = true; diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index bfde401..6fb5237 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -105,6 +105,30 @@ test("worker env budget counts required caller secret copies in service binding ); }); +test("worker env budget stores required caller secrets in a null-prototype map", () => { + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "caller", + nsSecrets: JSON.parse('{"__proto__":"secret"}'), + meta: { + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + requiredCallerSecrets: ["__proto__"], + }, + }, + }, + }); + const callerSecrets = /** @type {any} */ (estimated.PLATFORM).props.callerSecrets; + + assert.equal(Object.getPrototypeOf(callerSecrets), null); + assert.equal(Object.hasOwn(callerSecrets, "__proto__"), true); + assert.equal(callerSecrets.__proto__, "secret"); +}); + test("worker env budget counts configured assets CDN base", () => { const meta = { vars: { PAD: "" }, diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 0eaafc0..7be1ef5 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -22,6 +22,7 @@ function loadLogsTailHandler() { } async open() { this.openStarted = true; + this.socket = {}; this.openPromise = (async () => { const state = /** @type {any} */ (globalThis).__tailState; if (state.openBlocker) await state.openBlocker; @@ -31,6 +32,7 @@ function loadLogsTailHandler() { } async publish(channel, payload) { this.publishCalls.push([channel, payload]); } async xRead(...args) { this.xReadCalls.push(args); return null; } + hasOpenResources() { return Boolean(this.socket); } async close() { this.closed = true; } } const redisDbFromEnv = (env, name) => Number(env?.[name] || 0);`, @@ -266,6 +268,41 @@ test("logs tail closes session if watchdog fires while Redis open is pending", a await reader.cancel().catch(() => {}); }); +test("logs tail max-session watchdog closes a socket while Redis open is pending", async () => { + resetTailState(); + const state = /** @type {any} */ (globalThis).__tailState; + let releaseOpen = () => {}; + state.openBlocker = new Promise((resolve) => { + releaseOpen = () => resolve(undefined); + }); + + const { handle } = await loadLogsTailHandler(); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "30" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-open-socket-race", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + const pendingRead = readText(reader); + const session = /** @type {any} */ (globalThis).__tailSessions[0]; + await waitUntil("tail Redis open to assign socket", () => session.socket, { + timeoutMs: 500, intervalMs: 5, + }); + await Promise.all(waitUntilPromises); + + assert.equal(session.closed, true); + releaseOpen(); + await session.openPromise; + assert.match((await pendingRead).text, /session_expired/); + await reader.cancel().catch(() => {}); +}); + test("logs tail emits idle keepalives below common proxy idle timeouts", async () => { resetTailState(); const { handle } = await loadLogsTailHandler(); diff --git a/tests/unit/redis-session.test.js b/tests/unit/redis-session.test.js index 32322b8..084c066 100644 --- a/tests/unit/redis-session.test.js +++ b/tests/unit/redis-session.test.js @@ -532,6 +532,53 @@ test("RedisSession selects configured DB on its held socket", async () => { assert.equal(decode(socket._writes[1]), "*3\r\n$4\r\nHGET\r\n$1\r\nk\r\n$1\r\nf\r\n"); }); +test("RedisSession reports resources while SELECT is still pending", async () => { + let releaseSelect = () => {}; + const writer = { + /** @param {Uint8Array} _buf */ + async write(_buf) {}, + close() { writer.closed = true; }, + /** @type {boolean} */ + closed: false, + }; + const reader = { + read() { + return new Promise((resolve, reject) => { + releaseSelect = () => { + if (reader.released) { + reject(new Error("reader released")); + } else { + resolve({ done: false, value: bytes("+OK\r\n") }); + } + }; + }); + }, + releaseLock() { reader.released = true; }, + /** @type {boolean} */ + released: false, + }; + const socket = { + writable: { getWriter: () => writer }, + readable: { getReader: () => reader }, + close() { socket.closed = true; }, + /** @type {boolean} */ + closed: false, + }; + const { connect } = scriptedConnect(socket); + const session = new RedisSession("x", { connect, db: 1 }); + + assert.equal(session.hasOpenResources(), false); + const openPromise = session.open(); + assert.equal(session.hasOpenResources(), true); + await Promise.resolve(); + await session.close(); + assert.equal(writer.closed, true); + assert.equal(reader.released, true); + assert.equal(socket.closed, true); + releaseSelect(); + await assert.rejects(openPromise, /reader released/); +}); + test("RedisClient.set supports Valkey IFEQ and delIfEq", async () => { const socketA = makeFakeSocket([bytes("+OK\r\n")]); const socketB = makeFakeSocket([bytes(":1\r\n")]); From 3531849c8c4f17b4d88c096525e853c50c7c2462 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 2 Jul 2026 12:33:18 +0800 Subject: [PATCH 09/13] Adapt workerd 20260701 runtime contracts Split workerd experimental surface, add workerLoader env/code guards, bound log-tail cleanup without stream cancel, reject unsupported Python/experimental tenant metadata, and cap D1/DO runtime container memory for the 0701 SQLite behavior. Signed-off-by: Lu Zhang --- control/bundle.js | 20 +++- control/env-budget.js | 47 +++++++- control/handlers/logs-tail.js | 51 ++++++++ control/handlers/worker-secrets.js | 78 ++++++++++++- control/routing.js | 13 ++- do-runtime/alarm-shim-source.js | 3 +- do-runtime/config.capnp | 1 + do-runtime/load.js | 2 +- do-runtime/protocol.js | 9 ++ docs/compatibility.md | 11 +- docs/compatibility.zh.md | 11 +- docs/modules/control-auth.md | 17 ++- docs/modules/control-auth.zh.md | 7 +- docs/modules/d1.md | 5 + docs/modules/d1.zh.md | 2 + docs/modules/durable-objects.md | 6 + docs/modules/durable-objects.zh.md | 2 + docs/modules/log-tail-observability.md | 10 +- docs/modules/log-tail-observability.zh.md | 2 +- docs/modules/runtime.md | 22 +++- docs/modules/runtime.zh.md | 8 +- docs/source-map.md | 3 + docs/source-map.zh.md | 3 + runtime/config-system.capnp | 2 + runtime/config-user.capnp | 1 + runtime/lib.js | 16 ++- runtime/load.js | 2 +- scripts/scan-workerd-0701-metadata.mjs | 110 ++++++++++++++++++ shared/redis-session.js | 15 ++- shared/workerd-compat-flags.js | 62 ++++++++++ terraform/README.md | 15 +-- terraform/main.tf | 38 +++--- .../modules/compute/d1_runtime_service.tf | 1 + .../modules/compute/do_runtime_service.tf | 1 + terraform/modules/compute/variables.tf | 8 ++ terraform/variables.tf | 20 ++++ tests/helpers/load-control-lib.js | 2 + tests/helpers/load-do-protocol.js | 2 + tests/helpers/load-runtime-dispatch.js | 3 +- tests/helpers/load-shared-module.js | 9 ++ tests/integration/admin-api.test.js | 19 +++ tests/integration/http-features.test.js | 15 ++- tests/unit/control-env-budget.test.js | 45 ++++++- tests/unit/control-lib.test.js | 60 +++++++++- tests/unit/control-logs-tail.test.js | 41 ++++++- tests/unit/control-routing.test.js | 56 +++++++++ .../control-secret-envelope-handlers.test.js | 103 +++++++++++++++- tests/unit/do-alarm-shim.test.js | 36 ++++++ tests/unit/do-runtime-protocol.test.js | 16 +++ tests/unit/redis-session.test.js | 3 + tests/unit/runtime-kv-binding.test.js | 8 +- tests/unit/runtime-lib.test.js | 40 +++++-- tests/unit/runtime-queue-producer.test.js | 7 +- tests/unit/style-contracts.test.js | 20 ++++ 54 files changed, 1008 insertions(+), 101 deletions(-) create mode 100644 scripts/scan-workerd-0701-metadata.mjs create mode 100644 shared/workerd-compat-flags.js diff --git a/control/bundle.js b/control/bundle.js index 8e1c4f5..ed3ab19 100644 --- a/control/bundle.js +++ b/control/bundle.js @@ -12,6 +12,7 @@ import { isValidJsClassDeclarationName, validateModulePath, } from "shared-ns-pattern"; +import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; import { normalizeBindings, validateBindings } from "control-bindings"; import PACKAGE_JSON_SOURCE from "wdl-package-json-source"; @@ -131,11 +132,16 @@ export function normalizeModule(value) { return { type: "json", bytes: Buffer.from(JSON.stringify(record.json), "utf8") }; if (typeof record.cjs === "string") return { type: "cjs", bytes: Buffer.from(record.cjs, "utf8") }; - if (typeof record.py === "string") - return { type: "py", bytes: Buffer.from(record.py, "utf8") }; + if (typeof record.py === "string") { + throw new BundleConfigError( + 400, + "python_workers_unsupported", + "Python Workers modules are not supported by WDL" + ); + } } throw new Error( - "Unrecognized module value (expected string or {data_b64|wasm_b64|text|json|cjs|py})" + "Unrecognized module value (expected string or {data_b64|wasm_b64|text|json|cjs})" ); } @@ -159,6 +165,14 @@ function validateCompatibilityFlags(flags) { ); } } + const experimentalFlag = firstWorkerdExperimentalCompatFlag(flags); + if (experimentalFlag) { + throw new BundleConfigError( + 400, + "experimental_compat_flag_unsupported", + `compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` + ); + } } /** @param {unknown} workflows */ diff --git a/control/env-budget.js b/control/env-budget.js index abf24b9..d16632f 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -258,11 +258,44 @@ export function estimatedWorkerLoaderEnv({ return env; } +/** @param {string} value */ +function hasNonLatin1(value) { + for (let i = 0; i < value.length; i += 1) { + if (value.charCodeAt(i) > 0xff) return true; + } + return false; +} + +/** @param {string} value */ +function v8TwoByteStringPenalty(value) { + if (!hasNonLatin1(value)) return 0; + return Math.max(0, (2 * value.length) - Buffer.byteLength(value, "utf8")); +} + +/** @param {unknown} value */ +function v8StringPenalty(value) { + if (typeof value === "string") return v8TwoByteStringPenalty(value); + if (!value || typeof value !== "object") return 0; + let bytes = 0; + for (const [key, child] of Object.entries(value)) { + bytes += v8TwoByteStringPenalty(key); + bytes += v8StringPenalty(child); + } + return bytes; +} + +/** @param {unknown} value */ +export function estimatedWorkerLoaderEnvBytes(value) { + const json = JSON.stringify(value) ?? "null"; + return Buffer.byteLength(json, "utf8") + v8StringPenalty(value); +} + /** * @param {{ * ns: string, * worker?: string, * version?: string, + * sourceVersion?: string | null, * vars?: Record | null, * nsSecrets?: Record | null, * workerSecrets?: Record | null, @@ -274,6 +307,7 @@ export function assertWorkerLoaderUserEnvBudget({ ns, worker = undefined, version = ESTIMATED_VERSION, + sourceVersion = null, vars = null, nsSecrets = null, workerSecrets = null, @@ -282,8 +316,8 @@ export function assertWorkerLoaderUserEnvBudget({ }) { // workerd enforces the full workerLoader env as a Frankenvalue estimate. Control // mirrors the user strings plus runtime-injected binding/workflow env shapes as - // JSON and leaves headroom for native facade-object overhead and estimator drift. - const bytes = Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + // JSON, then accounts for V8's two-byte representation of non-Latin-1 strings. + const bytes = estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ ns, worker, version, @@ -292,14 +326,18 @@ export function assertWorkerLoaderUserEnvBudget({ workerSecrets, meta, assetsCdnBase, - })), "utf8"); + })); if (bytes > WORKER_LOADER_ENV_MAX_BYTES) { - const label = worker ? `${ns}/${worker}` : ns; + const label = worker + ? `${ns}/${worker}${sourceVersion ? `@${sourceVersion}` : ""}` + : ns; throw new WorkerEnvBudgetError( `estimated workerLoader env for ${label} serializes to ${bytes} bytes, exceeding WDL workerLoader env budget ${WORKER_LOADER_ENV_MAX_BYTES} bytes`, { namespace: ns, ...(worker ? { worker } : {}), + ...(sourceVersion ? { source_version: sourceVersion } : {}), + ...(sourceVersion && version ? { estimated_version: version } : {}), env_bytes: bytes, max_env_bytes: WORKER_LOADER_ENV_MAX_BYTES, upstream_max_env_bytes: UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, @@ -410,6 +448,7 @@ export async function assertWorkerVersionsUserEnvBudget({ ns, worker, version: estimatedVersion, + sourceVersion, vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) ? /** @type {Record} */ (meta.vars) : null, diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index 79bf8ff..75fea98 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -20,6 +20,8 @@ const TAIL_ACTIVATION_TTL_SECONDS = 30; const TAIL_ACTIVATION_MAX_ENTRIES = 10_000; const XREAD_BLOCK_MS = 10_000; const SSE_KEEPALIVE_MS = 5_000; +const LOG_TAIL_IDLE_PULL_GRACE_FACTOR = 3; +export const LOG_TAIL_IDLE_PULL_MS = SSE_KEEPALIVE_MS * LOG_TAIL_IDLE_PULL_GRACE_FACTOR; export const LOG_TAIL_MAX_SESSION_MS_DEFAULT = 15 * 60 * 1000; const MAX_WORKERS_PER_TAIL_SESSION = 50; const JSON_FIELD_BYTES = [0x6a, 0x73, 0x6f, 0x6e]; // "json" @@ -367,6 +369,8 @@ export async function handle({ request, env, ctx, ns, requestId }) { /** @type {ReadableStreamDefaultController | null} */ let streamController = null; let expiryLogged = false; + let idleLogged = false; + let lastPullAtMs = Date.now(); const { promise: cancelPromise, resolve: resolveCancel } = /** @type {PromiseWithResolvers} */ (Promise.withResolvers()); @@ -379,6 +383,15 @@ export async function handle({ request, env, ctx, ns, requestId }) { }), }); + const sessionIdleWarning = () => sseEvent({ + event: "tail_warning", + data: JSON.stringify({ + event: "tail_warning", + code: "session_idle", + message: "Tail session stopped receiving client reads; reconnecting closes the abandoned session.", + }), + }); + async function closeSessionIfOpen() { if (!sessionOpen && !session.hasOpenResources()) return; sessionClosePromise ??= (async () => { @@ -412,16 +425,53 @@ export async function handle({ request, env, ctx, ns, requestId }) { resolveCancel(); } + /** @param {ReadableStreamDefaultController | null} controller */ + function idleSession(controller) { + if (cancelled) return; + cancelled = true; + if (!idleLogged) { + idleLogged = true; + log("info", "tail_session_idle", { + request_id: requestId, namespace: ns, worker_count: workers.length, + idle_pull_ms: Date.now() - lastPullAtMs, + idle_limit_ms: LOG_TAIL_IDLE_PULL_MS, + }); + } + if (controller) { + try { controller.enqueue(utf8Encoder.encode(sessionIdleWarning())); } catch {} + try { controller.close(); } catch {} + } + resolveCancel(); + } + const expiryTimer = setTimeout(() => expireSession(streamController), maxSessionMs); if (typeof expiryTimer === "object" && typeof expiryTimer.unref === "function") { expiryTimer.unref(); } + /** @type {ReturnType | null} */ + let idleTimer = null; + function scheduleIdleWatchdog() { + const delayMs = Math.max(1, LOG_TAIL_IDLE_PULL_MS - (Date.now() - lastPullAtMs)); + idleTimer = setTimeout(() => { + if (cancelled) return; + if (Date.now() - lastPullAtMs >= LOG_TAIL_IDLE_PULL_MS) { + idleSession(streamController); + return; + } + scheduleIdleWatchdog(); + }, delayMs); + if (typeof idleTimer === "object" && typeof idleTimer.unref === "function") { + idleTimer.unref(); + } + } + scheduleIdleWatchdog(); // Pre-register cleanup. Per CLAUDE.md gotcha: scheduling waitUntil from // inside cancel races IoContext teardown — cancel only resolves the // promise, the actual close happens here. ctx.waitUntil(cancelPromise.then(async () => { clearTimeout(expiryTimer); + if (idleTimer) clearTimeout(idleTimer); await closeSessionIfOpen(); cleanupFinished = true; log("info", "tail_session_close", { @@ -442,6 +492,7 @@ export async function handle({ request, env, ctx, ns, requestId }) { const stream = new ReadableStream({ async pull(controller) { streamController = controller; + lastPullAtMs = Date.now(); if (cancelled) { try { controller.close(); } catch {} return; diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 2119542..b0e8c6b 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -63,6 +63,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI const key = subPath[0]; const invalidKey = invalidSecretMutationKeyResponse(key); if (invalidKey) return invalidKey; + const controlEnv = stringEnv(env); let storedValue = null; let putPlaintext = null; @@ -84,7 +85,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI redis, ns, name, key, method, value: storedValue, plaintext: putPlaintext, - controlEnv: stringEnv(env), + controlEnv, }); } catch (err) { if (err instanceof SecretAbort) { @@ -121,7 +122,20 @@ export async function handle({ request, env, method, ns, name, subPath, requestI /** @type {RoutingRedisClient} */ (redis), ns, name, - { log, requestId } + { + log, + requestId, + beforeStageCopy: ({ iso, currentVersion, newVersion }) => + assertWorkerSecretBumpEnvBudget({ + iso, + ns, + name, + currentVersion, + newVersion, + controlEnv, + ignoreWorkerSecretEnvelopeErrors: method === "DELETE", + }), + } ); log("info", method === "PUT" ? "secret_set" : "secret_deleted", { request_id: requestId, @@ -163,7 +177,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI else payload.deleted = true; return jsonResponse(200, payload); } - if (err instanceof RoutingError) { + if (err instanceof RoutingError || err instanceof WorkerEnvBudgetError) { // Secret already landed in our own MULTI; bump failure degrades // to a deferred reload — the secret is picked up on next natural // cold-load, or wiped by a concurrent whole-delete. @@ -300,3 +314,61 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n return { mutated: true }; }); } + +/** + * @param {{ + * iso: { + * watch: (...keys: string[]) => Promise, + * hGet: (key: string, field: string) => Promise, + * hGetAll: (key: string) => Promise>, + * }, + * ns: string, + * name: string, + * currentVersion: string, + * newVersion: string, + * controlEnv: Record, + * ignoreWorkerSecretEnvelopeErrors?: boolean, + * }} args + */ +async function assertWorkerSecretBumpEnvBudget({ + iso, + ns, + name, + currentVersion, + newVersion, + controlEnv, + ignoreWorkerSecretEnvelopeErrors = false, +}) { + const nsSecretsKey = `secrets:${ns}`; + const workerSecretsKey = `secrets:${ns}:${name}`; + await iso.watch(nsSecretsKey, workerSecretsKey); + + // Keep reads sequential: RedisSession is a single RESP stream. + const nsEncrypted = await iso.hGetAll(nsSecretsKey); + const workerEncrypted = await iso.hGetAll(workerSecretsKey); + const nsSecrets = await decryptSecretHash({ + encrypted: nsEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + }); + const workerSecrets = await decryptSecretHash({ + encrypted: workerEncrypted, + env: controlEnv, + hashKey: workerSecretsKey, + ignoreSecretEnvelopeErrors: ignoreWorkerSecretEnvelopeErrors, + }); + + await assertWorkerVersionsUserEnvBudget({ + redis: iso, + ns, + worker: name, + versions: [], + versionEstimates: [{ + sourceVersion: currentVersion, + estimatedVersion: newVersion, + }], + nsSecrets, + workerSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); +} diff --git a/control/routing.js b/control/routing.js index a5ea97d..d8e3583 100644 --- a/control/routing.js +++ b/control/routing.js @@ -63,6 +63,8 @@ function hostDeclarationsKey(host) { * @typedef {{ oldRoutes: RoutePattern[], oldQueueConsumers: QueueConsumer[], affectedHosts: Set, hostState: HostState }} PromoteObservedState * @typedef {{ newRouteKeys: Set, nsHostsAdd: string[], nsHostsRem: string[], cronKey: string, cronHash: Record, cronPlan: CronPlan, queuePlan: QueuePlan }} PromoteStagePlan * @typedef {{ log?: (level: string, event: string, fields: Record) => void, requestId?: string, ns?: string, workerName?: string }} LogContext + * @typedef {{ iso: RedisIso, currentVersion: string, newVersion: string, sourceMeta: BundleMeta }} BumpBeforeStageContext + * @typedef {LogContext & { beforeStageCopy?: (context: BumpBeforeStageContext) => Promise }} BumpOptions * @typedef {{ routes?: RoutePattern[], crons?: CronSpec[], queueConsumers?: QueueConsumer[], exports?: ExportSpec[], bindings?: unknown }} BundleMeta * @typedef {Record>} HostState * @typedef {import("shared-redis").RedisMulti} RedisMulti @@ -680,7 +682,7 @@ export async function promoteWithRoutes(redis, ns, workerName, newVersion, optio // "read active" and "promote new" can't silently roll content back. // Throws RoutingError(404) when no active version exists to copy; // RoutingError(409, "caller_deleting") if a whole-delete is in flight. -/** @param {RedisClient} redis @param {string} ns @param {string} workerName @param {LogContext} [options] */ +/** @param {RedisClient} redis @param {string} ns @param {string} workerName @param {BumpOptions} [options] */ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) { const logContext = { ...options, ns, workerName }; // Avoid burning a version number on the pre-deploy path. @@ -772,6 +774,15 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) throw new RoutingError(500, "invalid_generated_version", `bumpActiveAndPromote: bad new version tag ${newVersion}`); } + if (typeof options.beforeStageCopy === "function") { + await options.beforeStageCopy({ + iso, + currentVersion, + newVersion, + sourceMeta: srcMeta, + }); + } + const multi = iso.multi(); multi.copy(srcKey, dstKey, { REPLACE: true }); stageVersionFlip(multi, ns, workerName, newVersion, routes, affectedHosts); diff --git a/do-runtime/alarm-shim-source.js b/do-runtime/alarm-shim-source.js index f91d769..15b93b1 100644 --- a/do-runtime/alarm-shim-source.js +++ b/do-runtime/alarm-shim-source.js @@ -259,7 +259,8 @@ function quoteSqlIdentifier(name) { function sqlObjectDropStatement(row) { const type = String(row.type); const name = String(row.name); - if (name.startsWith("sqlite_") || name.startsWith("_cf_")) return null; + const lowerName = name.toLowerCase(); + if (lowerName.startsWith("sqlite_") || lowerName.startsWith("_cf_")) return null; if (type === "table") return "DROP TABLE IF EXISTS " + quoteSqlIdentifier(name); if (type === "view") return "DROP VIEW IF EXISTS " + quoteSqlIdentifier(name); if (type === "trigger") return "DROP TRIGGER IF EXISTS " + quoteSqlIdentifier(name); diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index 0625040..84a2a5e 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -90,6 +90,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-d1-timeout", esModule = embed "../shared/d1-timeout.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-owner-forwarder", esModule = embed "../shared/owner-forwarder.js"), diff --git a/do-runtime/load.js b/do-runtime/load.js index 6492fa9..3f15fe9 100644 --- a/do-runtime/load.js +++ b/do-runtime/load.js @@ -27,7 +27,7 @@ const NATIVE_DELETE_ALL_PRESERVES_ALARM_FLAG = "delete_all_preserves_alarm"; * @typedef {Record & { doStorageId?: unknown, className?: unknown }} RuntimeBindingSpec * @typedef {{ exports: Record }) => unknown> & { KV(options: { props: Record }): unknown, Assets(options: { props: Record }): unknown, QueueProducer(options: { props: Record }): unknown, D1Database(options: { props: Record }): unknown, R2Bucket(options: { props: Record }): unknown, ServiceBinding(options: { props: Record }): unknown, DurableObjectNamespace(options: { props: Record }): unknown, DoAlarmBinding(options: { props: Record }): unknown, InternalAuthBackend(options: { props: Record }): unknown } }} DoRuntimeContext * @typedef {{ name: string, spec: RuntimeBindingSpec }} DoBindingInput - * @typedef {string | { cjs: string } | { py: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue * @typedef {{ mainModule: string, modules: Record, compatibilityFlags?: string[], compatibilityDate?: string, env?: Record, globalOutbound?: unknown }} WorkerCode */ diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index 9ee16cf..a2bba28 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -13,6 +13,7 @@ import { import { DoRuntimeError } from "do-runtime-protocol-errors"; import { hostIdForObject, shardForObjectName } from "do-runtime-protocol-identity"; import { formatWorkerId } from "shared-worker-id"; +import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; import { BodyTooLargeError, readBoundedBytes as readRequestBoundedBytes, @@ -477,6 +478,14 @@ function normalizeWorkerCode(value) { const compatibilityFlags = Array.isArray(input.compatibilityFlags) ? input.compatibilityFlags.map((flag, index) => requireString(flag, `workerCode.compatibilityFlags[${index}]`, { maxBytes: 128 })) : ["nodejs_compat"]; + const experimentalFlag = firstWorkerdExperimentalCompatFlag(compatibilityFlags); + if (experimentalFlag) { + throw new DoRuntimeError( + 400, + "experimental_compat_flag_unsupported", + `workerCode.compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` + ); + } const env = input.env == null ? {} : requireRecord(input.env, "workerCode.env"); return { compatibilityDate, diff --git a/docs/compatibility.md b/docs/compatibility.md index a703cb7..19a1d40 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -31,8 +31,9 @@ Each row separates four compatibility claims: |---|---|---|---|---|---| | ES module Workers and `fetch()` | Supported | Module evaluation, request dispatch, `Response`/`Request`, service binding JSRPC machinery. | Dynamic `workerLoader`, immutable version ids, wrapper-generated `env`, gateway routing, request logs, and public/private outbound separation. | An uncaught tenant `fetch()` exception maps to platform `502 runtime_error`; exception detail goes to structured logs/live tail, not the client body. | WDL does not emulate every compatibility-date behavior change; workerd is configured with the platform's enabled flags. | | WebSocket upgrade | Supported | WebSocket API and 101 response handling inside workerd. | Gateway `GatewayWsHolder` Durable Object holds public sockets and forwards to runtime/do-runtime so long-lived 101s do not live on ordinary gateway request IoContexts. | WDL optimizes for preserving long-lived gateway-held sockets, while Cloudflare can rely on its global edge session model. | Gateway rolling still drops physical client sockets; application-level resume is not implemented. | -| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Control validates that `compatibility_date` is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; unsupported flags may only be stored or rejected depending on the path. | +| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Control validates that `compatibility_date` is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, and rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module` before deploy. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags. | | `nodejs_compat` | Partial | workerd-provided compatibility when the runtime service has the flag enabled. | CLI carries compatibility flags into metadata. | WDL exposes the workerd-enabled compatibility surface rather than a separate Node.js runtime. | This is not a full Node.js platform contract beyond workerd's enabled surface. | +| Python Workers modules | Not supported | workerd has an experimental Python Workers path. | Control rejects `py` module manifests with `python_workers_unsupported`; runtime and do-runtime also fail closed for retained metadata containing `py` modules. | WDL keeps tenant bundles JavaScript/WebAssembly/data only and does not permit cold-load-time Pyodide bootstrap. | Python Workers and mixed JS/Python bundles are unsupported. | ## Bindings And Storage @@ -41,14 +42,14 @@ Each row separates four compatibility claims: | KV namespace | Supported | A binding object can be exposed to user code through workerd entrypoint/JSRPC mechanics. | Runtime `KV` facade calls redis-proxy; redis-proxy stores values and metadata in DB 1 hash buckets `kvh:::b:`, with `v:` and `m:` fields, 512-byte key/list-prefix cap, 25 MiB value cap, batch raw-byte budget, TTL/EXAT, and prefix list cursors. | KV storage is Redis-backed and namespace-scoped in a WDL deployment. `cacheTtl` is not a storage freshness contract. | No Cloudflare global edge replication or eventual consistency model. | | R2 bucket | Supported | Fetch/stream primitives in workerd. | Runtime R2 facade signs S3-compatible requests with platform credentials; CLI parses `[[r2_buckets]]`. | Bucket lifecycle and placement are the S3-compatible backend's responsibility. | `preview_bucket_name` and `jurisdiction` are not supported. | | ASSETS | Partial | Worker can receive a platform-provided binding object. | CLI uploads assets to S3-compatible storage and runtime exposes the WDL `env.ASSETS.url(path)` helper for tokenized CDN URLs. | WDL assets are an S3/CDN helper model, not Cloudflare Pages asset hosting. | WDL does not provide a full Cloudflare Pages asset pipeline or a fetch-style assets binding contract. | -| D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. SQLite names under the reserved `_cf_` namespace are rejected by workerd. | -| Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. SQLite names under the reserved `_cf_` namespace are rejected by workerd. | +| D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | +| Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | | Queues producer/consumer | Partial | Worker queue handler API surface in loaded workers. | Runtime producer facade writes through redis-proxy DB 1; scheduler owns consumer dispatch, retry, DLQ, orphan, delayed queues, and batch splitting for oversized dispatch bodies. | `max_batch_timeout_ms` is configuration metadata, not a true aggregation window; dispatch concurrency is scheduler-owned. | `max_concurrency` is rejected. | | Cron triggers | Supported | `scheduled()` handler surface in workers. | Control stores cron config; scheduler owns indexed discovery, cron-slot buckets, due dispatch, and repairable projections. | Control and scheduler use JS/Rust `croner` engines and keep their behavior documented through tests and module docs. | New scheduler dispatch paths still require their own Redis lease/fence audit. | | Workflows | Partial | User workflow class code runs inside loaded workers when dispatched by runtime. | workflows owns DB 2 instance state, step/event/sleep commits, runtime-observed `step.do` DAG edges including `Promise.all` siblings, generation/run-token fencing, lifecycle APIs, progress callbacks, and scheduler tick. | WDL Workflows V2 has WDL-specific payload semantics and terminal failure rules; permanent `step.do` failure is terminal for the run even if caught, and one step may record at most 1000 dependency edges. | WDL Workflows V2 is not Cloudflare Workflows parity. `script_name`, cross-worker workflows, and Cloudflare's source-AST visualizer are unsupported. | | Service bindings | Supported | workerd service binding JSRPC and fetch dispatch. | WDL resolves target worker metadata, cold-loads immutable versions, propagates request ids where available, and enforces namespace/action ACL through control metadata. | Frozen-version service targets do not evict active siblings by design. | None currently documented. | | Platform bindings | Supported | workerd named entrypoint/JSRPC mechanics. | WDL expands ACL-checked platform bindings from control metadata. | Platform bindings are WDL-specific. | Platform bindings are not a Cloudflare tenant portability feature. | -| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding/workflow env values before deploy/secret mutation, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | +| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding/workflow env values before deploy/secret mutation, accounts for V8 two-byte string storage for non-Latin-1 strings, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | | Worker code size | Supported | Dynamic worker module bodies are accepted by `workerLoader` up to workerd's 64 MiB limit. | Control rejects module bodies over 64 MiB before allocating a version. | WDL's deploy JSON body limit is lower for ordinary inline deployments. | Large server-side bundle assembly paths must keep this guard. | ## Control Plane And Developer Tooling @@ -71,6 +72,8 @@ docs: | Workers AI, Vectorize, Analytics Engine, Browser Rendering, Hyperdrive, Email Workers | Not supported | No binding facade, control-plane metadata, or backing service exists in WDL. | | R2 multipart upload, customer-provided encryption keys, and Cloudflare-specific checksum behavior | Not supported | The current R2 facade targets S3-compatible object operations needed by WDL workers/assets. Advanced Cloudflare R2 behaviors need explicit design before being documented as compatible. | | Queue `contentType = "v8"` and per-consumer `max_concurrency` | Not supported | Queue messages support the documented `json`, `text`, and `bytes` content types; only `v8` is rejected. Dispatch concurrency remains scheduler-owned, and `max_concurrency` is rejected instead of silently ignored. | +| Upstream experimental compatibility flags | Not supported | Tenant `compatibility_flags` entries whose upstream workerd flag is marked `$experimental` are rejected at deploy and runtime decode. | +| Python Workers | Not supported | WDL rejects Python module manifests instead of letting workerd fail at cold-load. | | Durable Object cross-script bindings and migration rename/delete semantics | Not supported | WDL DO classes are same-worker only. Storage identity, owner routing, and delete cleanup are WDL-managed rather than Cloudflare migration-compatible. | | Cloudflare account API parity | Not supported | WDL exposes its own CLI/control API. Cloudflare API compatibility is not a stated goal. | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index 06ee309..8e72d51 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -24,8 +24,9 @@ |---|---|---|---|---|---| | ES module Workers 和 `fetch()` | Supported | Module evaluation、request dispatch、`Response`/`Request`、service binding JSRPC 机制。 | Dynamic `workerLoader`、immutable version id、wrapper 生成 `env`、gateway routing、request logs、public/private outbound 隔离。 | Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error`;异常细节进入结构化日志/live tail,不进客户端 body。 | WDL 不模拟每个 compatibility date 的历史行为;workerd 按平台启用的 flags 运行。 | | WebSocket upgrade | Supported | workerd 内的 WebSocket API 和 101 response 处理。 | Gateway `GatewayWsHolder` Durable Object 承载公网 socket 并转发到 runtime/do-runtime,避免长生命周期 101 留在普通 gateway request IoContext。 | WDL 优先保留 gateway-held 长连接;Cloudflare 依赖自己的全球 edge session 模型。 | Gateway rolling 仍会断开物理 client socket;应用级 resume 未实现。 | -| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | CLI/control 保存 bundle metadata。Control 会校验 `compatibility_date` 是真实的 `YYYY-MM-DD` 日期,且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期。 | WDL 把 compatibility metadata 当成部署期平台 metadata,不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为;不支持的 flags 会按路径存储或拒绝。 | +| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | CLI/control 保存 bundle metadata。Control 会校验 `compatibility_date` 是真实的 `YYYY-MM-DD` 日期,且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期;上游 `$experimental` enable flags(例如 `experimental` / `unsafe_module`)会在 deploy 前被拒绝。 | WDL 把 compatibility metadata 当成部署期平台 metadata,不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为;tenant worker 不能 opt into 上游 experimental-only flags。 | | `nodejs_compat` | Partial | runtime service 启用 flag 后由 workerd 提供的兼容 surface。 | CLI 把 compatibility flags 带入 metadata。 | WDL 暴露 workerd 已启用的兼容 surface,而不是单独 Node.js runtime。 | 不承诺超出 workerd 已启用 surface 的完整 Node.js 平台能力。 | +| Python Workers modules | Not supported | workerd 有 experimental Python Workers 路径。 | Control 用 `python_workers_unsupported` 拒绝 `py` module manifest;runtime 和 do-runtime 对 retained metadata 中的 `py` module 也 fail closed。 | WDL 保持 tenant bundle 只包含 JavaScript/WebAssembly/data,不允许 cold-load 时触发 Pyodide bootstrap。 | 不支持 Python Workers 和 JS/Python 混合 bundle。 | ## Bindings 和存储 @@ -34,14 +35,14 @@ | KV namespace | Supported | 通过 workerd entrypoint/JSRPC 机制把 binding object 暴露给用户代码。 | Runtime `KV` facade 调 redis-proxy;redis-proxy 在 DB 1 hash bucket `kvh:::b:` 存 value/metadata,字段为 `v:` 和 `m:`,支持 512-byte key/list-prefix cap、25 MiB value cap、batch raw-byte budget、TTL/EXAT、prefix list cursor。 | KV storage 是 WDL deployment 内的 Redis-backed、namespace-scoped 存储;`cacheTtl` 不是存储新鲜度合同。 | 没有 Cloudflare 全局边缘复制 / eventual consistency 模型。 | | R2 bucket | Supported | workerd 的 fetch/stream primitives。 | Runtime R2 facade 用平台 credentials 签 S3-compatible request;CLI 解析 `[[r2_buckets]]`。 | Bucket lifecycle/placement 由 S3-compatible backend 负责。 | 不支持 `preview_bucket_name` 和 `jurisdiction`。 | | ASSETS | Partial | Worker 可接收平台提供的 binding object。 | CLI 把 assets 上传到 S3-compatible storage;runtime 暴露 WDL 的 `env.ASSETS.url(path)` helper,用于生成 tokenized CDN URL。 | WDL assets 是 S3/CDN helper 模型,不是 Cloudflare Pages asset hosting。 | 不是完整 Cloudflare Pages assets pipeline,也不提供 fetch-style assets binding 合同。 | -| D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing,并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing,不是 Cloudflare global D1;metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。workerd 会拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | -| Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | 只支持同 worker class;不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。workerd 会拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | +| D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing,并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing,不是 Cloudflare global D1;metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | +| Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | 只支持同 worker class;不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | | Queues producer/consumer | Partial | loaded worker 中的 queue handler surface。 | Runtime producer facade 经 redis-proxy 写 DB 1;scheduler 负责 consumer dispatch、retry、DLQ、orphan、delayed queues,并会拆分过大的 dispatch body batch。 | `max_batch_timeout_ms` 是配置 metadata,不是真正 aggregation window;dispatch concurrency 由 scheduler 拥有。 | `max_concurrency` 被拒绝。 | | Cron triggers | Supported | worker 的 `scheduled()` handler surface。 | Control 存 cron config;scheduler 拥有 indexed discovery、cron-slot bucket、due dispatch 和可修复 projection。 | Control 和 scheduler 使用 JS/Rust `croner` 引擎,并通过测试和模块文档约束行为。 | 新增 scheduler dispatch 路径仍必须单独审计 Redis lease/fence。 | | Workflows | Partial | 用户 workflow class code 在 runtime dispatch 时运行在 loaded worker 内。 | workflows 拥有 DB 2 instance state、step/event/sleep commit、runtime-observed `step.do` DAG edges(包括 `Promise.all` sibling)、generation/run-token fence、lifecycle API、progress callback 和 scheduler tick。 | WDL Workflows V2 的 payload 语义和 terminal failure 规则由 WDL 定义;永久失败的 `step.do` 即使被捕获也会让 run terminal failed;单个 step 最多记录 1000 条 dependency edge。 | WDL Workflows V2 不是 Cloudflare Workflows parity;不支持 `script_name`、跨 worker workflow 和 Cloudflare 的源码 AST visualizer。 | | Service bindings | Supported | workerd service binding JSRPC 和 fetch dispatch。 | WDL 解析 target worker metadata,cold-load immutable version,在可用时传播 request id,并通过 control metadata 做 namespace/action ACL。 | Frozen-version service target 按设计不 evict active siblings。 | 当前无已记录缺口。 | | Platform bindings | Supported | workerd named entrypoint/JSRPC 机制。 | WDL 从 control metadata 展开 ACL-checked platform bindings。 | Platform bindings 是 WDL-specific。 | 不是 Cloudflare tenant portability feature。 | -| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 前按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets 加 runtime 注入的 binding/workflow env value;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | +| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 前按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets 加 runtime 注入的 binding/workflow env value,并计入非 Latin-1 字符串的 V8 two-byte storage;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | | Worker code size | Supported | Dynamic worker module bodies 可由 `workerLoader` 接受,直到 workerd 的 64 MiB 上限。 | Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies。 | WDL 的 deploy JSON body limit 对普通 inline deploy 更低。 | 大型 server-side bundle assembly 路径必须保留这道 guard。 | ## 控制面和开发工具 @@ -63,6 +64,8 @@ | Workers AI、Vectorize、Analytics Engine、Browser Rendering、Hyperdrive、Email Workers | Not supported | WDL 没有对应 binding facade、control-plane metadata 或后端服务。 | | R2 multipart upload、customer-provided encryption keys 和 Cloudflare-specific checksum 行为 | Not supported | 当前 R2 facade 面向 WDL worker/assets 所需的 S3-compatible object 操作。高级 Cloudflare R2 行为需要先设计,才能写成兼容合同。 | | Queue `contentType = "v8"` 和 per-consumer `max_concurrency` | Not supported | Queue message 支持文档化的 `json`、`text` 和 `bytes` content type;只有 `v8` 会被拒绝。Dispatch concurrency 仍由 scheduler 拥有,`max_concurrency` 会被拒绝,而不是静默忽略。 | +| 上游 experimental compatibility flags | Not supported | Tenant `compatibility_flags` 中属于上游 workerd `$experimental` 的 enable flag 会在 deploy 和 runtime decode 阶段被拒绝。 | +| Python Workers | Not supported | WDL 拒绝 Python module manifest,而不是让 workerd 在 cold-load 时失败。 | | Durable Object cross-script binding 和 migration rename/delete 语义 | Not supported | WDL DO class 仅支持 same-worker。Storage identity、owner routing 和 delete cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | | Cloudflare account API parity | Not supported | WDL 暴露自己的 CLI/control API。Cloudflare API 兼容不是目标。 | diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index cec8f97..5b16d1f 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -45,7 +45,7 @@ Worker lifecycle: |---|---|---| | `GET` | `/ns//workers` | Lists workers with namespace-owned state, including deploy-only, active, and secret-only workers. | | `GET` | `/ns//worker//versions` | Lists retained versions and active status. | -| `POST` | `/ns//worker//deploy` | Creates a new immutable version from shorthand code or full module manifest; routes, crons, queue consumers, service refs, platform refs, assets, vars, bindings, and `exports` are version metadata. | +| `POST` | `/ns//worker//deploy` | Creates a new immutable version from shorthand code or full module manifest; routes, crons, queue consumers, service refs, platform refs, assets, vars, bindings, and `exports` are version metadata. Python modules and upstream experimental compatibility flags are rejected before commit. | | `POST` | `/ns//worker//promote` | Promotes `{"version":"vN"}` through the WATCH/MULTI routing path. Host declaration failures are 403; live pattern conflicts are 409; exhausted transaction contention is 503. | | `DELETE` | `/ns//worker//versions/` | Deletes one retained non-active version after active-route, service-ref, lifecycle, and delete-lock blockers pass. Referrer redaction is principal-aware. | | `POST` | `/ns//worker//delete` | Whole-worker delete. `?dry_run=1` returns computed impact and blockers without writing. Redaction matches single-version delete. | @@ -78,7 +78,8 @@ Control lifecycle operations are split so each critical transition has one autho - Deploy parses the supported Wrangler/JSONC shape, validates bindings and routes, allocates the next immutable version through `worker:::next_version`, writes bundle metadata/modules/assets, then enters the same promote path used by - explicit promotion. + explicit promotion. Before allocation, deploy checks the 64 MiB workerd dynamic code + limit and the headroomed `workerLoader` env budget for the candidate metadata. - Promote is the only active-route flip. It WATCHes the delete lock, bundle metadata, D1 refs, service-binding target refs, queue consumer keys, host declarations, and pattern keys needed for the candidate. The EXEC updates active routes, host reverse indexes, @@ -89,6 +90,9 @@ Control lifecycle operations are split so each critical transition has one autho plaintext size and shape, encrypts it into a `WDL-ENC:` envelope before the Redis mutation/WATCH retry loop, and reuses the same envelope across retries. Runtime therefore sees a new immutable version id instead of mutable in-place secret changes. + Namespace-secret mutations WATCH the retained worker/version metadata they need to + re-estimate before commit; if concurrent metadata changes keep invalidating that view, + control returns `namespace_secret_mutation_contention`. - Version delete and whole-worker delete are fail-closed. They collect blockers from active routes, retained versions, service refs, D1 refs, workflow lifecycle checks, queue/cron projections, and delete locks before committing Redis lifecycle deletion. @@ -250,6 +254,15 @@ Auth-specific contract: - Control 5xx responses use generic/safe messages. Internal exception text, auth Redis diagnostics, backend messages, and provider errors belong in logs unless the endpoint explicitly owns a diagnostic response field. +- Deploy and secret mutations return `worker_code_too_large` when tenant module bodies + exceed the workerd 64 MiB dynamic code limit, and `worker_env_too_large` when the + estimated `workerLoader` env exceeds WDL's headroomed 1 MiB budget. `worker_env_too_large` + details include `namespace`, optional `worker`, `env_bytes`, `max_env_bytes`, + `upstream_max_env_bytes`, and `headroom_bytes`. When the blocker is an already-retained + version being re-estimated during a secret mutation, details also include + `source_version` and `estimated_version`, and the message identifies + `/@` so operators can find the retained version to delete + or redeploy. - Control never calls gateway directly. It writes Redis and publishes invalidation messages. - Control encrypts secret PUT values before entering Redis mutation loops. diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 9d935dd..6f2d916 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -34,7 +34,7 @@ Worker lifecycle: |---|---|---| | `GET` | `/ns//workers` | 列出有 namespace-owned state 的 worker,包括 deploy-only、active 和 secret-only worker。 | | `GET` | `/ns//worker//versions` | 列出 retained versions 和 active status。 | -| `POST` | `/ns//worker//deploy` | 从 shorthand code 或完整 module manifest 创建新的 immutable version;routes、crons、queue consumers、service refs、platform refs、assets、vars、bindings 和 `exports` 都是 version metadata。 | +| `POST` | `/ns//worker//deploy` | 从 shorthand code 或完整 module manifest 创建新的 immutable version;routes、crons、queue consumers、service refs、platform refs、assets、vars、bindings 和 `exports` 都是 version metadata。Python modules 和上游 experimental compatibility flags 会在 commit 前被拒绝。 | | `POST` | `/ns//worker//promote` | 通过 WATCH/MULTI routing path promote `{"version":"vN"}`。Host declaration 失败是 403;live pattern conflict 是 409;transaction contention 耗尽是 503。 | | `DELETE` | `/ns//worker//versions/` | 在 active-route、service-ref、lifecycle 和 delete-lock blocker 全部通过后删除一个 retained non-active version。Referrer redaction 按 principal 决定。 | | `POST` | `/ns//worker//delete` | Whole-worker delete。`?dry_run=1` 只返回 computed impact 和 blockers,不写入。Redaction 与 single-version delete 一致。 | @@ -64,9 +64,9 @@ Host、secret、data 和 auth 操作: Control lifecycle 操作会拆开处理,确保每个关键 transition 只有一个权威入口: -- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。 +- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会检查 workerd 64 MiB dynamic code limit 和候选 metadata 的 headroomed `workerLoader` env budget。 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 -- Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。 +- Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前,会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 - Worker delete lock value 和 `s3cleanup:` task id 必须由服务端生成随机值。`x-request-id` 只用于诊断,客户端或重试可能复用它;不能把它用作 lock token 或 cleanup-task id。 - Auth 不是一层约定俗成的 middleware。`parseControlRoute()` 分配 action,control 把 action 和 namespace 发给 auth,auth 用存储的 token record 对照 `shared/auth-roles.js` 评估权限。Dispatcher 代码不应自己从 URL prefix 推断权限。 @@ -135,6 +135,7 @@ Auth 子合同: - Details 可以增加字段,但不能覆盖 `error`、`message` 或 legacy `reason`。Auth reject reason 是 `error` machine code;日志可以把 `reason` 作为诊断上下文。 - Delegated issue 的 409 reason 有不同 retry 语义:`delegated_issue_busy` 表示 issuer/template lock 清除后可重试;`active_quota_exceeded` 在已有 delegated credential 过期或 revoke 前不应重试;`namespace_collision` 表示 Auth 已耗尽 configured candidate retry budget。 - Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志;除非 endpoint 明确拥有某个 diagnostic response field,否则不进入客户端 body。 +- Deploy 和 secret mutation 在 tenant module bodies 超过 workerd 64 MiB dynamic code limit 时返回 `worker_code_too_large`,在估算的 `workerLoader` env 超过 WDL headroomed 1 MiB budget 时返回 `worker_env_too_large`。`worker_env_too_large` details 包含 `namespace`、可选 `worker`、`env_bytes`、`max_env_bytes`、`upstream_max_env_bytes` 和 `headroom_bytes`。如果 blocker 是 secret mutation 重新估算到的既有 retained version,details 还会包含 `source_version` 和 `estimated_version`,message 会标出 `/@`,方便 operator 定位要删除或 redeploy 的 retained version。 - Control 不直接调用 gateway。它写 Redis 并 publish invalidation message。 - Control 在进入 Redis mutation loop 前加密 secret PUT value。加密/provider 失败会返回 control error,不写 plaintext fallback。 - Worker delete 先 commit Redis lifecycle state;异步 S3 cleanup enqueue 是 best-effort,失败时返回 warning。 diff --git a/docs/modules/d1.md b/docs/modules/d1.md index 29dceca..516efec 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -74,6 +74,11 @@ Runtime ownership: - Shared EFS/localDisk is safe only when the owner lease model preserves single-writer access. +workerd 2026-07-01 rejects SQLite object names under the reserved `_cf_` namespace +case-insensitively, including `ALTER TABLE ... RENAME TO _cf_*`. Existing databases +that somehow contain `_CF_*` or other case variants should treat those objects as +reserved upgrade debris rather than application-owned tables. + Key families: | Key | Type | Owner | Authority | Cleanup/delete semantics | diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index fc82b76..4909186 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -46,6 +46,8 @@ Runtime ownership: - Redis owner record 包含 task identity 和 monotonic generation。 - Shared EFS/localDisk 只有在 owner lease 模型保证 single-writer 时才安全。 +workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespace 下的 object name,包括 `ALTER TABLE ... RENAME TO _cf_*`。如果既有 database 中已经有 `_CF_*` 等大小写变体,应把它们视为 reserved upgrade debris,而不是应用拥有的表。 + Key families: | Key | Type | Owner | Authority | Cleanup/delete 语义 | diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 5542006..942ddfb 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -93,6 +93,12 @@ do-runtime and stores one internal job per pending row. Row tokens fence user-dr delete against stale backend delivery; Workflows run tokens fence dispatch retry and completion inside DB 2. +workerd 2026-07-01 rejects SQLite object names under the reserved `_cf_` namespace +case-insensitively. `ctx.storage.deleteAll()` skips those names case-insensitively as +well, so old storage created before the stricter check with variants such as `_CF_*` +does not make cleanup fail. Those legacy reserved-name objects remain inaccessible to +tenant SQL and should be treated as upgrade debris, not application tables. + `getAlarm()` performs alarm-scoped read repair: if SQLite has a pending alarm row but the Workflows DB 2 due index is missing, it idempotently rewrites the backend due index without adding Redis IO to ordinary DO fetches. Active and retained alarms keep their diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 6e2b04f..fec25a9 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -59,6 +59,8 @@ Ownership 按 shard 划分: Alarm state 存在 object SQLite。Workflows 接收 do-runtime 的 set/delete 请求,并为每个 pending row 保存一个 internal job。Row token 用于 fence 用户驱动 delete 和 stale backend delivery;Workflows run token 在 DB 2 内 fence dispatch retry 和 completion。 +workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespace 下的 object name。`ctx.storage.deleteAll()` 也会大小写不敏感地跳过这些名字,因此 0617 以前可能创建出的 `_CF_*` 这类大小写变体不会让 cleanup 失败。这些 legacy reserved-name object 对 tenant SQL 仍不可访问,应视为升级遗留物,而不是应用表。 + `getAlarm()` 会做 alarm-scoped read repair:如果 SQLite 中有 pending alarm row,但 Workflows DB 2 due index 缺失,它会幂等重写 backend due index,而不会给普通 DO fetch 增加 Redis IO。Active/retained alarm 保留调度时的 worker version;旧 version 删除后,只有 `doStorageId` 仍匹配时 alarm dispatch 才 retarget 到当前 active version。逻辑 worker 已消失或指向不同 `doStorageId` 时,alarm 会自清理。 ## Ownership / 并发 / 失败语义 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 8076303..e3317a3 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -86,9 +86,13 @@ pipe, not durable log storage. empty values fall back to 15 minutes. - workerd issue [#6832](https://github.com/cloudflare/workerd/issues/6832) means client disconnects may not call async response-body `ReadableStream.cancel()`. - Control therefore has an independent max-session watchdog; under heavy reconnect - churn, set `LOG_TAIL_MAX_SESSION_MS` lower to bound abandoned Redis tail sessions - until upstream restores a reliable disconnect hook. + Control therefore has independent watchdogs: a max-session watchdog for + reauthorization and an idle-pull watchdog that closes a session when the SSE body has + not been pulled for three keepalive intervals. Active clients naturally pull at the + keepalive cadence because each heartbeat frees queue space; abandoned clients stop + pulling and are cleaned up without waiting for the full session lifetime. A TCP + connection that stays open but whose application stops reading for that window may be + closed and should reconnect. - Tail events racing activation can be dropped. - High QPS or slow SSE readers can miss middle events due to stream caps. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index ceb3cd4..e9614fc 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -64,7 +64,7 @@ Tail streams 使用有界 `MAXLEN ~ 500`,并在写入时刷新 TTL。它们是 - 结构化 stdout 是持久平台日志的事实来源。 - 没有 active tailer 时,runtime 仍输出 stdout,但在本地 active-set miss cache 后跳过 per-event stream append work。 - Active tail session 是有时限的授权租约,必须通过正常 auth reconnect。`LOG_TAIL_MAX_SESSION_MS` 设置 control-side 最大时长;非法值或空值会回退到 15 分钟。 -- workerd issue [#6832](https://github.com/cloudflare/workerd/issues/6832) 会导致 client disconnect 不一定触发 async response-body `ReadableStream.cancel()`。Control 因此有独立的 max-session watchdog;如果 tail UI 重连抖动明显,可以调低 `LOG_TAIL_MAX_SESSION_MS`,在上游恢复可靠 disconnect hook 前约束遗弃 Redis tail session 的存活时间。 +- workerd issue [#6832](https://github.com/cloudflare/workerd/issues/6832) 会导致 client disconnect 不一定触发 async response-body `ReadableStream.cancel()`。Control 因此有独立 watchdog:max-session watchdog 负责 reauthorization 上界,idle-pull watchdog 在 SSE body 连续三个 keepalive 周期没有被 pull 时关闭 session。活跃客户端会因为每次 heartbeat 腾出 queue 空间而自然按 keepalive 粒度继续 pull;遗弃客户端会停止 pull,因此不需要等完整 session lifetime 才清理。TCP 连接还在但应用层长时间不读的客户端可能被关闭,应自行 reconnect。 - 与 activation race 的 tail event 可以丢失。 - 高 QPS 或慢 SSE reader 可能因为 stream cap 丢中间事件。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index b001b76..e726288 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -183,6 +183,9 @@ Data-plane bindings use their own storage: Runtime must treat Redis bundle metadata as control-authored, but still revalidates reserved runtime entrypoint and binding names when materializing older stored metadata. +It also fails closed if older metadata contains Python module entries or upstream +experimental compatibility flags, because those would otherwise reach workerd as +opaque cold-load failures under the current stock binary. ## Ownership / Concurrency / Failure Semantics @@ -240,6 +243,17 @@ when a matching active tail session exists. - Removing the broad loaded-worker `experimental` flag intentionally removes access to non-GA experimental-only tenant surfaces, such as irrevocable long-term stub storage. Do not re-enable it as a compatibility workaround without an explicit feature design. +- Control rejects upstream `$experimental` compatibility enable flags at deploy, and + runtime rejects retained metadata that still contains them. Disable-style flags such + as `no_*` are not part of that mirror unless upstream marks the enable flag itself + experimental. +- Python Workers modules are not supported. Control rejects new `py` module manifests, + and runtime/do-runtime reject retained metadata that contains them instead of letting + workerd fail later with a mixed JS/Python bundle error. +- Before rolling the 2026-07-01 workerd adaptation over an existing Redis DB 0, run + `node scripts/scan-workerd-0701-metadata.mjs` against the control Redis database to + find retained versions that still contain Python modules or upstream experimental + compatibility flags. - The runtime workerd processes still run with process-level `--experimental` because upstream workerd 2026-07-01 continues to gate `workerLoader` bindings on that switch. Do not re-add the `experimental` compatibility flag or `allowExperimental` to loaded @@ -248,7 +262,13 @@ when a matching active tail session exists. env at 1 MiB. Control rejects module bodies over 64 MiB before version allocation. Vars, namespace/worker secrets, and runtime-injected binding/workflow env values are prechecked against a headroomed `workerLoader` env budget because workerd's final - enforcement includes the full `env` estimate. + enforcement includes the full `env` estimate. The estimate starts from JSON bytes and + adds V8 two-byte string overhead for non-Latin-1 strings, so mixed ASCII plus CJK or + emoji secrets do not slip past control and fail later at cold-load. +- In current stock workerd, a client disconnect during an async `ReadableStream` + response body may not call the stream source's `cancel()` callback. Tenant streaming + and SSE workers should use their own heartbeat, timeout, or application close path + instead of relying on disconnect-driven `cancel()` as the only resource cleanup hook. - workerd upgrades can still change default or compatibility-flagged runtime surfaces; review the exposed surface, not only the loader/abort path. diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index a84da22..a283b86 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -74,7 +74,7 @@ Runtime 通过 `redis-proxy` 从 DB 0 读取不可变 bundle 和 metadata。Data - D1 和 DO binding 调用专门 runtime service。 - R2/ASSETS 使用 S3-compatible object storage。 -Runtime 可以把 Redis bundle metadata 视为 control-authored,但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。 +Runtime 可以把 Redis bundle metadata 视为 control-authored,但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含 Python module entry 或上游 experimental compatibility flag,也会 fail closed;否则这些形态会在当前 stock workerd binary 下变成 opaque cold-load failure。 ## Ownership / 并发 / 失败语义 @@ -106,8 +106,12 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - 如果 scheduler/workflows 依赖新的 `:8088` internal path 或 dispatch body,runtime 必须先滚。 - Runtime 不再为 loaded worker 开启 workerd 的宽泛 `experimental` flag。Historical-version eviction 仍然注入 `__WdlAbort__`,但当前 bundled workerd baseline 中 `abortIsolate()` 已经不需要该 flag。 - 移除 loaded worker 的宽泛 `experimental` flag 会有意收紧 tenant 对非 GA experimental-only surface 的访问,例如不可撤销的长期 stub storage。不要为了兼容绕过而重新打开它,除非先完成明确的功能设计。 +- Control 会在 deploy 时拒绝上游 `$experimental` compatibility enable flags,runtime 也会拒绝仍带有这些 flags 的 retained metadata。`no_*` 这类 disable-style flag 不属于这个 mirror,除非上游把对应 enable flag 本身标为 experimental。 +- Python Workers modules 不受支持。Control 会拒绝新的 `py` module manifest,runtime/do-runtime 会拒绝 retained metadata 中的 `py` module,而不是让 workerd 之后抛 mixed JS/Python bundle error。 +- 在已有 Redis DB 0 上 rollout 2026-07-01 workerd 适配前,先对 control Redis database 运行 `node scripts/scan-workerd-0701-metadata.mjs`,找出仍包含 Python modules 或上游 experimental compatibility flags 的 retained versions。 - Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 -- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会在 control plane 用留有 headroom 的 `workerLoader` env budget 预检,因为 workerd 的最终检查看的是完整 `env` estimate。 +- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会在 control plane 用留有 headroom 的 `workerLoader` env budget 预检,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 +- 当前 stock workerd 中,客户端在 async `ReadableStream` response body 中途断开时,不一定调用 stream source 的 `cancel()`。Tenant streaming/SSE worker 应使用自己的 heartbeat、timeout 或应用层 close path,不要把 disconnect-driven `cancel()` 当成唯一资源清理信号。 - workerd 升级仍可能改变默认或 compatibility-flagged runtime surface;升级时要审 exposed surface,而不只审 loader/abort path。 ## 保护该模块的测试 diff --git a/docs/source-map.md b/docs/source-map.md index 158c005..ee650dc 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -43,6 +43,7 @@ are outside this map unless they own runtime or deployable service behavior. | `control/topology.js` | Route, pattern, cron, queue consumer, and workflow declaration parsing for deploy metadata. | | `control/routing.js`, `control/routing/route-plan.js` | Promote, secret bump/promote, host reconcile WATCH/MULTI loops, and pure route/pattern planning helpers. | | `control/lifecycle-indexes.js` | Redis mutation helpers for worker lifecycle, cron, queue consumer, and referrer indexes. | +| `control/env-budget.js` | Control-plane estimate of workerd `workerLoader` env size for deploy and secret mutation guards. | | `control/d1-*` | D1 control metadata, store, lifecycle, migration, and d1-runtime client modules. | | `control/r2.js` | Control-plane R2 bucket/object API client for the configured S3-compatible store. | | `control/s3.js` | S3-compatible ASSETS upload helper. | @@ -65,6 +66,7 @@ are outside this map unless they own runtime or deployable service behavior. | `shared/bounded-body.js` | Shared bounded request body byte/text readers; each tier maps limit errors to its own HTTP error contract. | | `shared/ns-pattern.js` | Namespace, worker, binding, queue, KV id, module path, reserved object-key, and reserved namespace grammars. | | `shared/version.js` | Worker version formatting and bundle key helpers. | +| `shared/workerd-compat-flags.js` | Pinned mirror of upstream workerd experimental compatibility enable flags used to reject tenant metadata before cold-load. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | | `shared/d1-*.js`, `shared/sql-splitter.js` | D1 parameter, data-field, transport, timeout, query-wire, and SQL splitting utilities shared by runtime, d1-runtime, control, and tests. | @@ -102,6 +104,7 @@ are outside this map unless they own runtime or deployable service behavior. | `examples/` | Manual demos and reference projects. Tests should not silently depend on them unless the fixture graduates to `test-workers/`. | | `scripts/run-integration-tests.js` | Integration worker-pool runner. | | `scripts/compile-workerd-configs.js` | Compiles workerd Cap'n Proto configs into `dist/workerd-configs/*.bin`. | +| `scripts/scan-workerd-0701-metadata.mjs` | Read-only Redis metadata scanner for rollout checks before the workerd 2026-07-01 adaptation: reports retained versions using experimental flags or Python modules. | ## Infrastructure diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index d8ae6b6..aaa9aff 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -40,6 +40,7 @@ | `control/topology.js` | Deploy metadata 中 routes、patterns、cron、queue consumer 和 workflow declaration parsing。 | | `control/routing.js`、`control/routing/route-plan.js` | Promote、secret bump/promote、host reconcile WATCH/MULTI loops,以及纯 route/pattern planning helpers。 | | `control/lifecycle-indexes.js` | Worker lifecycle、cron、queue consumer 和 referrer indexes 的 Redis mutation helpers。 | +| `control/env-budget.js` | Deploy 和 secret mutation guard 使用的 workerd `workerLoader` env size 控制面估算。 | | `control/d1-*` | D1 control metadata、store、lifecycle、migration 和 d1-runtime client modules。 | | `control/r2.js` | 面向配置的 S3-compatible store 的 control-plane R2 bucket/object API client。 | | `control/s3.js` | S3-compatible ASSETS upload helper。 | @@ -62,6 +63,7 @@ | `shared/bounded-body.js` | 共享 bounded request body byte/text readers;各 tier 自己把 limit error 映射为对应 HTTP error contract。 | | `shared/ns-pattern.js` | Namespace、worker、binding、queue、KV id、module path、reserved object-key 和 reserved namespace grammars。 | | `shared/version.js` | Worker version formatting 和 bundle key helpers。 | +| `shared/workerd-compat-flags.js` | 上游 workerd experimental compatibility enable flags 的 pinned mirror,用于在 cold-load 前拒绝 tenant metadata。 | | `shared/queue-keys.js` | JavaScript queue key helpers,供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | | `shared/d1-*.js`、`shared/sql-splitter.js` | Runtime、d1-runtime、control 和 tests 共用的 D1 parameter、data-field、transport、timeout、query-wire 和 SQL splitting utilities。 | @@ -99,6 +101,7 @@ | `examples/` | 手工 demo 和 reference projects。测试不应悄悄依赖它们,除非 fixture 明确迁入 `test-workers/`。 | | `scripts/run-integration-tests.js` | Integration worker-pool runner。 | | `scripts/compile-workerd-configs.js` | 把 workerd Cap'n Proto configs 编译成 `dist/workerd-configs/*.bin`。 | +| `scripts/scan-workerd-0701-metadata.mjs` | workerd 2026-07-01 适配 rollout 前使用的只读 Redis metadata scanner,报告使用 experimental flags 或 Python modules 的 retained versions。 | ## Infrastructure diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 206f9ec..fef1bb8 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -92,6 +92,7 @@ const loaderWorker :Workerd.Worker = ( (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), @@ -199,6 +200,7 @@ const controlWorker :Workerd.Worker = ( (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), (name = "shared-version", esModule = embed "../shared/version.js"), diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index c2a4d9d..f9a4308 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -91,6 +91,7 @@ const loaderWorker :Workerd.Worker = ( (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), diff --git a/runtime/lib.js b/runtime/lib.js index a70afff..3822f83 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -1,5 +1,6 @@ -// Pure helpers for the runtime worker. No bare-specifier imports so this -// file can be loaded both as a workerd embedded module and by node --test. +// Pure helpers for the runtime worker. + +import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; const BASE64_CHUNK_SIZE = 0x8000; const utf8Encoder = new TextEncoder(); @@ -160,6 +161,11 @@ function mergeCompatFlags(userFlags, compatibilityDate) { `meta.compatibilityFlags entries must be non-empty strings, got ${JSON.stringify(f)}` ); } + if (firstWorkerdExperimentalCompatFlag([f])) { + throw new Error( + `meta.compatibilityFlags contains experimental workerd flag ${JSON.stringify(f)}, which WDL does not support for tenant workers` + ); + } out.push(f); } for (const f of platformFloorCompatFlags(compatibilityDate)) { @@ -183,14 +189,13 @@ function mergeCompatFlags(userFlags, compatibilityDate) { * exports?: { entrypoint?: unknown }[] | null, * [key: string]: unknown, * }} WorkerBundleMeta - * @typedef {string | { cjs: string } | { py: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue */ /** @type {[string, (bytes: Uint8Array) => WorkerModuleValue][]} */ const moduleDecoderEntries = [ ["module", (bytes) => utf8Decoder.decode(bytes)], ["cjs", (bytes) => ({ cjs: utf8Decoder.decode(bytes) })], - ["py", (bytes) => ({ py: utf8Decoder.decode(bytes) })], ["text", (bytes) => ({ text: utf8Decoder.decode(bytes) })], ["json", (bytes) => ({ json: JSON.parse(utf8Decoder.decode(bytes)) })], ["wasm", (bytes) => ({ wasm: bytes })], @@ -223,6 +228,9 @@ export function bundleToWorkerCode(hash) { for (const [path, info] of Object.entries(meta.modules)) { const bytes = hash[path]; if (!bytes) throw new Error(`Bundle missing module "${path}"`); + if (info.type === "py") { + throw new Error(`Module "${path}": Python Workers modules are not supported by WDL`); + } const decoder = typeof info.type === "string" ? moduleDecoders.get(info.type) : undefined; if (!decoder) throw new Error(`Module "${path}": unknown type "${info.type}"`); modules[path] = decoder(bytes); diff --git a/runtime/load.js b/runtime/load.js index ea32cdd..cbb27b9 100644 --- a/runtime/load.js +++ b/runtime/load.js @@ -46,7 +46,7 @@ const utf8Decoder = new TextDecoder(); /** * @typedef {{ bundle: Record, ns_secrets: Record, worker_secrets: Record }} RuntimeLoadPayload - * @typedef {string | { cjs: string } | { py: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue * @typedef {{ modules: Record, mainModule: string, [key: string]: unknown }} WorkerCodeShape * @typedef {Record & { type?: string, className?: unknown }} RuntimeBindingSpec * @typedef {{ binding?: unknown, className?: unknown }} RuntimeWorkflowSpec diff --git a/scripts/scan-workerd-0701-metadata.mjs b/scripts/scan-workerd-0701-metadata.mjs new file mode 100644 index 0000000..8ed0930 --- /dev/null +++ b/scripts/scan-workerd-0701-metadata.mjs @@ -0,0 +1,110 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { firstWorkerdExperimentalCompatFlag } from "../shared/workerd-compat-flags.js"; + +const redisUrl = redisUrlFromEnv(); +const pattern = "worker:*:*:v:*"; + +function redisUrlFromEnv() { + const raw = process.env.REDIS_URL || process.env.REDIS_ADDR || "redis://127.0.0.1:6379/0"; + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) return raw; + return `redis://${raw}`; +} + +/** @param {string[]} args */ +function redisCli(args) { + const result = spawnSync("redis-cli", ["-u", redisUrl, "--raw", ...args], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error(result.stderr.trim() || `redis-cli exited with status ${result.status}`); + } + return result.stdout; +} + +/** @param {string} key */ +function parseBundleKey(key) { + const match = /^worker:([^:]+):([^:]+):v:([1-9][0-9]*)$/.exec(key); + if (!match) return { namespace: null, worker: null, version: null }; + return { namespace: match[1], worker: match[2], version: `v${match[3]}` }; +} + +/** @param {unknown} meta */ +function pythonModules(meta) { + if (!meta || typeof meta !== "object" || Array.isArray(meta)) return []; + const modules = /** @type {{ modules?: unknown }} */ (meta).modules; + if (!modules || typeof modules !== "object" || Array.isArray(modules)) return []; + return Object.entries(/** @type {Record} */ (modules)) + .filter(([, value]) => + value && + typeof value === "object" && + !Array.isArray(value) && + /** @type {{ type?: unknown }} */ (value).type === "py" + ) + .map(([name]) => name); +} + +/** @param {unknown} meta */ +function experimentalFlag(meta) { + if (!meta || typeof meta !== "object" || Array.isArray(meta)) return null; + return firstWorkerdExperimentalCompatFlag(/** @type {{ compatibilityFlags?: unknown }} */ (meta).compatibilityFlags); +} + +const keys = redisCli(["--scan", "--pattern", pattern]) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + +/** @type {Array>} */ +const findings = []; +for (const key of keys) { + const identity = parseBundleKey(key); + const rawMeta = redisCli(["HGET", key, "__meta__"]).replace(/\n$/, ""); + if (!rawMeta) continue; + /** @type {unknown} */ + let meta; + try { + meta = JSON.parse(rawMeta); + } catch (err) { + findings.push({ + kind: "corrupt_meta", + key, + ...identity, + error: err instanceof Error ? err.message : String(err), + }); + continue; + } + + const flag = experimentalFlag(meta); + if (flag) { + findings.push({ + kind: "experimental_compat_flag", + key, + ...identity, + flag, + }); + } + for (const module of pythonModules(meta)) { + findings.push({ + kind: "python_worker_module", + key, + ...identity, + module, + }); + } +} + +for (const finding of findings) { + console.log(JSON.stringify(finding)); +} +if (findings.length === 0) { + console.error(`Scanned ${keys.length} worker bundle metadata keys; no workerd 0701 blockers found.`); +} else { + console.error(`Scanned ${keys.length} worker bundle metadata keys; found ${findings.length} workerd 0701 blocker(s).`); + process.exitCode = 1; +} diff --git a/shared/redis-session.js b/shared/redis-session.js index a9ab10d..3d5de2f 100644 --- a/shared/redis-session.js +++ b/shared/redis-session.js @@ -77,15 +77,22 @@ export class RedisSession { } hasOpenResources() { - return Boolean(this.socket || this.writer || this.reader || this.parser); + return !this._closed && Boolean(this.socket || this.writer || this.reader || this.parser); } async close() { if (this._closed) return; this._closed = true; - try { this.writer?.close(); } catch { /* already closed */ } - try { this.reader?.releaseLock(); } catch { /* already released */ } - try { this.socket?.close?.(); } catch { /* already closed */ } + const writer = this.writer; + const reader = this.reader; + const socket = this.socket; + this.writer = null; + this.reader = null; + this.parser = null; + this.socket = null; + try { writer?.close(); } catch { /* already closed */ } + try { reader?.releaseLock(); } catch { /* already released */ } + try { socket?.close?.(); } catch { /* already closed */ } } /** @param {RedisCommandEvent} event */ diff --git a/shared/workerd-compat-flags.js b/shared/workerd-compat-flags.js new file mode 100644 index 0000000..7dabeea --- /dev/null +++ b/shared/workerd-compat-flags.js @@ -0,0 +1,62 @@ +/* + * Mirrors workerd v1.20260701.1 src/workerd/io/compatibility-date.capnp. + * Regenerate on every workerd pin bump from an upstream workerd source checkout: + * + * node --input-type=module -e 'import fs from "node:fs"; const src=fs.readFileSync("src/workerd/io/compatibility-date.capnp","utf8"); const blocks=[]; let cur=[]; for (const line of src.split(/\n/)) { if (/^\s*[A-Za-z][A-Za-z0-9_]*\s+@\d+\s+:Bool/.test(line)) { if (cur.length) blocks.push(cur.join("\n")); cur=[line]; } else if (cur.length) cur.push(line); } if (cur.length) blocks.push(cur.join("\n")); const out=new Set(); for (const b of blocks) { if (!b.includes("$experimental")) continue; for (const m of b.matchAll(/\$compatEnableFlag\("([^"]+)"\)/g)) out.add(m[1]); } console.log([...out].sort().join("\n"));' + */ + +export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION = "1.20260701.1"; + +export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS = Object.freeze([ + "allow_insecure_inefficient_logged_eval", + "allow_irrevocable_stub_storage", + "auto_grpc_convert", + "cache_reload_enabled", + "connect_pass_through", + "durable_object_get_existing", + "durable_object_rename", + "enable_abortsignal_rpc", + "enable_ctx_version_metadata", + "enable_d1_with_sessions_api", + "enable_version_api", + "enable_web_file_system", + "experimental", + "increase_websocket_message_size", + "js_rpc", + "kv_direct_binding", + "memory_cache_delete", + "new_module_registry", + "nonclass_entrypoint_reuses_ctx_across_invocations", + "precise_timers", + "python_workers_development", + "python_workers_durable_objects", + "replica_routing", + "rtti_api", + "service_binding_extra_handlers", + "spec_compliant_property_attributes", + "streaming_tail_worker", + "streams_no_default_auto_allocate_chunk_size", + "tail_worker_user_spans", + "typescript_strip_types", + "unique_ctx_per_invocation", + "unsafe_module", + "unsupported_process_actual_platform", + "webgpu", + "workflows_step_rollback", +]); + +const WORKERD_EXPERIMENTAL_COMPAT_FLAG_SET = new Set(WORKERD_EXPERIMENTAL_COMPAT_FLAGS); + +/** @param {unknown} flag */ +export function isWorkerdExperimentalCompatFlag(flag) { + return typeof flag === "string" && WORKERD_EXPERIMENTAL_COMPAT_FLAG_SET.has(flag); +} + +/** @param {unknown} flags */ +export function firstWorkerdExperimentalCompatFlag(flags) { + if (!Array.isArray(flags)) return null; + for (const flag of flags) { + if (isWorkerdExperimentalCompatFlag(flag)) return flag; + } + return null; +} diff --git a/terraform/README.md b/terraform/README.md index 512a8be..e0b6523 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -225,13 +225,14 @@ The EC2 capacity provider uses a fixed Auto Scaling Group: - spread placement by AZ, then by instance ID - ENI trunking enabled with `awsvpcTrunking` -Task `cpu` and `memory` values are placement reservations on EC2 launch type, not -hard per-container caps. CPU is a cgroup share weight. Memory is an ECS placement -reservation because the containers do not set hard container memory limits. -This is especially relevant for D1 and Durable Object runtimes on newer workerd -releases, where SQLite's process hard heap is no longer capped at 512 MiB by -workerd itself; size task density and host memory with that shared-process -behavior in mind. +Task `cpu` and task-level `memory` values are placement reservations on EC2 launch type; +CPU is a cgroup share weight. D1 and Durable Object workerd containers also set explicit +container `memory` hard limits, defaulting to `runtime_memory` and overrideable with +`d1_runtime_container_memory` / `do_runtime_container_memory`. This matters on EC2 +because newer workerd releases no longer cap SQLite's process hard heap at 512 MiB; the +container cap keeps a runaway SQLite query from consuming arbitrary host memory. On +Fargate, task memory is already a hard task ceiling, so the container cap is mostly a +matching documentation of the intended ceiling. The launch template keeps IMDSv2 enabled for the ECS host agent, sets metadata hop limit to 1, and sets `ECS_AWSVPC_BLOCK_IMDS=true` so awsvpc tasks cannot read diff --git a/terraform/main.tf b/terraform/main.tf index 017248b..023ac44 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -40,24 +40,26 @@ module "compute" { secret_envelope_secret_arn = module.data.secret_envelope_secret_arn internal_auth_previous_token_secret_arn = var.internal_auth_previous_token_secret_arn - gateway_desired_count = var.gateway_desired_count - runtime_desired_count = var.runtime_desired_count - d1_runtime_desired_count = var.d1_runtime_desired_count - do_runtime_desired_count = var.do_runtime_desired_count - d1_test_hooks_enabled = var.d1_test_hooks_enabled - do_test_hooks_enabled = var.do_test_hooks_enabled - scheduler_desired_count = var.scheduler_desired_count - workflows_desired_count = var.workflows_desired_count - gateway_cpu = var.gateway_cpu - gateway_memory = var.gateway_memory - system_runtime_cpu = var.system_runtime_cpu - system_runtime_memory = var.system_runtime_memory - runtime_cpu = var.runtime_cpu - runtime_memory = var.runtime_memory - scheduler_cpu = var.scheduler_cpu - scheduler_memory = var.scheduler_memory - workflows_cpu = var.workflows_cpu - workflows_memory = var.workflows_memory + gateway_desired_count = var.gateway_desired_count + runtime_desired_count = var.runtime_desired_count + d1_runtime_desired_count = var.d1_runtime_desired_count + do_runtime_desired_count = var.do_runtime_desired_count + d1_test_hooks_enabled = var.d1_test_hooks_enabled + do_test_hooks_enabled = var.do_test_hooks_enabled + scheduler_desired_count = var.scheduler_desired_count + workflows_desired_count = var.workflows_desired_count + gateway_cpu = var.gateway_cpu + gateway_memory = var.gateway_memory + system_runtime_cpu = var.system_runtime_cpu + system_runtime_memory = var.system_runtime_memory + runtime_cpu = var.runtime_cpu + runtime_memory = var.runtime_memory + d1_runtime_container_memory = var.d1_runtime_container_memory + do_runtime_container_memory = var.do_runtime_container_memory + scheduler_cpu = var.scheduler_cpu + scheduler_memory = var.scheduler_memory + workflows_cpu = var.workflows_cpu + workflows_memory = var.workflows_memory log_level = var.log_level log_retention_days = var.log_retention_days diff --git a/terraform/modules/compute/d1_runtime_service.tf b/terraform/modules/compute/d1_runtime_service.tf index b819e46..d7d489e 100644 --- a/terraform/modules/compute/d1_runtime_service.tf +++ b/terraform/modules/compute/d1_runtime_service.tf @@ -27,6 +27,7 @@ resource "aws_ecs_task_definition" "d1_runtime" { image = var.workerd_image essential = true entryPoint = ["d1-supervisor"] + memory = coalesce(var.d1_runtime_container_memory, var.runtime_memory) portMappings = [{ name = "d1-http" diff --git a/terraform/modules/compute/do_runtime_service.tf b/terraform/modules/compute/do_runtime_service.tf index c11b2ad..30cea1d 100644 --- a/terraform/modules/compute/do_runtime_service.tf +++ b/terraform/modules/compute/do_runtime_service.tf @@ -50,6 +50,7 @@ resource "aws_ecs_task_definition" "do_runtime" { image = var.workerd_image essential = true entryPoint = ["do-supervisor"] + memory = coalesce(var.do_runtime_container_memory, var.runtime_memory) dependsOn = [{ containerName = "redis-proxy" diff --git a/terraform/modules/compute/variables.tf b/terraform/modules/compute/variables.tf index f2f3b9a..1866f2a 100644 --- a/terraform/modules/compute/variables.tf +++ b/terraform/modules/compute/variables.tf @@ -53,6 +53,14 @@ variable "system_runtime_cpu" { type = number } variable "system_runtime_memory" { type = number } variable "runtime_cpu" { type = number } variable "runtime_memory" { type = number } +variable "d1_runtime_container_memory" { + type = number + default = null +} +variable "do_runtime_container_memory" { + type = number + default = null +} variable "scheduler_cpu" { type = number } variable "scheduler_memory" { type = number } variable "workflows_cpu" { type = number } diff --git a/terraform/variables.tf b/terraform/variables.tf index b7d47dd..90397d1 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -201,6 +201,26 @@ variable "runtime_memory" { default = 768 } +variable "d1_runtime_container_memory" { + type = number + default = null + description = "Optional hard memory limit, in MiB, for the d1-runtime container. Defaults to runtime_memory." + validation { + condition = var.d1_runtime_container_memory == null || var.d1_runtime_container_memory > 0 + error_message = "d1_runtime_container_memory must be null or a positive number of MiB." + } +} + +variable "do_runtime_container_memory" { + type = number + default = null + description = "Optional hard memory limit, in MiB, for the do-runtime container. Defaults to runtime_memory." + validation { + condition = var.do_runtime_container_memory == null || var.do_runtime_container_memory > 0 + error_message = "do_runtime_container_memory must be null or a positive number of MiB." + } +} + variable "scheduler_desired_count" { type = number default = 1 diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index 290fe06..27a0344 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -17,6 +17,7 @@ const SHARED_NS_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_QUEUE_KEYS_URL = repositoryFileUrl("shared/queue-keys.js"); const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_WORKERD_COMPAT_FLAGS_URL = repositoryFileUrl("shared/workerd-compat-flags.js"); /** * Compile the control/* module graph against shared/* deps. Returns URLs so @@ -65,6 +66,7 @@ export async function compileControlGraph(opts = {}) { ); const bundleUrl = freshRepositoryModuleDataUrl("control/bundle.js", [ [/from "shared-ns-pattern"/g, `from ${JSON.stringify(SHARED_NS_URL)}`], + [/from "shared-workerd-compat-flags"/g, `from ${JSON.stringify(SHARED_WORKERD_COMPAT_FLAGS_URL)}`], [/from "control-bindings"/g, `from ${JSON.stringify(bindingsUrl)}`], [/from "wdl-package-json-source"/g, `from ${JSON.stringify(packageJsonSourceUrl)}`], ]); diff --git a/tests/helpers/load-do-protocol.js b/tests/helpers/load-do-protocol.js index fd81f52..ef49255 100644 --- a/tests/helpers/load-do-protocol.js +++ b/tests/helpers/load-do-protocol.js @@ -5,6 +5,7 @@ const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_WORKERD_COMPAT_FLAGS_URL = repositoryFileUrl("shared/workerd-compat-flags.js"); const DO_WIRE_GRAMMAR_URL = repositoryFileUrl("do-runtime/protocol/wire-grammar.js"); const DO_ERRORS_URL = repositoryModuleDataUrl("do-runtime/protocol/errors.js", [ [/from "shared-respond";/g, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], @@ -21,6 +22,7 @@ export function doProtocolDataUrl() { [/from "do-runtime-protocol-errors";/g, `from ${JSON.stringify(DO_ERRORS_URL)};`], [/from "do-runtime-protocol-identity";/g, `from ${JSON.stringify(DO_IDENTITY_URL)};`], [/from "shared-worker-id";/g, `from ${JSON.stringify(SHARED_WORKER_ID_URL)};`], + [/from "shared-workerd-compat-flags";/g, `from ${JSON.stringify(SHARED_WORKERD_COMPAT_FLAGS_URL)};`], [/from "shared-bounded-body";/g, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)};`], [/from "shared-internal-auth";/g, `from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)};`], ]); diff --git a/tests/helpers/load-runtime-dispatch.js b/tests/helpers/load-runtime-dispatch.js index 4cc9275..adf5154 100644 --- a/tests/helpers/load-runtime-dispatch.js +++ b/tests/helpers/load-runtime-dispatch.js @@ -7,6 +7,7 @@ import { moduleDataUrl, repositoryFileUrl, repositoryModuleDataUrl, + runtimeLibModuleDataUrl, } from "./load-shared-module.js"; import { runtimeProxyBindingStubUrl, sharedInternalAuthUrl } from "./runtime-proxy-stub.js"; @@ -15,7 +16,7 @@ const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); const RESPOND_URL = repositoryFileUrl("shared/respond.js"); const BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); -const RUNTIME_LIB_URL = repositoryFileUrl("runtime/lib.js"); +const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); const METRICS_MOCK_URL = moduleDataUrl(` export const metrics = { increment() {}, diff --git a/tests/helpers/load-shared-module.js b/tests/helpers/load-shared-module.js index 2b530e3..949dc48 100644 --- a/tests/helpers/load-shared-module.js +++ b/tests/helpers/load-shared-module.js @@ -11,6 +11,9 @@ const REPO_ROOT = path.resolve(__dirname, "../.."); const SHARED_ENV_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/env.js")).href; const SHARED_ERRORS_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/errors.js")).href; const SHARED_HEX_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/hex.js")).href; +const SHARED_WORKERD_COMPAT_FLAGS_URL = pathToFileURL( + path.resolve(REPO_ROOT, "shared/workerd-compat-flags.js") +).href; /** @typedef {Array<[RegExp | string, string]>} ModuleReplacements */ @@ -82,6 +85,12 @@ export function repositoryModuleDataUrl(relativePath, replacements = []) { return moduleDataUrl(readRepositoryModuleSource(relativePath, replacements)); } +export function runtimeLibModuleDataUrl() { + return repositoryModuleDataUrl("runtime/lib.js", importSpecifierReplacements({ + "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + })); +} + /** * Like repositoryModuleDataUrl but cache-busted per call. Returns the URL (not * the imported module) so loaders can chain it into a sibling's replacements. diff --git a/tests/integration/admin-api.test.js b/tests/integration/admin-api.test.js index 97c5463..a191dd9 100644 --- a/tests/integration/admin-api.test.js +++ b/tests/integration/admin-api.test.js @@ -68,6 +68,25 @@ test("deploy rejects malformed optional metadata instead of silently dropping it } }); +test("deploy rejects workerd 0701 unsupported bundle metadata before cold-load", async () => { + const experimental = await adminPost("/ns/test3/worker/unsupported/deploy", { + code: "export default { fetch() { return new Response('ok'); } };", + compatibilityFlags: ["unsafe_module"], + }); + assert.equal(experimental.status, 400); + assert.equal(experimental.json.error, "experimental_compat_flag_unsupported"); + + const python = await adminPost("/ns/test3/worker/unsupported/deploy", { + mainModule: "worker.js", + modules: { + "worker.js": "export default {};", + "mod.py": { py: "print(1)" }, + }, + }); + assert.equal(python.status, 400); + assert.equal(python.json.error, "python_workers_unsupported"); +}); + test("deploy rejects invalid namespace", async () => { const d = await adminPost("/ns/Bad_NS/worker/x/deploy", { code: "x" }); assert.equal(d.status, 400); diff --git a/tests/integration/http-features.test.js b/tests/integration/http-features.test.js index 64908c7..6722099 100644 --- a/tests/integration/http-features.test.js +++ b/tests/integration/http-features.test.js @@ -57,14 +57,17 @@ test("client disconnect response stream behavior is bounded on current workerd", await waitUntil("disconnect marker written", async () => { const r = await gatewayFetch(ns, `/w/poll?key=${encodeURIComponent(key)}`); const text = await r.text(); - // workerd #6832 means 2026-06-19+ no longer reliably calls - // ReadableStream.cancel() for async response bodies on client disconnect. - // Keep this test as a bounded-behavior regression anchor: stock workerd - // 2026-07-01 reports "ended-normally" after the orphaned stream completes. - if (text === "cancel" || text === "enqueue-threw" || text === "ended-normally") return true; + // workerd #6832 means 2026-06-19+ no longer calls ReadableStream.cancel() + // for this async response-body pattern on client disconnect. If "cancel" + // appears again, upstream likely fixed the bug and WDL should restore the + // strict cancel assertion and re-evaluate log-tail watchdog/documentation. + if (text === "cancel") { + throw new Error("upstream workerd #6832 appears fixed; restore strict cancel assertion and re-evaluate WDL stream watchdogs"); + } + if (text === "enqueue-threw" || text === "ended-normally") return true; if (text === "__null__") return false; throw new Error(`unexpected marker value ${JSON.stringify(text)}`); - }, { timeoutMs: 15000, intervalMs: 250 }); + }, { timeoutMs: 30000, intervalMs: 250 }); }); const STREAM_WORKER = ` diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 6fb5237..3f733df 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -19,6 +19,7 @@ const { assertWorkerVersionsUserEnvBudget, decryptSecretHash, estimatedWorkerLoaderEnv, + estimatedWorkerLoaderEnvBytes, } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ "shared-secret-envelope": secretEnvelopeUrl, "shared-version": sharedVersionUrl, @@ -49,7 +50,7 @@ test("worker env budget counts merged vars and secrets with worker-secret preced nsSecrets: { TOKEN: "ns", ONLY_NS: "n" }, workerSecrets: { TOKEN: "worker" }, }), - Buffer.byteLength(JSON.stringify(estimated), "utf8") + estimatedWorkerLoaderEnvBytes(estimated) ); }); @@ -105,6 +106,34 @@ test("worker env budget counts required caller secret copies in service binding ); }); +test("worker env budget accounts for V8 two-byte strings on mixed non-Latin-1 env", () => { + const mixed = `${"x".repeat(Math.floor(WORKER_LOADER_ENV_MAX_BYTES / 2) + 1)}中`; + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + vars: { BIG: mixed }, + }); + const jsonBytes = Buffer.byteLength(JSON.stringify(estimated), "utf8"); + const estimatedBytes = estimatedWorkerLoaderEnvBytes(estimated); + + assert.ok(jsonBytes <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(estimatedBytes > WORKER_LOADER_ENV_MAX_BYTES); + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "api", + vars: { BIG: mixed }, + }), + (err) => { + if (!(err instanceof WorkerEnvBudgetError)) return false; + const budgetErr = /** @type {WorkerEnvBudgetError} */ (err); + assert.equal(budgetErr.code, "worker_env_too_large"); + assert.equal(budgetErr.details.env_bytes, estimatedBytes); + return true; + } + ); +}); + test("worker env budget stores required caller secrets in a null-prototype map", () => { const estimated = estimatedWorkerLoaderEnv({ ns: "demo", @@ -139,13 +168,13 @@ test("worker env budget counts configured assets CDN base", () => { }; const assetsCdnBase = `https://${"assets-subdomain-".repeat(600)}example.test`; /** @param {number} padLength @param {string | null | undefined} cdnBase */ - const bytesWithPad = (padLength, cdnBase) => Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + const bytesWithPad = (padLength, cdnBase) => estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ ns: "demo", worker: "api", vars: { PAD: "x".repeat(padLength) }, meta, assetsCdnBase: cdnBase, - })), "utf8"); + })); const padLength = WORKER_LOADER_ENV_MAX_BYTES - bytesWithPad(0, assetsCdnBase) + 1; assert.ok(bytesWithPad(padLength, null) <= WORKER_LOADER_ENV_MAX_BYTES); @@ -309,13 +338,13 @@ test("worker env budget can estimate a source bundle under a future version stri }], }; /** @param {number} padLength @param {string} version */ - const bytesWithPad = (padLength, version) => Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + const bytesWithPad = (padLength, version) => estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ ns: "demo", worker: "api", version, vars: { PAD: "x".repeat(padLength) }, meta: baseMeta, - })), "utf8"); + })); const padLength = WORKER_LOADER_ENV_MAX_BYTES - bytesWithPad(0, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) + 1; assert.ok(bytesWithPad(padLength, "v1") <= WORKER_LOADER_ENV_MAX_BYTES); assert.ok(bytesWithPad(padLength, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) > WORKER_LOADER_ENV_MAX_BYTES); @@ -351,7 +380,11 @@ test("worker env budget can estimate a source bundle under a future version stri }), (err) => { assert.equal(err instanceof WorkerEnvBudgetError, true); - assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + const budgetErr = /** @type {WorkerEnvBudgetError} */ (err); + assert.equal(budgetErr.code, "worker_env_too_large"); + assert.match(budgetErr.message, /demo\/api@v1/); + assert.equal(budgetErr.details.source_version, "v1"); + assert.equal(budgetErr.details.estimated_version, WORKER_LOADER_ENV_VERSION_PLACEHOLDER); return true; } ); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 1ffdcf3..25409cc 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -3,6 +3,10 @@ import assert from "node:assert/strict"; import { loadControlLib } from "../helpers/load-control-lib.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { RESERVED_NS } from "../../shared/ns-pattern.js"; +import { + WORKERD_EXPERIMENTAL_COMPAT_FLAGS, + WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, +} from "../../shared/workerd-compat-flags.js"; const packageJson = /** @type {any} */ (readRepositoryJson("package.json")); const packageWorkerdDep = packageJson?.dependencies?.workerd; @@ -239,9 +243,21 @@ test("normalizeModule: json null is allowed", () => { assert.equal(r.bytes.toString(), "null"); }); -test("normalizeModule: cjs / py", () => { +test("normalizeModule: cjs", () => { assert.equal(normalizeModule({ cjs: "module.exports = {}" }).type, "cjs"); - assert.equal(normalizeModule({ py: "print(1)" }).type, "py"); +}); + +test("normalizeModule: py is rejected before workerd cold-load", () => { + assert.throws( + () => normalizeModule({ py: "print(1)" }), + (err) => { + if (!(err instanceof Error)) return false; + const coded = /** @type {Error & { code?: unknown, status?: unknown }} */ (err); + return coded.code === "python_workers_unsupported" && + coded.status === 400 && + /Python Workers modules are not supported by WDL/.test(coded.message); + } + ); }); test("normalizeModule: unknown shape throws", () => { @@ -689,6 +705,30 @@ test("prepareBundle: compatibilityFlags preserved", () => { assert.deepEqual(meta.compatibilityFlags, ["nodejs_compat"]); }); +test("prepareBundle: experimental workerd compatibility flags are rejected", () => { + assert.throws( + () => prepareBundle( + "w.js", + { "w.js": "x" }, + { compatibilityFlags: ["nodejs_compat", "unsafe_module"] } + ), + (err) => { + if (!(err instanceof Error)) return false; + const coded = /** @type {Error & { code?: unknown, status?: unknown }} */ (err); + return coded.code === "experimental_compat_flag_unsupported" && + coded.status === 400 && + /"unsafe_module"/.test(coded.message); + } + ); + assert.doesNotThrow(() => + prepareBundle( + "w.js", + { "w.js": "x" }, + { compatibilityFlags: ["nodejs_compat", "no_nodejs_compat"] } + ) + ); +}); + test("prepareBundle: compatibilityDate validates shape before commit", () => { assert.equal( prepareBundle("w.js", { "w.js": "x" }, { compatibilityDate: "2026-04-24" }).meta.compatibilityDate, @@ -710,6 +750,8 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( String(unsupported.getUTCMonth() + 1).padStart(2, "0"), String(unsupported.getUTCDate()).padStart(2, "0"), ].join("-"); + const afterUnsupported = new Date(unsupported); + afterUnsupported.setUTCDate(afterUnsupported.getUTCDate() + 1); assert.equal( validateCompatibilityDate("2026-06-20", new Date("2026-06-30T00:00:00Z")), @@ -720,7 +762,7 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( /must not be later than today UTC/ ); assert.throws( - () => validateCompatibilityDate(unsupportedDate, new Date("2026-12-31T00:00:00Z")), + () => validateCompatibilityDate(unsupportedDate, afterUnsupported), /newer than bundled workerd supports/ ); }); @@ -748,6 +790,18 @@ test("MAX_WORKER_COMPATIBILITY_DATE matches pinned workerd release plus seven da assert.equal(maxWorkerCompatibilityDateFromPackageJson(JSON.stringify({ dependencies: {} })), null); }); +test("workerd experimental compat flag mirror matches pinned workerd source version", () => { + const regenerate = "Regenerate shared/workerd-compat-flags.js from /home/sean/Projects/workerd/src/workerd/io/compatibility-date.capnp using the command in that file header."; + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, WORKERD_VERSION, regenerate); + assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("experimental")); + assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unsafe_module")); + assert.equal( + WORKERD_EXPERIMENTAL_COMPAT_FLAGS.some((flag) => flag.startsWith("no_")), + false, + "only enable flags from $experimental compatibility entries should be mirrored" + ); +}); + test("prepareBundle: compatibilityFlags rejected when not an array (would be silently dropped at runtime floor merge)", () => { assert.throws( () => prepareBundle("w.js", { "w.js": "x" }, { compatibilityFlags: "nodejs_compat" }), diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 7be1ef5..d39eb4e 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -3,8 +3,14 @@ import assert from "node:assert/strict"; import { importRepositoryModuleFresh } from "../helpers/load-shared-module.js"; import { delay, waitUntil } from "../helpers/timing.js"; -function loadLogsTailHandler() { - return importRepositoryModuleFresh("control/handlers/logs-tail.js", [ +/** @param {{ keepaliveMs?: number }} [options] */ +function loadLogsTailHandler(options = {}) { + /** @type {Array<[RegExp | string, string]>} */ + const replacements = []; + if (options.keepaliveMs) { + replacements.push([/const SSE_KEEPALIVE_MS = 5_000;/, `const SSE_KEEPALIVE_MS = ${options.keepaliveMs};`]); + } + replacements.push( [ /import \{ envValueOr \} from "shared-env";/, "const envValueOr = (value, fallback) => value == null || value === '' ? fallback : value;", @@ -61,7 +67,8 @@ function loadLogsTailHandler() { const requireControlLog = () => state.log; const controlTailRedis = () => state.dataRedis || state.redis;`, ], - ]); + ); + return importRepositoryModuleFresh("control/handlers/logs-tail.js", replacements); } function resetTailState() { @@ -228,6 +235,34 @@ test("logs tail max-session watchdog closes even without stream cancel", async ( await reader.cancel().catch(() => {}); }); +test("logs tail idle-pull watchdog closes abandoned streams before max-session", async () => { + resetTailState(); + const { handle } = await loadLogsTailHandler({ keepaliveMs: 5 }); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "1000" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-idle", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + assert.match((await readText(reader)).text, /tail-open/); + await waitUntil("tail idle watchdog to close session", () => + /** @type {any} */ (globalThis).__tailSessions[0]?.closed === true, { + timeoutMs: 500, intervalMs: 5, + }); + await Promise.all(waitUntilPromises); + + assert.equal(/** @type {any} */ (globalThis).__tailSessions.length, 1); + assert.equal(/** @type {any} */ (globalThis).__tailSessions[0].closed, true); + assert.ok(/** @type {any} */ (globalThis).__tailState.logs.some((/** @type {any} */ entry) => entry.event === "tail_session_idle")); + await reader.cancel().catch(() => {}); +}); + test("logs tail closes session if watchdog fires while Redis open is pending", async () => { resetTailState(); const state = /** @type {any} */ (globalThis).__tailState; diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index bff423b..e84ed43 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -414,6 +414,62 @@ test("bumpActiveAndPromote also rewrites full queue consumer projection", async }); }); +test("bumpActiveAndPromote runs beforeStageCopy inside the watched copy transaction", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { queueConsumers: [] }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + /** @type {unknown} */ + let seen = null; + + const result = await bumpActiveAndPromote(redis, "demo", "worker", { + /** @param {{ iso: any, currentVersion: string, newVersion: string, sourceMeta: any }} context */ + async beforeStageCopy(context) { + const { iso, currentVersion, newVersion, sourceMeta } = context; + await iso.watch("secrets:demo", "secrets:demo:worker"); + seen = { + currentVersion, + newVersion, + route: await iso.hGet("routes:demo", "worker"), + queueConsumers: sourceMeta.queueConsumers, + }; + }, + }); + + assert.equal(result.version, "v2"); + assert.deepEqual(seen, { + currentVersion: "v1", + newVersion: "v2", + route: "v1", + queueConsumers: [], + }); + assert.ok(redis.state.watchBatches.some((batch) => + batch.length === 2 && + batch.includes("secrets:demo") && + batch.includes("secrets:demo:worker") + )); + assert.ok(redis.state.hashes.has("bundle:demo:worker:v2")); +}); + +test("bumpActiveAndPromote does not copy or flip routes when beforeStageCopy rejects", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { queueConsumers: [] }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await assert.rejects( + bumpActiveAndPromote(redis, "demo", "worker", { + async beforeStageCopy() { + throw new Error("budget failed"); + }, + }), + /budget failed/ + ); + + assert.equal(redis.state.hashes.has("bundle:demo:worker:v2"), false); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); +}); + test("bumpActiveAndPromote rejects active routes that no longer declare their hosts", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 7c0fcd2..6ed65ef 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -489,8 +489,19 @@ export function stageWorkerVisible(multi, ns, name) { `); const routingStubUrl = moduleDataUrl(` export class RoutingError extends Error {} -export async function bumpActiveAndPromote() { - return { previousVersion: "v1", version: "v2" }; +export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) { + return await redis.session(async (iso) => { + const currentVersion = await iso.hGet(\`routes:\${ns}\`, workerName) || "v1"; + const currentNumber = Number(/^v(\\d+)$/.exec(currentVersion)?.[1] || 1); + const newVersion = \`v\${currentNumber + 1}\`; + await options.beforeStageCopy?.({ + iso, + currentVersion, + newVersion, + sourceMeta: {}, + }); + return { previousVersion: currentVersion, version: newVersion }; + }); } `); const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/worker-secrets.js"), [ @@ -508,6 +519,7 @@ const { WORKER_LOADER_ENV_MAX_BYTES, WORKER_LOADER_ENV_VERSION_PLACEHOLDER, estimatedWorkerLoaderEnv, + estimatedWorkerLoaderEnvBytes, } = await import(envBudgetUrl()); test("worker secret PUT encrypts before WATCH retries and reuses the envelope", async () => { @@ -640,14 +652,14 @@ test("worker secret PUT budgets the copied active bundle under a future version }], }; /** @param {number} padLength @param {string} version */ - const bytesWithPad = (padLength, version) => Buffer.byteLength(JSON.stringify(estimatedWorkerLoaderEnv({ + const bytesWithPad = (padLength, version) => estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ ns: "demo", worker: "api", version, vars: { PAD: "x".repeat(padLength) }, workerSecrets: { TOKEN: "plain-secret" }, meta: baseMeta, - })), "utf8"); + })); const padLength = WORKER_LOADER_ENV_MAX_BYTES - bytesWithPad(0, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) + 1; @@ -707,6 +719,89 @@ test("worker secret PUT budgets the copied active bundle under a future version } }); +test("worker secret PUT rechecks the active bundle copied by bump after the secret write", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const writesBefore = state.redis.writes.length; + let sessionCalls = 0; + let mutateExecCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => { + sessionCalls += 1; + return await fn({ + /** @param {...string} keys */ + async watch(...keys) { + state.redis.watchedKeys.push(...keys); + }, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return []; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") { + return sessionCalls === 1 ? "v1" : "v2"; + } + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { SAFE: "ok" } }); + } + if (key === "worker:demo:api:v:2" && field === "__meta__") { + return JSON.stringify({ vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES + 1) } }); + } + return null; + }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo:api") { + const written = state.redis.writes.at(-1); + return written?.key === key ? { [written.field]: written.value } : {}; + } + return {}; + }, + async zCard() { return 1; }, + async zRange() { return []; }, + multi() { + return { + /** @param {string} key @param {string} field @param {string} value */ + hSet(key, field, value) { + state.redis.writes.push({ key, field, value }); + }, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { mutateExecCalled = true; }, + }; + }, + }); + }; + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-bump-recheck", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(mutateExecCalled, true); + assert.equal(state.redis.writes.length, writesBefore + 1); + assert.equal(body.secretWritten, true); + assert.equal(body.reloadForced, false); + assert.equal(body.warnings[0].kind, "promote_failed"); + assert.match(body.warnings[0].reason, /workerLoader env/); + assert.ok(state.redis.watchedKeys.includes("secrets:demo")); + assert.ok(state.redis.watchedKeys.includes("secrets:demo:api")); + } finally { + state.redis.session = originalSession; + } +}); + test("worker secret DELETE skips decrypting the removed corrupt envelope", async () => { const { state } = await import(workerControlSharedUrl); const originalSession = state.redis.session; diff --git a/tests/unit/do-alarm-shim.test.js b/tests/unit/do-alarm-shim.test.js index 00692b9..8f43e0a 100644 --- a/tests/unit/do-alarm-shim.test.js +++ b/tests/unit/do-alarm-shim.test.js @@ -446,6 +446,42 @@ test("DO alarm shim: deleteAll clears alarm row and cancels backend schedule by assert.equal(kv.size, 0); }); +test("DO alarm shim: deleteAll skips _cf_ reserved SQL objects case-insensitively", async () => { + /** @type {string[]} */ + const dropped = []; + const storage = { + sql: { + /** @param {string} statement */ + exec(statement) { + if (statement.startsWith("CREATE TABLE")) return []; + if (statement.startsWith("SELECT scheduled_time")) return []; + if (statement.startsWith("SELECT type, name FROM sqlite_master")) { + return [ + { type: "table", name: "_CF_legacy" }, + { type: "index", name: "_Cf_legacy_idx" }, + { type: "table", name: "tenant_table" }, + ]; + } + if (statement.startsWith("PRAGMA foreign_keys")) return []; + if (statement.startsWith("DROP ")) { + dropped.push(statement); + return []; + } + throw new Error(`unexpected SQL: ${statement}`); + }, + }, + async list() { + return new Map(); + }, + async delete() {}, + }; + const wrapped = wrapStorage(storage, makeDoAlarmBinding([]), "Room", "alice"); + + await wrapped.deleteAll(); + + assert.deepEqual(dropped, ['DROP TABLE IF EXISTS "tenant_table"']); +}); + test("DO alarm shim: deleteAll deleteAlarm false preserves alarm row without backend cancel", async () => { /** @type {unknown[][]} */ const calls = []; diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 09a4cb6..60790cf 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -239,6 +239,22 @@ test("normalizes inline workerCode only for test hooks", () => { assert.equal("workerCode" in invoke ? "allowExperimental" in invoke.workerCode : undefined, false); }); +test("rejects experimental workerd compatibility flags in inline workerCode", () => { + assert.throws( + () => normalizeDoInvokeRequest({ + ...INLINE_BODY, + workerCode: { + ...INLINE_BODY.workerCode, + compatibilityFlags: ["nodejs_compat", "unsafe_module"], + }, + }, { allowInlineWorkerCode: true }), + (err) => err instanceof DoRuntimeError && + err.status === 400 && + err.code === "experimental_compat_flag_unsupported" && + /"unsafe_module"/.test(err.message) + ); +}); + test("builds forwarded Request for user durable object fetch", async () => { const invoke = withRequestBody(normalizeDoInvokeRequest({ ...BASE_BODY, diff --git a/tests/unit/redis-session.test.js b/tests/unit/redis-session.test.js index 084c066..113a46f 100644 --- a/tests/unit/redis-session.test.js +++ b/tests/unit/redis-session.test.js @@ -572,6 +572,7 @@ test("RedisSession reports resources while SELECT is still pending", async () => assert.equal(session.hasOpenResources(), true); await Promise.resolve(); await session.close(); + assert.equal(session.hasOpenResources(), false); assert.equal(writer.closed, true); assert.equal(reader.released, true); assert.equal(socket.closed, true); @@ -663,7 +664,9 @@ test("RedisSession.close is idempotent and blocks further commands", async () => // close() twice and assert the post-close command throws. const s = new RedisSession("x", { connect }); await s.open(); + assert.equal(s.hasOpenResources(), true); await s.close(); + assert.equal(s.hasOpenResources(), false); await s.close(); // idempotent — no throw await assert.rejects(() => s.get("k"), /session closed/); }); diff --git a/tests/unit/runtime-kv-binding.test.js b/tests/unit/runtime-kv-binding.test.js index 53c1c7a..3d1bc68 100644 --- a/tests/unit/runtime-kv-binding.test.js +++ b/tests/unit/runtime-kv-binding.test.js @@ -1,6 +1,10 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { + importRepositoryModule, + repositoryFileUrl, + runtimeLibModuleDataUrl, +} from "../helpers/load-shared-module.js"; import { installMockFetch, makeRecordingFetch } from "../helpers/mock-fetch.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { RUNTIME_METRICS_NOOP_URL } from "../helpers/mocks/runtime-metrics.js"; @@ -8,7 +12,7 @@ import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); -const RUNTIME_LIB_URL = repositoryFileUrl("runtime/lib.js"); +const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); /** @param {Array<[RegExp | string, string]>} [replacements] */ diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 021d60b..76b96bb 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -1,7 +1,9 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseBase64Json } from "../helpers/json-payload.js"; -import { +import { runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; + +const { toBytes, bundleToWorkerCode, buildAssetUrl, @@ -13,7 +15,7 @@ import { normalizeQueueDelaySeconds, normalizeQueuedDispatchBody, normalizeScheduledDispatchBody, -} from "../../runtime/lib.js"; +} = await import(runtimeLibModuleDataUrl()); const enc = new TextEncoder(); @@ -241,7 +243,7 @@ function mkBundle(meta, files) { return out; } -test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => { +test("bundleToWorkerCode: module + data + wasm + json + text + cjs", () => { const meta = { mainModule: "worker.js", modules: { @@ -251,7 +253,6 @@ test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => "config.json": { type: "json" }, "readme.txt": { type: "text" }, "commonjs.cjs": { type: "cjs" }, - "mod.py": { type: "py" }, }, }; const code = bundleToWorkerCode( @@ -262,7 +263,6 @@ test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => "config.json": enc.encode('{"k":1}'), "readme.txt": enc.encode("hello"), "commonjs.cjs": enc.encode("module.exports = 1"), - "mod.py": enc.encode("x = 1"), }) ); assert.equal(code.mainModule, "worker.js"); @@ -273,13 +273,23 @@ test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => const jsonModule = /** @type {{ json: unknown }} */ (code.modules["config.json"]); const textModule = /** @type {{ text: string }} */ (code.modules["readme.txt"]); const cjsModule = /** @type {{ cjs: string }} */ (code.modules["commonjs.cjs"]); - const pyModule = /** @type {{ py: string }} */ (code.modules["mod.py"]); assert.deepEqual(Array.from(dataModule.data), [1, 2]); assert.deepEqual(Array.from(wasmModule.wasm), [0, 97, 115, 109]); assert.deepEqual(jsonModule.json, { k: 1 }); assert.equal(textModule.text, "hello"); assert.equal(cjsModule.cjs, "module.exports = 1"); - assert.equal(pyModule.py, "x = 1"); +}); + +test("bundleToWorkerCode: py modules fail closed with WDL error", () => { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { mainModule: "worker.js", modules: { "worker.js": { type: "module" }, "mod.py": { type: "py" } } }, + { "worker.js": enc.encode("export default {}"), "mod.py": enc.encode("x = 1") } + ) + ), + /Module "mod\.py": Python Workers modules are not supported by WDL/ + ); }); test("bundleToWorkerCode: uses meta.compatibilityDate when set", () => { @@ -388,6 +398,22 @@ test("bundleToWorkerCode: throws (doesn't silently drop) on malformed compatibil ); }); +test("bundleToWorkerCode: experimental workerd compatibility flags fail closed", () => { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { + mainModule: "w.js", + compatibilityFlags: ["nodejs_compat", "unsafe_module"], + modules: { "w.js": { type: "module" } }, + }, + { "w.js": enc.encode("x") } + ) + ), + /meta\.compatibilityFlags contains experimental workerd flag "unsafe_module"/ + ); +}); + test("bundleToWorkerCode: missing __meta__ throws", () => { assert.throws( () => bundleToWorkerCode({ "worker.js": enc.encode("x") }), diff --git a/tests/unit/runtime-queue-producer.test.js b/tests/unit/runtime-queue-producer.test.js index cbbc29e..36aadf3 100644 --- a/tests/unit/runtime-queue-producer.test.js +++ b/tests/unit/runtime-queue-producer.test.js @@ -1,6 +1,9 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { + importRepositoryModule, + runtimeLibModuleDataUrl, +} from "../helpers/load-shared-module.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { RUNTIME_METRICS_NOOP_URL } from "../helpers/mocks/runtime-metrics.js"; import { withRecordingFetch } from "../helpers/mock-fetch.js"; @@ -10,7 +13,7 @@ import { parseJsonArrayRequestBody } from "../helpers/request-body.js"; import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); -const RUNTIME_LIB_URL = repositoryFileUrl("runtime/lib.js"); +const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); const IMMEDIATE_VISIBLE_AT = 0; // Arbitrary fixed Unix epoch ms for deterministic time-based queue assertions. const MOCK_TIMESTAMP_BASE_MS = 1_700_000_000_000; diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 1cb3963..35bdb65 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1885,6 +1885,26 @@ test("Terraform gates DO test hooks like D1 test hooks", () => { assert.match(doService, /!var\.do_test_hooks_enabled \|\| can\(regex\("\(\^\|-\)test\(\$\|-\)", var\.name\)\)/); }); +test("D1 and DO workerd containers keep explicit memory ceilings", () => { + const rootVars = readRepoFile("terraform/variables.tf"); + const moduleVars = readRepoFile("terraform/modules/compute/variables.tf"); + const main = readRepoFile("terraform/main.tf"); + const d1Service = readRepoFile("terraform/modules/compute/d1_runtime_service.tf"); + const doService = readRepoFile("terraform/modules/compute/do_runtime_service.tf"); + const d1Kube = readRepoFile("deploy/kubernetes/base/d1-runtime.yaml"); + const doKube = readRepoFile("deploy/kubernetes/base/do-runtime.yaml"); + + for (const name of ["d1_runtime_container_memory", "do_runtime_container_memory"]) { + assert.match(rootVars, new RegExp(`variable "${name}"`)); + assert.match(moduleVars, new RegExp(`variable "${name}"`)); + assert.match(main, new RegExp(`${name}\\s+=\\s+var\\.${name}`)); + } + assert.match(d1Service, /memory\s+=\s+coalesce\(var\.d1_runtime_container_memory, var\.runtime_memory\)/); + assert.match(doService, /memory\s+=\s+coalesce\(var\.do_runtime_container_memory, var\.runtime_memory\)/); + assert.match(d1Kube, /name: d1-runtime[\s\S]*?limits:\n\s+memory: 1Gi/); + assert.match(doKube, /name: do-runtime[\s\S]*?limits:\n\s+memory: 1Gi/); +}); + test("EC2 capacity hosts block awsvpc task access to host IMDS", () => { const terraformCapacity = readRepoFile("terraform/modules/compute/ec2_capacity.tf"); From 725e40e2dff292cfdcf63ffe7cd4770979bffa40 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 2 Jul 2026 17:17:19 +0800 Subject: [PATCH 10/13] Close workerd 0701 rollout gaps Fix the experimental flag mirror and add read-only rollout scanning for retained metadata blockers, including missing metadata, Python modules, experimental flags, and env-size risks without decrypting secrets. Document the TLS, log-tail, runtime scanner, and Terraform memory boundaries for the 2026-07-01 workerd adaptation. Signed-off-by: Lu Zhang --- control/handlers/logs-tail.js | 2 +- docs/compatibility.md | 9 + docs/compatibility.zh.md | 2 + docs/modules/control-auth.md | 10 +- docs/modules/log-tail-observability.md | 12 +- docs/modules/log-tail-observability.zh.md | 2 +- docs/modules/runtime.md | 15 +- docs/modules/runtime.zh.md | 2 +- docs/source-map.md | 2 +- docs/source-map.zh.md | 2 +- runtime/lib.js | 4 +- scripts/scan-workerd-0701-metadata.mjs | 324 ++++++++++++++++-- shared/workerd-compat-flags.js | 4 +- terraform/README.md | 17 +- tests/unit/control-lib.test.js | 8 +- tests/unit/scan-workerd-0701-metadata.test.js | 150 ++++++++ 16 files changed, 498 insertions(+), 67 deletions(-) create mode 100644 tests/unit/scan-workerd-0701-metadata.test.js diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index 75fea98..530f451 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -1,7 +1,7 @@ // SSE handler for `wdl tail`. Pull-based ReadableStream + pre-registered // ctx.waitUntil(cancelPromise) keeps cleanup outside the cancel callback. // workerd >= 2026-06-19 no longer reliably calls cancel() on client -// disconnect, so max-session cleanup also has an independent watchdog. +// disconnect, so max-session and idle-pull cleanup use independent watchdogs. import { RedisSession, redisDbFromEnv } from "shared-redis"; import { envValueOr } from "shared-env"; diff --git a/docs/compatibility.md b/docs/compatibility.md index 19a1d40..a3883eb 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -35,6 +35,15 @@ Each row separates four compatibility claims: | `nodejs_compat` | Partial | workerd-provided compatibility when the runtime service has the flag enabled. | CLI carries compatibility flags into metadata. | WDL exposes the workerd-enabled compatibility surface rather than a separate Node.js runtime. | This is not a full Node.js platform contract beyond workerd's enabled surface. | | Python Workers modules | Not supported | workerd has an experimental Python Workers path. | Control rejects `py` module manifests with `python_workers_unsupported`; runtime and do-runtime also fail closed for retained metadata containing `py` modules. | WDL keeps tenant bundles JavaScript/WebAssembly/data only and does not permit cold-load-time Pyodide bootstrap. | Python Workers and mixed JS/Python bundles are unsupported. | +Node.js TLS behavior follows the bundled workerd binary. With workerd 2026-07-01, +workers whose compatibility date is at least 2026-06-16 get +`throw_on_not_implemented_tls_options`: unsupported `node:tls` options such as +`checkServerIdentity` now throw `ERR_OPTION_NOT_IMPLEMENTED` instead of being silently +ignored. This can affect already-deployed versions whose date is in the 2026-06-16 +through 2026-06-24 window when WDL upgrades from the 2026-06-17 workerd pin. Separately, +workerd's `servername` / expected-certificate-hostname behavior changed outside any +compatibility flag, so certificate hostname validation can change for all dates. + ## Bindings And Storage | Surface | Status | What workerd provides | Stronger / added in WDL | Different from Cloudflare | Not implemented / gaps | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index 8e72d51..1cdec2f 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -28,6 +28,8 @@ | `nodejs_compat` | Partial | runtime service 启用 flag 后由 workerd 提供的兼容 surface。 | CLI 把 compatibility flags 带入 metadata。 | WDL 暴露 workerd 已启用的兼容 surface,而不是单独 Node.js runtime。 | 不承诺超出 workerd 已启用 surface 的完整 Node.js 平台能力。 | | Python Workers modules | Not supported | workerd 有 experimental Python Workers 路径。 | Control 用 `python_workers_unsupported` 拒绝 `py` module manifest;runtime 和 do-runtime 对 retained metadata 中的 `py` module 也 fail closed。 | WDL 保持 tenant bundle 只包含 JavaScript/WebAssembly/data,不允许 cold-load 时触发 Pyodide bootstrap。 | 不支持 Python Workers 和 JS/Python 混合 bundle。 | +Node.js TLS 行为跟随 bundled workerd binary。升级到 workerd 2026-07-01 后,compatibility date 不早于 2026-06-16 的 worker 会拿到 `throw_on_not_implemented_tls_options`:`node:tls` 中尚未实现的选项(例如 `checkServerIdentity`)会从“静默忽略”变为抛 `ERR_OPTION_NOT_IMPLEMENTED`。从 2026-06-17 workerd pin 升级时,这会影响已经部署且日期在 2026-06-16 到 2026-06-24 之间的版本。另外,workerd 的 `servername` / expected-certificate-hostname 行为变化不受任何 compatibility flag 门控,因此证书 hostname 校验可能对所有日期变化。 + ## Bindings 和存储 | Surface | 状态 | workerd 提供什么 | WDL 增强 / 新增什么 | 与 Cloudflare 的模型差异 | 未实现 / 缺口 | diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index 5b16d1f..6f2ddcc 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -256,11 +256,11 @@ Auth-specific contract: explicitly owns a diagnostic response field. - Deploy and secret mutations return `worker_code_too_large` when tenant module bodies exceed the workerd 64 MiB dynamic code limit, and `worker_env_too_large` when the - estimated `workerLoader` env exceeds WDL's headroomed 1 MiB budget. `worker_env_too_large` - details include `namespace`, optional `worker`, `env_bytes`, `max_env_bytes`, - `upstream_max_env_bytes`, and `headroom_bytes`. When the blocker is an already-retained - version being re-estimated during a secret mutation, details also include - `source_version` and `estimated_version`, and the message identifies + estimated `workerLoader` env exceeds WDL's headroomed 1 MiB budget. + `worker_env_too_large` details include `namespace`, optional `worker`, `env_bytes`, + `max_env_bytes`, `upstream_max_env_bytes`, and `headroom_bytes`. When the blocker is an + already-retained version being re-estimated during a secret mutation, details also + include `source_version` and `estimated_version`, and the message identifies `/@` so operators can find the retained version to delete or redeploy. - Control never calls gateway directly. It writes Redis and publishes invalidation diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index e3317a3..86484e1 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -84,11 +84,13 @@ pipe, not durable log storage. - Active tail sessions are time-bounded authorization leases and must reconnect through normal auth. `LOG_TAIL_MAX_SESSION_MS` sets the control-side maximum; invalid or empty values fall back to 15 minutes. -- workerd issue [#6832](https://github.com/cloudflare/workerd/issues/6832) means - client disconnects may not call async response-body `ReadableStream.cancel()`. - Control therefore has independent watchdogs: a max-session watchdog for - reauthorization and an idle-pull watchdog that closes a session when the SSE body has - not been pulled for three keepalive intervals. Active clients naturally pull at the +- Current stock workerd behavior, tracked upstream as + [#6832](https://github.com/cloudflare/workerd/issues/6832), does not reliably call + async response-body `ReadableStream.cancel()` on client disconnect. WDL treats this + as a permanent compatibility boundary: Control has independent watchdogs, not a + temporary workaround waiting on upstream. The max-session watchdog bounds + reauthorization, and the idle-pull watchdog closes a session when the SSE body has not + been pulled for three keepalive intervals. Active clients naturally pull at the keepalive cadence because each heartbeat frees queue space; abandoned clients stop pulling and are cleaned up without waiting for the full session lifetime. A TCP connection that stays open but whose application stops reading for that window may be diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index e9614fc..1892098 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -64,7 +64,7 @@ Tail streams 使用有界 `MAXLEN ~ 500`,并在写入时刷新 TTL。它们是 - 结构化 stdout 是持久平台日志的事实来源。 - 没有 active tailer 时,runtime 仍输出 stdout,但在本地 active-set miss cache 后跳过 per-event stream append work。 - Active tail session 是有时限的授权租约,必须通过正常 auth reconnect。`LOG_TAIL_MAX_SESSION_MS` 设置 control-side 最大时长;非法值或空值会回退到 15 分钟。 -- workerd issue [#6832](https://github.com/cloudflare/workerd/issues/6832) 会导致 client disconnect 不一定触发 async response-body `ReadableStream.cancel()`。Control 因此有独立 watchdog:max-session watchdog 负责 reauthorization 上界,idle-pull watchdog 在 SSE body 连续三个 keepalive 周期没有被 pull 时关闭 session。活跃客户端会因为每次 heartbeat 腾出 queue 空间而自然按 keepalive 粒度继续 pull;遗弃客户端会停止 pull,因此不需要等完整 session lifetime 才清理。TCP 连接还在但应用层长时间不读的客户端可能被关闭,应自行 reconnect。 +- 当前 stock workerd 的行为(上游 issue [#6832](https://github.com/cloudflare/workerd/issues/6832) 跟踪)不会可靠地在 client disconnect 时触发 async response-body `ReadableStream.cancel()`。WDL 把它当作永久兼容边界处理:Control 的独立 watchdog 不是等待上游修复的临时 workaround。max-session watchdog 负责 reauthorization 上界,idle-pull watchdog 在 SSE body 连续三个 keepalive 周期没有被 pull 时关闭 session。活跃客户端会因为每次 heartbeat 腾出 queue 空间而自然按 keepalive 粒度继续 pull;遗弃客户端会停止 pull,因此不需要等完整 session lifetime 才清理。TCP 连接还在但应用层长时间不读的客户端可能被关闭,应自行 reconnect。 - 与 activation race 的 tail event 可以丢失。 - 高 QPS 或慢 SSE reader 可能因为 stream cap 丢中间事件。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index e726288..6432271 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -250,10 +250,17 @@ when a matching active tail session exists. - Python Workers modules are not supported. Control rejects new `py` module manifests, and runtime/do-runtime reject retained metadata that contains them instead of letting workerd fail later with a mixed JS/Python bundle error. -- Before rolling the 2026-07-01 workerd adaptation over an existing Redis DB 0, run - `node scripts/scan-workerd-0701-metadata.mjs` against the control Redis database to - find retained versions that still contain Python modules or upstream experimental - compatibility flags. +- Before rolling the 2026-07-01 workerd adaptation over an existing Redis DB 0, install + `redis-cli` and run `node scripts/scan-workerd-0701-metadata.mjs` against the control + Redis database to find retained versions that are missing metadata, contain Python + modules or upstream experimental compatibility flags, or exceed the headroomed + `workerLoader` env budget under the 2026-07-01 estimator. The scanner is read-only and + does not decrypt secrets; it uses encrypted envelope length as a conservative + two-byte string upper bound. Treat `worker_env_too_large` findings involving secrets + as rollout blockers to review precisely, for example by redeploying or using the + normal secret mutation API path that runs the exact decrypting estimator. Set + `ASSETS_CDN_BASE` when scanning deployments whose assets CDN base is longer than the + default placeholder. - The runtime workerd processes still run with process-level `--experimental` because upstream workerd 2026-07-01 continues to gate `workerLoader` bindings on that switch. Do not re-add the `experimental` compatibility flag or `allowExperimental` to loaded diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index a283b86..d50e7a1 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -108,7 +108,7 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - 移除 loaded worker 的宽泛 `experimental` flag 会有意收紧 tenant 对非 GA experimental-only surface 的访问,例如不可撤销的长期 stub storage。不要为了兼容绕过而重新打开它,除非先完成明确的功能设计。 - Control 会在 deploy 时拒绝上游 `$experimental` compatibility enable flags,runtime 也会拒绝仍带有这些 flags 的 retained metadata。`no_*` 这类 disable-style flag 不属于这个 mirror,除非上游把对应 enable flag 本身标为 experimental。 - Python Workers modules 不受支持。Control 会拒绝新的 `py` module manifest,runtime/do-runtime 会拒绝 retained metadata 中的 `py` module,而不是让 workerd 之后抛 mixed JS/Python bundle error。 -- 在已有 Redis DB 0 上 rollout 2026-07-01 workerd 适配前,先对 control Redis database 运行 `node scripts/scan-workerd-0701-metadata.mjs`,找出仍包含 Python modules 或上游 experimental compatibility flags 的 retained versions。 +- 在已有 Redis DB 0 上 rollout 2026-07-01 workerd 适配前,先安装 `redis-cli`,再对 control Redis database 运行 `node scripts/scan-workerd-0701-metadata.mjs`,找出缺少 metadata、仍包含 Python modules 或上游 experimental compatibility flags,或按 2026-07-01 估算器会超过 headroomed `workerLoader` env budget 的 retained versions。这个 scanner 只读且不解密 secret;它用 encrypted envelope length 作为 conservative two-byte string 上界。涉及 secret 的 `worker_env_too_large` finding 应作为 rollout blocker 精确复核,例如 redeploy,或走正常 secret mutation API path 触发会解密的精确估算。扫描 assets CDN base 比默认 placeholder 更长的部署时设置 `ASSETS_CDN_BASE`。 - Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 - 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会在 control plane 用留有 headroom 的 `workerLoader` env budget 预检,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 - 当前 stock workerd 中,客户端在 async `ReadableStream` response body 中途断开时,不一定调用 stream source 的 `cancel()`。Tenant streaming/SSE worker 应使用自己的 heartbeat、timeout 或应用层 close path,不要把 disconnect-driven `cancel()` 当成唯一资源清理信号。 diff --git a/docs/source-map.md b/docs/source-map.md index ee650dc..9842096 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -104,7 +104,7 @@ are outside this map unless they own runtime or deployable service behavior. | `examples/` | Manual demos and reference projects. Tests should not silently depend on them unless the fixture graduates to `test-workers/`. | | `scripts/run-integration-tests.js` | Integration worker-pool runner. | | `scripts/compile-workerd-configs.js` | Compiles workerd Cap'n Proto configs into `dist/workerd-configs/*.bin`. | -| `scripts/scan-workerd-0701-metadata.mjs` | Read-only Redis metadata scanner for rollout checks before the workerd 2026-07-01 adaptation: reports retained versions using experimental flags or Python modules. | +| `scripts/scan-workerd-0701-metadata.mjs` | Read-only Redis metadata scanner for rollout checks before the workerd 2026-07-01 adaptation: reports retained versions with missing metadata, experimental flags, Python modules, or oversized estimated env. | ## Infrastructure diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index aaa9aff..8b13635 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -101,7 +101,7 @@ | `examples/` | 手工 demo 和 reference projects。测试不应悄悄依赖它们,除非 fixture 明确迁入 `test-workers/`。 | | `scripts/run-integration-tests.js` | Integration worker-pool runner。 | | `scripts/compile-workerd-configs.js` | 把 workerd Cap'n Proto configs 编译成 `dist/workerd-configs/*.bin`。 | -| `scripts/scan-workerd-0701-metadata.mjs` | workerd 2026-07-01 适配 rollout 前使用的只读 Redis metadata scanner,报告使用 experimental flags 或 Python modules 的 retained versions。 | +| `scripts/scan-workerd-0701-metadata.mjs` | workerd 2026-07-01 适配 rollout 前使用的只读 Redis metadata scanner,报告缺失 metadata、使用 experimental flags、Python modules 或估算 env 超限的 retained versions。 | ## Infrastructure diff --git a/runtime/lib.js b/runtime/lib.js index 3822f83..8e3e0f8 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -1,6 +1,6 @@ // Pure helpers for the runtime worker. -import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; +import { isWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; const BASE64_CHUNK_SIZE = 0x8000; const utf8Encoder = new TextEncoder(); @@ -161,7 +161,7 @@ function mergeCompatFlags(userFlags, compatibilityDate) { `meta.compatibilityFlags entries must be non-empty strings, got ${JSON.stringify(f)}` ); } - if (firstWorkerdExperimentalCompatFlag([f])) { + if (isWorkerdExperimentalCompatFlag(f)) { throw new Error( `meta.compatibilityFlags contains experimental workerd flag ${JSON.stringify(f)}, which WDL does not support for tenant workers` ); diff --git a/scripts/scan-workerd-0701-metadata.mjs b/scripts/scan-workerd-0701-metadata.mjs index 8ed0930..497ce1d 100644 --- a/scripts/scan-workerd-0701-metadata.mjs +++ b/scripts/scan-workerd-0701-metadata.mjs @@ -1,24 +1,69 @@ #!/usr/bin/env node import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { firstWorkerdExperimentalCompatFlag } from "../shared/workerd-compat-flags.js"; -const redisUrl = redisUrlFromEnv(); -const pattern = "worker:*:*:v:*"; +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, ".."); +const REDIS_PATTERN = "worker:*:*:v:*"; +const TWO_BYTE_SECRET_CHAR = "\u0100"; -function redisUrlFromEnv() { - const raw = process.env.REDIS_URL || process.env.REDIS_ADDR || "redis://127.0.0.1:6379/0"; +const { + estimatedWorkerLoaderEnv, + estimatedWorkerLoaderEnvBytes, + WORKER_LOADER_ENV_MAX_BYTES, + UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_HEADROOM_BYTES, +} = await importControlEnvBudget(); + +/** @param {string} relativePath */ +function repoFileUrl(relativePath) { + return pathToFileURL(path.resolve(REPO_ROOT, relativePath)).href; +} + +/** + * @param {string} relativePath + * @param {Array<[RegExp | string, string]>} [replacements] + */ +function repoModuleDataUrl(relativePath, replacements = []) { + let source = readFileSync(path.resolve(REPO_ROOT, relativePath), "utf8"); + for (const [pattern, replacement] of replacements) { + source = source.replace(pattern, replacement); + } + return `data:text/javascript,${encodeURIComponent(source)}`; +} + +async function importControlEnvBudget() { + return await import(repoModuleDataUrl("control/env-budget.js", [ + [/from "shared-secret-envelope";/, `from ${JSON.stringify(repoFileUrl("shared/secret-envelope.js"))};`], + [/from "shared-version";/, `from ${JSON.stringify(repoFileUrl("shared/version.js"))};`], + ])); +} + +export function redisUrlFromEnv(env = process.env) { + const raw = env.REDIS_URL || env.REDIS_ADDR || "redis://127.0.0.1:6379/0"; if (/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) return raw; return `redis://${raw}`; } -/** @param {string[]} args */ -function redisCli(args) { - const result = spawnSync("redis-cli", ["-u", redisUrl, "--raw", ...args], { +/** + * @param {string[]} args + * @param {{ redisUrl?: string, spawn?: typeof spawnSync }} [options] + */ +export function redisCli(args, { redisUrl = redisUrlFromEnv(), spawn = spawnSync } = {}) { + const result = spawn("redis-cli", ["-u", redisUrl, "--raw", ...args], { encoding: "utf8", maxBuffer: 64 * 1024 * 1024, }); if (result.error) { + if (/** @type {NodeJS.ErrnoException} */ (result.error).code === "ENOENT") { + throw new Error( + "redis-cli not found. Install redis-cli/redis-tools before running scripts/scan-workerd-0701-metadata.mjs; local compose users can inspect Redis with `docker compose exec -T redis redis-cli`." + ); + } throw result.error; } if (result.status !== 0) { @@ -27,15 +72,30 @@ function redisCli(args) { return result.stdout; } +/** @param {string} raw */ +export function parseRedisHash(raw) { + const lines = raw.split("\n"); + if (lines.at(-1) === "") lines.pop(); + if (lines.length % 2 !== 0) { + throw new Error("redis-cli HGETALL returned an odd number of raw lines"); + } + /** @type {Record} */ + const out = Object.create(null); + for (let i = 0; i < lines.length; i += 2) { + out[lines[i]] = lines[i + 1]; + } + return out; +} + /** @param {string} key */ -function parseBundleKey(key) { +export function parseBundleKey(key) { const match = /^worker:([^:]+):([^:]+):v:([1-9][0-9]*)$/.exec(key); if (!match) return { namespace: null, worker: null, version: null }; return { namespace: match[1], worker: match[2], version: `v${match[3]}` }; } /** @param {unknown} meta */ -function pythonModules(meta) { +export function pythonModules(meta) { if (!meta || typeof meta !== "object" || Array.isArray(meta)) return []; const modules = /** @type {{ modules?: unknown }} */ (meta).modules; if (!modules || typeof modules !== "object" || Array.isArray(modules)) return []; @@ -50,36 +110,146 @@ function pythonModules(meta) { } /** @param {unknown} meta */ -function experimentalFlag(meta) { +export function experimentalFlag(meta) { if (!meta || typeof meta !== "object" || Array.isArray(meta)) return null; return firstWorkerdExperimentalCompatFlag(/** @type {{ compatibilityFlags?: unknown }} */ (meta).compatibilityFlags); } -const keys = redisCli(["--scan", "--pattern", pattern]) - .split("\n") - .map((line) => line.trim()) - .filter(Boolean); - -/** @type {Array>} */ -const findings = []; -for (const key of keys) { +/** + * @param {string} key + * @param {string} rawMeta + */ +function parseMetadataRecord(key, rawMeta) { const identity = parseBundleKey(key); - const rawMeta = redisCli(["HGET", key, "__meta__"]).replace(/\n$/, ""); - if (!rawMeta) continue; + if (!rawMeta) { + return { + identity, + meta: null, + findings: [{ kind: "missing_meta", key, ...identity }], + }; + } + /** @type {unknown} */ let meta; try { meta = JSON.parse(rawMeta); } catch (err) { - findings.push({ - kind: "corrupt_meta", - key, - ...identity, - error: err instanceof Error ? err.message : String(err), - }); - continue; + return { + identity, + meta: null, + findings: [{ + kind: "corrupt_meta", + key, + ...identity, + error: err instanceof Error ? err.message : String(err), + }], + }; } + if (!meta || typeof meta !== "object" || Array.isArray(meta)) { + return { + identity, + meta: null, + findings: [{ kind: "corrupt_meta", key, ...identity, error: "__meta__ must be a JSON object" }], + }; + } + return { + identity, + meta: /** @type {Record} */ (meta), + findings: [], + }; +} + +/** @param {Record} encrypted */ +export function secretUpperBoundStrings(encrypted) { + /** @type {Record} */ + const out = Object.create(null); + for (const [key, value] of Object.entries(encrypted || {})) { + if (typeof value !== "string") continue; + out[key] = TWO_BYTE_SECRET_CHAR.repeat(Math.min(value.length, UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES)); + } + return out; +} +/** + * @param {{ + * identity: { namespace: string | null, worker: string | null, version: string | null }, + * meta: Record, + * nsSecretsEncrypted?: Record, + * workerSecretsEncrypted?: Record, + * assetsCdnBase?: string | null, + * }} args + */ +export function estimateBundleEnvBytes({ + identity, + meta, + nsSecretsEncrypted = {}, + workerSecretsEncrypted = {}, + assetsCdnBase = null, +}) { + if (!identity.namespace || !identity.worker || !identity.version) return null; + const env = estimatedWorkerLoaderEnv({ + ns: identity.namespace, + worker: identity.worker, + version: identity.version, + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) + ? /** @type {Record} */ (meta.vars) + : null, + nsSecrets: secretUpperBoundStrings(nsSecretsEncrypted), + workerSecrets: secretUpperBoundStrings(workerSecretsEncrypted), + meta, + assetsCdnBase, + }); + return estimatedWorkerLoaderEnvBytes(env); +} + +/** + * @param {{ + * key: string, + * rawMeta: string, + * nsSecretsEncrypted?: Record, + * workerSecretsEncrypted?: Record, + * assetsCdnBase?: string | null, + * }} args + */ +export function findingsForBundleMetadata({ + key, + rawMeta, + nsSecretsEncrypted = {}, + workerSecretsEncrypted = {}, + assetsCdnBase = null, +}) { + const { identity, meta, findings: parseFindings } = parseMetadataRecord(key, rawMeta); + if (!meta) return parseFindings; + return findingsForParsedBundleMetadata({ + key, + identity, + meta, + nsSecretsEncrypted, + workerSecretsEncrypted, + assetsCdnBase, + }); +} + +/** + * @param {{ + * key: string, + * identity: { namespace: string | null, worker: string | null, version: string | null }, + * meta: Record, + * nsSecretsEncrypted?: Record, + * workerSecretsEncrypted?: Record, + * assetsCdnBase?: string | null, + * }} args + */ +function findingsForParsedBundleMetadata({ + key, + identity, + meta, + nsSecretsEncrypted = {}, + workerSecretsEncrypted = {}, + assetsCdnBase = null, +}) { + /** @type {Array>} */ + const findings = []; const flag = experimentalFlag(meta); if (flag) { findings.push({ @@ -97,14 +267,100 @@ for (const key of keys) { module, }); } + + const envBytes = estimateBundleEnvBytes({ + identity, + meta, + nsSecretsEncrypted, + workerSecretsEncrypted, + assetsCdnBase, + }); + if (envBytes !== null && envBytes > WORKER_LOADER_ENV_MAX_BYTES) { + findings.push({ + kind: "worker_env_too_large", + key, + ...identity, + env_bytes: envBytes, + max_env_bytes: WORKER_LOADER_ENV_MAX_BYTES, + upstream_max_env_bytes: UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + headroom_bytes: WORKER_LOADER_ENV_HEADROOM_BYTES, + secret_value_estimate: "encrypted_envelope_length_as_two_byte_string", + }); + } + return findings; } -for (const finding of findings) { - console.log(JSON.stringify(finding)); +/** + * @param {{ redis: (args: string[]) => string, assetsCdnBase?: string | null }} args + */ +export function scanWorkerd0701Metadata({ redis, assetsCdnBase = null }) { + const keys = redis(["--scan", "--pattern", REDIS_PATTERN]) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + /** @type {Map>} */ + const secretHashCache = new Map(); + /** @param {string} key */ + const readSecretHash = (key) => { + let cached = secretHashCache.get(key); + if (!cached) { + cached = parseRedisHash(redis(["HGETALL", key])); + secretHashCache.set(key, cached); + } + return cached; + }; + + /** @type {Array>} */ + const findings = []; + for (const key of keys) { + const rawMeta = redis(["HGET", key, "__meta__"]).replace(/\n$/, ""); + const { identity, meta, findings: parseFindings } = parseMetadataRecord(key, rawMeta); + if (!meta) { + findings.push(...parseFindings); + continue; + } + const nsSecretsEncrypted = identity.namespace + ? readSecretHash(`secrets:${identity.namespace}`) + : {}; + const workerSecretsEncrypted = identity.namespace && identity.worker + ? readSecretHash(`secrets:${identity.namespace}:${identity.worker}`) + : {}; + findings.push(...findingsForParsedBundleMetadata({ + key, + identity, + meta, + nsSecretsEncrypted, + workerSecretsEncrypted, + assetsCdnBase, + })); + } + return { keysScanned: keys.length, findings }; +} + +export async function runCli() { + const redisUrl = redisUrlFromEnv(); + const { keysScanned, findings } = scanWorkerd0701Metadata({ + redis: (args) => redisCli(args, { redisUrl }), + assetsCdnBase: process.env.ASSETS_CDN_BASE || null, + }); + for (const finding of findings) { + console.log(JSON.stringify(finding)); + } + if (findings.length === 0) { + console.error(`Scanned ${keysScanned} worker bundle metadata keys; no workerd 0701 blockers found.`); + } else { + console.error(`Scanned ${keysScanned} worker bundle metadata keys; found ${findings.length} workerd 0701 blocker(s).`); + process.exitCode = 1; + } +} + +function isMainModule() { + return process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); } -if (findings.length === 0) { - console.error(`Scanned ${keys.length} worker bundle metadata keys; no workerd 0701 blockers found.`); -} else { - console.error(`Scanned ${keys.length} worker bundle metadata keys; found ${findings.length} workerd 0701 blocker(s).`); - process.exitCode = 1; + +if (isMainModule()) { + runCli().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; + }); } diff --git a/shared/workerd-compat-flags.js b/shared/workerd-compat-flags.js index 7dabeea..cde615d 100644 --- a/shared/workerd-compat-flags.js +++ b/shared/workerd-compat-flags.js @@ -2,7 +2,7 @@ * Mirrors workerd v1.20260701.1 src/workerd/io/compatibility-date.capnp. * Regenerate on every workerd pin bump from an upstream workerd source checkout: * - * node --input-type=module -e 'import fs from "node:fs"; const src=fs.readFileSync("src/workerd/io/compatibility-date.capnp","utf8"); const blocks=[]; let cur=[]; for (const line of src.split(/\n/)) { if (/^\s*[A-Za-z][A-Za-z0-9_]*\s+@\d+\s+:Bool/.test(line)) { if (cur.length) blocks.push(cur.join("\n")); cur=[line]; } else if (cur.length) cur.push(line); } if (cur.length) blocks.push(cur.join("\n")); const out=new Set(); for (const b of blocks) { if (!b.includes("$experimental")) continue; for (const m of b.matchAll(/\$compatEnableFlag\("([^"]+)"\)/g)) out.add(m[1]); } console.log([...out].sort().join("\n"));' + * node --input-type=module -e 'import fs from "node:fs"; const src=fs.readFileSync("src/workerd/io/compatibility-date.capnp","utf8"); const blocks=[]; let cur=[]; for (const line of src.split(/\n/)) { if (/^\s*[A-Za-z][A-Za-z0-9_]*\s+@\d+\s*:\s*Bool/.test(line)) { if (cur.length) blocks.push(cur.join("\n")); cur=[line]; } else if (cur.length) cur.push(line); } if (cur.length) blocks.push(cur.join("\n")); const out=new Set(); for (const b of blocks) { if (!b.includes("$experimental")) continue; for (const m of b.matchAll(/\$compatEnableFlag\("([^"]+)"\)/g)) out.add(m[1]); } console.log([...out].sort().join("\n"));' */ export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION = "1.20260701.1"; @@ -26,7 +26,6 @@ export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS = Object.freeze([ "kv_direct_binding", "memory_cache_delete", "new_module_registry", - "nonclass_entrypoint_reuses_ctx_across_invocations", "precise_timers", "python_workers_development", "python_workers_durable_objects", @@ -38,7 +37,6 @@ export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS = Object.freeze([ "streams_no_default_auto_allocate_chunk_size", "tail_worker_user_spans", "typescript_strip_types", - "unique_ctx_per_invocation", "unsafe_module", "unsupported_process_actual_platform", "webgpu", diff --git a/terraform/README.md b/terraform/README.md index e0b6523..ac993c2 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -225,14 +225,15 @@ The EC2 capacity provider uses a fixed Auto Scaling Group: - spread placement by AZ, then by instance ID - ENI trunking enabled with `awsvpcTrunking` -Task `cpu` and task-level `memory` values are placement reservations on EC2 launch type; -CPU is a cgroup share weight. D1 and Durable Object workerd containers also set explicit -container `memory` hard limits, defaulting to `runtime_memory` and overrideable with -`d1_runtime_container_memory` / `do_runtime_container_memory`. This matters on EC2 -because newer workerd releases no longer cap SQLite's process hard heap at 512 MiB; the -container cap keeps a runaway SQLite query from consuming arbitrary host memory. On -Fargate, task memory is already a hard task ceiling, so the container cap is mostly a -matching documentation of the intended ceiling. +Task-level `memory` is a task cgroup ceiling on this EC2-only stack, but it does not +separate one container's memory from another container in the same task. CPU remains a +cgroup share weight. D1 and Durable Object stateful runtime containers also set +explicit container `memory` hard limits, defaulting to `runtime_memory` and +overrideable with `d1_runtime_container_memory` / `do_runtime_container_memory`. This +matters because newer workerd releases no longer cap SQLite's process hard heap at 512 +MiB; the container cap keeps a runaway SQLite query inside the stateful runtime +container budget. The supervisor is PID 1 in that same container, while DO's +redis-proxy sidecar keeps separate task headroom. The launch template keeps IMDSv2 enabled for the ECS host agent, sets metadata hop limit to 1, and sets `ECS_AWSVPC_BLOCK_IMDS=true` so awsvpc tasks cannot read diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 25409cc..dd72ddc 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -791,10 +791,16 @@ test("MAX_WORKER_COMPATIBILITY_DATE matches pinned workerd release plus seven da }); test("workerd experimental compat flag mirror matches pinned workerd source version", () => { - const regenerate = "Regenerate shared/workerd-compat-flags.js from /home/sean/Projects/workerd/src/workerd/io/compatibility-date.capnp using the command in that file header."; + const regenerate = "Regenerate shared/workerd-compat-flags.js from an upstream workerd checkout with the command in that file header."; assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, WORKERD_VERSION, regenerate); assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("experimental")); assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unsafe_module")); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.length, 33); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unique_ctx_per_invocation"), false); + assert.equal( + WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("nonclass_entrypoint_reuses_ctx_across_invocations"), + false + ); assert.equal( WORKERD_EXPERIMENTAL_COMPAT_FLAGS.some((flag) => flag.startsWith("no_")), false, diff --git a/tests/unit/scan-workerd-0701-metadata.test.js b/tests/unit/scan-workerd-0701-metadata.test.js new file mode 100644 index 0000000..fc70583 --- /dev/null +++ b/tests/unit/scan-workerd-0701-metadata.test.js @@ -0,0 +1,150 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + estimateBundleEnvBytes, + findingsForBundleMetadata, + parseRedisHash, + redisCli, + scanWorkerd0701Metadata, + secretUpperBoundStrings, +} from "../../scripts/scan-workerd-0701-metadata.mjs"; +import { WORKERD_EXPERIMENTAL_COMPAT_FLAGS } from "../../shared/workerd-compat-flags.js"; + +test("workerd experimental compat flag mirror excludes GA context reuse flags", () => { + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("experimental"), true); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unsafe_module"), true); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.length, 33); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unique_ctx_per_invocation"), false); + assert.equal( + WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("nonclass_entrypoint_reuses_ctx_across_invocations"), + false + ); +}); + +test("workerd 0701 scanner reports missing metadata, py modules, and experimental flags", () => { + assert.deepEqual( + findingsForBundleMetadata({ key: "worker:demo:api:v:1", rawMeta: "" }), + [{ + kind: "missing_meta", + key: "worker:demo:api:v:1", + namespace: "demo", + worker: "api", + version: "v1", + }] + ); + + const findings = findingsForBundleMetadata({ + key: "worker:demo:api:v:1", + rawMeta: JSON.stringify({ + compatibilityFlags: ["unsafe_module"], + modules: { "worker.py": { type: "py" } }, + }), + }); + assert.equal(findings.some((finding) => finding.kind === "experimental_compat_flag"), true); + assert.equal(findings.some((finding) => finding.kind === "python_worker_module"), true); +}); + +test("workerd 0701 scanner reports retained env that would exceed workerLoader budget", () => { + const encryptedEnvelope = `WDL-ENC:${"a".repeat(600_000)}`; + const { findings } = scanWorkerd0701Metadata({ + redis(args) { + if (args[0] === "--scan") return "worker:demo:api:v:1\n"; + if (args[0] === "HGET") return `${JSON.stringify({ vars: { SMALL: "ok" } })}\n`; + if (args[0] === "HGETALL" && args[1] === "secrets:demo") return `BIG\n${encryptedEnvelope}\n`; + if (args[0] === "HGETALL" && args[1] === "secrets:demo:api") return ""; + throw new Error(`unexpected redis args ${JSON.stringify(args)}`); + }, + }); + + const finding = findings.find((entry) => entry.kind === "worker_env_too_large"); + assert.ok(finding); + assert.equal(finding.namespace, "demo"); + assert.equal(finding.worker, "api"); + assert.equal(finding.version, "v1"); + assert.equal(finding.secret_value_estimate, "encrypted_envelope_length_as_two_byte_string"); +}); + +test("workerd 0701 scanner parses raw redis hashes and rejects odd replies", () => { + assert.deepEqual(Object.fromEntries(Object.entries(parseRedisHash("A\n1\nB\n2\n"))), { A: "1", B: "2" }); + assert.deepEqual(Object.fromEntries(Object.entries(parseRedisHash(""))), {}); + assert.throws(() => parseRedisHash("A\n1\nB\n"), /odd number/); +}); + +test("workerd 0701 scanner uses two-byte secret upper bounds capped at upstream max", () => { + const out = secretUpperBoundStrings({ + ASCII: "WDL-ENC:abc", + HUGE: "x".repeat(2 * 1024 * 1024), + }); + assert.equal(out.ASCII.length, "WDL-ENC:abc".length); + assert.equal(out.ASCII.charCodeAt(0), 0x100); + assert.equal(out.HUGE.length, 1024 * 1024); +}); + +test("workerd 0701 scanner estimates assets env with the supplied CDN base", () => { + const meta = { + assets: { prefix: "assets/demo/api/v1/" }, + bindings: { ASSETS: { type: "assets" } }, + }; + const identity = { namespace: "demo", worker: "api", version: "v1" }; + const shortBytes = estimateBundleEnvBytes({ identity, meta, assetsCdnBase: "https://a.invalid" }); + const longBytes = estimateBundleEnvBytes({ + identity, + meta, + assetsCdnBase: "https://very-long-assets-hostname.example.invalid", + }); + assert.ok(longBytes > shortBytes); +}); + +test("workerd 0701 scanner reports missing metadata before reading secret hashes", () => { + let secretReads = 0; + const { findings } = scanWorkerd0701Metadata({ + redis(args) { + if (args[0] === "--scan") return "worker:demo:api:v:1\n"; + if (args[0] === "HGET") return ""; + if (args[0] === "HGETALL") { + secretReads += 1; + throw new Error("secret hash should not be read for a bundle missing __meta__"); + } + throw new Error(`unexpected redis args ${JSON.stringify(args)}`); + }, + }); + + assert.equal(secretReads, 0); + assert.deepEqual(findings.map((finding) => finding.kind), ["missing_meta"]); +}); + +test("workerd 0701 scanner reports corrupt metadata before reading secret hashes", () => { + let secretReads = 0; + const { findings } = scanWorkerd0701Metadata({ + redis(args) { + if (args[0] === "--scan") return "worker:demo:api:v:1\n"; + if (args[0] === "HGET") return "not-json\n"; + if (args[0] === "HGETALL") { + secretReads += 1; + throw new Error("secret hash should not be read for corrupt __meta__"); + } + throw new Error(`unexpected redis args ${JSON.stringify(args)}`); + }, + }); + + assert.equal(secretReads, 0); + assert.deepEqual(findings.map((finding) => finding.kind), ["corrupt_meta"]); +}); + +test("workerd 0701 scanner fails clearly when redis-cli is missing", () => { + const error = Object.assign(new Error("spawn redis-cli ENOENT"), { code: "ENOENT" }); + assert.throws( + () => redisCli(["PING"], { + redisUrl: "redis://unit", + spawn() { + return /** @type {any} */ ({ + error, + status: null, + stdout: "", + stderr: "", + }); + }, + }), + /redis-cli not found/ + ); +}); From 29d2af630bae8ff59799a84c8c41c7bba5d506fe Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 2 Jul 2026 18:25:47 +0800 Subject: [PATCH 11/13] Close workerd 0701 PR feedback Document the two-phase env-budget deploy check, pin the Durable Object env estimate to the real runtime shape, and leave explicit task memory headroom for the DO redis-proxy sidecar. Add regression coverage for the DO env estimate and Terraform memory contract. Signed-off-by: Lu Zhang --- control/env-budget.js | 2 ++ control/handlers/deploy.js | 2 ++ docs/compatibility.md | 2 +- docs/compatibility.zh.md | 2 +- docs/modules/control-auth.md | 4 +++- docs/modules/control-auth.zh.md | 2 +- docs/modules/runtime.md | 13 ++++++------ docs/modules/runtime.zh.md | 4 ++-- terraform/README.md | 13 ++++++------ .../modules/compute/do_runtime_service.tf | 7 ++++++- terraform/modules/compute/locals.tf | 9 +++++++-- terraform/variables.tf | 2 +- tests/unit/control-env-budget.test.js | 20 +++++++++++++++++++ tests/unit/style-contracts.test.js | 7 ++++++- 14 files changed, 66 insertions(+), 23 deletions(-) diff --git a/control/env-budget.js b/control/env-budget.js index d16632f..f37bc7a 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -137,6 +137,8 @@ function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, asset binding: name, className: stringOrFallback(spec.className), }; + // DO bindings intentionally mirror runtime's default/factory output shape: + // props are top-level fields, while hostProxy carries the JSRPC namespace. return { __wdlBinding: "do", ...props, diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 2b15b4f..7e675d7 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -788,6 +788,8 @@ export async function handle({ request, env, ns, name, requestId }) { } try { + // Fast pre-allocation check for obvious env-budget failures. commitWithWatch() + // repeats this after materializing watched metadata such as resolved D1 ids. await validateCommittedEnvBudget({ redis, controlEnv, diff --git a/docs/compatibility.md b/docs/compatibility.md index a3883eb..b6a8860 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -58,7 +58,7 @@ compatibility flag, so certificate hostname validation can change for all dates. | Workflows | Partial | User workflow class code runs inside loaded workers when dispatched by runtime. | workflows owns DB 2 instance state, step/event/sleep commits, runtime-observed `step.do` DAG edges including `Promise.all` siblings, generation/run-token fencing, lifecycle APIs, progress callbacks, and scheduler tick. | WDL Workflows V2 has WDL-specific payload semantics and terminal failure rules; permanent `step.do` failure is terminal for the run even if caught, and one step may record at most 1000 dependency edges. | WDL Workflows V2 is not Cloudflare Workflows parity. `script_name`, cross-worker workflows, and Cloudflare's source-AST visualizer are unsupported. | | Service bindings | Supported | workerd service binding JSRPC and fetch dispatch. | WDL resolves target worker metadata, cold-loads immutable versions, propagates request ids where available, and enforces namespace/action ACL through control metadata. | Frozen-version service targets do not evict active siblings by design. | None currently documented. | | Platform bindings | Supported | workerd named entrypoint/JSRPC mechanics. | WDL expands ACL-checked platform bindings from control metadata. | Platform bindings are WDL-specific. | Platform bindings are not a Cloudflare tenant portability feature. | -| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding/workflow env values before deploy/secret mutation, accounts for V8 two-byte string storage for non-Latin-1 strings, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | +| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding/workflow env values during deploy/secret mutation, accounts for V8 two-byte string storage for non-Latin-1 strings, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | | Worker code size | Supported | Dynamic worker module bodies are accepted by `workerLoader` up to workerd's 64 MiB limit. | Control rejects module bodies over 64 MiB before allocating a version. | WDL's deploy JSON body limit is lower for ordinary inline deployments. | Large server-side bundle assembly paths must keep this guard. | ## Control Plane And Developer Tooling diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index 1cdec2f..decb162 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -44,7 +44,7 @@ Node.js TLS 行为跟随 bundled workerd binary。升级到 workerd 2026-07-01 | Workflows | Partial | 用户 workflow class code 在 runtime dispatch 时运行在 loaded worker 内。 | workflows 拥有 DB 2 instance state、step/event/sleep commit、runtime-observed `step.do` DAG edges(包括 `Promise.all` sibling)、generation/run-token fence、lifecycle API、progress callback 和 scheduler tick。 | WDL Workflows V2 的 payload 语义和 terminal failure 规则由 WDL 定义;永久失败的 `step.do` 即使被捕获也会让 run terminal failed;单个 step 最多记录 1000 条 dependency edge。 | WDL Workflows V2 不是 Cloudflare Workflows parity;不支持 `script_name`、跨 worker workflow 和 Cloudflare 的源码 AST visualizer。 | | Service bindings | Supported | workerd service binding JSRPC 和 fetch dispatch。 | WDL 解析 target worker metadata,cold-load immutable version,在可用时传播 request id,并通过 control metadata 做 namespace/action ACL。 | Frozen-version service target 按设计不 evict active siblings。 | 当前无已记录缺口。 | | Platform bindings | Supported | workerd named entrypoint/JSRPC 机制。 | WDL 从 control metadata 展开 ACL-checked platform bindings。 | Platform bindings 是 WDL-specific。 | 不是 Cloudflare tenant portability feature。 | -| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 前按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets 加 runtime 注入的 binding/workflow env value,并计入非 Latin-1 字符串的 V8 two-byte storage;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | +| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 过程中按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets 加 runtime 注入的 binding/workflow env value,并计入非 Latin-1 字符串的 V8 two-byte storage;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | | Worker code size | Supported | Dynamic worker module bodies 可由 `workerLoader` 接受,直到 workerd 的 64 MiB 上限。 | Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies。 | WDL 的 deploy JSON body limit 对普通 inline deploy 更低。 | 大型 server-side bundle assembly 路径必须保留这道 guard。 | ## 控制面和开发工具 diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index 6f2ddcc..5eae949 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -79,7 +79,9 @@ Control lifecycle operations are split so each critical transition has one autho allocates the next immutable version through `worker:::next_version`, writes bundle metadata/modules/assets, then enters the same promote path used by explicit promotion. Before allocation, deploy checks the 64 MiB workerd dynamic code - limit and the headroomed `workerLoader` env budget for the candidate metadata. + limit and runs a fast headroomed `workerLoader` env-budget check for the candidate + metadata. The watched commit path rechecks the env budget after metadata + materialization, such as resolved D1 database ids, before writing the version. - Promote is the only active-route flip. It WATCHes the delete lock, bundle metadata, D1 refs, service-binding target refs, queue consumer keys, host declarations, and pattern keys needed for the candidate. The EXEC updates active routes, host reverse indexes, diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 6f2d916..cddc645 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -64,7 +64,7 @@ Host、secret、data 和 auth 操作: Control lifecycle 操作会拆开处理,确保每个关键 transition 只有一个权威入口: -- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会检查 workerd 64 MiB dynamic code limit 和候选 metadata 的 headroomed `workerLoader` env budget。 +- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会检查 workerd 64 MiB dynamic code limit,并对候选 metadata 做一次快速 headroomed `workerLoader` env-budget 检查。真正的 Redis WATCH commit 会在 metadata materialization(例如 D1 database id 解析)之后再次检查 env budget,然后才写入 version。 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 - Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前,会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 6432271..b037d57 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -83,7 +83,7 @@ services. Runtime therefore treats bindings as adapters: materialization merges in fixed precedence: bundle vars, then namespace secrets, then worker secrets. A worker-level secret with the same name wins over a namespace-level secret, which wins over a var. Control enforces a headroomed estimate of workerd's - `workerLoader` serialized env budget before deploys and secret mutations. That + `workerLoader` serialized env budget during deploys and secret mutations. That estimate includes user vars/secrets plus runtime-injected binding/workflow env values, including required caller secret copies in platform/service binding props, so an over-large env fails in the control plane instead of during runtime cold-load. @@ -267,11 +267,12 @@ when a matching active tail session exists. WorkerCode unless another upstream API explicitly requires it. - Upstream workerd 2026-07-01 caps dynamic worker code at 64 MiB and serialized dynamic env at 1 MiB. Control rejects module bodies over 64 MiB before version allocation. - Vars, namespace/worker secrets, and runtime-injected binding/workflow env values are - prechecked against a headroomed `workerLoader` env budget because workerd's final - enforcement includes the full `env` estimate. The estimate starts from JSON bytes and - adds V8 two-byte string overhead for non-Latin-1 strings, so mixed ASCII plus CJK or - emoji secrets do not slip past control and fail later at cold-load. + Vars, namespace/worker secrets, and runtime-injected binding/workflow env values get + a fast deploy precheck and an authoritative watched-commit check against a headroomed + `workerLoader` env budget because workerd's final enforcement includes the full `env` + estimate after metadata materialization. The estimate starts from JSON bytes and adds + V8 two-byte string overhead for non-Latin-1 strings, so mixed ASCII plus CJK or emoji + secrets do not slip past control and fail later at cold-load. - In current stock workerd, a client disconnect during an async `ReadableStream` response body may not call the stream source's `cancel()` callback. Tenant streaming and SSE workers should use their own heartbeat, timeout, or application close path diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index d50e7a1..f003de9 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -40,7 +40,7 @@ Tenant 可见 binding 包括 KV、R2、D1、Durable Objects、Queues、ASSETS、 workerd 提供 isolate、module evaluation、named entrypoint 和 JSRPC 机制。Cloudflare 生产平台通常在外部服务里实现的 binding 后端,则由 WDL 自己补齐。因此 runtime 把 binding 当作 adapter: - KV、queue producer 这类纯数据 binding 调 colocated redis-proxy sidecar。Loaded worker 看到 Cloudflare-shaped object,但 method call 会先通过 workerd JSRPC 回到 runtime,再经 HTTP 调 redis-proxy。 -- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 前用留有 headroom 的预算估算完整 workerLoader env,包括用户 vars/secrets、runtime 注入的 binding/workflow env value,以及 platform/service binding props 中复制的 required caller secret,避免超限配置拖到 runtime cold-load 才失败。 +- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 过程中用留有 headroom 的预算估算完整 workerLoader env,包括用户 vars/secrets、runtime 注入的 binding/workflow env value,以及 platform/service binding props 中复制的 required caller secret,避免超限配置拖到 runtime cold-load 才失败。 - D1、Durable Objects、Workflows 这类 stateful binding 调专门 backend service。Hidden backend Fetcher 留在 runtime 内部,并在 tenant code 观察 `env` 前被删除。 - R2 是 S3-compatible object-storage adapter:runtime 使用平台 credential 签名请求,并发送到配置的 endpoint。 - ASSETS 是 deploy artifact URL helper:control 在 deploy 时把 assets 上传到 S3-compatible storage;runtime 读取 `__meta__.assets` 和 `ASSETS_CDN_BASE`,只暴露 `env.ASSETS.url(path)` 用来生成 tokenized CDN URL。 @@ -110,7 +110,7 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - Python Workers modules 不受支持。Control 会拒绝新的 `py` module manifest,runtime/do-runtime 会拒绝 retained metadata 中的 `py` module,而不是让 workerd 之后抛 mixed JS/Python bundle error。 - 在已有 Redis DB 0 上 rollout 2026-07-01 workerd 适配前,先安装 `redis-cli`,再对 control Redis database 运行 `node scripts/scan-workerd-0701-metadata.mjs`,找出缺少 metadata、仍包含 Python modules 或上游 experimental compatibility flags,或按 2026-07-01 估算器会超过 headroomed `workerLoader` env budget 的 retained versions。这个 scanner 只读且不解密 secret;它用 encrypted envelope length 作为 conservative two-byte string 上界。涉及 secret 的 `worker_env_too_large` finding 应作为 rollout blocker 精确复核,例如 redeploy,或走正常 secret mutation API path 触发会解密的精确估算。扫描 assets CDN base 比默认 placeholder 更长的部署时设置 `ASSETS_CDN_BASE`。 - Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 -- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会在 control plane 用留有 headroom 的 `workerLoader` env budget 预检,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 +- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会先做快速 deploy 预检,并在 Redis WATCH commit 内用 materialization 后的 metadata 做权威检查,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 - 当前 stock workerd 中,客户端在 async `ReadableStream` response body 中途断开时,不一定调用 stream source 的 `cancel()`。Tenant streaming/SSE worker 应使用自己的 heartbeat、timeout 或应用层 close path,不要把 disconnect-driven `cancel()` 当成唯一资源清理信号。 - workerd 升级仍可能改变默认或 compatibility-flagged runtime surface;升级时要审 exposed surface,而不只审 loader/abort path。 diff --git a/terraform/README.md b/terraform/README.md index ac993c2..5bf8cf9 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -228,12 +228,13 @@ The EC2 capacity provider uses a fixed Auto Scaling Group: Task-level `memory` is a task cgroup ceiling on this EC2-only stack, but it does not separate one container's memory from another container in the same task. CPU remains a cgroup share weight. D1 and Durable Object stateful runtime containers also set -explicit container `memory` hard limits, defaulting to `runtime_memory` and -overrideable with `d1_runtime_container_memory` / `do_runtime_container_memory`. This -matters because newer workerd releases no longer cap SQLite's process hard heap at 512 -MiB; the container cap keeps a runaway SQLite query inside the stateful runtime -container budget. The supervisor is PID 1 in that same container, while DO's -redis-proxy sidecar keeps separate task headroom. +explicit container `memory` hard limits. D1 defaults to `runtime_memory`; DO defaults +to `runtime_memory - 128 MiB` so the colocated redis-proxy sidecar keeps task-level +headroom. Both are overrideable with `d1_runtime_container_memory` / +`do_runtime_container_memory`, but the DO override must still stay below +`runtime_memory`. This matters because newer workerd releases no longer cap SQLite's +process hard heap at 512 MiB; the container cap keeps a runaway SQLite query inside the +stateful runtime container budget. The supervisor is PID 1 in that same container. The launch template keeps IMDSv2 enabled for the ECS host agent, sets metadata hop limit to 1, and sets `ECS_AWSVPC_BLOCK_IMDS=true` so awsvpc tasks cannot read diff --git a/terraform/modules/compute/do_runtime_service.tf b/terraform/modules/compute/do_runtime_service.tf index 30cea1d..ed96e3a 100644 --- a/terraform/modules/compute/do_runtime_service.tf +++ b/terraform/modules/compute/do_runtime_service.tf @@ -50,7 +50,7 @@ resource "aws_ecs_task_definition" "do_runtime" { image = var.workerd_image essential = true entryPoint = ["do-supervisor"] - memory = coalesce(var.do_runtime_container_memory, var.runtime_memory) + memory = local.do_runtime_container_memory dependsOn = [{ containerName = "redis-proxy" @@ -105,6 +105,11 @@ resource "aws_ecs_task_definition" "do_runtime" { ]) lifecycle { + precondition { + condition = local.do_runtime_container_memory > 0 && local.do_runtime_container_memory < var.runtime_memory + error_message = "do_runtime_container_memory must leave positive task-level memory headroom for the redis-proxy sidecar." + } + precondition { condition = !var.do_test_hooks_enabled || can(regex("(^|-)test($|-)", var.name)) error_message = "do_test_hooks_enabled may only be enabled for test-named compute stacks." diff --git a/terraform/modules/compute/locals.tf b/terraform/modules/compute/locals.tf index 7af0153..d38af5d 100644 --- a/terraform/modules/compute/locals.tf +++ b/terraform/modules/compute/locals.tf @@ -1,6 +1,11 @@ locals { - redis_addr = "${var.valkey_host}:${var.valkey_port}" - data_redis_url = "redis://${local.redis_addr}/1" + redis_addr = "${var.valkey_host}:${var.valkey_port}" + data_redis_url = "redis://${local.redis_addr}/1" + do_redis_proxy_memory_headroom = 128 + do_runtime_container_memory = coalesce( + var.do_runtime_container_memory, + var.runtime_memory - local.do_redis_proxy_memory_headroom, + ) # PLATFORM_DOMAIN is the base hostname gateway uses to split ns from host # (regex builds "."). platform_domain carries a leading diff --git a/terraform/variables.tf b/terraform/variables.tf index 90397d1..be21b76 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -214,7 +214,7 @@ variable "d1_runtime_container_memory" { variable "do_runtime_container_memory" { type = number default = null - description = "Optional hard memory limit, in MiB, for the do-runtime container. Defaults to runtime_memory." + description = "Optional hard memory limit, in MiB, for the do-runtime container. Defaults to runtime_memory minus 128 MiB of redis-proxy sidecar headroom." validation { condition = var.do_runtime_container_memory == null || var.do_runtime_container_memory > 0 error_message = "do_runtime_container_memory must be null or a positive number of MiB." diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 3f733df..91889b4 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -211,6 +211,26 @@ test("worker env budget includes do-runtime alarm binding for Durable Object wor }, }); + assert.deepEqual(estimated.ROOM, { + __wdlBinding: "do", + ns: "demo", + worker: "api", + version: "v12", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + hostProxy: { + __wdlBinding: "do-host-proxy", + props: { + ns: "demo", + worker: "api", + version: "v12", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, + }, + }); assert.deepEqual(estimated.__WDL_DO_ALARMS__, { __wdlBinding: "do-alarms", props: { diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 35bdb65..5a5d049 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1889,6 +1889,7 @@ test("D1 and DO workerd containers keep explicit memory ceilings", () => { const rootVars = readRepoFile("terraform/variables.tf"); const moduleVars = readRepoFile("terraform/modules/compute/variables.tf"); const main = readRepoFile("terraform/main.tf"); + const locals = readRepoFile("terraform/modules/compute/locals.tf"); const d1Service = readRepoFile("terraform/modules/compute/d1_runtime_service.tf"); const doService = readRepoFile("terraform/modules/compute/do_runtime_service.tf"); const d1Kube = readRepoFile("deploy/kubernetes/base/d1-runtime.yaml"); @@ -1900,7 +1901,11 @@ test("D1 and DO workerd containers keep explicit memory ceilings", () => { assert.match(main, new RegExp(`${name}\\s+=\\s+var\\.${name}`)); } assert.match(d1Service, /memory\s+=\s+coalesce\(var\.d1_runtime_container_memory, var\.runtime_memory\)/); - assert.match(doService, /memory\s+=\s+coalesce\(var\.do_runtime_container_memory, var\.runtime_memory\)/); + assert.match(locals, /do_redis_proxy_memory_headroom\s+=\s+128/); + assert.match(locals, /do_runtime_container_memory\s+=\s+coalesce\(\s*var\.do_runtime_container_memory,\s*var\.runtime_memory - local\.do_redis_proxy_memory_headroom,\s*\)/); + assert.match(doService, /memory\s+=\s+local\.do_runtime_container_memory/); + assert.match(doService, /local\.do_runtime_container_memory > 0 && local\.do_runtime_container_memory < var\.runtime_memory/); + assert.match(doService, /redis-proxy sidecar/); assert.match(d1Kube, /name: d1-runtime[\s\S]*?limits:\n\s+memory: 1Gi/); assert.match(doKube, /name: do-runtime[\s\S]*?limits:\n\s+memory: 1Gi/); }); From 6fd4ae7b202cda16af55a58d0f80f8509162fd78 Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Thu, 2 Jul 2026 21:54:00 +0800 Subject: [PATCH 12/13] Address workerd 0701 review feedback Make deploy env-budget rejection authoritative after version allocation, keep DELETE secret repair paths from being blocked by unrelated corrupt envelopes, and update docs/tests for the revised contracts. Signed-off-by: Lu Zhang --- control/handlers/deploy.js | 15 +- control/handlers/ns-secrets.js | 11 +- control/handlers/worker-secrets.js | 11 +- docs/modules/control-auth.md | 11 +- docs/modules/control-auth.zh.md | 4 +- docs/modules/runtime.md | 8 +- docs/modules/runtime.zh.md | 2 +- tests/unit/control-deploy-watch.test.js | 48 +++-- .../control-secret-envelope-handlers.test.js | 169 ++++++++++++++++++ 9 files changed, 249 insertions(+), 30 deletions(-) diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 7e675d7..7af2261 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -788,8 +788,10 @@ export async function handle({ request, env, ns, name, requestId }) { } try { - // Fast pre-allocation check for obvious env-budget failures. commitWithWatch() - // repeats this after materializing watched metadata such as resolved D1 ids. + // Advisory pre-allocation pass that decrypts current secret envelopes before + // assets/version side effects. It uses a conservative version placeholder, so + // commitWithWatch() is the only env-budget rejection point after version + // allocation and watched metadata materialization such as resolved D1 ids. await validateCommittedEnvBudget({ redis, controlEnv, @@ -798,9 +800,12 @@ export async function handle({ request, env, ns, name, requestId }) { meta: prepared.meta, }); } catch (err) { - if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); - if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); - throw err; + if (err instanceof WorkerEnvBudgetError) { + // Do not reject here: the placeholder can over-estimate DO/workflow-heavy + // bundles. The watched commit check below uses the real version and will + // reject true budget failures before the version is written. + } else if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + else throw err; } const num = await redis.incr(`worker:${ns}:${name}:next_version`); diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index c58b2a3..7552fba 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -33,9 +33,16 @@ class NamespaceSecretAbort extends ControlAbort {} * controlEnv: Record, * nsName: string, * nsSecrets: Record, + * ignoreWorkerSecretEnvelopeErrors?: boolean, * }} args */ -async function validateNamespaceSecretBudget({ redis, controlEnv, nsName, nsSecrets }) { +async function validateNamespaceSecretBudget({ + redis, + controlEnv, + nsName, + nsSecrets, + ignoreWorkerSecretEnvelopeErrors = false, +}) { const activeRoutes = await redis.hGetAll(routesKey(nsName)); const indexedWorkers = await redis.sMembers(workersIndexKey(nsName)); const workerNames = new Set([ @@ -61,6 +68,7 @@ async function validateNamespaceSecretBudget({ redis, controlEnv, nsName, nsSecr encrypted: workerEncrypted, env: controlEnv, hashKey: workerSecretsKey, + ignoreSecretEnvelopeErrors: ignoreWorkerSecretEnvelopeErrors, }); await assertWorkerVersionsUserEnvBudget({ redis, @@ -135,6 +143,7 @@ async function mutateNamespaceSecret({ controlEnv, nsName, nsSecrets, + ignoreWorkerSecretEnvelopeErrors: method === "DELETE", }); const multi = iso.multi(); diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index b0e8c6b..23ada8e 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -134,6 +134,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI newVersion, controlEnv, ignoreWorkerSecretEnvelopeErrors: method === "DELETE", + ignoreNamespaceSecretEnvelopeErrors: method === "DELETE", }), } ); @@ -266,7 +267,12 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n const workerBudgetEncrypted = { ...workerEncrypted }; delete workerBudgetEncrypted[key]; const [nsSecrets, workerSecrets] = await Promise.all([ - decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: nsSecretsKey }), + decryptSecretHash({ + encrypted: nsEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", + }), decryptSecretHash({ encrypted: workerBudgetEncrypted, env: controlEnv, @@ -328,6 +334,7 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n * newVersion: string, * controlEnv: Record, * ignoreWorkerSecretEnvelopeErrors?: boolean, + * ignoreNamespaceSecretEnvelopeErrors?: boolean, * }} args */ async function assertWorkerSecretBumpEnvBudget({ @@ -338,6 +345,7 @@ async function assertWorkerSecretBumpEnvBudget({ newVersion, controlEnv, ignoreWorkerSecretEnvelopeErrors = false, + ignoreNamespaceSecretEnvelopeErrors = false, }) { const nsSecretsKey = `secrets:${ns}`; const workerSecretsKey = `secrets:${ns}:${name}`; @@ -350,6 +358,7 @@ async function assertWorkerSecretBumpEnvBudget({ encrypted: nsEncrypted, env: controlEnv, hashKey: nsSecretsKey, + ignoreSecretEnvelopeErrors: ignoreNamespaceSecretEnvelopeErrors, }); const workerSecrets = await decryptSecretHash({ encrypted: workerEncrypted, diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index 5eae949..44de508 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -79,9 +79,10 @@ Control lifecycle operations are split so each critical transition has one autho allocates the next immutable version through `worker:::next_version`, writes bundle metadata/modules/assets, then enters the same promote path used by explicit promotion. Before allocation, deploy checks the 64 MiB workerd dynamic code - limit and runs a fast headroomed `workerLoader` env-budget check for the candidate - metadata. The watched commit path rechecks the env budget after metadata - materialization, such as resolved D1 database ids, before writing the version. + limit and runs an advisory pass over the candidate metadata and current secret + envelopes. The watched commit path is the authoritative headroomed `workerLoader` + env-budget check after version allocation and metadata materialization, such as + resolved D1 database ids, before writing the version. - Promote is the only active-route flip. It WATCHes the delete lock, bundle metadata, D1 refs, service-binding target refs, queue consumer keys, host declarations, and pattern keys needed for the candidate. The EXEC updates active routes, host reverse indexes, @@ -92,6 +93,10 @@ Control lifecycle operations are split so each critical transition has one autho plaintext size and shape, encrypts it into a `WDL-ENC:` envelope before the Redis mutation/WATCH retry loop, and reuses the same envelope across retries. Runtime therefore sees a new immutable version id instead of mutable in-place secret changes. + Secret DELETE remains a repair surface: when estimating the post-delete env, it skips + corrupt envelopes in the other secret scope instead of letting an unrelated bad + namespace or worker secret block deletion. Secret PUT still fails closed on corrupt + retained envelopes. Namespace-secret mutations WATCH the retained worker/version metadata they need to re-estimate before commit; if concurrent metadata changes keep invalidating that view, control returns `namespace_secret_mutation_contention`. diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index cddc645..78f78c6 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -64,9 +64,9 @@ Host、secret、data 和 auth 操作: Control lifecycle 操作会拆开处理,确保每个关键 transition 只有一个权威入口: -- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会检查 workerd 64 MiB dynamic code limit,并对候选 metadata 做一次快速 headroomed `workerLoader` env-budget 检查。真正的 Redis WATCH commit 会在 metadata materialization(例如 D1 database id 解析)之后再次检查 env budget,然后才写入 version。 +- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会检查 workerd 64 MiB dynamic code limit,并对候选 metadata 和当前 secret envelope 做一次 advisory pass。真正的 Redis WATCH commit 会在分配真实 version 并完成 metadata materialization(例如 D1 database id 解析)之后用 headroomed `workerLoader` env-budget 做权威检查,然后才写入 version。 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 -- Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 +- Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Secret DELETE 仍是 repair surface:估算删除后的 env 时,会跳过另一层 secret scope 里的坏 envelope,避免无关的 namespace 或 worker 坏 secret 阻塞删除;Secret PUT 仍会对 retained corrupt envelope fail closed。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前,会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 - Worker delete lock value 和 `s3cleanup:` task id 必须由服务端生成随机值。`x-request-id` 只用于诊断,客户端或重试可能复用它;不能把它用作 lock token 或 cleanup-task id。 - Auth 不是一层约定俗成的 middleware。`parseControlRoute()` 分配 action,control 把 action 和 namespace 发给 auth,auth 用存储的 token record 对照 `shared/auth-roles.js` 评估权限。Dispatcher 代码不应自己从 URL prefix 推断权限。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index b037d57..6be6815 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -268,11 +268,11 @@ when a matching active tail session exists. - Upstream workerd 2026-07-01 caps dynamic worker code at 64 MiB and serialized dynamic env at 1 MiB. Control rejects module bodies over 64 MiB before version allocation. Vars, namespace/worker secrets, and runtime-injected binding/workflow env values get - a fast deploy precheck and an authoritative watched-commit check against a headroomed + an advisory deploy pass and an authoritative watched-commit check against a headroomed `workerLoader` env budget because workerd's final enforcement includes the full `env` - estimate after metadata materialization. The estimate starts from JSON bytes and adds - V8 two-byte string overhead for non-Latin-1 strings, so mixed ASCII plus CJK or emoji - secrets do not slip past control and fail later at cold-load. + estimate after version allocation and metadata materialization. The estimate starts + from JSON bytes and adds V8 two-byte string overhead for non-Latin-1 strings, so mixed + ASCII plus CJK or emoji secrets do not slip past control and fail later at cold-load. - In current stock workerd, a client disconnect during an async `ReadableStream` response body may not call the stream source's `cancel()` callback. Tenant streaming and SSE workers should use their own heartbeat, timeout, or application close path diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index f003de9..5916195 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -110,7 +110,7 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - Python Workers modules 不受支持。Control 会拒绝新的 `py` module manifest,runtime/do-runtime 会拒绝 retained metadata 中的 `py` module,而不是让 workerd 之后抛 mixed JS/Python bundle error。 - 在已有 Redis DB 0 上 rollout 2026-07-01 workerd 适配前,先安装 `redis-cli`,再对 control Redis database 运行 `node scripts/scan-workerd-0701-metadata.mjs`,找出缺少 metadata、仍包含 Python modules 或上游 experimental compatibility flags,或按 2026-07-01 估算器会超过 headroomed `workerLoader` env budget 的 retained versions。这个 scanner 只读且不解密 secret;它用 encrypted envelope length 作为 conservative two-byte string 上界。涉及 secret 的 `worker_env_too_large` finding 应作为 rollout blocker 精确复核,例如 redeploy,或走正常 secret mutation API path 触发会解密的精确估算。扫描 assets CDN base 比默认 placeholder 更长的部署时设置 `ASSETS_CDN_BASE`。 - Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 -- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会先做快速 deploy 预检,并在 Redis WATCH commit 内用 materialization 后的 metadata 做权威检查,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 +- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会先做 advisory deploy pass,并在 Redis WATCH commit 内用真实 version 和 materialization 后的 metadata 做权威检查,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 - 当前 stock workerd 中,客户端在 async `ReadableStream` response body 中途断开时,不一定调用 stream source 的 `cancel()`。Tenant streaming/SSE worker 应使用自己的 heartbeat、timeout 或应用层 close path,不要把 disconnect-driven `cancel()` 当成唯一资源清理信号。 - workerd 升级仍可能改变默认或 compatibility-flagged runtime surface;升级时要审 exposed surface,而不只审 loader/abort path。 diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index a4878ab..761e62e 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -29,6 +29,7 @@ const CONTROL_DEPLOY_TEST_STATE = { parsedQueueConsumers: null, watchedKeys: null, envBudgetError: false, + envBudgetFailuresRemaining: 0, envBudgetCalls: [], redis: null, logs: [], @@ -54,6 +55,7 @@ function resetControlDeployTestState() { CONTROL_DEPLOY_TEST_STATE.parsedQueueConsumers = null; CONTROL_DEPLOY_TEST_STATE.watchedKeys = null; CONTROL_DEPLOY_TEST_STATE.envBudgetError = false; + CONTROL_DEPLOY_TEST_STATE.envBudgetFailuresRemaining = 0; CONTROL_DEPLOY_TEST_STATE.envBudgetCalls = []; CONTROL_DEPLOY_TEST_STATE.redis = null; CONTROL_DEPLOY_TEST_STATE.logs = []; @@ -81,14 +83,19 @@ export async function recordS3CleanupIntent(intent) { const controlBundleUrl = moduleDataUrl(` export function deepFreeze(value) { return value; } export function prepareBundle(mainModule, modules, options = {}) { - return /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle || { - meta: { - mainModule, - modules, - bindings: options.bindings, - exports: options.exports, - workflows: options.workflows, - }, + if (/** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle) { + return /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle; + } + const meta = { + mainModule, + modules, + bindings: options.bindings, + exports: options.exports, + workflows: options.workflows, + }; + if (options.vars !== undefined) meta.vars = options.vars; + return { + meta, normalized: Object.entries(modules || {}), }; } @@ -211,6 +218,10 @@ export class WorkerEnvBudgetError extends Error { } export function assertWorkerLoaderUserEnvBudget() { /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.push(Array.from(arguments)[0] || {}); + if (/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining > 0) { + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining -= 1; + throw new WorkerEnvBudgetError("env too large"); + } if (/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError) { throw new WorkerEnvBudgetError("env too large"); } @@ -645,11 +656,13 @@ test("deploy handler rejects assets without S3 before allocating a version", asy } }); -test("deploy handler rejects workerLoader env budget violations before allocating a version", async () => { +test("deploy handler treats pre-allocation workerLoader env budget failures as advisory", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; - /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError = true; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining = 1; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls = []; + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; const session = makeSession(); let incrCalled = false; @@ -662,6 +675,10 @@ test("deploy handler rejects workerLoader env budget violations before allocatin async hGetAll(key) { return await session.hGetAll(key); }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(session); + }, }; try { @@ -680,11 +697,16 @@ test("deploy handler rejects workerLoader env budget violations before allocatin requestId: "rid-env-budget", }); - assert.equal((await readJsonResponse(response, 400)).error, "worker_env_too_large"); - assert.equal(incrCalled, false); + const body = await readJsonResponse(response, 201); + assert.equal(body.version, "v1"); + assert.equal(incrCalled, true); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.length, 2); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].version, undefined); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[1].version, "v1"); + assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta.vars, { BIG: "x" }); } finally { /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; - /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError = false; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining = 0; /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; } }); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 6ed65ef..bd79146 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -395,6 +395,50 @@ test("namespace secret DELETE skips other corrupt namespace envelopes for repair } }); +test("namespace secret DELETE skips corrupt worker envelopes for repair", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + zRange: state.redis.zRange, + }; + const deletesBefore = state.redis.deletes.length; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo", + fieldName: "TOKEN", + }); + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: encrypted }; + if (key === "routes:demo") return {}; + if (key === "secrets:demo:api") return { BAD: "WDL-ENC:not-json" }; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? ["api"] : []; + state.redis.zRange = async () => []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-worker-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.deletes.length, deletesBefore + 1); + assert.deepEqual(state.redis.deletes.at(-1), { key: "secrets:demo", field: "TOKEN" }); + } finally { + Object.assign(state.redis, original); + } +}); + test("namespace secret PUT still fails closed on other corrupt namespace envelopes", async () => { const { state } = await import(controlSharedUrl); const original = { @@ -911,6 +955,131 @@ test("worker secret DELETE skips other corrupt worker envelopes for repair", asy } }); +test("worker secret DELETE skips corrupt namespace envelopes for repair", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let execCalled = false; + let deletedField = null; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["TOKEN"]; }, + async hGet() { return null; }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo") return { BAD: "WDL-ENC:not-json" }; + if (key === "secrets:demo:api") return { TOKEN: encrypted }; + return {}; + }, + async zCard() { return 0; }, + async zRange() { return []; }, + multi() { + return { + hSet() {}, + /** @param {string} _key @param {string} field */ + hDel(_key, field) { deletedField = field; }, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-ns-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(execCalled, true); + assert.equal(deletedField, "TOKEN"); + } finally { + state.redis.session = originalSession; + } +}); + +test("worker secret DELETE skips corrupt namespace envelopes during bump repair", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let sessionCalls = 0; + let execCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => { + sessionCalls += 1; + return await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["TOKEN"]; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") return "v1"; + if (key === "worker:demo:api:v:1" && field === "__meta__") return JSON.stringify({ vars: { SAFE: "ok" } }); + return null; + }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo") return { BAD: "WDL-ENC:not-json" }; + if (key === "secrets:demo:api" && sessionCalls === 1) return { TOKEN: encrypted }; + return {}; + }, + async zCard() { return 1; }, + async zRange() { return []; }, + multi() { + return { + hSet() {}, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + }; + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-bump-ns-corrupt", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(execCalled, true); + assert.equal(body.deleted, true); + assert.equal(body.previousVersion, "v1"); + assert.equal(body.version, "v2"); + } finally { + state.redis.session = originalSession; + } +}); + test("worker secret PUT still fails closed on other corrupt worker envelopes", async () => { const { state } = await import(workerControlSharedUrl); const originalSession = state.redis.session; From 019f1d3ceb52f664e95d7ce56ecbb0350a0cc16e Mon Sep 17 00:00:00 2001 From: Lu Zhang Date: Fri, 3 Jul 2026 09:06:55 +0800 Subject: [PATCH 13/13] Complete workerd 0701 adaptation guards Signed-off-by: Lu Zhang --- control/env-budget.js | 37 +- control/handlers/deploy.js | 21 +- control/handlers/ns-secrets.js | 18 +- control/handlers/worker-secrets.js | 129 +++--- control/worker-code-budget.js | 54 +++ do-runtime/config.capnp | 1 + docs/compatibility.md | 2 +- docs/compatibility.zh.md | 2 +- docs/modules/cli.md | 2 +- docs/modules/cli.zh.md | 2 +- docs/modules/control-auth.md | 16 +- docs/modules/control-auth.zh.md | 4 +- docs/modules/runtime.md | 5 +- docs/modules/runtime.zh.md | 2 +- docs/source-map.md | 2 + docs/source-map.zh.md | 2 + runtime/config-system.capnp | 18 + runtime/config-user.capnp | 1 + runtime/load.js | 240 +----------- runtime/load/code-budget.js | 370 ++++++++++++++++++ ...ract-workerd-experimental-compat-flags.mjs | 50 +++ scripts/scan-workerd-0701-metadata.mjs | 15 +- shared/workerd-compat-flags.js | 3 +- terraform/README.md | 9 +- .../modules/compute/d1_runtime_service.tf | 10 +- terraform/modules/compute/locals.tf | 4 + tests/helpers/load-control-lib.js | 7 +- tests/helpers/load-do-protocol.js | 22 +- tests/unit/control-deploy-watch.test.js | 49 ++- tests/unit/control-env-budget.test.js | 48 ++- tests/unit/control-lib.test.js | 6 +- .../control-secret-envelope-handlers.test.js | 253 ++++++------ tests/unit/runtime-load.test.js | 73 +++- tests/unit/style-contracts.test.js | 10 +- .../unit/workerd-compat-flags-extract.test.js | 20 + 35 files changed, 1046 insertions(+), 461 deletions(-) create mode 100644 control/worker-code-budget.js create mode 100644 runtime/load/code-budget.js create mode 100644 scripts/extract-workerd-experimental-compat-flags.mjs create mode 100644 tests/unit/workerd-compat-flags-extract.test.js diff --git a/control/env-budget.js b/control/env-budget.js index f37bc7a..63a44fc 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,4 +1,5 @@ import { SecretEnvelopeError, decryptSecretValue } from "shared-secret-envelope"; +import { errorMessage } from "shared-errors"; import { bundleKey } from "shared-version"; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; @@ -334,7 +335,8 @@ export function assertWorkerLoaderUserEnvBudget({ ? `${ns}/${worker}${sourceVersion ? `@${sourceVersion}` : ""}` : ns; throw new WorkerEnvBudgetError( - `estimated workerLoader env for ${label} serializes to ${bytes} bytes, exceeding WDL workerLoader env budget ${WORKER_LOADER_ENV_MAX_BYTES} bytes`, + `estimated workerLoader env for ${label} serializes to ${bytes} bytes, ` + + `exceeding WDL workerLoader env budget ${WORKER_LOADER_ENV_MAX_BYTES} bytes`, { namespace: ns, ...(worker ? { worker } : {}), @@ -431,21 +433,26 @@ export async function assertWorkerVersionsUserEnvBudget({ // whose command protocol is single-flight even though secret decryption is not. for (const { sourceVersion, estimatedVersion } of uniqueChecks) { const rawMeta = await redis.hGet(bundleKey(ns, worker, sourceVersion), "__meta__"); - /** @type {Record} */ - let meta = {}; - if (typeof rawMeta === "string") { - try { - const parsed = JSON.parse(rawMeta); - meta = parsed && typeof parsed === "object" && !Array.isArray(parsed) - ? /** @type {Record} */ (parsed) - : {}; - } catch (err) { - throw new Error( - `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ${err instanceof Error ? err.message : String(err)}`, - { cause: err } - ); - } + if (typeof rawMeta !== "string") { + throw new Error(`bundle metadata missing for ${ns}/${worker}@${sourceVersion}`); + } + /** @type {unknown} */ + let parsed; + try { + parsed = JSON.parse(rawMeta); + } catch (err) { + throw new Error( + `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ${errorMessage(err)}`, + { cause: err } + ); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error( + `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ` + + "__meta__ must be a JSON object" + ); } + const meta = /** @type {Record} */ (parsed); assertWorkerLoaderUserEnvBudget({ ns, worker, diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 7af2261..2b86810 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -47,11 +47,14 @@ import { assertWorkerLoaderUserEnvBudget, decryptSecretHash, } from "control-env-budget"; +import { + WORKER_LOADER_CODE_MAX_BYTES, + estimatePreparedWorkerLoaderCodeBytes, +} from "control-worker-code-budget"; import { SecretEnvelopeError } from "shared-secret-envelope"; const MAX_COMMIT_ATTEMPTS = 5; const DEPLOY_JSON_BODY_MAX_BYTES = 32 * 1024 * 1024; -const WORKER_LOADER_CODE_MAX_BYTES = 64 * 1024 * 1024; const DEPLOY_ASSET_UPLOAD_CONCURRENCY = 8; class DeployAbort extends ControlAbort {} @@ -153,24 +156,22 @@ function deployRequestErrorFromUnknown(err) { ); } -/** @param {string | Uint8Array} bytes */ -function moduleBodyByteLength(bytes) { - return typeof bytes === "string" ? Buffer.byteLength(bytes, "utf8") : bytes.byteLength; -} - /** * @param {{ prepared: PreparedBundle, ns: string, name: string }} args */ function validateWorkerLoaderCodeBudget({ prepared, ns, name }) { - let totalBytes = 0; - for (const [, bytes] of prepared.normalized) { - totalBytes += moduleBodyByteLength(bytes); + let totalBytes; + try { + totalBytes = estimatePreparedWorkerLoaderCodeBytes(prepared); + } catch (err) { + throw invalidDeployRequest(errMessage(err)); } if (totalBytes > WORKER_LOADER_CODE_MAX_BYTES) { throw new DeployRequestError( 413, "worker_code_too_large", - `module bodies for ${ns}/${name} total ${totalBytes} bytes, exceeding workerd workerLoader code limit ${WORKER_LOADER_CODE_MAX_BYTES} bytes` + `final WorkerCode for ${ns}/${name} totals ${totalBytes} bytes, ` + + `exceeding workerd workerLoader code limit ${WORKER_LOADER_CODE_MAX_BYTES} bytes` ); } } diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 7552fba..eed7795 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -27,6 +27,14 @@ const MAX_NS_SECRET_ATTEMPTS = 5; class NamespaceSecretAbort extends ControlAbort {} +/** @param {unknown} err */ +function namespaceSecretMutationErrorResponse(err) { + if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); + if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); + if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + return null; +} + /** * @param {{ * redis: import("shared-redis").RedisSession, @@ -197,9 +205,8 @@ export async function handle({ request, env, method, nsName, secretKey, requestI plaintext: put.plaintext, }); } catch (err) { - if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); - if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); - if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + const response = namespaceSecretMutationErrorResponse(err); + if (response) return response; throw err; } log("info", "ns_secret_set", { request_id: requestId, namespace: nsName, key: secretKey }); @@ -223,9 +230,8 @@ export async function handle({ request, env, method, nsName, secretKey, requestI method: "DELETE", }); } catch (err) { - if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); - if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); - if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + const response = namespaceSecretMutationErrorResponse(err); + if (response) return response; throw err; } log("info", "ns_secret_deleted", { diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 23ada8e..6a0c8c8 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -34,6 +34,26 @@ const MAX_SECRET_ATTEMPTS = 5; /** * @typedef {import("shared-redis").RedisClient} RedisClient * @typedef {import("control-routing").RedisClient} RoutingRedisClient + * @typedef {{ + * namespace: string, + * name: string, + * key: string, + * version: string, + * previousVersion: string, + * set?: boolean, + * deleted?: boolean, + * }} SecretMutationVersionPayload + * @typedef {{ + * namespace: string, + * name: string, + * key: string, + * secretWritten: boolean, + * reloadForced: boolean, + * effect: string, + * warnings: { kind: string, reason: string, nextPickup: string }[], + * set?: boolean, + * deleted?: boolean, + * }} SecretMutationDeferredPayload */ class SecretAbort extends ControlAbort {} @@ -146,7 +166,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI previous_version: result.previousVersion, new_version: result.version, }); - /** @type {{ namespace: string, name: string, key: string, version: string, previousVersion: string, set?: boolean, deleted?: boolean }} */ + /** @type {SecretMutationVersionPayload} */ const payload = { namespace: ns, name, @@ -190,7 +210,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI status: err.status, ...formatError(err), }); - /** @type {{ namespace: string, name: string, key: string, secretWritten: boolean, reloadForced: boolean, effect: string, warnings: { kind: string, reason: string, nextPickup: string }[], set?: boolean, deleted?: boolean }} */ + /** @type {SecretMutationDeferredPayload} */ const payload = { namespace: ns, name, @@ -219,7 +239,16 @@ export async function handle({ request, env, method, ns, name, subPath, requestI // env-budget checks must cover active and retained versions before writing a // new secret shape. /** - * @param {{ redis: RedisClient, ns: string, name: string, key: string, method: string, value: string | null, plaintext?: string | null, controlEnv: Record }} args + * @param {{ + * redis: RedisClient, + * ns: string, + * name: string, + * key: string, + * method: "PUT" | "DELETE", + * value: string | null, + * plaintext?: string | null, + * controlEnv: Record, + * }} args */ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = null, controlEnv }) { const secretsKey = `secrets:${ns}:${name}`; @@ -232,7 +261,13 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n }); }, }, async (iso) => { - const watches = [deleteLockKey(ns, name), nsSecretsKey, secretsKey, routesKey(ns), workerVersionsKey(ns, name)]; + const watches = [ + deleteLockKey(ns, name), + nsSecretsKey, + secretsKey, + routesKey(ns), + workerVersionsKey(ns, name), + ]; await iso.watch(...watches); const callerLock = await iso.get(deleteLockKey(ns, name)); @@ -257,52 +292,50 @@ async function mutateSecret({ redis, ns, name, key, method, value, plaintext = n } } - const multi = iso.multi(); - if (method === "PUT" || method === "DELETE") { - if (method === "PUT" && typeof plaintext !== "string") throw new Error("PUT secret plaintext missing"); - const activeVersion = await iso.hGet(routesKey(ns), name); - const retainedVersions = await iso.zRange(workerVersionsKey(ns, name), 0, -1); - const nsEncrypted = await iso.hGetAll(nsSecretsKey); - const workerEncrypted = await iso.hGetAll(secretsKey); - const workerBudgetEncrypted = { ...workerEncrypted }; - delete workerBudgetEncrypted[key]; - const [nsSecrets, workerSecrets] = await Promise.all([ - decryptSecretHash({ - encrypted: nsEncrypted, - env: controlEnv, - hashKey: nsSecretsKey, - ignoreSecretEnvelopeErrors: method === "DELETE", - }), - decryptSecretHash({ - encrypted: workerBudgetEncrypted, - env: controlEnv, - hashKey: secretsKey, - ignoreSecretEnvelopeErrors: method === "DELETE", - }), - ]); - if (method === "PUT") { - workerSecrets[key] = /** @type {string} */ (plaintext); - } - await assertWorkerVersionsUserEnvBudget({ - redis: iso, - ns, - worker: name, - versions: [ - ...retainedVersions, - ...(typeof activeVersion === "string" && activeVersion ? [activeVersion] : []), - ], - versionEstimates: typeof activeVersion === "string" && activeVersion - ? [{ - sourceVersion: activeVersion, - estimatedVersion: WORKER_LOADER_ENV_VERSION_PLACEHOLDER, - }] - : [], - nsSecrets, - workerSecrets, - assetsCdnBase: controlEnv.ASSETS_CDN_BASE, - }); + if (method === "PUT" && typeof plaintext !== "string") throw new Error("PUT secret plaintext missing"); + const activeVersion = await iso.hGet(routesKey(ns), name); + const retainedVersions = await iso.zRange(workerVersionsKey(ns, name), 0, -1); + const nsEncrypted = await iso.hGetAll(nsSecretsKey); + const workerEncrypted = await iso.hGetAll(secretsKey); + const workerBudgetEncrypted = { ...workerEncrypted }; + delete workerBudgetEncrypted[key]; + const [nsSecrets, workerSecrets] = await Promise.all([ + decryptSecretHash({ + encrypted: nsEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", + }), + decryptSecretHash({ + encrypted: workerBudgetEncrypted, + env: controlEnv, + hashKey: secretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", + }), + ]); + if (method === "PUT") { + workerSecrets[key] = /** @type {string} */ (plaintext); } + await assertWorkerVersionsUserEnvBudget({ + redis: iso, + ns, + worker: name, + versions: [ + ...retainedVersions, + ...(typeof activeVersion === "string" && activeVersion ? [activeVersion] : []), + ], + versionEstimates: typeof activeVersion === "string" && activeVersion + ? [{ + sourceVersion: activeVersion, + estimatedVersion: WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + }] + : [], + nsSecrets, + workerSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); + const multi = iso.multi(); if (method === "PUT") { if (typeof value !== "string") throw new Error("PUT secret value missing"); multi.hSet(secretsKey, key, value); diff --git a/control/worker-code-budget.js b/control/worker-code-budget.js new file mode 100644 index 0000000..682f0c9 --- /dev/null +++ b/control/worker-code-budget.js @@ -0,0 +1,54 @@ +import { + WORKER_LOADER_CODE_MAX_BYTES, + estimateFinalWorkerLoaderCodeBytes, +} from "runtime-load-code-budget"; +import D1_CLIENT_SOURCE from "runtime-d1-client-source"; +import D1_DATA_FIELD_SOURCE from "runtime-d1-data-field-source"; +import D1_PARAMS_SOURCE from "runtime-d1-params-source"; +import SQL_SPLITTER_SOURCE from "runtime-sql-splitter-source"; +import D1_TRANSPORT_SOURCE from "runtime-d1-transport-source"; +import R2_CLIENT_SOURCE from "runtime-r2-client-source"; +import R2_UTILS_SOURCE from "runtime-r2-utils-source"; +import DO_CLIENT_SOURCE from "runtime-do-client-source"; +import DO_TRANSPORT_SOURCE from "runtime-do-transport-source"; +import OWNER_ENDPOINT_SOURCE from "runtime-owner-endpoint-source"; +import OWNER_HINT_CACHE_SOURCE from "runtime-owner-hint-cache-source"; +import REQUEST_ID_SOURCE from "runtime-request-id-source"; +import WORKFLOWS_CLIENT_SOURCE from "runtime-workflows-client-source"; + +export { WORKER_LOADER_CODE_MAX_BYTES }; + +const RUNTIME_INJECTION_SOURCES = Object.freeze({ + d1ClientSource: D1_CLIENT_SOURCE, + d1DataFieldSource: D1_DATA_FIELD_SOURCE, + d1ParamsSource: D1_PARAMS_SOURCE, + sqlSplitterSource: SQL_SPLITTER_SOURCE, + d1TransportSource: D1_TRANSPORT_SOURCE, + r2ClientSource: R2_CLIENT_SOURCE, + r2UtilsSource: R2_UTILS_SOURCE, + doClientSource: DO_CLIENT_SOURCE, + doTransportSource: DO_TRANSPORT_SOURCE, + ownerEndpointSource: OWNER_ENDPOINT_SOURCE, + ownerHintCacheSource: OWNER_HINT_CACHE_SOURCE, + requestIdSource: REQUEST_ID_SOURCE, + workflowsClientSource: WORKFLOWS_CLIENT_SOURCE, +}); + +/** + * @param {{ + * meta: { mainModule?: unknown, modules?: Record | null, [key: string]: unknown }, + * normalized: Array<[string, string | Uint8Array]>, + * }} prepared + */ +export function estimatePreparedWorkerLoaderCodeBytes(prepared) { + const mainModule = prepared.meta.mainModule; + if (typeof mainModule !== "string" || !mainModule) { + throw new Error("WorkerCode budget requires prepared meta.mainModule"); + } + return estimateFinalWorkerLoaderCodeBytes({ + mainModule, + normalized: prepared.normalized, + meta: prepared.meta, + runtimeSources: RUNTIME_INJECTION_SOURCES, + }); +} diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index 84a2a5e..34799ba 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -46,6 +46,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "runtime-lib", esModule = embed "../runtime/lib.js"), (name = "runtime-load", esModule = embed "../runtime/load.js"), (name = "runtime-load-env-build", esModule = embed "../runtime/load/env-build.js"), + (name = "runtime-load-code-budget", esModule = embed "../runtime/load/code-budget.js"), (name = "runtime-load-module-rewrite", esModule = embed "../runtime/load/module-rewrite.js"), (name = "runtime-load-wrapper-generate", esModule = embed "../runtime/load/wrapper-generate.js"), (name = "runtime-bindings-proxy", esModule = embed "../runtime/bindings/proxy.js"), diff --git a/docs/compatibility.md b/docs/compatibility.md index b6a8860..8ce9ec8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -59,7 +59,7 @@ compatibility flag, so certificate hostname validation can change for all dates. | Service bindings | Supported | workerd service binding JSRPC and fetch dispatch. | WDL resolves target worker metadata, cold-loads immutable versions, propagates request ids where available, and enforces namespace/action ACL through control metadata. | Frozen-version service targets do not evict active siblings by design. | None currently documented. | | Platform bindings | Supported | workerd named entrypoint/JSRPC mechanics. | WDL expands ACL-checked platform bindings from control metadata. | Platform bindings are WDL-specific. | Platform bindings are not a Cloudflare tenant portability feature. | | Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding/workflow env values during deploy/secret mutation, accounts for V8 two-byte string storage for non-Latin-1 strings, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | -| Worker code size | Supported | Dynamic worker module bodies are accepted by `workerLoader` up to workerd's 64 MiB limit. | Control rejects module bodies over 64 MiB before allocating a version. | WDL's deploy JSON body limit is lower for ordinary inline deployments. | Large server-side bundle assembly paths must keep this guard. | +| Worker code size | Supported | Dynamic worker module bodies are accepted by `workerLoader` up to workerd's 64 MiB limit. | Control estimates final WorkerCode, including runtime-injected wrapper/client modules and workflow import rewrites, before allocating a version. | WDL's deploy JSON body limit is lower for ordinary inline deployments. | Large server-side bundle assembly paths must keep this guard. | ## Control Plane And Developer Tooling diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index decb162..d0e2a6a 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -45,7 +45,7 @@ Node.js TLS 行为跟随 bundled workerd binary。升级到 workerd 2026-07-01 | Service bindings | Supported | workerd service binding JSRPC 和 fetch dispatch。 | WDL 解析 target worker metadata,cold-load immutable version,在可用时传播 request id,并通过 control metadata 做 namespace/action ACL。 | Frozen-version service target 按设计不 evict active siblings。 | 当前无已记录缺口。 | | Platform bindings | Supported | workerd named entrypoint/JSRPC 机制。 | WDL 从 control metadata 展开 ACL-checked platform bindings。 | Platform bindings 是 WDL-specific。 | 不是 Cloudflare tenant portability feature。 | | Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 过程中按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets 加 runtime 注入的 binding/workflow env value,并计入非 Latin-1 字符串的 V8 two-byte storage;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | -| Worker code size | Supported | Dynamic worker module bodies 可由 `workerLoader` 接受,直到 workerd 的 64 MiB 上限。 | Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies。 | WDL 的 deploy JSON body limit 对普通 inline deploy 更低。 | 大型 server-side bundle assembly 路径必须保留这道 guard。 | +| Worker code size | Supported | Dynamic worker module bodies 可由 `workerLoader` 接受,直到 workerd 的 64 MiB 上限。 | Control 会在分配 version 前估算最终 WorkerCode,包含 runtime 注入的 wrapper/client modules 和 workflow import rewrite。 | WDL 的 deploy JSON body limit 对普通 inline deploy 更低。 | 大型 server-side bundle assembly 路径必须保留这道 guard。 | ## 控制面和开发工具 diff --git a/docs/modules/cli.md b/docs/modules/cli.md index 7816063..551e907 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -148,7 +148,7 @@ Supported config surfaces: | Field | WDL behavior | |---|---| -| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit; module bodies must fit workerd's 64 MiB `workerLoader` code limit. | +| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit; final WorkerCode, including runtime-injected modules, must fit workerd's 64 MiB `workerLoader` code limit. | | `[vars]` | String, number, and boolean values are accepted and stringified into `env`; vars, namespace/worker secrets, and runtime-injected binding/workflow env values must fit WDL's headroomed workerd 1 MiB `workerLoader` env budget. | | `[[kv_namespaces]]` | `id` is a platform-local KV namespace id, not a Cloudflare UUID. | | `[[r2_buckets]]` | `binding` plus `bucket_name` become a namespace-scoped virtual R2 bucket under the platform S3 bucket. | diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index ce83d2f..8b2bc00 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -91,7 +91,7 @@ WDL 遵循 Wrangler selected-env 继承规则: | 字段 | WDL 行为 | |---|---| -| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`;module bodies 必须落在 workerd 的 64 MiB `workerLoader` code limit 内。 | +| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`;包含 runtime 注入模块后的最终 WorkerCode 必须落在 workerd 64 MiB `workerLoader` code limit 内。 | | `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 必须落在 WDL 留有 headroom 的 workerd 1 MiB `workerLoader` env budget 内。 | | `[[kv_namespaces]]` | `id` 是 platform-local KV namespace id,不是 Cloudflare UUID。 | | `[[r2_buckets]]` | `binding` 加 `bucket_name` 映射为平台 S3 bucket 下的 namespace-scoped virtual R2 bucket。 | diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index 44de508..8b1f1b5 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -78,11 +78,12 @@ Control lifecycle operations are split so each critical transition has one autho - Deploy parses the supported Wrangler/JSONC shape, validates bindings and routes, allocates the next immutable version through `worker:::next_version`, writes bundle metadata/modules/assets, then enters the same promote path used by - explicit promotion. Before allocation, deploy checks the 64 MiB workerd dynamic code - limit and runs an advisory pass over the candidate metadata and current secret - envelopes. The watched commit path is the authoritative headroomed `workerLoader` - env-budget check after version allocation and metadata materialization, such as - resolved D1 database ids, before writing the version. + explicit promotion. Before allocation, deploy estimates final WorkerCode under + workerd's 64 MiB limit, including runtime-injected wrapper/client modules and workflow + import rewrites, and runs an advisory pass over the candidate metadata and current + secret envelopes. The watched commit path is the authoritative headroomed + `workerLoader` env-budget check after version allocation and metadata materialization, + such as resolved D1 database ids, before writing the version. - Promote is the only active-route flip. It WATCHes the delete lock, bundle metadata, D1 refs, service-binding target refs, queue consumer keys, host declarations, and pattern keys needed for the candidate. The EXEC updates active routes, host reverse indexes, @@ -262,8 +263,9 @@ Auth-specific contract: diagnostics, backend messages, and provider errors belong in logs unless the endpoint explicitly owns a diagnostic response field. - Deploy and secret mutations return `worker_code_too_large` when tenant module bodies - exceed the workerd 64 MiB dynamic code limit, and `worker_env_too_large` when the - estimated `workerLoader` env exceeds WDL's headroomed 1 MiB budget. + plus runtime-injected modules exceed workerd's 64 MiB dynamic code limit, and + `worker_env_too_large` when the estimated `workerLoader` env exceeds WDL's headroomed + 1 MiB budget. `worker_env_too_large` details include `namespace`, optional `worker`, `env_bytes`, `max_env_bytes`, `upstream_max_env_bytes`, and `headroom_bytes`. When the blocker is an already-retained version being re-estimated during a secret mutation, details also diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 78f78c6..72e8202 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -64,7 +64,7 @@ Host、secret、data 和 auth 操作: Control lifecycle 操作会拆开处理,确保每个关键 transition 只有一个权威入口: -- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会检查 workerd 64 MiB dynamic code limit,并对候选 metadata 和当前 secret envelope 做一次 advisory pass。真正的 Redis WATCH commit 会在分配真实 version 并完成 metadata materialization(例如 D1 database id 解析)之后用 headroomed `workerLoader` env-budget 做权威检查,然后才写入 version。 +- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会按 workerd 64 MiB 上限估算最终 WorkerCode,包含 runtime 注入的 wrapper/client modules 和 workflow import rewrite,并对候选 metadata 和当前 secret envelope 做一次 advisory pass。真正的 Redis WATCH commit 会在分配真实 version 并完成 metadata materialization(例如 D1 database id 解析)之后用 headroomed `workerLoader` env-budget 做权威检查,然后才写入 version。 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 - Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Secret DELETE 仍是 repair surface:估算删除后的 env 时,会跳过另一层 secret scope 里的坏 envelope,避免无关的 namespace 或 worker 坏 secret 阻塞删除;Secret PUT 仍会对 retained corrupt envelope fail closed。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前,会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 @@ -135,7 +135,7 @@ Auth 子合同: - Details 可以增加字段,但不能覆盖 `error`、`message` 或 legacy `reason`。Auth reject reason 是 `error` machine code;日志可以把 `reason` 作为诊断上下文。 - Delegated issue 的 409 reason 有不同 retry 语义:`delegated_issue_busy` 表示 issuer/template lock 清除后可重试;`active_quota_exceeded` 在已有 delegated credential 过期或 revoke 前不应重试;`namespace_collision` 表示 Auth 已耗尽 configured candidate retry budget。 - Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志;除非 endpoint 明确拥有某个 diagnostic response field,否则不进入客户端 body。 -- Deploy 和 secret mutation 在 tenant module bodies 超过 workerd 64 MiB dynamic code limit 时返回 `worker_code_too_large`,在估算的 `workerLoader` env 超过 WDL headroomed 1 MiB budget 时返回 `worker_env_too_large`。`worker_env_too_large` details 包含 `namespace`、可选 `worker`、`env_bytes`、`max_env_bytes`、`upstream_max_env_bytes` 和 `headroom_bytes`。如果 blocker 是 secret mutation 重新估算到的既有 retained version,details 还会包含 `source_version` 和 `estimated_version`,message 会标出 `/@`,方便 operator 定位要删除或 redeploy 的 retained version。 +- Deploy 和 secret mutation 在 tenant module bodies 加 runtime 注入模块后超过 workerd 64 MiB dynamic code limit 时返回 `worker_code_too_large`,在估算的 `workerLoader` env 超过 WDL headroomed 1 MiB budget 时返回 `worker_env_too_large`。`worker_env_too_large` details 包含 `namespace`、可选 `worker`、`env_bytes`、`max_env_bytes`、`upstream_max_env_bytes` 和 `headroom_bytes`。如果 blocker 是 secret mutation 重新估算到的既有 retained version,details 还会包含 `source_version` 和 `estimated_version`,message 会标出 `/@`,方便 operator 定位要删除或 redeploy 的 retained version。 - Control 不直接调用 gateway。它写 Redis 并 publish invalidation message。 - Control 在进入 Redis mutation loop 前加密 secret PUT value。加密/provider 失败会返回 control error,不写 plaintext fallback。 - Worker delete 先 commit Redis lifecycle state;异步 S3 cleanup enqueue 是 best-effort,失败时返回 warning。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 6be6815..5b5f834 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -266,8 +266,9 @@ when a matching active tail session exists. Do not re-add the `experimental` compatibility flag or `allowExperimental` to loaded WorkerCode unless another upstream API explicitly requires it. - Upstream workerd 2026-07-01 caps dynamic worker code at 64 MiB and serialized dynamic - env at 1 MiB. Control rejects module bodies over 64 MiB before version allocation. - Vars, namespace/worker secrets, and runtime-injected binding/workflow env values get + env at 1 MiB. Control estimates final WorkerCode before version allocation, including + runtime-injected wrapper/client modules and workflow import rewrites. Vars, + namespace/worker secrets, and runtime-injected binding/workflow env values get an advisory deploy pass and an authoritative watched-commit check against a headroomed `workerLoader` env budget because workerd's final enforcement includes the full `env` estimate after version allocation and metadata materialization. The estimate starts diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 5916195..0098fa6 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -110,7 +110,7 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - Python Workers modules 不受支持。Control 会拒绝新的 `py` module manifest,runtime/do-runtime 会拒绝 retained metadata 中的 `py` module,而不是让 workerd 之后抛 mixed JS/Python bundle error。 - 在已有 Redis DB 0 上 rollout 2026-07-01 workerd 适配前,先安装 `redis-cli`,再对 control Redis database 运行 `node scripts/scan-workerd-0701-metadata.mjs`,找出缺少 metadata、仍包含 Python modules 或上游 experimental compatibility flags,或按 2026-07-01 估算器会超过 headroomed `workerLoader` env budget 的 retained versions。这个 scanner 只读且不解密 secret;它用 encrypted envelope length 作为 conservative two-byte string 上界。涉及 secret 的 `worker_env_too_large` finding 应作为 rollout blocker 精确复核,例如 redeploy,或走正常 secret mutation API path 触发会解密的精确估算。扫描 assets CDN base 比默认 placeholder 更长的部署时设置 `ASSETS_CDN_BASE`。 - Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 -- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前拒绝超过 64 MiB 的 module bodies;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会先做 advisory deploy pass,并在 Redis WATCH commit 内用真实 version 和 materialization 后的 metadata 做权威检查,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 +- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前估算最终 WorkerCode,包含 runtime 注入的 wrapper/client modules 和 workflow import rewrite;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会先做 advisory deploy pass,并在 Redis WATCH commit 内用真实 version 和 materialization 后的 metadata 做权威检查,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 - 当前 stock workerd 中,客户端在 async `ReadableStream` response body 中途断开时,不一定调用 stream source 的 `cancel()`。Tenant streaming/SSE worker 应使用自己的 heartbeat、timeout 或应用层 close path,不要把 disconnect-driven `cancel()` 当成唯一资源清理信号。 - workerd 升级仍可能改变默认或 compatibility-flagged runtime surface;升级时要审 exposed surface,而不只审 loader/abort path。 diff --git a/docs/source-map.md b/docs/source-map.md index 9842096..585386e 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -44,6 +44,7 @@ are outside this map unless they own runtime or deployable service behavior. | `control/routing.js`, `control/routing/route-plan.js` | Promote, secret bump/promote, host reconcile WATCH/MULTI loops, and pure route/pattern planning helpers. | | `control/lifecycle-indexes.js` | Redis mutation helpers for worker lifecycle, cron, queue consumer, and referrer indexes. | | `control/env-budget.js` | Control-plane estimate of workerd `workerLoader` env size for deploy and secret mutation guards. | +| `control/worker-code-budget.js` | Control-plane final WorkerCode size estimate for deploy guards, sharing runtime wrapper/module injection rules. | | `control/d1-*` | D1 control metadata, store, lifecycle, migration, and d1-runtime client modules. | | `control/r2.js` | Control-plane R2 bucket/object API client for the configured S3-compatible store. | | `control/s3.js` | S3-compatible ASSETS upload helper. | @@ -104,6 +105,7 @@ are outside this map unless they own runtime or deployable service behavior. | `examples/` | Manual demos and reference projects. Tests should not silently depend on them unless the fixture graduates to `test-workers/`. | | `scripts/run-integration-tests.js` | Integration worker-pool runner. | | `scripts/compile-workerd-configs.js` | Compiles workerd Cap'n Proto configs into `dist/workerd-configs/*.bin`. | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | Pin-bump flag extractor. | | `scripts/scan-workerd-0701-metadata.mjs` | Read-only Redis metadata scanner for rollout checks before the workerd 2026-07-01 adaptation: reports retained versions with missing metadata, experimental flags, Python modules, or oversized estimated env. | ## Infrastructure diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index 8b13635..52b7819 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -41,6 +41,7 @@ | `control/routing.js`、`control/routing/route-plan.js` | Promote、secret bump/promote、host reconcile WATCH/MULTI loops,以及纯 route/pattern planning helpers。 | | `control/lifecycle-indexes.js` | Worker lifecycle、cron、queue consumer 和 referrer indexes 的 Redis mutation helpers。 | | `control/env-budget.js` | Deploy 和 secret mutation guard 使用的 workerd `workerLoader` env size 控制面估算。 | +| `control/worker-code-budget.js` | Deploy guard 使用的最终 WorkerCode size 控制面估算,复用 runtime wrapper/module injection 规则。 | | `control/d1-*` | D1 control metadata、store、lifecycle、migration 和 d1-runtime client modules。 | | `control/r2.js` | 面向配置的 S3-compatible store 的 control-plane R2 bucket/object API client。 | | `control/s3.js` | S3-compatible ASSETS upload helper。 | @@ -101,6 +102,7 @@ | `examples/` | 手工 demo 和 reference projects。测试不应悄悄依赖它们,除非 fixture 明确迁入 `test-workers/`。 | | `scripts/run-integration-tests.js` | Integration worker-pool runner。 | | `scripts/compile-workerd-configs.js` | 把 workerd Cap'n Proto configs 编译成 `dist/workerd-configs/*.bin`。 | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | pin bump flag 提取脚本。 | | `scripts/scan-workerd-0701-metadata.mjs` | workerd 2026-07-01 适配 rollout 前使用的只读 Redis metadata scanner,报告缺失 metadata、使用 experimental flags、Python modules 或估算 env 超限的 retained versions。 | ## Infrastructure diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index fef1bb8..911e060 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -48,6 +48,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-load", esModule = embed "load.js"), (name = "runtime-load-env-build", esModule = embed "load/env-build.js"), + (name = "runtime-load-code-budget", esModule = embed "load/code-budget.js"), (name = "runtime-load-module-rewrite", esModule = embed "load/module-rewrite.js"), (name = "runtime-load-wrapper-generate", esModule = embed "load/wrapper-generate.js"), (name = "runtime-metrics", esModule = embed "metrics.js"), @@ -160,6 +161,7 @@ const controlWorker :Workerd.Worker = ( (name = "control-bindings", esModule = embed "../control/bindings.js"), (name = "control-topology", esModule = embed "../control/topology.js"), (name = "control-env-budget", esModule = embed "../control/env-budget.js"), + (name = "control-worker-code-budget", esModule = embed "../control/worker-code-budget.js"), (name = "control-lifecycle-indexes", esModule = embed "../control/lifecycle-indexes.js"), (name = "control-routing", esModule = embed "../control/routing.js"), (name = "control-routing-route-plan", esModule = embed "../control/routing/route-plan.js"), @@ -218,6 +220,22 @@ const controlWorker :Workerd.Worker = ( (name = "shared-env", esModule = embed "../shared/env.js"), (name = "shared-d1-timeout", esModule = embed "../shared/d1-timeout.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), + (name = "runtime-load-code-budget", esModule = embed "load/code-budget.js"), + (name = "runtime-load-module-rewrite", esModule = embed "load/module-rewrite.js"), + (name = "runtime-load-wrapper-generate", esModule = embed "load/wrapper-generate.js"), + (name = "runtime-d1-client-source", text = embed "d1-client.js"), + (name = "runtime-d1-data-field-source", text = embed "../shared/d1-data-field.js"), + (name = "runtime-d1-params-source", text = embed "../shared/d1-params.js"), + (name = "runtime-sql-splitter-source", text = embed "../shared/sql-splitter.js"), + (name = "runtime-d1-transport-source", text = embed "../shared/d1-transport.js"), + (name = "runtime-r2-client-source", text = embed "r2-client.js"), + (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), + (name = "runtime-do-client-source", text = embed "do-client.js"), + (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), + (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), + (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), + (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), (name = "wdl-package-json-source", text = embed "../package.json"), (name = "runtime-r2-utils", esModule = embed "r2-utils.js"), # Pre-bundled via `npm run build:vendor`; workerd embed can't walk node_modules. diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index f9a4308..a34494a 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -47,6 +47,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-load", esModule = embed "load.js"), (name = "runtime-load-env-build", esModule = embed "load/env-build.js"), + (name = "runtime-load-code-budget", esModule = embed "load/code-budget.js"), (name = "runtime-load-module-rewrite", esModule = embed "load/module-rewrite.js"), (name = "runtime-load-wrapper-generate", esModule = embed "load/wrapper-generate.js"), (name = "runtime-metrics", esModule = embed "metrics.js"), diff --git a/runtime/load.js b/runtime/load.js index cbb27b9..91fe8d6 100644 --- a/runtime/load.js +++ b/runtime/load.js @@ -5,10 +5,6 @@ import { bundleToWorkerCode } from "runtime-lib"; import { formatError } from "shared-observability"; import { withInternalAuth } from "shared-internal-auth"; import { discardResponseBody } from "shared-respond"; -import { - WDL_RESERVED_ENTRYPOINT_RE, - isValidJsClassDeclarationName, -} from "shared-ns-pattern"; import { parseRuntimeLoadWorkerId } from "shared-worker-id"; import D1_CLIENT_SOURCE from "runtime-d1-client-source"; import D1_DATA_FIELD_SOURCE from "runtime-d1-data-field-source"; @@ -24,16 +20,9 @@ import OWNER_HINT_CACHE_SOURCE from "runtime-owner-hint-cache-source"; import REQUEST_ID_SOURCE from "runtime-request-id-source"; import WORKFLOWS_CLIENT_SOURCE from "runtime-workflows-client-source"; import { - HOST_BINDING_RESERVED_MODULES, - HOST_BINDING_RESERVED_MODULE_NAMES, - WORKFLOWS_MODULE_NAME, - WORKFLOWS_MODULE_SOURCE, - rewriteCloudflareWorkflowsImports, -} from "runtime-load-module-rewrite"; -import { - generateAbortShimWrapperModule, - generateHostBindingWrapperModule, -} from "runtime-load-wrapper-generate"; + analyzeRuntimeMeta, + injectRuntimeModulesForHostBindings, +} from "runtime-load-code-budget"; import { buildWorkerEnv } from "runtime-load-env-build"; export { buildWorkerEnv } from "runtime-load-env-build"; @@ -81,95 +70,7 @@ const utf8Decoder = new TextDecoder(); * }} RuntimeLoaderMetrics * @typedef {(options: { props: Record }) => unknown} RuntimeEntrypointFactory * @typedef {{ exports: Record & { KV: RuntimeEntrypointFactory, Assets: RuntimeEntrypointFactory, QueueProducer: RuntimeEntrypointFactory, D1Database: RuntimeEntrypointFactory, R2Bucket: RuntimeEntrypointFactory, ServiceBinding: RuntimeEntrypointFactory, DurableObjectNamespace: RuntimeEntrypointFactory, InternalAuthBackend: RuntimeEntrypointFactory } }} RuntimeContext - * @typedef {[name: string, source: string]} RuntimeModuleInjection - */ - -/** @type {RuntimeModuleInjection} */ -const REQUEST_ID_MODULE_INJECTION = ["_wdl-request-id.js", REQUEST_ID_SOURCE]; -const D1_DATA_FIELD_MODULE_NAME = "_wdl-d1-data-field.js"; -const D1_TRANSPORT_INJECTED_SOURCE = D1_TRANSPORT_SOURCE.replace( - /from "shared-d1-data-field";/, - `from "./${D1_DATA_FIELD_MODULE_NAME}";` -); -/** @type {RuntimeModuleInjection[]} */ -const D1_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - [D1_DATA_FIELD_MODULE_NAME, D1_DATA_FIELD_SOURCE], - ["_wdl-d1-params.js", D1_PARAMS_SOURCE], - ["_wdl-sql-splitter.js", SQL_SPLITTER_SOURCE], - ["_wdl-d1-transport.js", D1_TRANSPORT_INJECTED_SOURCE], - ["_wdl-d1-client.js", D1_CLIENT_SOURCE], -]; -/** @type {RuntimeModuleInjection[]} */ -const R2_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - ["_wdl-r2-utils.js", R2_UTILS_SOURCE], - ["_wdl-r2-client.js", R2_CLIENT_SOURCE], -]; -/** @type {RuntimeModuleInjection[]} */ -const DO_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - ["_wdl-do-transport.js", DO_TRANSPORT_SOURCE], - ["_wdl-owner-endpoint.js", OWNER_ENDPOINT_SOURCE], - ["_wdl-owner-hint-cache.js", OWNER_HINT_CACHE_SOURCE], - ["_wdl-do-client.js", DO_CLIENT_SOURCE], -]; -/** @type {RuntimeModuleInjection[]} */ -const WORKFLOWS_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - ["_wdl-workflows-client.js", WORKFLOWS_CLIENT_SOURCE], -]; - -/** - * @typedef {{ - * type: string, - * modules: RuntimeModuleInjection[], - * addBinding(plan: Pick, name: string): void, - * bindingNames(plan: Pick): string[], - * }} HostFacadeBindingDefinition - */ - -/** @type {HostFacadeBindingDefinition[]} */ -const HOST_FACADE_BINDING_DEFINITIONS = [ - { - type: "d1", - modules: D1_MODULE_INJECTIONS, - addBinding(plan, name) { plan.d1Bindings.push(name); }, - bindingNames(plan) { return plan.d1Bindings; }, - }, - { - type: "r2", - modules: R2_MODULE_INJECTIONS, - addBinding(plan, name) { plan.r2Bindings.push(name); }, - bindingNames(plan) { return plan.r2Bindings; }, - }, - { - type: "do", - modules: DO_MODULE_INJECTIONS, - addBinding(plan, name) { plan.doBindings.push(name); }, - bindingNames(plan) { return plan.doBindings; }, - }, -]; - -/** @param {WorkerCodeShape} workerCode @param {RuntimeModuleInjection[]} modules */ -function injectRuntimeModules(workerCode, modules) { - for (const [name, source] of modules) workerCode.modules[name] = source; -} - -/** - * @param {RuntimeMetaPlan} plan - * @param {RuntimeBindingSpec} spec - * @param {string} name */ -function addHostFacadeBinding(plan, spec, name) { - const definition = HOST_FACADE_BINDING_DEFINITIONS.find((entry) => entry.type === spec?.type); - if (definition) definition.addBinding(plan, name); -} - -/** @param {RuntimeMetaPlan} plan */ -function hasHostFacadeBindings(plan) { - return HOST_FACADE_BINDING_DEFINITIONS.some((entry) => entry.bindingNames(plan).length > 0); -} /** * Keep the internal auth token in the host loader realm. Generated tenant @@ -280,128 +181,25 @@ function redisProxyUrl(env) { return String(env.REDIS_PROXY_URL).replace(/\/+$/, ""); } -/** @param {RuntimeBundleMeta} meta */ -function d1ExportedEntrypointNames(meta) { - /** @type {string[]} */ - const out = []; - for (const entry of meta.exports || []) { - const name = entry?.entrypoint; - if (!name || name === "default") continue; - if (typeof name !== "string") { - throw new Error(`Host binding wrapper requires exported entrypoint names to be strings, got ${JSON.stringify(name)}`); - } - if (!isValidJsClassDeclarationName(name)) { - throw new Error(`Host binding wrapper requires exported entrypoint names to be valid JS class declaration names, got ${JSON.stringify(name)}`); - } - if (WDL_RESERVED_ENTRYPOINT_RE.test(name)) { - throw new Error(`Exported entrypoint targets reserved runtime entrypoint "${name}" (redeploy worker)`); - } - out.push(name); - } - return out; -} - -/** @param {RuntimeBundleMeta} meta @param {Array<[string, RuntimeBindingSpec]>} bindingEntries @param {RuntimeWorkflowSpec[]} workflows */ -function hostWrappedClassNames(meta, bindingEntries, workflows) { - const out = new Set(d1ExportedEntrypointNames(meta)); - for (const [, spec] of bindingEntries) { - if (spec?.type === "do" && typeof spec.className === "string" && spec.className) { - if (!isValidJsClassDeclarationName(spec.className)) { - throw new Error(`Host binding wrapper requires Durable Object class names to be valid JS class declaration names, got ${JSON.stringify(spec.className)}`); - } - if (WDL_RESERVED_ENTRYPOINT_RE.test(spec.className)) { - throw new Error(`Durable Object binding targets reserved runtime entrypoint "${spec.className}" (redeploy worker)`); - } - out.add(spec.className); - } - } - for (const workflow of workflows) { - const className = workflow?.className; - if (typeof className === "string" && className) { - if (!isValidJsClassDeclarationName(className)) { - throw new Error(`Host binding wrapper requires Workflow class names to be valid JS class declaration names, got ${JSON.stringify(className)}`); - } - if (WDL_RESERVED_ENTRYPOINT_RE.test(className)) { - throw new Error(`Workflow binding targets reserved runtime entrypoint "${className}" (redeploy worker)`); - } - out.add(className); - } - } - return [...out]; -} - -/** @param {RuntimeBundleMeta} meta @returns {RuntimeMetaPlan} */ -export function analyzeRuntimeMeta(meta) { - const bindingEntries = Object.entries(meta.bindings || {}); - const workflows = Array.isArray(meta.workflows) ? meta.workflows : []; - /** @type {RuntimeMetaPlan} */ - const plan = { - bindingEntries, - workflows, - d1Bindings: [], - r2Bindings: [], - doBindings: [], - workflowBindings: Object.create(null), - hostWrappedClassNames: [], - needsDoBackend: false, - needsWorkflowsBackend: false, - needsHostBindingWrapper: false, - }; - for (const [name, spec] of bindingEntries) { - addHostFacadeBinding(plan, spec, name); - } - for (const workflow of workflows) { - if (typeof workflow?.binding === "string" && workflow.binding) plan.workflowBindings[workflow.binding] = workflow; - } - plan.needsDoBackend = plan.doBindings.length > 0; - plan.needsWorkflowsBackend = Object.keys(plan.workflowBindings).length > 0; - plan.needsHostBindingWrapper = hasHostFacadeBindings(plan) || plan.needsWorkflowsBackend; - if (plan.needsHostBindingWrapper) { - plan.hostWrappedClassNames = hostWrappedClassNames(meta, bindingEntries, workflows); - } - return plan; -} +const RUNTIME_INJECTION_SOURCES = Object.freeze({ + d1ClientSource: D1_CLIENT_SOURCE, + d1DataFieldSource: D1_DATA_FIELD_SOURCE, + d1ParamsSource: D1_PARAMS_SOURCE, + sqlSplitterSource: SQL_SPLITTER_SOURCE, + d1TransportSource: D1_TRANSPORT_SOURCE, + r2ClientSource: R2_CLIENT_SOURCE, + r2UtilsSource: R2_UTILS_SOURCE, + doClientSource: DO_CLIENT_SOURCE, + doTransportSource: DO_TRANSPORT_SOURCE, + ownerEndpointSource: OWNER_ENDPOINT_SOURCE, + ownerHintCacheSource: OWNER_HINT_CACHE_SOURCE, + requestIdSource: REQUEST_ID_SOURCE, + workflowsClientSource: WORKFLOWS_CLIENT_SOURCE, +}); /** @param {WorkerCodeShape} workerCode @param {RuntimeBundleMeta} meta @param {RuntimeMetaPlan} [plan] */ export function wrapWorkerCodeForHostBindings(workerCode, meta, plan = analyzeRuntimeMeta(meta)) { - const { d1Bindings, r2Bindings, doBindings, workflowBindings } = plan; - const originalMain = workerCode.mainModule; - if (typeof originalMain !== "string" || !originalMain) { - throw new Error("Host binding wrapper requires a string mainModule"); - } - rewriteCloudflareWorkflowsImports(workerCode); - if ( - HOST_BINDING_RESERVED_MODULES.has(originalMain) || - [...HOST_BINDING_RESERVED_MODULES].some((name) => workerCode.modules[name]) - ) { - throw new Error( - `Host binding wrapper requires reserved module names ${HOST_BINDING_RESERVED_MODULE_NAMES.join(", ")}` - ); - } - // Every loaded worker needs the __WdlAbort__ entrypoint for historical - // version eviction. Host facade bindings need the heavier request-context - // wrapper that swaps loaded-isolate facades into env. - for (const definition of HOST_FACADE_BINDING_DEFINITIONS) { - if (definition.bindingNames(plan).length > 0) { - injectRuntimeModules(workerCode, definition.modules); - } - } - if (plan.needsWorkflowsBackend) { - injectRuntimeModules(workerCode, WORKFLOWS_MODULE_INJECTIONS); - } - workerCode.modules[WORKFLOWS_MODULE_NAME] = WORKFLOWS_MODULE_SOURCE; - workerCode.modules["_wdl-wrapper.js"] = plan.needsHostBindingWrapper - ? generateHostBindingWrapperModule( - originalMain, - d1Bindings, - r2Bindings, - doBindings, - workflowBindings, - plan.hostWrappedClassNames - ) - : generateAbortShimWrapperModule(originalMain); - workerCode.mainModule = "_wdl-wrapper.js"; - return workerCode; + return injectRuntimeModulesForHostBindings(workerCode, meta, RUNTIME_INJECTION_SOURCES, plan); } /** @param {Record} env @param {string} ns @param {string} worker @param {string} version */ diff --git a/runtime/load/code-budget.js b/runtime/load/code-budget.js new file mode 100644 index 0000000..39361bf --- /dev/null +++ b/runtime/load/code-budget.js @@ -0,0 +1,370 @@ +import { + WDL_RESERVED_ENTRYPOINT_RE, + isValidJsClassDeclarationName, +} from "shared-ns-pattern"; +import { + HOST_BINDING_RESERVED_MODULES, + HOST_BINDING_RESERVED_MODULE_NAMES, + WORKFLOWS_MODULE_NAME, + WORKFLOWS_MODULE_SOURCE, + rewriteCloudflareWorkflowsImports, +} from "runtime-load-module-rewrite"; +import { + generateAbortShimWrapperModule, + generateHostBindingWrapperModule, +} from "runtime-load-wrapper-generate"; + +export const WORKER_LOADER_CODE_MAX_BYTES = 64 * 1024 * 1024; + +const D1_DATA_FIELD_MODULE_NAME = "_wdl-d1-data-field.js"; + +/** + * @typedef {string | Uint8Array} NormalizedModuleBody + * @typedef {[name: string, body: NormalizedModuleBody]} NormalizedModule + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {{ modules: Record, mainModule: string, [key: string]: unknown }} WorkerCodeShape + * @typedef {Record & { type?: string, className?: unknown }} RuntimeBindingSpec + * @typedef {{ binding?: unknown, className?: unknown }} RuntimeWorkflowSpec + * @typedef {{ entrypoint?: unknown }} RuntimeExportSpec + * @typedef {{ bindings?: Record | null, workflows?: RuntimeWorkflowSpec[] | null, exports?: RuntimeExportSpec[] | null, modules?: Record | null }} RuntimeBundleMeta + * @typedef {{ + * bindingEntries: Array<[string, RuntimeBindingSpec]>, + * workflows: RuntimeWorkflowSpec[], + * d1Bindings: string[], + * r2Bindings: string[], + * doBindings: string[], + * workflowBindings: Record, + * hostWrappedClassNames: string[], + * needsDoBackend: boolean, + * needsWorkflowsBackend: boolean, + * needsHostBindingWrapper: boolean, + * }} RuntimeMetaPlan + * @typedef {{ + * d1ClientSource: string, + * d1DataFieldSource: string, + * d1ParamsSource: string, + * sqlSplitterSource: string, + * d1TransportSource: string, + * r2ClientSource: string, + * r2UtilsSource: string, + * doClientSource: string, + * doTransportSource: string, + * ownerEndpointSource: string, + * ownerHintCacheSource: string, + * requestIdSource: string, + * workflowsClientSource: string, + * }} RuntimeInjectionSources + * @typedef {[name: string, source: string]} RuntimeModuleInjection + */ + +/** @param {string | Uint8Array} body */ +export function moduleBodyByteLength(body) { + return typeof body === "string" ? Buffer.byteLength(body, "utf8") : body.byteLength; +} + +/** @param {RuntimeInjectionSources} sources */ +function runtimeModuleInjections(sources) { + /** @type {RuntimeModuleInjection} */ + const requestIdModuleInjection = ["_wdl-request-id.js", sources.requestIdSource]; + const d1TransportInjectedSource = sources.d1TransportSource.replace( + /from "shared-d1-data-field";/, + `from "./${D1_DATA_FIELD_MODULE_NAME}";` + ); + /** @type {RuntimeModuleInjection[]} */ + const d1ModuleInjections = [ + requestIdModuleInjection, + [D1_DATA_FIELD_MODULE_NAME, sources.d1DataFieldSource], + ["_wdl-d1-params.js", sources.d1ParamsSource], + ["_wdl-sql-splitter.js", sources.sqlSplitterSource], + ["_wdl-d1-transport.js", d1TransportInjectedSource], + ["_wdl-d1-client.js", sources.d1ClientSource], + ]; + /** @type {RuntimeModuleInjection[]} */ + const r2ModuleInjections = [ + requestIdModuleInjection, + ["_wdl-r2-utils.js", sources.r2UtilsSource], + ["_wdl-r2-client.js", sources.r2ClientSource], + ]; + /** @type {RuntimeModuleInjection[]} */ + const doModuleInjections = [ + requestIdModuleInjection, + ["_wdl-do-transport.js", sources.doTransportSource], + ["_wdl-owner-endpoint.js", sources.ownerEndpointSource], + ["_wdl-owner-hint-cache.js", sources.ownerHintCacheSource], + ["_wdl-do-client.js", sources.doClientSource], + ]; + /** @type {RuntimeModuleInjection[]} */ + const workflowsModuleInjections = [ + requestIdModuleInjection, + ["_wdl-workflows-client.js", sources.workflowsClientSource], + ]; + return { + d1ModuleInjections, + r2ModuleInjections, + doModuleInjections, + workflowsModuleInjections, + }; +} + +/** + * @typedef {{ + * type: string, + * modules: RuntimeModuleInjection[], + * addBinding(plan: Pick, name: string): void, + * bindingNames(plan: Pick): string[], + * }} HostFacadeBindingDefinition + */ + +/** @param {RuntimeInjectionSources} sources @returns {HostFacadeBindingDefinition[]} */ +function hostFacadeBindingDefinitions(sources) { + const { + d1ModuleInjections, + r2ModuleInjections, + doModuleInjections, + } = runtimeModuleInjections(sources); + return [ + { + type: "d1", + modules: d1ModuleInjections, + addBinding(plan, name) { plan.d1Bindings.push(name); }, + bindingNames(plan) { return plan.d1Bindings; }, + }, + { + type: "r2", + modules: r2ModuleInjections, + addBinding(plan, name) { plan.r2Bindings.push(name); }, + bindingNames(plan) { return plan.r2Bindings; }, + }, + { + type: "do", + modules: doModuleInjections, + addBinding(plan, name) { plan.doBindings.push(name); }, + bindingNames(plan) { return plan.doBindings; }, + }, + ]; +} + +/** + * @param {RuntimeMetaPlan} plan + * @param {RuntimeBindingSpec} spec + * @param {string} name + */ +function addHostFacadeBinding(plan, spec, name) { + switch (spec?.type) { + case "d1": + plan.d1Bindings.push(name); + return; + case "r2": + plan.r2Bindings.push(name); + return; + case "do": + plan.doBindings.push(name); + return; + } +} + +/** @param {RuntimeMetaPlan} plan */ +function hasHostFacadeBindings(plan) { + return plan.d1Bindings.length > 0 || plan.r2Bindings.length > 0 || plan.doBindings.length > 0; +} + +/** @param {RuntimeBundleMeta} meta */ +function d1ExportedEntrypointNames(meta) { + /** @type {string[]} */ + const out = []; + for (const entry of meta.exports || []) { + const name = entry?.entrypoint; + if (!name || name === "default") continue; + if (typeof name !== "string") { + throw new Error( + `Host binding wrapper requires exported entrypoint names to be strings, got ${JSON.stringify(name)}` + ); + } + if (!isValidJsClassDeclarationName(name)) { + throw new Error( + `Host binding wrapper requires exported entrypoint names to be valid JS class declaration names, got ${JSON.stringify(name)}` + ); + } + if (WDL_RESERVED_ENTRYPOINT_RE.test(name)) { + throw new Error(`Exported entrypoint targets reserved runtime entrypoint "${name}" (redeploy worker)`); + } + out.push(name); + } + return out; +} + +/** + * @param {RuntimeBundleMeta} meta + * @param {Array<[string, RuntimeBindingSpec]>} bindingEntries + * @param {RuntimeWorkflowSpec[]} workflows + */ +function hostWrappedClassNames(meta, bindingEntries, workflows) { + const out = new Set(d1ExportedEntrypointNames(meta)); + for (const [, spec] of bindingEntries) { + if (spec?.type === "do" && typeof spec.className === "string" && spec.className) { + if (!isValidJsClassDeclarationName(spec.className)) { + throw new Error( + `Host binding wrapper requires Durable Object class names to be valid JS class declaration names, got ${JSON.stringify(spec.className)}` + ); + } + if (WDL_RESERVED_ENTRYPOINT_RE.test(spec.className)) { + throw new Error(`Durable Object binding targets reserved runtime entrypoint "${spec.className}" (redeploy worker)`); + } + out.add(spec.className); + } + } + for (const workflow of workflows) { + const className = workflow?.className; + if (typeof className === "string" && className) { + if (!isValidJsClassDeclarationName(className)) { + throw new Error( + `Host binding wrapper requires Workflow class names to be valid JS class declaration names, got ${JSON.stringify(className)}` + ); + } + if (WDL_RESERVED_ENTRYPOINT_RE.test(className)) { + throw new Error(`Workflow binding targets reserved runtime entrypoint "${className}" (redeploy worker)`); + } + out.add(className); + } + } + return [...out]; +} + +/** @param {RuntimeBundleMeta} meta @returns {RuntimeMetaPlan} */ +export function analyzeRuntimeMeta(meta) { + const bindingEntries = Object.entries(meta.bindings || {}); + const workflows = Array.isArray(meta.workflows) ? meta.workflows : []; + /** @type {RuntimeMetaPlan} */ + const plan = { + bindingEntries, + workflows, + d1Bindings: [], + r2Bindings: [], + doBindings: [], + workflowBindings: Object.create(null), + hostWrappedClassNames: [], + needsDoBackend: false, + needsWorkflowsBackend: false, + needsHostBindingWrapper: false, + }; + for (const [name, spec] of bindingEntries) { + addHostFacadeBinding(plan, spec, name); + } + for (const workflow of workflows) { + if (typeof workflow?.binding === "string" && workflow.binding) plan.workflowBindings[workflow.binding] = workflow; + } + plan.needsDoBackend = plan.doBindings.length > 0; + plan.needsWorkflowsBackend = Object.keys(plan.workflowBindings).length > 0; + plan.needsHostBindingWrapper = hasHostFacadeBindings(plan) || plan.needsWorkflowsBackend; + if (plan.needsHostBindingWrapper) { + plan.hostWrappedClassNames = hostWrappedClassNames(meta, bindingEntries, workflows); + } + return plan; +} + +/** + * @param {string} mainModule + * @param {RuntimeBundleMeta} meta + * @param {RuntimeInjectionSources} runtimeSources + * @param {RuntimeMetaPlan} [plan] + */ +export function runtimeInjectedModuleSources(mainModule, meta, runtimeSources, plan = analyzeRuntimeMeta(meta)) { + /** @type {Map} */ + const out = new Map(); + /** @param {RuntimeModuleInjection[]} modules */ + const addModules = (modules) => { + for (const [name, source] of modules) out.set(name, source); + }; + for (const definition of hostFacadeBindingDefinitions(runtimeSources)) { + if (definition.bindingNames(plan).length > 0) addModules(definition.modules); + } + if (plan.needsWorkflowsBackend) { + addModules(runtimeModuleInjections(runtimeSources).workflowsModuleInjections); + } + out.set(WORKFLOWS_MODULE_NAME, WORKFLOWS_MODULE_SOURCE); + out.set( + "_wdl-wrapper.js", + plan.needsHostBindingWrapper + ? generateHostBindingWrapperModule( + mainModule, + plan.d1Bindings, + plan.r2Bindings, + plan.doBindings, + plan.workflowBindings, + plan.hostWrappedClassNames + ) + : generateAbortShimWrapperModule(mainModule) + ); + return [...out]; +} + +/** + * @param {WorkerCodeShape} workerCode + * @param {RuntimeBundleMeta} meta + * @param {RuntimeInjectionSources} runtimeSources + * @param {RuntimeMetaPlan} [plan] + */ +export function injectRuntimeModulesForHostBindings( + workerCode, + meta, + runtimeSources, + plan = analyzeRuntimeMeta(meta) +) { + const originalMain = workerCode.mainModule; + if (typeof originalMain !== "string" || !originalMain) { + throw new Error("Host binding wrapper requires a string mainModule"); + } + rewriteCloudflareWorkflowsImports(workerCode); + if ( + HOST_BINDING_RESERVED_MODULES.has(originalMain) || + [...HOST_BINDING_RESERVED_MODULES].some((name) => workerCode.modules[name]) + ) { + throw new Error( + `Host binding wrapper requires reserved module names ${HOST_BINDING_RESERVED_MODULE_NAMES.join(", ")}` + ); + } + for (const [name, source] of runtimeInjectedModuleSources(originalMain, meta, runtimeSources, plan)) { + workerCode.modules[name] = source; + } + workerCode.mainModule = "_wdl-wrapper.js"; + return workerCode; +} + +/** @param {Record | null | undefined} modules @param {string} name */ +function moduleType(modules, name) { + const type = modules?.[name]?.type; + return typeof type === "string" ? type : ""; +} + +/** + * @param {{ + * mainModule: string, + * normalized: NormalizedModule[], + * meta: RuntimeBundleMeta, + * runtimeSources: RuntimeInjectionSources, + * }} args + */ +export function estimateFinalWorkerLoaderCodeBytes({ mainModule, normalized, meta, runtimeSources }) { + /** @type {Record} */ + const jsModules = Object.create(null); + /** @type {Map} */ + const userModuleBytes = new Map(); + for (const [name, body] of normalized) { + const type = moduleType(meta.modules, name); + if (type === "module" || type === "cjs") { + jsModules[name] = typeof body === "string" ? body : new TextDecoder().decode(body); + } else { + userModuleBytes.set(name, moduleBodyByteLength(body)); + } + } + rewriteCloudflareWorkflowsImports({ modules: jsModules }); + for (const [name, source] of Object.entries(jsModules)) { + userModuleBytes.set(name, Buffer.byteLength(source, "utf8")); + } + let total = 0; + for (const bytes of userModuleBytes.values()) total += bytes; + for (const [, source] of runtimeInjectedModuleSources(mainModule, meta, runtimeSources)) { + total += Buffer.byteLength(source, "utf8"); + } + return total; +} diff --git a/scripts/extract-workerd-experimental-compat-flags.mjs b/scripts/extract-workerd-experimental-compat-flags.mjs new file mode 100644 index 0000000..ae93822 --- /dev/null +++ b/scripts/extract-workerd-experimental-compat-flags.mjs @@ -0,0 +1,50 @@ +#!/usr/bin/env node + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const DEFAULT_COMPATIBILITY_DATE_CAPNP = "src/workerd/io/compatibility-date.capnp"; + +/** @param {string} source */ +export function extractExperimentalCompatFlags(source) { + /** @type {string[]} */ + const blocks = []; + /** @type {string[]} */ + let currentBlock = []; + for (const line of source.split(/\n/)) { + if (/^\s*[A-Za-z][A-Za-z0-9_]*\s+@\d+\s*:\s*Bool/.test(line)) { + if (currentBlock.length) blocks.push(currentBlock.join("\n")); + currentBlock = [line]; + } else if (currentBlock.length) { + currentBlock.push(line); + } + } + if (currentBlock.length) blocks.push(currentBlock.join("\n")); + + const flags = new Set(); + for (const block of blocks) { + if (!block.includes("$experimental")) continue; + for (const match of block.matchAll(/\$compatEnableFlag\("([^"]+)"\)/g)) { + flags.add(match[1]); + } + } + return [...flags].sort(); +} + +/** @param {string[]} [argv] */ +export function runCli(argv = process.argv.slice(2)) { + const input = argv[0] || DEFAULT_COMPATIBILITY_DATE_CAPNP; + const source = readFileSync(input, "utf8"); + for (const flag of extractExperimentalCompatFlags(source)) { + console.log(flag); + } +} + +function isMainModule() { + return process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); +} + +if (isMainModule()) { + runCli(); +} diff --git a/scripts/scan-workerd-0701-metadata.mjs b/scripts/scan-workerd-0701-metadata.mjs index 497ce1d..901a96f 100644 --- a/scripts/scan-workerd-0701-metadata.mjs +++ b/scripts/scan-workerd-0701-metadata.mjs @@ -39,6 +39,7 @@ function repoModuleDataUrl(relativePath, replacements = []) { async function importControlEnvBudget() { return await import(repoModuleDataUrl("control/env-budget.js", [ [/from "shared-secret-envelope";/, `from ${JSON.stringify(repoFileUrl("shared/secret-envelope.js"))};`], + [/from "shared-errors";/, `from ${JSON.stringify(repoFileUrl("shared/errors.js"))};`], [/from "shared-version";/, `from ${JSON.stringify(repoFileUrl("shared/version.js"))};`], ])); } @@ -61,7 +62,9 @@ export function redisCli(args, { redisUrl = redisUrlFromEnv(), spawn = spawnSync if (result.error) { if (/** @type {NodeJS.ErrnoException} */ (result.error).code === "ENOENT") { throw new Error( - "redis-cli not found. Install redis-cli/redis-tools before running scripts/scan-workerd-0701-metadata.mjs; local compose users can inspect Redis with `docker compose exec -T redis redis-cli`." + "redis-cli not found. Install redis-cli/redis-tools before running " + + "scripts/scan-workerd-0701-metadata.mjs; local compose users can inspect " + + "Redis with `docker compose exec -T redis redis-cli`." ); } throw result.error; @@ -347,9 +350,15 @@ export async function runCli() { console.log(JSON.stringify(finding)); } if (findings.length === 0) { - console.error(`Scanned ${keysScanned} worker bundle metadata keys; no workerd 0701 blockers found.`); + console.error( + `Scanned ${keysScanned} worker bundle metadata keys; ` + + "no workerd 0701 blockers found." + ); } else { - console.error(`Scanned ${keysScanned} worker bundle metadata keys; found ${findings.length} workerd 0701 blocker(s).`); + console.error( + `Scanned ${keysScanned} worker bundle metadata keys; ` + + `found ${findings.length} workerd 0701 blocker(s).` + ); process.exitCode = 1; } } diff --git a/shared/workerd-compat-flags.js b/shared/workerd-compat-flags.js index cde615d..33e72b9 100644 --- a/shared/workerd-compat-flags.js +++ b/shared/workerd-compat-flags.js @@ -2,7 +2,8 @@ * Mirrors workerd v1.20260701.1 src/workerd/io/compatibility-date.capnp. * Regenerate on every workerd pin bump from an upstream workerd source checkout: * - * node --input-type=module -e 'import fs from "node:fs"; const src=fs.readFileSync("src/workerd/io/compatibility-date.capnp","utf8"); const blocks=[]; let cur=[]; for (const line of src.split(/\n/)) { if (/^\s*[A-Za-z][A-Za-z0-9_]*\s+@\d+\s*:\s*Bool/.test(line)) { if (cur.length) blocks.push(cur.join("\n")); cur=[line]; } else if (cur.length) cur.push(line); } if (cur.length) blocks.push(cur.join("\n")); const out=new Set(); for (const b of blocks) { if (!b.includes("$experimental")) continue; for (const m of b.matchAll(/\$compatEnableFlag\("([^"]+)"\)/g)) out.add(m[1]); } console.log([...out].sort().join("\n"));' + * node scripts/extract-workerd-experimental-compat-flags.mjs \ + * /path/to/workerd/src/workerd/io/compatibility-date.capnp */ export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION = "1.20260701.1"; diff --git a/terraform/README.md b/terraform/README.md index 5bf8cf9..229423e 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -231,10 +231,11 @@ cgroup share weight. D1 and Durable Object stateful runtime containers also set explicit container `memory` hard limits. D1 defaults to `runtime_memory`; DO defaults to `runtime_memory - 128 MiB` so the colocated redis-proxy sidecar keeps task-level headroom. Both are overrideable with `d1_runtime_container_memory` / -`do_runtime_container_memory`, but the DO override must still stay below -`runtime_memory`. This matters because newer workerd releases no longer cap SQLite's -process hard heap at 512 MiB; the container cap keeps a runaway SQLite query inside the -stateful runtime container budget. The supervisor is PID 1 in that same container. +`do_runtime_container_memory`; D1 must be no greater than `runtime_memory`, and DO must +still stay below `runtime_memory` to leave sidecar headroom. This matters because newer +workerd releases no longer cap SQLite's process hard heap at 512 MiB; the container cap +keeps a runaway SQLite query inside the stateful runtime container budget. The +supervisor is PID 1 in that same container. The launch template keeps IMDSv2 enabled for the ECS host agent, sets metadata hop limit to 1, and sets `ECS_AWSVPC_BLOCK_IMDS=true` so awsvpc tasks cannot read diff --git a/terraform/modules/compute/d1_runtime_service.tf b/terraform/modules/compute/d1_runtime_service.tf index d7d489e..37a2df2 100644 --- a/terraform/modules/compute/d1_runtime_service.tf +++ b/terraform/modules/compute/d1_runtime_service.tf @@ -27,7 +27,7 @@ resource "aws_ecs_task_definition" "d1_runtime" { image = var.workerd_image essential = true entryPoint = ["d1-supervisor"] - memory = coalesce(var.d1_runtime_container_memory, var.runtime_memory) + memory = local.d1_runtime_container_memory portMappings = [{ name = "d1-http" @@ -75,6 +75,14 @@ resource "aws_ecs_task_definition" "d1_runtime" { }]) lifecycle { + precondition { + condition = ( + local.d1_runtime_container_memory > 0 && + local.d1_runtime_container_memory <= var.runtime_memory + ) + error_message = "d1_runtime_container_memory must be positive and no greater than runtime_memory." + } + precondition { condition = !var.d1_test_hooks_enabled || can(regex("(^|-)test($|-)", var.name)) error_message = "d1_test_hooks_enabled may only be enabled for test-named compute stacks." diff --git a/terraform/modules/compute/locals.tf b/terraform/modules/compute/locals.tf index d38af5d..44802a6 100644 --- a/terraform/modules/compute/locals.tf +++ b/terraform/modules/compute/locals.tf @@ -2,6 +2,10 @@ locals { redis_addr = "${var.valkey_host}:${var.valkey_port}" data_redis_url = "redis://${local.redis_addr}/1" do_redis_proxy_memory_headroom = 128 + d1_runtime_container_memory = coalesce( + var.d1_runtime_container_memory, + var.runtime_memory, + ) do_runtime_container_memory = coalesce( var.do_runtime_container_memory, var.runtime_memory - local.do_redis_proxy_memory_headroom, diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index 27a0344..9effdbd 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -3,6 +3,7 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { freshRepositoryModuleDataUrl, + importSpecifierReplacements, moduleDataUrl, readRepositoryFile, repositoryFileUrl, @@ -65,8 +66,10 @@ export async function compileControlGraph(opts = {}) { `export default ${JSON.stringify(readRepositoryFile("package.json"))};` ); const bundleUrl = freshRepositoryModuleDataUrl("control/bundle.js", [ - [/from "shared-ns-pattern"/g, `from ${JSON.stringify(SHARED_NS_URL)}`], - [/from "shared-workerd-compat-flags"/g, `from ${JSON.stringify(SHARED_WORKERD_COMPAT_FLAGS_URL)}`], + ...importSpecifierReplacements({ + "shared-ns-pattern": SHARED_NS_URL, + "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + }), [/from "control-bindings"/g, `from ${JSON.stringify(bindingsUrl)}`], [/from "wdl-package-json-source"/g, `from ${JSON.stringify(packageJsonSourceUrl)}`], ]); diff --git a/tests/helpers/load-do-protocol.js b/tests/helpers/load-do-protocol.js index ef49255..1ef16ee 100644 --- a/tests/helpers/load-do-protocol.js +++ b/tests/helpers/load-do-protocol.js @@ -1,4 +1,8 @@ -import { repositoryFileUrl, repositoryModuleDataUrl } from "./load-shared-module.js"; +import { + importSpecifierReplacements, + repositoryFileUrl, + repositoryModuleDataUrl, +} from "./load-shared-module.js"; const SHARED_FNV_URL = repositoryFileUrl("shared/fnv1a32.js"); const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); @@ -18,13 +22,15 @@ const DO_IDENTITY_URL = repositoryModuleDataUrl("do-runtime/protocol/identity.js export function doProtocolDataUrl() { return repositoryModuleDataUrl("do-runtime/protocol.js", [ - [/from "do-runtime-protocol-wire-grammar";/g, `from ${JSON.stringify(DO_WIRE_GRAMMAR_URL)};`], - [/from "do-runtime-protocol-errors";/g, `from ${JSON.stringify(DO_ERRORS_URL)};`], - [/from "do-runtime-protocol-identity";/g, `from ${JSON.stringify(DO_IDENTITY_URL)};`], - [/from "shared-worker-id";/g, `from ${JSON.stringify(SHARED_WORKER_ID_URL)};`], - [/from "shared-workerd-compat-flags";/g, `from ${JSON.stringify(SHARED_WORKERD_COMPAT_FLAGS_URL)};`], - [/from "shared-bounded-body";/g, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)};`], - [/from "shared-internal-auth";/g, `from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)};`], + ...importSpecifierReplacements({ + "do-runtime-protocol-wire-grammar": DO_WIRE_GRAMMAR_URL, + "do-runtime-protocol-errors": DO_ERRORS_URL, + "do-runtime-protocol-identity": DO_IDENTITY_URL, + "shared-worker-id": SHARED_WORKER_ID_URL, + "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + "shared-bounded-body": SHARED_BOUNDED_BODY_URL, + "shared-internal-auth": SHARED_INTERNAL_AUTH_URL, + }), ]); } diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 761e62e..0b3f5cc 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -6,7 +6,11 @@ import { } from "../helpers/control-handler-harness.js"; import { compileControlGraph } from "../helpers/load-control-lib.js"; import { + importSpecifierReplacements, moduleDataUrl, + readRepositoryFile, + readRepositoryModuleSource, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { readJsonResponse } from "../helpers/response-json.js"; @@ -239,6 +243,39 @@ export class SecretEnvelopeError extends Error { } `); +/** @param {string} path */ +function textModuleUrl(path) { + return moduleDataUrl(`export default ${JSON.stringify(readRepositoryFile(path))};`); +} + +const runtimeLoadCodeBudgetUrl = moduleDataUrl(readRepositoryModuleSource( + "runtime/load/code-budget.js", + importSpecifierReplacements({ + "shared-ns-pattern": repositoryFileUrl("shared/ns-pattern.js"), + "runtime-load-module-rewrite": repositoryFileUrl("runtime/load/module-rewrite.js"), + "runtime-load-wrapper-generate": repositoryFileUrl("runtime/load/wrapper-generate.js"), + }) +)); +const controlWorkerCodeBudgetUrl = moduleDataUrl(readRepositoryModuleSource( + "control/worker-code-budget.js", + importSpecifierReplacements({ + "runtime-load-code-budget": runtimeLoadCodeBudgetUrl, + "runtime-d1-client-source": textModuleUrl("runtime/d1-client.js"), + "runtime-d1-data-field-source": textModuleUrl("shared/d1-data-field.js"), + "runtime-d1-params-source": textModuleUrl("shared/d1-params.js"), + "runtime-sql-splitter-source": textModuleUrl("shared/sql-splitter.js"), + "runtime-d1-transport-source": textModuleUrl("shared/d1-transport.js"), + "runtime-r2-client-source": textModuleUrl("runtime/r2-client.js"), + "runtime-r2-utils-source": textModuleUrl("runtime/r2-utils.js"), + "runtime-do-client-source": textModuleUrl("runtime/do-client.js"), + "runtime-do-transport-source": textModuleUrl("runtime/_wdl-do-transport.js"), + "runtime-owner-endpoint-source": textModuleUrl("runtime/_wdl-owner-endpoint.js"), + "runtime-owner-hint-cache-source": textModuleUrl("runtime/_wdl-owner-hint-cache.js"), + "runtime-request-id-source": textModuleUrl("runtime/_wdl-request-id.js"), + "runtime-workflows-client-source": textModuleUrl("runtime/workflows-client.js"), + }) +)); + const { commitWithWatch, handle } = await importControlHandler("control/handlers/deploy.js", { globalName: "__controlDeployTestState", extraSharedSource: controlSharedExtraSource, @@ -256,6 +293,7 @@ const { commitWithWatch, handle } = await importControlHandler("control/handlers "shared-assets-token": sharedAssetsUrl, "control-d1-store": d1StoreUrl, "control-env-budget": controlEnvBudgetUrl, + "control-worker-code-budget": controlWorkerCodeBudgetUrl, "shared-secret-envelope": secretEnvelopeUrl, }, }); @@ -711,15 +749,18 @@ test("deploy handler treats pre-allocation workerLoader env budget failures as a } }); -test("deploy handler rejects workerLoader code size violations before allocating a version", async () => { +test("deploy handler counts runtime-generated wrapper code before allocating a version", async () => { + const oversizedEntrypoint = `Entrypoint${"A".repeat(600 * 1024)}`; /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = { meta: { mainModule: "worker.js", modules: { "worker.js": { type: "module" } }, + bindings: { DB: { type: "d1", databaseId: "db" } }, + exports: [{ entrypoint: oversizedEntrypoint }], }, - normalized: [["worker.js", new Uint8Array(64 * 1024 * 1024 + 1)]], + normalized: [["worker.js", new Uint8Array(64 * 1024 * 1024 - 512 * 1024)]], }; let incrCalled = false; @@ -745,7 +786,9 @@ test("deploy handler rejects workerLoader code size violations before allocating requestId: "rid-code-budget", }); - assert.equal((await readJsonResponse(response, 413)).error, "worker_code_too_large"); + const body = await readJsonResponse(response, 413); + assert.equal(body.error, "worker_code_too_large"); + assert.match(body.message, /final WorkerCode/); assert.equal(incrCalled, false); } finally { /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 91889b4..a042f28 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -8,6 +8,7 @@ import { import { encryptSecretValue } from "../../shared/secret-envelope.js"; const secretEnvelopeUrl = repositoryFileUrl("shared/secret-envelope.js"); +const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const { UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, @@ -22,6 +23,7 @@ const { estimatedWorkerLoaderEnvBytes, } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ "shared-secret-envelope": secretEnvelopeUrl, + "shared-errors": sharedErrorsUrl, "shared-version": sharedVersionUrl, })); @@ -289,7 +291,9 @@ test("worker env budget checks every retained worker version", async () => { async hGet(key, field) { assert.equal(field, "__meta__"); if (key === "worker:demo:api:v:1") return JSON.stringify({ vars: { SMALL: "ok" } }); - if (key === "worker:demo:api:v:2") return JSON.stringify({ vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES) } }); + if (key === "worker:demo:api:v:2") { + return JSON.stringify({ vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES) } }); + } return null; }, }; @@ -430,3 +434,45 @@ test("worker env budget reports bundle metadata parse context", async () => { /invalid bundle metadata for demo\/api@v1/ ); }); + +test("worker env budget fails closed when retained bundle metadata is missing", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return null; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + /bundle metadata missing for demo\/api@v1/ + ); +}); + +test("worker env budget fails closed when retained bundle metadata is not an object", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return "[]"; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + /__meta__ must be a JSON object/ + ); +}); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index dd72ddc..c48b63a 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -791,7 +791,11 @@ test("MAX_WORKER_COMPATIBILITY_DATE matches pinned workerd release plus seven da }); test("workerd experimental compat flag mirror matches pinned workerd source version", () => { - const regenerate = "Regenerate shared/workerd-compat-flags.js from an upstream workerd checkout with the command in that file header."; + const regenerate = [ + "Regenerate shared/workerd-compat-flags.js from an upstream workerd checkout:", + "node scripts/extract-workerd-experimental-compat-flags.mjs", + "/path/to/workerd/src/workerd/io/compatibility-date.capnp", + ].join(" "); assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, WORKERD_VERSION, regenerate); assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("experimental")); assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unsafe_module")); diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index bd79146..b4681e0 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -2,16 +2,102 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; import { decryptSecretValue, encryptSecretValue, isSecretEnvelope } from "../../shared/secret-envelope.js"; -import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { + applyModuleReplacements, + moduleDataUrl, + readRepositoryFile, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { installMockProperty } from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; const SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const env = { SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", }; +/** @returns {Record} */ +function emptySecretHash() { + return {}; +} + +/** + * @param {string} key + * @param {string} field + */ +function defaultWorkerSecretBundleMeta(key, field) { + return /^worker:[^:]+:[^:]+:v:[1-9][0-9]*$/.test(key) && field === "__meta__" + ? "{}" + : null; +} + +/** + * @param {{ + * hKeys?: string[], + * hGet?: (key: string, field: string) => string | null | Promise, + * hGetAll?: (key: string) => Record | Promise>, + * zCard?: number, + * zRange?: string[], + * onHSet?: (key: string, field: string, value: string) => void, + * onHDel?: (key: string, field: string) => void, + * onExec?: () => void | Promise, + * }} options + */ +function makeWorkerSecretSession({ + hKeys = [], + hGet = defaultWorkerSecretBundleMeta, + hGetAll = emptySecretHash, + zCard = 0, + zRange = [], + onHSet = () => {}, + onHDel = () => {}, + onExec = () => {}, +} = {}) { + return { + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return hKeys; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { return await hGet(key, field); }, + /** @param {string} key */ + async hGetAll(key) { return await hGetAll(key); }, + async zCard() { return zCard; }, + async zRange() { return zRange; }, + multi() { + return { + /** @param {string} key @param {string} field @param {string} value */ + hSet(key, field, value) { onHSet(key, field, value); }, + /** @param {string} key @param {string} field */ + hDel(key, field) { onHDel(key, field); }, + sAdd() {}, + sRem() {}, + async exec() { await onExec(); }, + }; + }, + }; +} + +/** + * @param {{ redis: Record }} state + * @param {ReturnType} session + * @param {() => unknown | Promise} callback + */ +async function withWorkerSecretSession(state, session, callback) { + const restore = installMockProperty(state.redis, "session", + async (/** @type {(session: ReturnType) => unknown | Promise} */ fn) => + await fn(session) + ); + try { + return await callback(); + } finally { + restore(); + } +} + const validateSecretKeyStubSource = ` const SECRET_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; const WDL_RESERVED_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; @@ -38,6 +124,7 @@ function secretPutUrl(controlSharedUrl, controlLibUrl) { function envBudgetUrl() { const source = applyModuleReplacements(readRepositoryFile("control/env-budget.js"), [ [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], + [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], ]); return moduleDataUrl(source); @@ -54,7 +141,11 @@ export const state = { deletes: [], watchedKeys: [], async hKeys() { return []; }, - async hGet() { return null; }, + async hGet(key, field) { + return /^worker:[^:]+:[^:]+:v:[1-9][0-9]*$/.test(key) && field === "__meta__" + ? "{}" + : null; + }, async hGetAll() { return {}; }, async sMembers() { return []; }, async zRange() { return []; }, @@ -495,7 +586,11 @@ export const state = { async unwatch() {}, async get() { return null; }, async hKeys() { return []; }, - async hGet() { return null; }, + async hGet(key, field) { + return /^worker:[^:]+:[^:]+:v:[1-9][0-9]*$/.test(key) && field === "__meta__" + ? "{}" + : null; + }, async hGetAll() { return {}; }, async zCard() { return 0; }, async zRange() { return []; }, @@ -548,10 +643,11 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) }); } `); +const workerSecretPutUrl = secretPutUrl(workerControlSharedUrl, workerLibStubUrl); const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/worker-secrets.js"), [ [/from "control-shared";/, `from ${JSON.stringify(workerControlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(workerLibStubUrl)};`], - [/from "control-handlers-secret-put";/, `from ${JSON.stringify(secretPutUrl(workerControlSharedUrl, workerLibStubUrl))};`], + [/from "control-handlers-secret-put";/, `from ${JSON.stringify(workerSecretPutUrl)};`], [/from "control-lifecycle-indexes";/, `from ${JSON.stringify(lifecycleStubUrl)};`], [/from "control-routing";/, `from ${JSON.stringify(routingStubUrl)};`], [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], @@ -848,36 +944,18 @@ test("worker secret PUT rechecks the active bundle copied by bump after the secr test("worker secret DELETE skips decrypting the removed corrupt envelope", async () => { const { state } = await import(workerControlSharedUrl); - const originalSession = state.redis.session; let execCalled = false; + /** @type {string | null} */ let deletedField = null; - /** @param {(session: unknown) => Promise} fn */ - state.redis.session = async (fn) => await fn({ - async watch() {}, - async unwatch() {}, - async get() { return null; }, - async hKeys() { return ["TOKEN"]; }, - async hGet() { return null; }, - /** @param {string} key */ - async hGetAll(key) { + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["TOKEN"], + hGetAll(key) { if (key === "secrets:demo:api") return { TOKEN: "WDL-ENC:not-json" }; - return {}; - }, - async zCard() { return 0; }, - async zRange() { return []; }, - multi() { - return { - hSet() {}, - /** @param {string} _key @param {string} field */ - hDel(_key, field) { deletedField = field; }, - sAdd() {}, - sRem() {}, - async exec() { execCalled = true; }, - }; + return emptySecretHash(); }, - }); - - try { + onHDel(_key, field) { deletedField = field; }, + onExec() { execCalled = true; }, + }), async () => { const response = await workerHandle({ request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { method: "DELETE", @@ -893,48 +971,28 @@ test("worker secret DELETE skips decrypting the removed corrupt envelope", async assert.equal(response.status, 200); assert.equal(execCalled, true); assert.equal(deletedField, "TOKEN"); - } finally { - state.redis.session = originalSession; - } + }); }); test("worker secret DELETE skips other corrupt worker envelopes for repair", async () => { const { state } = await import(workerControlSharedUrl); - const originalSession = state.redis.session; const encrypted = await encryptSecretValue("plain", { env, hashKey: "secrets:demo:api", fieldName: "TOKEN", }); let execCalled = false; + /** @type {string | null} */ let deletedField = null; - /** @param {(session: unknown) => Promise} fn */ - state.redis.session = async (fn) => await fn({ - async watch() {}, - async unwatch() {}, - async get() { return null; }, - async hKeys() { return ["TOKEN", "BAD"]; }, - async hGet() { return null; }, - /** @param {string} key */ - async hGetAll(key) { + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["TOKEN", "BAD"], + hGetAll(key) { if (key === "secrets:demo:api") return { TOKEN: encrypted, BAD: "WDL-ENC:not-json" }; - return {}; + return emptySecretHash(); }, - async zCard() { return 0; }, - async zRange() { return []; }, - multi() { - return { - hSet() {}, - /** @param {string} _key @param {string} field */ - hDel(_key, field) { deletedField = field; }, - sAdd() {}, - sRem() {}, - async exec() { execCalled = true; }, - }; - }, - }); - - try { + onHDel(_key, field) { deletedField = field; }, + onExec() { execCalled = true; }, + }), async () => { const response = await workerHandle({ request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { method: "DELETE", @@ -950,49 +1008,29 @@ test("worker secret DELETE skips other corrupt worker envelopes for repair", asy assert.equal(response.status, 200); assert.equal(execCalled, true); assert.equal(deletedField, "TOKEN"); - } finally { - state.redis.session = originalSession; - } + }); }); test("worker secret DELETE skips corrupt namespace envelopes for repair", async () => { const { state } = await import(workerControlSharedUrl); - const originalSession = state.redis.session; const encrypted = await encryptSecretValue("plain", { env, hashKey: "secrets:demo:api", fieldName: "TOKEN", }); let execCalled = false; + /** @type {string | null} */ let deletedField = null; - /** @param {(session: unknown) => Promise} fn */ - state.redis.session = async (fn) => await fn({ - async watch() {}, - async unwatch() {}, - async get() { return null; }, - async hKeys() { return ["TOKEN"]; }, - async hGet() { return null; }, - /** @param {string} key */ - async hGetAll(key) { + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["TOKEN"], + hGetAll(key) { if (key === "secrets:demo") return { BAD: "WDL-ENC:not-json" }; if (key === "secrets:demo:api") return { TOKEN: encrypted }; - return {}; + return emptySecretHash(); }, - async zCard() { return 0; }, - async zRange() { return []; }, - multi() { - return { - hSet() {}, - /** @param {string} _key @param {string} field */ - hDel(_key, field) { deletedField = field; }, - sAdd() {}, - sRem() {}, - async exec() { execCalled = true; }, - }; - }, - }); - - try { + onHDel(_key, field) { deletedField = field; }, + onExec() { execCalled = true; }, + }), async () => { const response = await workerHandle({ request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { method: "DELETE", @@ -1008,9 +1046,7 @@ test("worker secret DELETE skips corrupt namespace envelopes for repair", async assert.equal(response.status, 200); assert.equal(execCalled, true); assert.equal(deletedField, "TOKEN"); - } finally { - state.redis.session = originalSession; - } + }); }); test("worker secret DELETE skips corrupt namespace envelopes during bump repair", async () => { @@ -1082,34 +1118,15 @@ test("worker secret DELETE skips corrupt namespace envelopes during bump repair" test("worker secret PUT still fails closed on other corrupt worker envelopes", async () => { const { state } = await import(workerControlSharedUrl); - const originalSession = state.redis.session; let hSetCalled = false; - /** @param {(session: unknown) => Promise} fn */ - state.redis.session = async (fn) => await fn({ - async watch() {}, - async unwatch() {}, - async get() { return null; }, - async hKeys() { return ["BAD"]; }, - async hGet() { return null; }, - /** @param {string} key */ - async hGetAll(key) { + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["BAD"], + hGetAll(key) { if (key === "secrets:demo:api") return { BAD: "WDL-ENC:not-json" }; - return {}; - }, - async zCard() { return 0; }, - async zRange() { return []; }, - multi() { - return { - hSet() { hSetCalled = true; }, - hDel() {}, - sAdd() {}, - sRem() {}, - async exec() {}, - }; + return emptySecretHash(); }, - }); - - try { + onHSet() { hSetCalled = true; }, + }), async () => { const response = await workerHandle({ request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { method: "PUT", @@ -1126,7 +1143,5 @@ test("worker secret PUT still fails closed on other corrupt worker envelopes", a const body = await readJsonResponse(response, 503); assert.equal(body.error, "invalid_envelope"); assert.equal(hSetCalled, false); - } finally { - state.redis.session = originalSession; - } + }); }); diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index 6c970e8..1281199 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -96,6 +96,10 @@ const src = readRepositoryModuleSource("runtime/load.js", [ /import WORKFLOWS_CLIENT_SOURCE from "runtime-workflows-client-source";/, 'const WORKFLOWS_CLIENT_SOURCE = "export class Workflow { constructor(metadata) { this.metadata = metadata; } }";' ], + [ + /from "runtime-load-code-budget";/, + 'from "./load/code-budget.js";' + ], [ /from "runtime-load-module-rewrite";/, 'from "./load/module-rewrite.js";' @@ -118,10 +122,12 @@ const ENV_BUILD_SOURCE = readRepositoryModuleSource("runtime/load/env-build.js", })); mkdirSync(LOAD_TEST_SUBMODULE_DIR, { recursive: true }); writeFileSync(path.join(LOAD_TEST_RUNTIME_DIR, "load.js"), src); -for (const name of ["env-build.js", "module-rewrite.js", "wrapper-generate.js"]) { +for (const name of ["env-build.js", "code-budget.js", "module-rewrite.js", "wrapper-generate.js"]) { const moduleSource = name === "env-build.js" ? ENV_BUILD_SOURCE : readRepositoryModuleSource(`runtime/load/${name}`, importSpecifierReplacements({ + "runtime-load-module-rewrite": pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "module-rewrite.js")).href, + "runtime-load-wrapper-generate": pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "wrapper-generate.js")).href, "shared-ns-pattern": SHARED_NS_PATTERN_URL, })); writeFileSync( @@ -132,6 +138,7 @@ for (const name of ["env-build.js", "module-rewrite.js", "wrapper-generate.js"]) after(() => removeTempDir(LOAD_TEST_DIR)); const mod = await import(pathToFileURL(path.join(LOAD_TEST_RUNTIME_DIR, "load.js")).href); +const codeBudgetMod = await import(pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "code-budget.js")).href); const { buildWorkerEnv, createLoaderCallback, @@ -139,6 +146,10 @@ const { runtimeLoadContentTypeMatches, wrapWorkerCodeForHostBindings, } = mod; +const { + estimateFinalWorkerLoaderCodeBytes, + injectRuntimeModulesForHostBindings, +} = codeBudgetMod; const RUNTIME_LOAD_MAGIC = "WDLLOAD!"; const RUNTIME_LOAD_CONTENT_TYPE = "application/vnd.wdl.runtime-load"; @@ -1375,6 +1386,66 @@ test("createLoaderCallback: aborts hung redis proxy runtime load requests", asyn ); }); +const TEST_RUNTIME_INJECTION_SOURCES = Object.freeze({ + d1ClientSource: "export class D1Database {}", + d1DataFieldSource: "export function setDataField() {}", + d1ParamsSource: "export function normalizeD1Param(value) { return value; }", + sqlSplitterSource: "export function splitSqlStatements(sql) { return [{ sql }]; }", + d1TransportSource: 'import { setDataField } from "shared-d1-data-field"; export function decode() { setDataField(); }', + r2ClientSource: "export class R2Bucket {}", + r2UtilsSource: "export const R2_OBJECT_MAX_BUFFER_BYTES = 1;", + doClientSource: "export class DurableObjectNamespace {}", + doTransportSource: "export function requestSpec() {}", + ownerEndpointSource: "export function validOwnerEndpointForService() { return true; }", + ownerHintCacheSource: "export function createOwnerHintCache() { return {}; }", + requestIdSource: "export function requestIdFromOptions() { return null; }", + workflowsClientSource: "export class Workflow {}", +}); + +/** @param {{ modules: Record }} workerCode */ +function workerCodeModuleBytes(workerCode) { + let total = 0; + for (const value of Object.values(workerCode.modules)) { + assert.equal(typeof value, "string"); + total += Buffer.byteLength(/** @type {string} */ (value), "utf8"); + } + return total; +} + +test("workerLoader code estimator matches runtime wrapper injection exactly", () => { + const entrypoint = `Api${"A".repeat(1024)}`; + const source = 'import { WorkflowEntrypoint } from "cloudflare:workflows"; export default {};'; + const meta = { + mainModule: "src/worker.js", + modules: { "src/worker.js": { type: "module" } }, + bindings: { DB: { type: "d1", databaseId: "main" } }, + exports: [{ entrypoint }], + workflows: [{ binding: "FLOW", name: "flow", className: "FlowHandler" }], + }; + /** @type {{ mainModule: string, modules: Record }} */ + const workerCode = { + mainModule: meta.mainModule, + modules: { "src/worker.js": source }, + }; + + injectRuntimeModulesForHostBindings(workerCode, meta, TEST_RUNTIME_INJECTION_SOURCES); + + assert.equal( + estimateFinalWorkerLoaderCodeBytes({ + mainModule: meta.mainModule, + normalized: [["src/worker.js", Buffer.from(source)]], + meta, + runtimeSources: TEST_RUNTIME_INJECTION_SOURCES, + }), + workerCodeModuleBytes(workerCode) + ); + assert.match( + /** @type {string} */ (workerCode.modules["src/worker.js"]), + /"\.\.\/_wdl-cloudflare-workflows\.js"/ + ); + assert.match(/** @type {string} */ (workerCode.modules["_wdl-wrapper.js"]), new RegExp(entrypoint)); +}); + test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserves original main module", () => { const workerCode = { mainModule: "src/worker.js", diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 5a5d049..da89433 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1900,7 +1900,15 @@ test("D1 and DO workerd containers keep explicit memory ceilings", () => { assert.match(moduleVars, new RegExp(`variable "${name}"`)); assert.match(main, new RegExp(`${name}\\s+=\\s+var\\.${name}`)); } - assert.match(d1Service, /memory\s+=\s+coalesce\(var\.d1_runtime_container_memory, var\.runtime_memory\)/); + assert.match( + locals, + /d1_runtime_container_memory\s+=\s+coalesce\(\s*var\.d1_runtime_container_memory,\s*var\.runtime_memory,\s*\)/ + ); + assert.match(d1Service, /memory\s+=\s+local\.d1_runtime_container_memory/); + assert.match( + d1Service, + /local\.d1_runtime_container_memory > 0 &&\s*local\.d1_runtime_container_memory <= var\.runtime_memory/ + ); assert.match(locals, /do_redis_proxy_memory_headroom\s+=\s+128/); assert.match(locals, /do_runtime_container_memory\s+=\s+coalesce\(\s*var\.do_runtime_container_memory,\s*var\.runtime_memory - local\.do_redis_proxy_memory_headroom,\s*\)/); assert.match(doService, /memory\s+=\s+local\.do_runtime_container_memory/); diff --git a/tests/unit/workerd-compat-flags-extract.test.js b/tests/unit/workerd-compat-flags-extract.test.js new file mode 100644 index 0000000..9c42bba --- /dev/null +++ b/tests/unit/workerd-compat-flags-extract.test.js @@ -0,0 +1,20 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { extractExperimentalCompatFlags } from "../../scripts/extract-workerd-experimental-compat-flags.mjs"; + +test("workerd experimental compat flag extractor mirrors only experimental enable flags", () => { + const flags = extractExperimentalCompatFlags(` +struct CompatibilityFlags { + gaFlag @1 :Bool $compatEnableFlag("unique_ctx_per_invocation"); + experimentalFlag @2 :Bool + $experimental + $compatEnableFlag("experimental_one") + $compatDisableFlag("no_experimental_one"); + spacedExperimentalFlag @3 : Bool + $compatEnableFlag("experimental_two") + $experimental; +} +`); + assert.deepEqual(flags, ["experimental_one", "experimental_two"]); +});