diff --git a/CLAUDE.md b/CLAUDE.md index 01bc50b..d5f796c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -73,7 +73,10 @@ shared crate `wdl-rust-common`. Runtime receives plaintext only in the internal load envelope after redis-proxy decrypts during `/runtime/load`. Env materializes in fixed precedence — vars, then namespace secrets, then worker secrets — so a worker secret shadows a namespace secret - shadows a var on the same key. + shadows a var on the same key. Control must keep the estimated full workerLoader env + — user vars/secrets plus runtime-injected binding/workflow env values such as + required caller secret copies — within WDL's headroomed workerd serialized env budget + before deploy/secret mutation, not let that fail later during cold-load. - **DB split is intentional.** DB 0 is control metadata, DB 1 is data-plane KV/queue/log streams, DB 2 is Workflows. See `docs/redis-key-layout.md`. - **D1/DO correctness comes from owner lease + generation fence.** Service DNS only @@ -165,8 +168,10 @@ output capture, or fixture loading. host-only helpers in module functions or WeakMaps. - `workerd serve` emits raw console stdout in addition to structured tail events. The structured JSON log is the platform source of truth. -- Mid-response disconnects do not reliably trip `request.signal`; use - `ReadableStream.cancel` plus pre-registered `ctx.waitUntil` for cleanup. +- On current workerd, mid-response disconnects do not reliably trip `request.signal` + or async response-body `ReadableStream.cancel`; do not use either as the only + cleanup signal. Bound streaming responses with an independent timeout or an + explicit app-level heartbeat/close path. - Forwarding WebSocket `101` responses requires preserving `response.webSocket`; use the shared response helper instead of wrapping with `new Response(body, init)`. - workerd/capnp wiring traps apply to every tier's `*.capnp`, not just runtime: `workerd diff --git a/Dockerfile.workerd b/Dockerfile.workerd index 883d1c7..ca12156 100644 --- a/Dockerfile.workerd +++ b/Dockerfile.workerd @@ -86,7 +86,7 @@ RUN ["/usr/local/bin/workerd", "--version"] # ECS task definitions pick the entry/command: # gateway : entryPoint ["workerd"] -# command ["serve", "-b", "/app/dist/workerd-configs/gateway.bin", "--experimental"] +# command ["serve", "-b", "/app/dist/workerd-configs/gateway.bin"] # user-runtime : entryPoint ["workerd"] # command ["serve", "-b", "/app/dist/workerd-configs/user-runtime.bin", "--experimental"] # system-runtime : entryPoint ["workerd"] @@ -95,5 +95,6 @@ RUN ["/usr/local/bin/workerd", "--version"] # stopTimeout >= 20s # env D1_OWNER_TTL_SECONDS=120 D1_PROBE_TIMEOUT_MS=500 D1_QUERY_TIMEOUT_MS=30000 D1_SHUTDOWN_TIMEOUT_MS=15000 # do-runtime : entryPoint ["do-supervisor"] +# child workerd args include "--experimental" for workerLoader # stopTimeout >= 20s # env DO_OWNER_TTL_SECONDS=120 DO_DRAIN_IN_FLIGHT_TIMEOUT_MS=8000 diff --git a/control/bundle.js b/control/bundle.js index 8e1c4f5..ed3ab19 100644 --- a/control/bundle.js +++ b/control/bundle.js @@ -12,6 +12,7 @@ import { isValidJsClassDeclarationName, validateModulePath, } from "shared-ns-pattern"; +import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; import { normalizeBindings, validateBindings } from "control-bindings"; import PACKAGE_JSON_SOURCE from "wdl-package-json-source"; @@ -131,11 +132,16 @@ export function normalizeModule(value) { return { type: "json", bytes: Buffer.from(JSON.stringify(record.json), "utf8") }; if (typeof record.cjs === "string") return { type: "cjs", bytes: Buffer.from(record.cjs, "utf8") }; - if (typeof record.py === "string") - return { type: "py", bytes: Buffer.from(record.py, "utf8") }; + if (typeof record.py === "string") { + throw new BundleConfigError( + 400, + "python_workers_unsupported", + "Python Workers modules are not supported by WDL" + ); + } } throw new Error( - "Unrecognized module value (expected string or {data_b64|wasm_b64|text|json|cjs|py})" + "Unrecognized module value (expected string or {data_b64|wasm_b64|text|json|cjs})" ); } @@ -159,6 +165,14 @@ function validateCompatibilityFlags(flags) { ); } } + const experimentalFlag = firstWorkerdExperimentalCompatFlag(flags); + if (experimentalFlag) { + throw new BundleConfigError( + 400, + "experimental_compat_flag_unsupported", + `compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` + ); + } } /** @param {unknown} workflows */ diff --git a/control/env-budget.js b/control/env-budget.js new file mode 100644 index 0000000..63a44fc --- /dev/null +++ b/control/env-budget.js @@ -0,0 +1,470 @@ +import { SecretEnvelopeError, decryptSecretValue } from "shared-secret-envelope"; +import { errorMessage } from "shared-errors"; +import { bundleKey } from "shared-version"; + +const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; +const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; +const DO_ALARMS_BINDING = "__WDL_DO_ALARMS__"; +const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; +const ESTIMATED_ASSETS_CDN_BASE = "https://assets.invalid"; +const ESTIMATED_VERSION = "v0000000000"; +const ESTIMATED_DO_STORAGE_ID = "do_00000000000000000000000000000000"; +const ESTIMATED_WORKFLOW_KEY = "wf_00000000000000000000000000000000"; + +export const WORKER_LOADER_ENV_VERSION_PLACEHOLDER = ESTIMATED_VERSION; +export const UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES = 1024 * 1024; +export const WORKER_LOADER_ENV_HEADROOM_BYTES = 8 * 1024; +export const WORKER_LOADER_ENV_MAX_BYTES = + UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES - WORKER_LOADER_ENV_HEADROOM_BYTES; + +export class WorkerEnvBudgetError extends Error { + /** + * @param {string} message + * @param {Record} [details] + */ + constructor(message, details = {}) { + super(message); + this.status = 400; + this.code = "worker_env_too_large"; + this.details = details; + } +} + +/** @param {Record | null | undefined} source */ +function stringRecord(source) { + /** @type {Record} */ + const out = Object.create(null); + for (const [key, value] of Object.entries(source || {})) { + if (typeof value === "string") out[key] = value; + } + return out; +} + +/** @param {unknown} value @returns {Record | null} */ +function objectRecord(value) { + return value && typeof value === "object" && !Array.isArray(value) + ? /** @type {Record} */ (value) + : null; +} + +/** @param {unknown} value */ +function stringOrFallback(value, fallback = "") { + return typeof value === "string" ? value : fallback; +} + +/** + * @param {{ + * requiredCallerSecrets?: unknown, + * nsSecrets: Record, + * workerSecrets: Record, + * }} args + */ +function callerSecretsForBinding({ requiredCallerSecrets, nsSecrets, workerSecrets }) { + if (!Array.isArray(requiredCallerSecrets) || requiredCallerSecrets.length === 0) return undefined; + /** @type {Record} */ + const callerSecrets = Object.create(null); + for (const key of requiredCallerSecrets) { + if (typeof key !== "string") continue; + if (Object.hasOwn(workerSecrets, key)) { + callerSecrets[key] = workerSecrets[key]; + } else if (Object.hasOwn(nsSecrets, key)) { + callerSecrets[key] = nsSecrets[key]; + } + } + return callerSecrets; +} + +/** + * @param {{ + * name: string, + * spec: Record, + * meta: Record, + * ns: string, + * worker: string, + * version: string, + * assetsCdnBase: string, + * nsSecrets: Record, + * workerSecrets: Record, + * }} args + */ +function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, assetsCdnBase, nsSecrets, workerSecrets }) { + switch (spec.type) { + case "kv": + return { + __wdlBinding: "kv", + props: { ns, id: stringOrFallback(spec.id) }, + }; + case "assets": + return { + __wdlBinding: "assets", + props: { + cdnBase: assetsCdnBase, + prefix: stringOrFallback(objectRecord(meta.assets)?.prefix), + }, + }; + case "queue": + return { + __wdlBinding: "queue", + props: { + ns, + id: stringOrFallback(spec.id), + deliveryDelaySeconds: spec.deliveryDelaySeconds ?? 0, + }, + }; + case "d1": + return { + __wdlBinding: "d1", + props: { + ns, + databaseId: stringOrFallback(spec.databaseId), + binding: name, + }, + }; + case "r2": + return { + __wdlBinding: "r2", + props: { + ns, + bucketName: stringOrFallback(spec.bucketName), + binding: name, + }, + }; + case "do": { + const props = { + ns, + worker, + version, + doStorageId: stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID), + binding: name, + className: stringOrFallback(spec.className), + }; + // DO bindings intentionally mirror runtime's default/factory output shape: + // props are top-level fields, while hostProxy carries the JSRPC namespace. + return { + __wdlBinding: "do", + ...props, + hostProxy: { __wdlBinding: "do-host-proxy", props }, + }; + } + case "service": { + const callerSecrets = callerSecretsForBinding({ + requiredCallerSecrets: spec.requiredCallerSecrets, + nsSecrets, + workerSecrets, + }); + return { + __wdlBinding: "service", + props: { + targetNs: stringOrFallback(spec.ns, ns), + targetWorker: stringOrFallback(spec.service), + targetVersion: stringOrFallback(spec.version), + targetEntrypoint: typeof spec.entrypoint === "string" ? spec.entrypoint : null, + callerNs: ns, + ...(callerSecrets ? { callerSecrets } : {}), + }, + }; + } + default: + return { __wdlBinding: stringOrFallback(spec.type, "unknown") }; + } +} + +/** + * @param {{ + * ns: string, + * worker?: string, + * version?: string, + * vars?: Record | null, + * nsSecrets?: Record | null, + * workerSecrets?: Record | null, + * meta?: Record | null, + * assetsCdnBase?: string | null, + * }} args + */ +export function estimatedWorkerLoaderEnv({ + ns, + worker = "", + version = ESTIMATED_VERSION, + vars = null, + nsSecrets = null, + workerSecrets = null, + meta = null, + assetsCdnBase = ESTIMATED_ASSETS_CDN_BASE, +}) { + const nsSecretStrings = stringRecord(nsSecrets); + const workerSecretStrings = stringRecord(workerSecrets); + /** @type {Record} */ + const env = { + ...stringRecord(vars), + ...nsSecretStrings, + ...workerSecretStrings, + }; + const metaRecord = objectRecord(meta); + if (!metaRecord || !worker) return env; + + let hasDoBinding = false; + let doAlarmStorageId = ESTIMATED_DO_STORAGE_ID; + let hasWorkflowBinding = false; + const workflows = Array.isArray(metaRecord.workflows) ? metaRecord.workflows : []; + for (const workflow of workflows) { + const record = objectRecord(workflow); + if (!record) continue; + const binding = stringOrFallback(record.binding); + if (!binding) continue; + hasWorkflowBinding = true; + env[binding] = { + ns, + worker, + version, + name: stringOrFallback(record.name), + binding, + className: stringOrFallback(record.className), + workflowKey: stringOrFallback(record.workflowKey, ESTIMATED_WORKFLOW_KEY), + }; + } + + const bindings = objectRecord(metaRecord.bindings); + if (bindings) { + for (const [name, rawSpec] of Object.entries(bindings)) { + const spec = objectRecord(rawSpec); + if (!spec) continue; + env[name] = estimatedBindingEnvValue({ + name, + spec, + meta: metaRecord, + ns, + worker, + version, + assetsCdnBase: typeof assetsCdnBase === "string" && assetsCdnBase + ? assetsCdnBase + : ESTIMATED_ASSETS_CDN_BASE, + nsSecrets: nsSecretStrings, + workerSecrets: workerSecretStrings, + }); + if (spec.type === "do") { + hasDoBinding = true; + doAlarmStorageId = stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID); + } + } + } + if (hasDoBinding) { + env[DO_BACKEND_BINDING] = { __wdlBinding: "internal", name: "DO_BACKEND" }; + env[DO_OWNER_NETWORK_BINDING] = { __wdlBinding: "internal", name: "DO_OWNER_NETWORK" }; + env[DO_ALARMS_BINDING] = { + __wdlBinding: "do-alarms", + props: { ns, worker, version, doStorageId: doAlarmStorageId }, + }; + } + if (hasWorkflowBinding) { + env[WORKFLOWS_BACKEND_BINDING] = { __wdlBinding: "internal", name: "WORKFLOWS_BACKEND" }; + } + return env; +} + +/** @param {string} value */ +function hasNonLatin1(value) { + for (let i = 0; i < value.length; i += 1) { + if (value.charCodeAt(i) > 0xff) return true; + } + return false; +} + +/** @param {string} value */ +function v8TwoByteStringPenalty(value) { + if (!hasNonLatin1(value)) return 0; + return Math.max(0, (2 * value.length) - Buffer.byteLength(value, "utf8")); +} + +/** @param {unknown} value */ +function v8StringPenalty(value) { + if (typeof value === "string") return v8TwoByteStringPenalty(value); + if (!value || typeof value !== "object") return 0; + let bytes = 0; + for (const [key, child] of Object.entries(value)) { + bytes += v8TwoByteStringPenalty(key); + bytes += v8StringPenalty(child); + } + return bytes; +} + +/** @param {unknown} value */ +export function estimatedWorkerLoaderEnvBytes(value) { + const json = JSON.stringify(value) ?? "null"; + return Buffer.byteLength(json, "utf8") + v8StringPenalty(value); +} + +/** + * @param {{ + * ns: string, + * worker?: string, + * version?: string, + * sourceVersion?: string | null, + * vars?: Record | null, + * nsSecrets?: Record | null, + * workerSecrets?: Record | null, + * meta?: Record | null, + * assetsCdnBase?: string | null, + * }} args + */ +export function assertWorkerLoaderUserEnvBudget({ + ns, + worker = undefined, + version = ESTIMATED_VERSION, + sourceVersion = null, + vars = null, + nsSecrets = null, + workerSecrets = null, + meta = null, + assetsCdnBase = ESTIMATED_ASSETS_CDN_BASE, +}) { + // workerd enforces the full workerLoader env as a Frankenvalue estimate. Control + // mirrors the user strings plus runtime-injected binding/workflow env shapes as + // JSON, then accounts for V8's two-byte representation of non-Latin-1 strings. + const bytes = estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ + ns, + worker, + version, + vars, + nsSecrets, + workerSecrets, + meta, + assetsCdnBase, + })); + if (bytes > WORKER_LOADER_ENV_MAX_BYTES) { + const label = worker + ? `${ns}/${worker}${sourceVersion ? `@${sourceVersion}` : ""}` + : ns; + throw new WorkerEnvBudgetError( + `estimated workerLoader env for ${label} serializes to ${bytes} bytes, ` + + `exceeding WDL workerLoader env budget ${WORKER_LOADER_ENV_MAX_BYTES} bytes`, + { + namespace: ns, + ...(worker ? { worker } : {}), + ...(sourceVersion ? { source_version: sourceVersion } : {}), + ...(sourceVersion && version ? { estimated_version: version } : {}), + env_bytes: bytes, + max_env_bytes: WORKER_LOADER_ENV_MAX_BYTES, + upstream_max_env_bytes: UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + headroom_bytes: WORKER_LOADER_ENV_HEADROOM_BYTES, + } + ); + } + return bytes; +} + +/** + * @param {{ + * encrypted: Record, + * env: Record, + * hashKey: string, + * ignoreSecretEnvelopeErrors?: boolean, + * }} args + */ +export async function decryptSecretHash({ encrypted, env, hashKey, ignoreSecretEnvelopeErrors = false }) { + const entries = await Promise.all( + Object.entries(encrypted || {}) + .filter((entry) => typeof entry[1] === "string") + .map(async ([fieldName, value]) => { + try { + return [ + fieldName, + await decryptSecretValue(/** @type {string} */ (value), { env, hashKey, fieldName }), + ]; + } catch (err) { + if (ignoreSecretEnvelopeErrors && err instanceof SecretEnvelopeError) return null; + throw err; + } + }) + ); + /** @type {Record} */ + const out = Object.create(null); + for (const entry of entries) { + if (!entry) continue; + const [fieldName, value] = entry; + out[fieldName] = value; + } + return out; +} + +/** + * @param {{ + * redis: { hGet(key: string, field: string): Promise }, + * ns: string, + * worker: string, + * versions: Iterable, + * versionEstimates?: Iterable<{ sourceVersion: string, estimatedVersion: string }>, + * nsSecrets?: Record | null, + * workerSecrets?: Record | null, + * assetsCdnBase?: string | null, + * }} args + */ +export async function assertWorkerVersionsUserEnvBudget({ + redis, + ns, + worker, + versions, + versionEstimates = [], + nsSecrets = null, + workerSecrets = null, + assetsCdnBase = ESTIMATED_ASSETS_CDN_BASE, +}) { + const checks = [ + ...[...versions] + .filter((version) => typeof version === "string" && version) + .map((version) => ({ sourceVersion: version, estimatedVersion: version })), + ...[...versionEstimates] + .filter((entry) => + entry && + typeof entry.sourceVersion === "string" && + entry.sourceVersion && + typeof entry.estimatedVersion === "string" && + entry.estimatedVersion + ), + ]; + const uniqueChecks = [...new Map( + checks.map((entry) => [`${entry.sourceVersion}\0${entry.estimatedVersion}`, entry]) + ).values()]; + if (uniqueChecks.length === 0) { + assertWorkerLoaderUserEnvBudget({ ns, worker, nsSecrets, workerSecrets, assetsCdnBase }); + return; + } + + // Keep bundle metadata reads sequential: callers may pass a RedisSession, + // whose command protocol is single-flight even though secret decryption is not. + for (const { sourceVersion, estimatedVersion } of uniqueChecks) { + const rawMeta = await redis.hGet(bundleKey(ns, worker, sourceVersion), "__meta__"); + if (typeof rawMeta !== "string") { + throw new Error(`bundle metadata missing for ${ns}/${worker}@${sourceVersion}`); + } + /** @type {unknown} */ + let parsed; + try { + parsed = JSON.parse(rawMeta); + } catch (err) { + throw new Error( + `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ${errorMessage(err)}`, + { cause: err } + ); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new Error( + `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ` + + "__meta__ must be a JSON object" + ); + } + const meta = /** @type {Record} */ (parsed); + assertWorkerLoaderUserEnvBudget({ + ns, + worker, + version: estimatedVersion, + sourceVersion, + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) + ? /** @type {Record} */ (meta.vars) + : null, + nsSecrets, + workerSecrets, + meta, + assetsCdnBase, + }); + } +} diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index 2829fe3..2b86810 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -1,7 +1,7 @@ import { jsonResponse, jsonError, readJsonBody, formatError, requireControlLog, requireControlRedis, - errMessage, prefixedId, + errMessage, prefixedId, stringEnv, getControlS3, runOptimistic, stageBundleCommit, buildS3CleanupTaskId, recordS3CleanupIntent, @@ -42,6 +42,16 @@ import { isReservedNs, isValidRouteNs, ROUTES_ALLOWED_RESERVED_NS } from "shared import { putAsset, inferContentType } from "control-s3"; import { generateAssetsToken, assetsPrefixFor } from "shared-assets-token"; import { resolveDatabaseRefFrom } from "control-d1-store"; +import { + WorkerEnvBudgetError, + assertWorkerLoaderUserEnvBudget, + decryptSecretHash, +} from "control-env-budget"; +import { + WORKER_LOADER_CODE_MAX_BYTES, + estimatePreparedWorkerLoaderCodeBytes, +} from "control-worker-code-budget"; +import { SecretEnvelopeError } from "shared-secret-envelope"; const MAX_COMMIT_ATTEMPTS = 5; const DEPLOY_JSON_BODY_MAX_BYTES = 32 * 1024 * 1024; @@ -146,6 +156,26 @@ function deployRequestErrorFromUnknown(err) { ); } +/** + * @param {{ prepared: PreparedBundle, ns: string, name: string }} args + */ +function validateWorkerLoaderCodeBudget({ prepared, ns, name }) { + let totalBytes; + try { + totalBytes = estimatePreparedWorkerLoaderCodeBytes(prepared); + } catch (err) { + throw invalidDeployRequest(errMessage(err)); + } + if (totalBytes > WORKER_LOADER_CODE_MAX_BYTES) { + throw new DeployRequestError( + 413, + "worker_code_too_large", + `final WorkerCode for ${ns}/${name} totals ${totalBytes} bytes, ` + + `exceeding workerd workerLoader code limit ${WORKER_LOADER_CODE_MAX_BYTES} bytes` + ); + } +} + /** @param {BindingMap} [bindings] */ function hasDurableObjectBinding(bindings = {}) { return Object.values(bindings).some((spec) => spec?.type === "do"); @@ -575,6 +605,30 @@ async function runDeployPreflight({ redis, ns, name, deployRequest }) { }); } +/** + * @param {{ redis: RedisClient | RedisSession, controlEnv: Record, ns: string, name: string, meta: PreparedMeta | CommittedMeta, version?: string }} args + */ +async function validateCommittedEnvBudget({ redis, controlEnv, ns, name, meta, version = undefined }) { + const nsEncrypted = await redis.hGetAll(`secrets:${ns}`); + const workerEncrypted = await redis.hGetAll(`secrets:${ns}:${name}`); + const [nsSecrets, workerSecrets] = await Promise.all([ + decryptSecretHash({ encrypted: nsEncrypted, env: controlEnv, hashKey: `secrets:${ns}` }), + decryptSecretHash({ encrypted: workerEncrypted, env: controlEnv, hashKey: `secrets:${ns}:${name}` }), + ]); + assertWorkerLoaderUserEnvBudget({ + ns, + worker: name, + version, + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) + ? /** @type {Record} */ (meta.vars) + : null, + nsSecrets, + workerSecrets, + meta, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); +} + /** * @param {{ deployRequest: DeployRequest, ns: string, name: string, mergedBindings: BindingMap }} args * @returns {{ response: Response, committed?: never } | { response?: never, committed: CommittedBundle }} @@ -643,16 +697,17 @@ async function uploadDeployAssetsBeforeCommit({ * requestId: string, * warnings: DeployWarning[], * log: ControlLogger, + * controlEnv: Record, * }} args * @returns {Promise<{ response: Response, commitDurationMs?: never } | { response?: never, commitDurationMs: number }>} */ async function commitPreparedDeploy({ - redis, ns, name, version, prepared, outgoingRefs, d1Refs, uploadedPrefix, requestId, warnings, log, + redis, ns, name, version, prepared, outgoingRefs, d1Refs, uploadedPrefix, requestId, warnings, log, controlEnv, }) { const commitStartedAt = Date.now(); try { await commitWithWatch({ - redis, ns, name, version, prepared, outgoingRefs, d1Refs, + redis, ns, name, version, prepared, outgoingRefs, d1Refs, controlEnv, }); } catch (err) { if (uploadedPrefix) { @@ -674,6 +729,8 @@ async function commitPreparedDeploy({ }); return { response: controlAbortResponse(err, warnings.length ? { warnings } : {}) }; } + if (err instanceof WorkerEnvBudgetError) return { response: codedErrorResponse(err, err.code) }; + if (err instanceof SecretEnvelopeError) return { response: jsonError(503, err.code, err.message) }; throw err; } return { commitDurationMs: Date.now() - commitStartedAt }; @@ -718,11 +775,40 @@ export async function handle({ request, env, ns, name, requestId }) { outgoingRefs, d1Refs, } = candidate.committed; + const controlEnv = stringEnv(env); + + try { + validateWorkerLoaderCodeBudget({ prepared, ns, name }); + } catch (err) { + if (err instanceof DeployRequestError) return deployRequestErrorResponse(err); + throw err; + } if (parsed.deployRequest.assetsToUpload && !s3) { return deployAssetsS3NotConfiguredResponse(); } + try { + // Advisory pre-allocation pass that decrypts current secret envelopes before + // assets/version side effects. It uses a conservative version placeholder, so + // commitWithWatch() is the only env-budget rejection point after version + // allocation and watched metadata materialization such as resolved D1 ids. + await validateCommittedEnvBudget({ + redis, + controlEnv, + ns, + name, + meta: prepared.meta, + }); + } catch (err) { + if (err instanceof WorkerEnvBudgetError) { + // Do not reject here: the placeholder can over-estimate DO/workflow-heavy + // bundles. The watched commit check below uses the real version and will + // reject true budget failures before the version is written. + } else if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + else throw err; + } + const num = await redis.incr(`worker:${ns}:${name}:next_version`); const version = formatVersion(num); @@ -751,6 +837,7 @@ export async function handle({ request, env, ns, name, requestId }) { requestId, warnings, log, + controlEnv, }); if (commitResult.response) return commitResult.response; @@ -812,10 +899,10 @@ async function scheduleDeployAbortCleanup({ } /** - * @param {{ redis: RedisClient, ns: string, name: string, version: string, prepared: PreparedBundle, outgoingRefs: OutgoingRef[], d1Refs: DeployD1Ref[] }} args + * @param {{ redis: RedisClient, ns: string, name: string, version: string, prepared: PreparedBundle, outgoingRefs: OutgoingRef[], d1Refs: DeployD1Ref[], controlEnv?: Record | null }} args */ export async function commitWithWatch({ - redis, ns, name, version, prepared, outgoingRefs, d1Refs, + redis, ns, name, version, prepared, outgoingRefs, d1Refs, controlEnv = null, }) { const vNum = parseVersion(version); if (vNum == null) throw new Error(`commitWithWatch: bad version ${version}`); @@ -829,6 +916,9 @@ export async function commitWithWatch({ }, }, async (iso) => { await watchCommitKeys(iso, { ns, name, prepared, outgoingRefs, d1Refs }); + if (controlEnv) { + await iso.watch(`secrets:${ns}`, `secrets:${ns}:${name}`); + } const resolvedD1Refs = await resolveD1RefsForCommit(iso, { ns, d1Refs }); await validateCallerNotDeleting(iso, { ns, name }); @@ -844,6 +934,16 @@ export async function commitWithWatch({ prepared, resolvedD1Refs, }); + if (controlEnv) { + await validateCommittedEnvBudget({ + redis: iso, + controlEnv, + ns, + name, + meta: committedMeta, + version, + }); + } const multi = iso.multi(); stageDeployCommit(multi, { diff --git a/control/handlers/logs-tail.js b/control/handlers/logs-tail.js index 4888b9f..530f451 100644 --- a/control/handlers/logs-tail.js +++ b/control/handlers/logs-tail.js @@ -1,7 +1,7 @@ // SSE handler for `wdl tail`. Pull-based ReadableStream + pre-registered -// ctx.waitUntil(cancelPromise) so the cancel callback fires reliably on -// HTTP client disconnect. Push-style intervals have historically failed to -// propagate disconnect cleanup reliably here. +// ctx.waitUntil(cancelPromise) keeps cleanup outside the cancel callback. +// workerd >= 2026-06-19 no longer reliably calls cancel() on client +// disconnect, so max-session and idle-pull cleanup use independent watchdogs. import { RedisSession, redisDbFromEnv } from "shared-redis"; import { envValueOr } from "shared-env"; @@ -20,6 +20,8 @@ const TAIL_ACTIVATION_TTL_SECONDS = 30; const TAIL_ACTIVATION_MAX_ENTRIES = 10_000; const XREAD_BLOCK_MS = 10_000; const SSE_KEEPALIVE_MS = 5_000; +const LOG_TAIL_IDLE_PULL_GRACE_FACTOR = 3; +export const LOG_TAIL_IDLE_PULL_MS = SSE_KEEPALIVE_MS * LOG_TAIL_IDLE_PULL_GRACE_FACTOR; export const LOG_TAIL_MAX_SESSION_MS_DEFAULT = 15 * 60 * 1000; const MAX_WORKERS_PER_TAIL_SESSION = 50; const JSON_FIELD_BYTES = [0x6a, 0x73, 0x6f, 0x6e]; // "json" @@ -360,22 +362,118 @@ export async function handle({ request, env, ctx, ns, requestId }) { db: redisDbFromEnv(env, "DATA_REDIS_DB"), }); let sessionOpen = false; + let cleanupFinished = false; + /** @type {Promise | null} */ + let sessionClosePromise = null; let cancelled = false; + /** @type {ReadableStreamDefaultController | null} */ + let streamController = null; + let expiryLogged = false; + let idleLogged = false; + let lastPullAtMs = Date.now(); const { promise: cancelPromise, resolve: resolveCancel } = /** @type {PromiseWithResolvers} */ (Promise.withResolvers()); + const sessionExpiredWarning = () => sseEvent({ + event: "tail_warning", + data: JSON.stringify({ + event: "tail_warning", + code: "session_expired", + message: "Tail session reached its maximum lifetime; reconnecting for reauthorization.", + }), + }); + + const sessionIdleWarning = () => sseEvent({ + event: "tail_warning", + data: JSON.stringify({ + event: "tail_warning", + code: "session_idle", + message: "Tail session stopped receiving client reads; reconnecting closes the abandoned session.", + }), + }); + + async function closeSessionIfOpen() { + if (!sessionOpen && !session.hasOpenResources()) return; + sessionClosePromise ??= (async () => { + try { + await session.close(); + } catch (err) { + log("warn", "tail_session_close_failed", { + request_id: requestId, namespace: ns, + error_message: errMessage(err), + }); + } + })(); + await sessionClosePromise; + } + + /** @param {ReadableStreamDefaultController | null} controller */ + function expireSession(controller) { + if (cancelled) return; + cancelled = true; + if (!expiryLogged) { + expiryLogged = true; + log("info", "tail_session_expired", { + request_id: requestId, namespace: ns, worker_count: workers.length, + max_session_ms: maxSessionMs, + }); + } + if (controller) { + try { controller.enqueue(utf8Encoder.encode(sessionExpiredWarning())); } catch {} + try { controller.close(); } catch {} + } + resolveCancel(); + } + + /** @param {ReadableStreamDefaultController | null} controller */ + function idleSession(controller) { + if (cancelled) return; + cancelled = true; + if (!idleLogged) { + idleLogged = true; + log("info", "tail_session_idle", { + request_id: requestId, namespace: ns, worker_count: workers.length, + idle_pull_ms: Date.now() - lastPullAtMs, + idle_limit_ms: LOG_TAIL_IDLE_PULL_MS, + }); + } + if (controller) { + try { controller.enqueue(utf8Encoder.encode(sessionIdleWarning())); } catch {} + try { controller.close(); } catch {} + } + resolveCancel(); + } + + const expiryTimer = setTimeout(() => expireSession(streamController), maxSessionMs); + if (typeof expiryTimer === "object" && typeof expiryTimer.unref === "function") { + expiryTimer.unref(); + } + /** @type {ReturnType | null} */ + let idleTimer = null; + function scheduleIdleWatchdog() { + const delayMs = Math.max(1, LOG_TAIL_IDLE_PULL_MS - (Date.now() - lastPullAtMs)); + idleTimer = setTimeout(() => { + if (cancelled) return; + if (Date.now() - lastPullAtMs >= LOG_TAIL_IDLE_PULL_MS) { + idleSession(streamController); + return; + } + scheduleIdleWatchdog(); + }, delayMs); + if (typeof idleTimer === "object" && typeof idleTimer.unref === "function") { + idleTimer.unref(); + } + } + scheduleIdleWatchdog(); + // Pre-register cleanup. Per CLAUDE.md gotcha: scheduling waitUntil from // inside cancel races IoContext teardown — cancel only resolves the // promise, the actual close happens here. ctx.waitUntil(cancelPromise.then(async () => { - try { - if (sessionOpen) await session.close(); - } catch (err) { - log("warn", "tail_session_close_failed", { - request_id: requestId, namespace: ns, - error_message: errMessage(err), - }); - } + clearTimeout(expiryTimer); + if (idleTimer) clearTimeout(idleTimer); + await closeSessionIfOpen(); + cleanupFinished = true; log("info", "tail_session_close", { request_id: requestId, namespace: ns, worker_count: workers.length, }); @@ -393,7 +491,12 @@ export async function handle({ request, env, ctx, ns, requestId }) { const stream = new ReadableStream({ async pull(controller) { - if (cancelled) return; + streamController = controller; + lastPullAtMs = Date.now(); + if (cancelled) { + try { controller.close(); } catch {} + return; + } try { if (!bootstrapped) { // Open BEFORE flipping bootstrapped so a session.open() throw @@ -401,26 +504,17 @@ export async function handle({ request, env, ctx, ns, requestId }) { // it and fail on the first Redis stream read. await session.open(); sessionOpen = true; + if (cancelled || cleanupFinished) { + await closeSessionIfOpen(); + try { controller.close(); } catch {} + return; + } bootstrapped = true; } const remainingMs = maxSessionMs - (Date.now() - sessionStartedAtMs); if (remainingMs <= 0) { - log("info", "tail_session_expired", { - request_id: requestId, namespace: ns, worker_count: workers.length, - max_session_ms: maxSessionMs, - }); - controller.enqueue(utf8Encoder.encode(sseEvent({ - event: "tail_warning", - data: JSON.stringify({ - event: "tail_warning", - code: "session_expired", - message: "Tail session reached its maximum lifetime; reconnecting for reauthorization.", - }), - }))); - cancelled = true; - resolveCancel(); - controller.close(); + expireSession(controller); return; } @@ -458,6 +552,10 @@ export async function handle({ request, env, ctx, ns, requestId }) { session, Math.min(XREAD_BLOCK_MS, SSE_KEEPALIVE_MS, remainingMs), ); + if (cancelled) { + try { controller.close(); } catch {} + return; + } if (!batch) { // Timed out with no events → SSE comment so intermediaries know @@ -477,6 +575,10 @@ export async function handle({ request, env, ctx, ns, requestId }) { }))); } } catch (err) { + if (cancelled) { + try { controller.close(); } catch {} + return; + } log("error", "tail_pull_failed", { request_id: requestId, namespace: ns, error_message: errMessage(err), diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index 1dcf57a..eed7795 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -3,11 +3,167 @@ import { jsonError, requireControlLog, requireControlRedis, + stringEnv, + codedErrorResponse, + runOptimistic, + ControlAbort, + controlAbortResponse, } from "control-shared"; import { invalidSecretMutationKeyResponse, readEncryptedSecretPutValue, } from "control-handlers-secret-put"; +import { routesKey, workerVersionsKey } from "shared-version"; +import { workersIndexKey } from "control-lib"; +import { + WorkerEnvBudgetError, + assertWorkerLoaderUserEnvBudget, + assertWorkerVersionsUserEnvBudget, + decryptSecretHash, +} from "control-env-budget"; +import { SecretEnvelopeError } from "shared-secret-envelope"; + +const MAX_NS_SECRET_ATTEMPTS = 5; + +class NamespaceSecretAbort extends ControlAbort {} + +/** @param {unknown} err */ +function namespaceSecretMutationErrorResponse(err) { + if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); + if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); + if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + return null; +} + +/** + * @param {{ + * redis: import("shared-redis").RedisSession, + * controlEnv: Record, + * nsName: string, + * nsSecrets: Record, + * ignoreWorkerSecretEnvelopeErrors?: boolean, + * }} args + */ +async function validateNamespaceSecretBudget({ + redis, + controlEnv, + nsName, + nsSecrets, + ignoreWorkerSecretEnvelopeErrors = false, +}) { + const activeRoutes = await redis.hGetAll(routesKey(nsName)); + const indexedWorkers = await redis.sMembers(workersIndexKey(nsName)); + const workerNames = new Set([ + ...indexedWorkers.filter((worker) => typeof worker === "string" && worker), + ...Object.keys(activeRoutes), + ]); + if (workerNames.size === 0) { + assertWorkerLoaderUserEnvBudget({ + ns: nsName, + nsSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); + return; + } + + for (const worker of workerNames) { + const workerSecretsKey = `secrets:${nsName}:${worker}`; + await redis.watch(workerVersionsKey(nsName, worker), workerSecretsKey); + const activeVersion = activeRoutes[worker]; + const retainedVersions = await redis.zRange(workerVersionsKey(nsName, worker), 0, -1); + const workerEncrypted = await redis.hGetAll(workerSecretsKey); + const workerSecrets = await decryptSecretHash({ + encrypted: workerEncrypted, + env: controlEnv, + hashKey: workerSecretsKey, + ignoreSecretEnvelopeErrors: ignoreWorkerSecretEnvelopeErrors, + }); + await assertWorkerVersionsUserEnvBudget({ + redis, + ns: nsName, + worker, + versions: [ + ...retainedVersions, + ...(typeof activeVersion === "string" && activeVersion ? [activeVersion] : []), + ], + nsSecrets, + workerSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); + } +} + +/** + * @param {{ + * redis: import("shared-redis").RedisClient, + * env: Record, + * nsName: string, + * secretKey: string, + * method: "PUT" | "DELETE", + * encrypted?: string | null, + * plaintext?: string | null, + * }} args + */ +async function mutateNamespaceSecret({ + redis, + env, + nsName, + secretKey, + method, + encrypted = null, + plaintext = null, +}) { + const controlEnv = stringEnv(env); + const nsSecretsKey = `secrets:${nsName}`; + return await runOptimistic(redis, { + attempts: MAX_NS_SECRET_ATTEMPTS, + onExhausted: () => { + throw new NamespaceSecretAbort(503, "namespace_secret_mutation_contention", { + message: `exhausted ${MAX_NS_SECRET_ATTEMPTS} retries; retry later`, + }); + }, + }, async (iso) => { + await iso.watch(nsSecretsKey, routesKey(nsName), workersIndexKey(nsName)); + + const existingEncrypted = await iso.hGetAll(nsSecretsKey); + if (method === "DELETE" && !Object.hasOwn(existingEncrypted, secretKey)) { + return { mutated: false }; + } + + const budgetEncrypted = { ...existingEncrypted }; + delete budgetEncrypted[secretKey]; + const nsSecrets = await decryptSecretHash({ + encrypted: budgetEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", + }); + if (method === "PUT") { + if (typeof encrypted !== "string") throw new Error("PUT namespace secret encrypted value missing"); + if (typeof plaintext !== "string") throw new Error("PUT namespace secret plaintext missing"); + nsSecrets[secretKey] = plaintext; + } else { + delete nsSecrets[secretKey]; + } + + await validateNamespaceSecretBudget({ + redis: iso, + controlEnv, + nsName, + nsSecrets, + ignoreWorkerSecretEnvelopeErrors: method === "DELETE", + }); + + const multi = iso.multi(); + if (method === "PUT") { + multi.hSet(nsSecretsKey, secretKey, /** @type {string} */ (encrypted)); + } else { + multi.hDel(nsSecretsKey, secretKey); + } + await multi.exec(); + return { mutated: true }; + }); +} /** * @param {{ @@ -38,8 +194,21 @@ export async function handle({ request, env, method, nsName, secretKey, requestI fieldName: secretKey, }); if ("response" in put) return put.response; - const encrypted = put.encrypted; - await redis.hSet(nsSecretsKey, secretKey, encrypted); + try { + await mutateNamespaceSecret({ + redis, + env, + nsName, + secretKey, + method: "PUT", + encrypted: put.encrypted, + plaintext: put.plaintext, + }); + } catch (err) { + const response = namespaceSecretMutationErrorResponse(err); + if (response) return response; + throw err; + } log("info", "ns_secret_set", { request_id: requestId, namespace: nsName, key: secretKey }); return jsonResponse(200, { namespace: nsName, @@ -51,14 +220,27 @@ export async function handle({ request, env, method, nsName, secretKey, requestI if (method === "DELETE" && secretKey !== undefined) { const invalidKey = invalidSecretMutationKeyResponse(secretKey); if (invalidKey) return invalidKey; - const removed = Number(await redis.hDel(nsSecretsKey, secretKey)) > 0; + let result; + try { + result = await mutateNamespaceSecret({ + redis, + env, + nsName, + secretKey, + method: "DELETE", + }); + } catch (err) { + const response = namespaceSecretMutationErrorResponse(err); + if (response) return response; + throw err; + } log("info", "ns_secret_deleted", { request_id: requestId, namespace: nsName, key: secretKey, - existed: removed, + existed: result.mutated, }); - return jsonResponse(200, { namespace: nsName, key: secretKey, deleted: removed }); + return jsonResponse(200, { namespace: nsName, key: secretKey, deleted: result.mutated }); } return jsonError(405, "method_not_allowed", "Method not allowed for /secrets"); } diff --git a/control/handlers/secret-put.js b/control/handlers/secret-put.js index 274d219..548db58 100644 --- a/control/handlers/secret-put.js +++ b/control/handlers/secret-put.js @@ -29,7 +29,7 @@ export function invalidSecretMutationKeyResponse(key) { * hashKey: string, * fieldName: string, * }} args - * @returns {Promise<{ response: Response } | { encrypted: string }>} + * @returns {Promise<{ response: Response } | { encrypted: string, plaintext: string }>} */ export async function readEncryptedSecretPutValue({ request, env, hashKey, fieldName }) { const parsed = await readJsonBody(request, { @@ -46,6 +46,7 @@ export async function readEncryptedSecretPutValue({ request, env, hashKey, field } try { return { + plaintext: body.value, encrypted: await encryptSecretValue(body.value, { env: stringEnv(env), hashKey, fieldName }), }; } catch (err) { diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index dc1646f..6a0c8c8 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -7,6 +7,8 @@ import { requireControlLog, requireControlRedis, runOptimistic, + stringEnv, + codedErrorResponse, } from "control-shared"; import { deleteLockKey, @@ -19,12 +21,39 @@ import { } from "control-handlers-secret-put"; import { stageWorkerHidden, stageWorkerVisible } from "control-lifecycle-indexes"; import { bumpActiveAndPromote, RoutingError } from "control-routing"; +import { + WorkerEnvBudgetError, + WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + assertWorkerVersionsUserEnvBudget, + decryptSecretHash, +} from "control-env-budget"; +import { SecretEnvelopeError } from "shared-secret-envelope"; const MAX_SECRET_ATTEMPTS = 5; /** * @typedef {import("shared-redis").RedisClient} RedisClient * @typedef {import("control-routing").RedisClient} RoutingRedisClient + * @typedef {{ + * namespace: string, + * name: string, + * key: string, + * version: string, + * previousVersion: string, + * set?: boolean, + * deleted?: boolean, + * }} SecretMutationVersionPayload + * @typedef {{ + * namespace: string, + * name: string, + * key: string, + * secretWritten: boolean, + * reloadForced: boolean, + * effect: string, + * warnings: { kind: string, reason: string, nextPickup: string }[], + * set?: boolean, + * deleted?: boolean, + * }} SecretMutationDeferredPayload */ class SecretAbort extends ControlAbort {} @@ -54,8 +83,10 @@ export async function handle({ request, env, method, ns, name, subPath, requestI const key = subPath[0]; const invalidKey = invalidSecretMutationKeyResponse(key); if (invalidKey) return invalidKey; + const controlEnv = stringEnv(env); let storedValue = null; + let putPlaintext = null; if (method === "PUT") { const put = await readEncryptedSecretPutValue({ request, @@ -65,6 +96,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI }); if ("response" in put) return put.response; storedValue = put.encrypted; + putPlaintext = put.plaintext; } let mutationResult; @@ -72,6 +104,8 @@ export async function handle({ request, env, method, ns, name, subPath, requestI mutationResult = await mutateSecret({ redis, ns, name, key, method, value: storedValue, + plaintext: putPlaintext, + controlEnv, }); } catch (err) { if (err instanceof SecretAbort) { @@ -86,6 +120,8 @@ export async function handle({ request, env, method, ns, name, subPath, requestI }); return controlAbortResponse(err); } + if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); + if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); throw err; } @@ -106,7 +142,21 @@ export async function handle({ request, env, method, ns, name, subPath, requestI /** @type {RoutingRedisClient} */ (redis), ns, name, - { log, requestId } + { + log, + requestId, + beforeStageCopy: ({ iso, currentVersion, newVersion }) => + assertWorkerSecretBumpEnvBudget({ + iso, + ns, + name, + currentVersion, + newVersion, + controlEnv, + ignoreWorkerSecretEnvelopeErrors: method === "DELETE", + ignoreNamespaceSecretEnvelopeErrors: method === "DELETE", + }), + } ); log("info", method === "PUT" ? "secret_set" : "secret_deleted", { request_id: requestId, @@ -116,7 +166,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI previous_version: result.previousVersion, new_version: result.version, }); - /** @type {{ namespace: string, name: string, key: string, version: string, previousVersion: string, set?: boolean, deleted?: boolean }} */ + /** @type {SecretMutationVersionPayload} */ const payload = { namespace: ns, name, @@ -148,7 +198,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI else payload.deleted = true; return jsonResponse(200, payload); } - if (err instanceof RoutingError) { + if (err instanceof RoutingError || err instanceof WorkerEnvBudgetError) { // Secret already landed in our own MULTI; bump failure degrades // to a deferred reload — the secret is picked up on next natural // cold-load, or wiped by a concurrent whole-delete. @@ -160,7 +210,7 @@ export async function handle({ request, env, method, ns, name, subPath, requestI status: err.status, ...formatError(err), }); - /** @type {{ namespace: string, name: string, key: string, secretWritten: boolean, reloadForced: boolean, effect: string, warnings: { kind: string, reason: string, nextPickup: string }[], set?: boolean, deleted?: boolean }} */ + /** @type {SecretMutationDeferredPayload} */ const payload = { namespace: ns, name, @@ -185,13 +235,24 @@ export async function handle({ request, env, method, ns, name, subPath, requestI return jsonError(405, "method_not_allowed", "Method not allowed for /secrets"); } -// DELETE extends WATCH to routes + worker-versions so the -// "last key → SREM workers:" branch can trust its preconditions. +// Secret mutations watch routes, worker-versions, and both secret hashes because +// env-budget checks must cover active and retained versions before writing a +// new secret shape. /** - * @param {{ redis: RedisClient, ns: string, name: string, key: string, method: string, value: string | null }} args + * @param {{ + * redis: RedisClient, + * ns: string, + * name: string, + * key: string, + * method: "PUT" | "DELETE", + * value: string | null, + * plaintext?: string | null, + * controlEnv: Record, + * }} args */ -async function mutateSecret({ redis, ns, name, key, method, value }) { +async function mutateSecret({ redis, ns, name, key, method, value, plaintext = null, controlEnv }) { const secretsKey = `secrets:${ns}:${name}`; + const nsSecretsKey = `secrets:${ns}`; return await runOptimistic(redis, { attempts: MAX_SECRET_ATTEMPTS, onExhausted: () => { @@ -200,10 +261,13 @@ async function mutateSecret({ redis, ns, name, key, method, value }) { }); }, }, async (iso) => { - const watches = [deleteLockKey(ns, name), secretsKey]; - if (method === "DELETE") { - watches.push(routesKey(ns), workerVersionsKey(ns, name)); - } + const watches = [ + deleteLockKey(ns, name), + nsSecretsKey, + secretsKey, + routesKey(ns), + workerVersionsKey(ns, name), + ]; await iso.watch(...watches); const callerLock = await iso.get(deleteLockKey(ns, name)); @@ -228,6 +292,49 @@ async function mutateSecret({ redis, ns, name, key, method, value }) { } } + if (method === "PUT" && typeof plaintext !== "string") throw new Error("PUT secret plaintext missing"); + const activeVersion = await iso.hGet(routesKey(ns), name); + const retainedVersions = await iso.zRange(workerVersionsKey(ns, name), 0, -1); + const nsEncrypted = await iso.hGetAll(nsSecretsKey); + const workerEncrypted = await iso.hGetAll(secretsKey); + const workerBudgetEncrypted = { ...workerEncrypted }; + delete workerBudgetEncrypted[key]; + const [nsSecrets, workerSecrets] = await Promise.all([ + decryptSecretHash({ + encrypted: nsEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", + }), + decryptSecretHash({ + encrypted: workerBudgetEncrypted, + env: controlEnv, + hashKey: secretsKey, + ignoreSecretEnvelopeErrors: method === "DELETE", + }), + ]); + if (method === "PUT") { + workerSecrets[key] = /** @type {string} */ (plaintext); + } + await assertWorkerVersionsUserEnvBudget({ + redis: iso, + ns, + worker: name, + versions: [ + ...retainedVersions, + ...(typeof activeVersion === "string" && activeVersion ? [activeVersion] : []), + ], + versionEstimates: typeof activeVersion === "string" && activeVersion + ? [{ + sourceVersion: activeVersion, + estimatedVersion: WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + }] + : [], + nsSecrets, + workerSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); + const multi = iso.multi(); if (method === "PUT") { if (typeof value !== "string") throw new Error("PUT secret value missing"); @@ -246,3 +353,64 @@ async function mutateSecret({ redis, ns, name, key, method, value }) { return { mutated: true }; }); } + +/** + * @param {{ + * iso: { + * watch: (...keys: string[]) => Promise, + * hGet: (key: string, field: string) => Promise, + * hGetAll: (key: string) => Promise>, + * }, + * ns: string, + * name: string, + * currentVersion: string, + * newVersion: string, + * controlEnv: Record, + * ignoreWorkerSecretEnvelopeErrors?: boolean, + * ignoreNamespaceSecretEnvelopeErrors?: boolean, + * }} args + */ +async function assertWorkerSecretBumpEnvBudget({ + iso, + ns, + name, + currentVersion, + newVersion, + controlEnv, + ignoreWorkerSecretEnvelopeErrors = false, + ignoreNamespaceSecretEnvelopeErrors = false, +}) { + const nsSecretsKey = `secrets:${ns}`; + const workerSecretsKey = `secrets:${ns}:${name}`; + await iso.watch(nsSecretsKey, workerSecretsKey); + + // Keep reads sequential: RedisSession is a single RESP stream. + const nsEncrypted = await iso.hGetAll(nsSecretsKey); + const workerEncrypted = await iso.hGetAll(workerSecretsKey); + const nsSecrets = await decryptSecretHash({ + encrypted: nsEncrypted, + env: controlEnv, + hashKey: nsSecretsKey, + ignoreSecretEnvelopeErrors: ignoreNamespaceSecretEnvelopeErrors, + }); + const workerSecrets = await decryptSecretHash({ + encrypted: workerEncrypted, + env: controlEnv, + hashKey: workerSecretsKey, + ignoreSecretEnvelopeErrors: ignoreWorkerSecretEnvelopeErrors, + }); + + await assertWorkerVersionsUserEnvBudget({ + redis: iso, + ns, + worker: name, + versions: [], + versionEstimates: [{ + sourceVersion: currentVersion, + estimatedVersion: newVersion, + }], + nsSecrets, + workerSecrets, + assetsCdnBase: controlEnv.ASSETS_CDN_BASE, + }); +} diff --git a/control/routing.js b/control/routing.js index a5ea97d..d8e3583 100644 --- a/control/routing.js +++ b/control/routing.js @@ -63,6 +63,8 @@ function hostDeclarationsKey(host) { * @typedef {{ oldRoutes: RoutePattern[], oldQueueConsumers: QueueConsumer[], affectedHosts: Set, hostState: HostState }} PromoteObservedState * @typedef {{ newRouteKeys: Set, nsHostsAdd: string[], nsHostsRem: string[], cronKey: string, cronHash: Record, cronPlan: CronPlan, queuePlan: QueuePlan }} PromoteStagePlan * @typedef {{ log?: (level: string, event: string, fields: Record) => void, requestId?: string, ns?: string, workerName?: string }} LogContext + * @typedef {{ iso: RedisIso, currentVersion: string, newVersion: string, sourceMeta: BundleMeta }} BumpBeforeStageContext + * @typedef {LogContext & { beforeStageCopy?: (context: BumpBeforeStageContext) => Promise }} BumpOptions * @typedef {{ routes?: RoutePattern[], crons?: CronSpec[], queueConsumers?: QueueConsumer[], exports?: ExportSpec[], bindings?: unknown }} BundleMeta * @typedef {Record>} HostState * @typedef {import("shared-redis").RedisMulti} RedisMulti @@ -680,7 +682,7 @@ export async function promoteWithRoutes(redis, ns, workerName, newVersion, optio // "read active" and "promote new" can't silently roll content back. // Throws RoutingError(404) when no active version exists to copy; // RoutingError(409, "caller_deleting") if a whole-delete is in flight. -/** @param {RedisClient} redis @param {string} ns @param {string} workerName @param {LogContext} [options] */ +/** @param {RedisClient} redis @param {string} ns @param {string} workerName @param {BumpOptions} [options] */ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) { const logContext = { ...options, ns, workerName }; // Avoid burning a version number on the pre-deploy path. @@ -772,6 +774,15 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) throw new RoutingError(500, "invalid_generated_version", `bumpActiveAndPromote: bad new version tag ${newVersion}`); } + if (typeof options.beforeStageCopy === "function") { + await options.beforeStageCopy({ + iso, + currentVersion, + newVersion, + sourceMeta: srcMeta, + }); + } + const multi = iso.multi(); multi.copy(srcKey, dstKey, { REPLACE: true }); stageVersionFlip(multi, ns, workerName, newVersion, routes, affectedHosts); diff --git a/control/worker-code-budget.js b/control/worker-code-budget.js new file mode 100644 index 0000000..682f0c9 --- /dev/null +++ b/control/worker-code-budget.js @@ -0,0 +1,54 @@ +import { + WORKER_LOADER_CODE_MAX_BYTES, + estimateFinalWorkerLoaderCodeBytes, +} from "runtime-load-code-budget"; +import D1_CLIENT_SOURCE from "runtime-d1-client-source"; +import D1_DATA_FIELD_SOURCE from "runtime-d1-data-field-source"; +import D1_PARAMS_SOURCE from "runtime-d1-params-source"; +import SQL_SPLITTER_SOURCE from "runtime-sql-splitter-source"; +import D1_TRANSPORT_SOURCE from "runtime-d1-transport-source"; +import R2_CLIENT_SOURCE from "runtime-r2-client-source"; +import R2_UTILS_SOURCE from "runtime-r2-utils-source"; +import DO_CLIENT_SOURCE from "runtime-do-client-source"; +import DO_TRANSPORT_SOURCE from "runtime-do-transport-source"; +import OWNER_ENDPOINT_SOURCE from "runtime-owner-endpoint-source"; +import OWNER_HINT_CACHE_SOURCE from "runtime-owner-hint-cache-source"; +import REQUEST_ID_SOURCE from "runtime-request-id-source"; +import WORKFLOWS_CLIENT_SOURCE from "runtime-workflows-client-source"; + +export { WORKER_LOADER_CODE_MAX_BYTES }; + +const RUNTIME_INJECTION_SOURCES = Object.freeze({ + d1ClientSource: D1_CLIENT_SOURCE, + d1DataFieldSource: D1_DATA_FIELD_SOURCE, + d1ParamsSource: D1_PARAMS_SOURCE, + sqlSplitterSource: SQL_SPLITTER_SOURCE, + d1TransportSource: D1_TRANSPORT_SOURCE, + r2ClientSource: R2_CLIENT_SOURCE, + r2UtilsSource: R2_UTILS_SOURCE, + doClientSource: DO_CLIENT_SOURCE, + doTransportSource: DO_TRANSPORT_SOURCE, + ownerEndpointSource: OWNER_ENDPOINT_SOURCE, + ownerHintCacheSource: OWNER_HINT_CACHE_SOURCE, + requestIdSource: REQUEST_ID_SOURCE, + workflowsClientSource: WORKFLOWS_CLIENT_SOURCE, +}); + +/** + * @param {{ + * meta: { mainModule?: unknown, modules?: Record | null, [key: string]: unknown }, + * normalized: Array<[string, string | Uint8Array]>, + * }} prepared + */ +export function estimatePreparedWorkerLoaderCodeBytes(prepared) { + const mainModule = prepared.meta.mainModule; + if (typeof mainModule !== "string" || !mainModule) { + throw new Error("WorkerCode budget requires prepared meta.mainModule"); + } + return estimateFinalWorkerLoaderCodeBytes({ + mainModule, + normalized: prepared.normalized, + meta: prepared.meta, + runtimeSources: RUNTIME_INJECTION_SOURCES, + }); +} diff --git a/deploy/kubernetes/base/gateway.yaml b/deploy/kubernetes/base/gateway.yaml index 4744508..8e0c544 100644 --- a/deploy/kubernetes/base/gateway.yaml +++ b/deploy/kubernetes/base/gateway.yaml @@ -41,7 +41,6 @@ spec: - serve - -b - /app/dist/workerd-configs/gateway.bin - - --experimental envFrom: - configMapRef: name: wdl-config diff --git a/do-runtime/alarm-shim-source.js b/do-runtime/alarm-shim-source.js index f91d769..15b93b1 100644 --- a/do-runtime/alarm-shim-source.js +++ b/do-runtime/alarm-shim-source.js @@ -259,7 +259,8 @@ function quoteSqlIdentifier(name) { function sqlObjectDropStatement(row) { const type = String(row.type); const name = String(row.name); - if (name.startsWith("sqlite_") || name.startsWith("_cf_")) return null; + const lowerName = name.toLowerCase(); + if (lowerName.startsWith("sqlite_") || lowerName.startsWith("_cf_")) return null; if (type === "table") return "DROP TABLE IF EXISTS " + quoteSqlIdentifier(name); if (type === "view") return "DROP VIEW IF EXISTS " + quoteSqlIdentifier(name); if (type === "trigger") return "DROP TRIGGER IF EXISTS " + quoteSqlIdentifier(name); diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index 6b19b1e..34799ba 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -46,6 +46,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "runtime-lib", esModule = embed "../runtime/lib.js"), (name = "runtime-load", esModule = embed "../runtime/load.js"), (name = "runtime-load-env-build", esModule = embed "../runtime/load/env-build.js"), + (name = "runtime-load-code-budget", esModule = embed "../runtime/load/code-budget.js"), (name = "runtime-load-module-rewrite", esModule = embed "../runtime/load/module-rewrite.js"), (name = "runtime-load-wrapper-generate", esModule = embed "../runtime/load/wrapper-generate.js"), (name = "runtime-bindings-proxy", esModule = embed "../runtime/bindings/proxy.js"), @@ -90,6 +91,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-d1-timeout", esModule = embed "../shared/d1-timeout.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-owner-forwarder", esModule = embed "../shared/owner-forwarder.js"), @@ -115,7 +117,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "@wdl-dev/aws-sigv4", esModule = embed "../shared/vendor/aws-sigv4.js"), ], compatibilityDate = "2026-04-24", - compatibilityFlags = ["nodejs_compat", "experimental"], + compatibilityFlags = ["nodejs_compat"], globalOutbound = "internal-network", # The do-runtime host actor owns many user DO facets behind one stable # storage shard. Keep it resident so short idle gaps do not force workerd to diff --git a/do-runtime/load.js b/do-runtime/load.js index bf5c145..3f15fe9 100644 --- a/do-runtime/load.js +++ b/do-runtime/load.js @@ -27,8 +27,8 @@ const NATIVE_DELETE_ALL_PRESERVES_ALARM_FLAG = "delete_all_preserves_alarm"; * @typedef {Record & { doStorageId?: unknown, className?: unknown }} RuntimeBindingSpec * @typedef {{ exports: Record }) => unknown> & { KV(options: { props: Record }): unknown, Assets(options: { props: Record }): unknown, QueueProducer(options: { props: Record }): unknown, D1Database(options: { props: Record }): unknown, R2Bucket(options: { props: Record }): unknown, ServiceBinding(options: { props: Record }): unknown, DurableObjectNamespace(options: { props: Record }): unknown, DoAlarmBinding(options: { props: Record }): unknown, InternalAuthBackend(options: { props: Record }): unknown } }} DoRuntimeContext * @typedef {{ name: string, spec: RuntimeBindingSpec }} DoBindingInput - * @typedef {string | { cjs: string } | { py: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue - * @typedef {{ mainModule: string, modules: Record, compatibilityFlags?: string[], compatibilityDate?: string, allowExperimental?: boolean, env?: Record, globalOutbound?: unknown }} WorkerCode + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {{ mainModule: string, modules: Record, compatibilityFlags?: string[], compatibilityDate?: string, env?: Record, globalOutbound?: unknown }} WorkerCode */ /** @param {DoEnv} env */ @@ -256,7 +256,6 @@ export async function loadDoWorkerCode(env, ctx, invoke, requestId = null) { const { meta, ...codeBase } = bundleToWorkerCode(loaded.bundle); const workerCode = { ...codeBase, - allowExperimental: true, env: buildDoEnv( meta, loaded.ns_secrets, diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index 504db12..a2bba28 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -13,6 +13,7 @@ import { import { DoRuntimeError } from "do-runtime-protocol-errors"; import { hostIdForObject, shardForObjectName } from "do-runtime-protocol-identity"; import { formatWorkerId } from "shared-worker-id"; +import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; import { BodyTooLargeError, readBoundedBytes as readRequestBoundedBytes, @@ -477,11 +478,18 @@ function normalizeWorkerCode(value) { const compatibilityFlags = Array.isArray(input.compatibilityFlags) ? input.compatibilityFlags.map((flag, index) => requireString(flag, `workerCode.compatibilityFlags[${index}]`, { maxBytes: 128 })) : ["nodejs_compat"]; + const experimentalFlag = firstWorkerdExperimentalCompatFlag(compatibilityFlags); + if (experimentalFlag) { + throw new DoRuntimeError( + 400, + "experimental_compat_flag_unsupported", + `workerCode.compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` + ); + } const env = input.env == null ? {} : requireRecord(input.env, "workerCode.env"); return { compatibilityDate, compatibilityFlags, - allowExperimental: input.allowExperimental !== false, mainModule, modules: normalizedModules, env, diff --git a/docker-compose.yml b/docker-compose.yml index 7e8ea43..d2f34c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -418,7 +418,7 @@ services: - ./dist:/app/dist:ro entrypoint: ["workerd"] command: - ["serve", "-b", "/app/dist/workerd-configs/gateway-local.bin", "--experimental"] + ["serve", "-b", "/app/dist/workerd-configs/gateway-local.bin"] ports: - "${WDL_GATEWAY_HOST_PORT:-8080}:8080" environment: diff --git a/docs/compatibility.md b/docs/compatibility.md index c70955d..8ce9ec8 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -31,8 +31,18 @@ Each row separates four compatibility claims: |---|---|---|---|---|---| | ES module Workers and `fetch()` | Supported | Module evaluation, request dispatch, `Response`/`Request`, service binding JSRPC machinery. | Dynamic `workerLoader`, immutable version ids, wrapper-generated `env`, gateway routing, request logs, and public/private outbound separation. | An uncaught tenant `fetch()` exception maps to platform `502 runtime_error`; exception detail goes to structured logs/live tail, not the client body. | WDL does not emulate every compatibility-date behavior change; workerd is configured with the platform's enabled flags. | | WebSocket upgrade | Supported | WebSocket API and 101 response handling inside workerd. | Gateway `GatewayWsHolder` Durable Object holds public sockets and forwards to runtime/do-runtime so long-lived 101s do not live on ordinary gateway request IoContexts. | WDL optimizes for preserving long-lived gateway-held sockets, while Cloudflare can rely on its global edge session model. | Gateway rolling still drops physical client sockets; application-level resume is not implemented. | -| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Control validates that `compatibility_date` is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; unsupported flags may only be stored or rejected depending on the path. | +| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Control validates that `compatibility_date` is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, and rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module` before deploy. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags. | | `nodejs_compat` | Partial | workerd-provided compatibility when the runtime service has the flag enabled. | CLI carries compatibility flags into metadata. | WDL exposes the workerd-enabled compatibility surface rather than a separate Node.js runtime. | This is not a full Node.js platform contract beyond workerd's enabled surface. | +| Python Workers modules | Not supported | workerd has an experimental Python Workers path. | Control rejects `py` module manifests with `python_workers_unsupported`; runtime and do-runtime also fail closed for retained metadata containing `py` modules. | WDL keeps tenant bundles JavaScript/WebAssembly/data only and does not permit cold-load-time Pyodide bootstrap. | Python Workers and mixed JS/Python bundles are unsupported. | + +Node.js TLS behavior follows the bundled workerd binary. With workerd 2026-07-01, +workers whose compatibility date is at least 2026-06-16 get +`throw_on_not_implemented_tls_options`: unsupported `node:tls` options such as +`checkServerIdentity` now throw `ERR_OPTION_NOT_IMPLEMENTED` instead of being silently +ignored. This can affect already-deployed versions whose date is in the 2026-06-16 +through 2026-06-24 window when WDL upgrades from the 2026-06-17 workerd pin. Separately, +workerd's `servername` / expected-certificate-hostname behavior changed outside any +compatibility flag, so certificate hostname validation can change for all dates. ## Bindings And Storage @@ -41,14 +51,15 @@ Each row separates four compatibility claims: | KV namespace | Supported | A binding object can be exposed to user code through workerd entrypoint/JSRPC mechanics. | Runtime `KV` facade calls redis-proxy; redis-proxy stores values and metadata in DB 1 hash buckets `kvh:::b:`, with `v:` and `m:` fields, 512-byte key/list-prefix cap, 25 MiB value cap, batch raw-byte budget, TTL/EXAT, and prefix list cursors. | KV storage is Redis-backed and namespace-scoped in a WDL deployment. `cacheTtl` is not a storage freshness contract. | No Cloudflare global edge replication or eventual consistency model. | | R2 bucket | Supported | Fetch/stream primitives in workerd. | Runtime R2 facade signs S3-compatible requests with platform credentials; CLI parses `[[r2_buckets]]`. | Bucket lifecycle and placement are the S3-compatible backend's responsibility. | `preview_bucket_name` and `jurisdiction` are not supported. | | ASSETS | Partial | Worker can receive a platform-provided binding object. | CLI uploads assets to S3-compatible storage and runtime exposes the WDL `env.ASSETS.url(path)` helper for tokenized CDN URLs. | WDL assets are an S3/CDN helper model, not Cloudflare Pages asset hosting. | WDL does not provide a full Cloudflare Pages asset pipeline or a fetch-style assets binding contract. | -| D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. | -| Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. | +| D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | +| Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | | Queues producer/consumer | Partial | Worker queue handler API surface in loaded workers. | Runtime producer facade writes through redis-proxy DB 1; scheduler owns consumer dispatch, retry, DLQ, orphan, delayed queues, and batch splitting for oversized dispatch bodies. | `max_batch_timeout_ms` is configuration metadata, not a true aggregation window; dispatch concurrency is scheduler-owned. | `max_concurrency` is rejected. | | Cron triggers | Supported | `scheduled()` handler surface in workers. | Control stores cron config; scheduler owns indexed discovery, cron-slot buckets, due dispatch, and repairable projections. | Control and scheduler use JS/Rust `croner` engines and keep their behavior documented through tests and module docs. | New scheduler dispatch paths still require their own Redis lease/fence audit. | | Workflows | Partial | User workflow class code runs inside loaded workers when dispatched by runtime. | workflows owns DB 2 instance state, step/event/sleep commits, runtime-observed `step.do` DAG edges including `Promise.all` siblings, generation/run-token fencing, lifecycle APIs, progress callbacks, and scheduler tick. | WDL Workflows V2 has WDL-specific payload semantics and terminal failure rules; permanent `step.do` failure is terminal for the run even if caught, and one step may record at most 1000 dependency edges. | WDL Workflows V2 is not Cloudflare Workflows parity. `script_name`, cross-worker workflows, and Cloudflare's source-AST visualizer are unsupported. | | Service bindings | Supported | workerd service binding JSRPC and fetch dispatch. | WDL resolves target worker metadata, cold-loads immutable versions, propagates request ids where available, and enforces namespace/action ACL through control metadata. | Frozen-version service targets do not evict active siblings by design. | None currently documented. | | Platform bindings | Supported | workerd named entrypoint/JSRPC mechanics. | WDL expands ACL-checked platform bindings from control metadata. | Platform bindings are WDL-specific. | Platform bindings are not a Cloudflare tenant portability feature. | -| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | +| Vars and secrets | Supported | Env values can be materialized into worker `env`. | Control stores worker vars/secrets metadata, encrypts secret values as `WDL-ENC:` envelopes at rest, redis-proxy decrypts them during cold-load, enforces a headroomed workerd 1 MiB `workerLoader` serialized env budget for user vars/secrets plus runtime-injected binding/workflow env values during deploy/secret mutation, accounts for V8 two-byte string storage for non-Latin-1 strings, and immutable versions are promoted when worker secrets change. | Secrets are platform-managed by WDL; the at-rest envelope provider is a deployment concern. | WDL secrets are not Cloudflare account secrets. | +| Worker code size | Supported | Dynamic worker module bodies are accepted by `workerLoader` up to workerd's 64 MiB limit. | Control estimates final WorkerCode, including runtime-injected wrapper/client modules and workflow import rewrites, before allocating a version. | WDL's deploy JSON body limit is lower for ordinary inline deployments. | Large server-side bundle assembly paths must keep this guard. | ## Control Plane And Developer Tooling @@ -70,6 +81,8 @@ docs: | Workers AI, Vectorize, Analytics Engine, Browser Rendering, Hyperdrive, Email Workers | Not supported | No binding facade, control-plane metadata, or backing service exists in WDL. | | R2 multipart upload, customer-provided encryption keys, and Cloudflare-specific checksum behavior | Not supported | The current R2 facade targets S3-compatible object operations needed by WDL workers/assets. Advanced Cloudflare R2 behaviors need explicit design before being documented as compatible. | | Queue `contentType = "v8"` and per-consumer `max_concurrency` | Not supported | Queue messages support the documented `json`, `text`, and `bytes` content types; only `v8` is rejected. Dispatch concurrency remains scheduler-owned, and `max_concurrency` is rejected instead of silently ignored. | +| Upstream experimental compatibility flags | Not supported | Tenant `compatibility_flags` entries whose upstream workerd flag is marked `$experimental` are rejected at deploy and runtime decode. | +| Python Workers | Not supported | WDL rejects Python module manifests instead of letting workerd fail at cold-load. | | Durable Object cross-script bindings and migration rename/delete semantics | Not supported | WDL DO classes are same-worker only. Storage identity, owner routing, and delete cleanup are WDL-managed rather than Cloudflare migration-compatible. | | Cloudflare account API parity | Not supported | WDL exposes its own CLI/control API. Cloudflare API compatibility is not a stated goal. | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index 36e3e28..d0e2a6a 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -24,8 +24,11 @@ |---|---|---|---|---|---| | ES module Workers 和 `fetch()` | Supported | Module evaluation、request dispatch、`Response`/`Request`、service binding JSRPC 机制。 | Dynamic `workerLoader`、immutable version id、wrapper 生成 `env`、gateway routing、request logs、public/private outbound 隔离。 | Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error`;异常细节进入结构化日志/live tail,不进客户端 body。 | WDL 不模拟每个 compatibility date 的历史行为;workerd 按平台启用的 flags 运行。 | | WebSocket upgrade | Supported | workerd 内的 WebSocket API 和 101 response 处理。 | Gateway `GatewayWsHolder` Durable Object 承载公网 socket 并转发到 runtime/do-runtime,避免长生命周期 101 留在普通 gateway request IoContext。 | WDL 优先保留 gateway-held 长连接;Cloudflare 依赖自己的全球 edge session 模型。 | Gateway rolling 仍会断开物理 client socket;应用级 resume 未实现。 | -| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | CLI/control 保存 bundle metadata。Control 会校验 `compatibility_date` 是真实的 `YYYY-MM-DD` 日期,且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期。 | WDL 把 compatibility metadata 当成部署期平台 metadata,不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为;不支持的 flags 会按路径存储或拒绝。 | +| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | CLI/control 保存 bundle metadata。Control 会校验 `compatibility_date` 是真实的 `YYYY-MM-DD` 日期,且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期;上游 `$experimental` enable flags(例如 `experimental` / `unsafe_module`)会在 deploy 前被拒绝。 | WDL 把 compatibility metadata 当成部署期平台 metadata,不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为;tenant worker 不能 opt into 上游 experimental-only flags。 | | `nodejs_compat` | Partial | runtime service 启用 flag 后由 workerd 提供的兼容 surface。 | CLI 把 compatibility flags 带入 metadata。 | WDL 暴露 workerd 已启用的兼容 surface,而不是单独 Node.js runtime。 | 不承诺超出 workerd 已启用 surface 的完整 Node.js 平台能力。 | +| Python Workers modules | Not supported | workerd 有 experimental Python Workers 路径。 | Control 用 `python_workers_unsupported` 拒绝 `py` module manifest;runtime 和 do-runtime 对 retained metadata 中的 `py` module 也 fail closed。 | WDL 保持 tenant bundle 只包含 JavaScript/WebAssembly/data,不允许 cold-load 时触发 Pyodide bootstrap。 | 不支持 Python Workers 和 JS/Python 混合 bundle。 | + +Node.js TLS 行为跟随 bundled workerd binary。升级到 workerd 2026-07-01 后,compatibility date 不早于 2026-06-16 的 worker 会拿到 `throw_on_not_implemented_tls_options`:`node:tls` 中尚未实现的选项(例如 `checkServerIdentity`)会从“静默忽略”变为抛 `ERR_OPTION_NOT_IMPLEMENTED`。从 2026-06-17 workerd pin 升级时,这会影响已经部署且日期在 2026-06-16 到 2026-06-24 之间的版本。另外,workerd 的 `servername` / expected-certificate-hostname 行为变化不受任何 compatibility flag 门控,因此证书 hostname 校验可能对所有日期变化。 ## Bindings 和存储 @@ -34,14 +37,15 @@ | KV namespace | Supported | 通过 workerd entrypoint/JSRPC 机制把 binding object 暴露给用户代码。 | Runtime `KV` facade 调 redis-proxy;redis-proxy 在 DB 1 hash bucket `kvh:::b:` 存 value/metadata,字段为 `v:` 和 `m:`,支持 512-byte key/list-prefix cap、25 MiB value cap、batch raw-byte budget、TTL/EXAT、prefix list cursor。 | KV storage 是 WDL deployment 内的 Redis-backed、namespace-scoped 存储;`cacheTtl` 不是存储新鲜度合同。 | 没有 Cloudflare 全局边缘复制 / eventual consistency 模型。 | | R2 bucket | Supported | workerd 的 fetch/stream primitives。 | Runtime R2 facade 用平台 credentials 签 S3-compatible request;CLI 解析 `[[r2_buckets]]`。 | Bucket lifecycle/placement 由 S3-compatible backend 负责。 | 不支持 `preview_bucket_name` 和 `jurisdiction`。 | | ASSETS | Partial | Worker 可接收平台提供的 binding object。 | CLI 把 assets 上传到 S3-compatible storage;runtime 暴露 WDL 的 `env.ASSETS.url(path)` helper,用于生成 tokenized CDN URL。 | WDL assets 是 S3/CDN helper 模型,不是 Cloudflare Pages asset hosting。 | 不是完整 Cloudflare Pages assets pipeline,也不提供 fetch-style assets binding 合同。 | -| D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing,并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing,不是 Cloudflare global D1;metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。 | -| Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | 只支持同 worker class;不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。 | +| D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing,并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing,不是 Cloudflare global D1;metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | +| Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | 只支持同 worker class;不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | | Queues producer/consumer | Partial | loaded worker 中的 queue handler surface。 | Runtime producer facade 经 redis-proxy 写 DB 1;scheduler 负责 consumer dispatch、retry、DLQ、orphan、delayed queues,并会拆分过大的 dispatch body batch。 | `max_batch_timeout_ms` 是配置 metadata,不是真正 aggregation window;dispatch concurrency 由 scheduler 拥有。 | `max_concurrency` 被拒绝。 | | Cron triggers | Supported | worker 的 `scheduled()` handler surface。 | Control 存 cron config;scheduler 拥有 indexed discovery、cron-slot bucket、due dispatch 和可修复 projection。 | Control 和 scheduler 使用 JS/Rust `croner` 引擎,并通过测试和模块文档约束行为。 | 新增 scheduler dispatch 路径仍必须单独审计 Redis lease/fence。 | | Workflows | Partial | 用户 workflow class code 在 runtime dispatch 时运行在 loaded worker 内。 | workflows 拥有 DB 2 instance state、step/event/sleep commit、runtime-observed `step.do` DAG edges(包括 `Promise.all` sibling)、generation/run-token fence、lifecycle API、progress callback 和 scheduler tick。 | WDL Workflows V2 的 payload 语义和 terminal failure 规则由 WDL 定义;永久失败的 `step.do` 即使被捕获也会让 run terminal failed;单个 step 最多记录 1000 条 dependency edge。 | WDL Workflows V2 不是 Cloudflare Workflows parity;不支持 `script_name`、跨 worker workflow 和 Cloudflare 的源码 AST visualizer。 | | Service bindings | Supported | workerd service binding JSRPC 和 fetch dispatch。 | WDL 解析 target worker metadata,cold-load immutable version,在可用时传播 request id,并通过 control metadata 做 namespace/action ACL。 | Frozen-version service target 按设计不 evict active siblings。 | 当前无已记录缺口。 | | Platform bindings | Supported | workerd named entrypoint/JSRPC 机制。 | WDL 从 control metadata 展开 ACL-checked platform bindings。 | Platform bindings 是 WDL-specific。 | 不是 Cloudflare tenant portability feature。 | -| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密,worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | +| Vars 和 secrets | Supported | Env values 可 materialize 到 worker `env`。 | Control 存 worker vars/secrets metadata,把 secret value 加密成 at-rest `WDL-ENC:` envelope;redis-proxy 在 cold-load 时解密;deploy/secret mutation 过程中按留有 headroom 的 workerd 1 MiB `workerLoader` serialized env budget 校验用户 vars/secrets 加 runtime 注入的 binding/workflow env value,并计入非 Latin-1 字符串的 V8 two-byte storage;worker secret 改变时 promote immutable version。 | Secrets 由 WDL 平台管理;at-rest envelope provider 是 WDL 部署关注点。 | 不是 Cloudflare account secrets。 | +| Worker code size | Supported | Dynamic worker module bodies 可由 `workerLoader` 接受,直到 workerd 的 64 MiB 上限。 | Control 会在分配 version 前估算最终 WorkerCode,包含 runtime 注入的 wrapper/client modules 和 workflow import rewrite。 | WDL 的 deploy JSON body limit 对普通 inline deploy 更低。 | 大型 server-side bundle assembly 路径必须保留这道 guard。 | ## 控制面和开发工具 @@ -62,6 +66,8 @@ | Workers AI、Vectorize、Analytics Engine、Browser Rendering、Hyperdrive、Email Workers | Not supported | WDL 没有对应 binding facade、control-plane metadata 或后端服务。 | | R2 multipart upload、customer-provided encryption keys 和 Cloudflare-specific checksum 行为 | Not supported | 当前 R2 facade 面向 WDL worker/assets 所需的 S3-compatible object 操作。高级 Cloudflare R2 行为需要先设计,才能写成兼容合同。 | | Queue `contentType = "v8"` 和 per-consumer `max_concurrency` | Not supported | Queue message 支持文档化的 `json`、`text` 和 `bytes` content type;只有 `v8` 会被拒绝。Dispatch concurrency 仍由 scheduler 拥有,`max_concurrency` 会被拒绝,而不是静默忽略。 | +| 上游 experimental compatibility flags | Not supported | Tenant `compatibility_flags` 中属于上游 workerd `$experimental` 的 enable flag 会在 deploy 和 runtime decode 阶段被拒绝。 | +| Python Workers | Not supported | WDL 拒绝 Python module manifest,而不是让 workerd 在 cold-load 时失败。 | | Durable Object cross-script binding 和 migration rename/delete 语义 | Not supported | WDL DO class 仅支持 same-worker。Storage identity、owner routing 和 delete cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | | Cloudflare account API parity | Not supported | WDL 暴露自己的 CLI/control API。Cloudflare API 兼容不是目标。 | diff --git a/docs/modules/cli.md b/docs/modules/cli.md index 79f0e2d..551e907 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -148,8 +148,8 @@ Supported config surfaces: | Field | WDL behavior | |---|---| -| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit. | -| `[vars]` | String, number, and boolean values are accepted and stringified into `env`. | +| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit; final WorkerCode, including runtime-injected modules, must fit workerd's 64 MiB `workerLoader` code limit. | +| `[vars]` | String, number, and boolean values are accepted and stringified into `env`; vars, namespace/worker secrets, and runtime-injected binding/workflow env values must fit WDL's headroomed workerd 1 MiB `workerLoader` env budget. | | `[[kv_namespaces]]` | `id` is a platform-local KV namespace id, not a Cloudflare UUID. | | `[[r2_buckets]]` | `binding` plus `bucket_name` become a namespace-scoped virtual R2 bucket under the platform S3 bucket. | | `[assets]` | `directory` contents upload to S3-compatible assets storage and auto-inject `ASSETS`. | diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index 5abf147..8b2bc00 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -91,8 +91,8 @@ WDL 遵循 Wrangler selected-env 继承规则: | 字段 | WDL 行为 | |---|---| -| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`。 | -| `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`。 | +| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`;包含 runtime 注入模块后的最终 WorkerCode 必须落在 workerd 64 MiB `workerLoader` code limit 内。 | +| `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 必须落在 WDL 留有 headroom 的 workerd 1 MiB `workerLoader` env budget 内。 | | `[[kv_namespaces]]` | `id` 是 platform-local KV namespace id,不是 Cloudflare UUID。 | | `[[r2_buckets]]` | `binding` 加 `bucket_name` 映射为平台 S3 bucket 下的 namespace-scoped virtual R2 bucket。 | | `[assets]` | `directory` 内容上传到 S3-compatible assets storage,并 auto-inject `ASSETS`。 | diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index cec8f97..8b1f1b5 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -45,7 +45,7 @@ Worker lifecycle: |---|---|---| | `GET` | `/ns//workers` | Lists workers with namespace-owned state, including deploy-only, active, and secret-only workers. | | `GET` | `/ns//worker//versions` | Lists retained versions and active status. | -| `POST` | `/ns//worker//deploy` | Creates a new immutable version from shorthand code or full module manifest; routes, crons, queue consumers, service refs, platform refs, assets, vars, bindings, and `exports` are version metadata. | +| `POST` | `/ns//worker//deploy` | Creates a new immutable version from shorthand code or full module manifest; routes, crons, queue consumers, service refs, platform refs, assets, vars, bindings, and `exports` are version metadata. Python modules and upstream experimental compatibility flags are rejected before commit. | | `POST` | `/ns//worker//promote` | Promotes `{"version":"vN"}` through the WATCH/MULTI routing path. Host declaration failures are 403; live pattern conflicts are 409; exhausted transaction contention is 503. | | `DELETE` | `/ns//worker//versions/` | Deletes one retained non-active version after active-route, service-ref, lifecycle, and delete-lock blockers pass. Referrer redaction is principal-aware. | | `POST` | `/ns//worker//delete` | Whole-worker delete. `?dry_run=1` returns computed impact and blockers without writing. Redaction matches single-version delete. | @@ -78,7 +78,12 @@ Control lifecycle operations are split so each critical transition has one autho - Deploy parses the supported Wrangler/JSONC shape, validates bindings and routes, allocates the next immutable version through `worker:::next_version`, writes bundle metadata/modules/assets, then enters the same promote path used by - explicit promotion. + explicit promotion. Before allocation, deploy estimates final WorkerCode under + workerd's 64 MiB limit, including runtime-injected wrapper/client modules and workflow + import rewrites, and runs an advisory pass over the candidate metadata and current + secret envelopes. The watched commit path is the authoritative headroomed + `workerLoader` env-budget check after version allocation and metadata materialization, + such as resolved D1 database ids, before writing the version. - Promote is the only active-route flip. It WATCHes the delete lock, bundle metadata, D1 refs, service-binding target refs, queue consumer keys, host declarations, and pattern keys needed for the candidate. The EXEC updates active routes, host reverse indexes, @@ -89,6 +94,13 @@ Control lifecycle operations are split so each critical transition has one autho plaintext size and shape, encrypts it into a `WDL-ENC:` envelope before the Redis mutation/WATCH retry loop, and reuses the same envelope across retries. Runtime therefore sees a new immutable version id instead of mutable in-place secret changes. + Secret DELETE remains a repair surface: when estimating the post-delete env, it skips + corrupt envelopes in the other secret scope instead of letting an unrelated bad + namespace or worker secret block deletion. Secret PUT still fails closed on corrupt + retained envelopes. + Namespace-secret mutations WATCH the retained worker/version metadata they need to + re-estimate before commit; if concurrent metadata changes keep invalidating that view, + control returns `namespace_secret_mutation_contention`. - Version delete and whole-worker delete are fail-closed. They collect blockers from active routes, retained versions, service refs, D1 refs, workflow lifecycle checks, queue/cron projections, and delete locks before committing Redis lifecycle deletion. @@ -250,6 +262,16 @@ Auth-specific contract: - Control 5xx responses use generic/safe messages. Internal exception text, auth Redis diagnostics, backend messages, and provider errors belong in logs unless the endpoint explicitly owns a diagnostic response field. +- Deploy and secret mutations return `worker_code_too_large` when tenant module bodies + plus runtime-injected modules exceed workerd's 64 MiB dynamic code limit, and + `worker_env_too_large` when the estimated `workerLoader` env exceeds WDL's headroomed + 1 MiB budget. + `worker_env_too_large` details include `namespace`, optional `worker`, `env_bytes`, + `max_env_bytes`, `upstream_max_env_bytes`, and `headroom_bytes`. When the blocker is an + already-retained version being re-estimated during a secret mutation, details also + include `source_version` and `estimated_version`, and the message identifies + `/@` so operators can find the retained version to delete + or redeploy. - Control never calls gateway directly. It writes Redis and publishes invalidation messages. - Control encrypts secret PUT values before entering Redis mutation loops. diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index 9d935dd..72e8202 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -34,7 +34,7 @@ Worker lifecycle: |---|---|---| | `GET` | `/ns//workers` | 列出有 namespace-owned state 的 worker,包括 deploy-only、active 和 secret-only worker。 | | `GET` | `/ns//worker//versions` | 列出 retained versions 和 active status。 | -| `POST` | `/ns//worker//deploy` | 从 shorthand code 或完整 module manifest 创建新的 immutable version;routes、crons、queue consumers、service refs、platform refs、assets、vars、bindings 和 `exports` 都是 version metadata。 | +| `POST` | `/ns//worker//deploy` | 从 shorthand code 或完整 module manifest 创建新的 immutable version;routes、crons、queue consumers、service refs、platform refs、assets、vars、bindings 和 `exports` 都是 version metadata。Python modules 和上游 experimental compatibility flags 会在 commit 前被拒绝。 | | `POST` | `/ns//worker//promote` | 通过 WATCH/MULTI routing path promote `{"version":"vN"}`。Host declaration 失败是 403;live pattern conflict 是 409;transaction contention 耗尽是 503。 | | `DELETE` | `/ns//worker//versions/` | 在 active-route、service-ref、lifecycle 和 delete-lock blocker 全部通过后删除一个 retained non-active version。Referrer redaction 按 principal 决定。 | | `POST` | `/ns//worker//delete` | Whole-worker delete。`?dry_run=1` 只返回 computed impact 和 blockers,不写入。Redaction 与 single-version delete 一致。 | @@ -64,9 +64,9 @@ Host、secret、data 和 auth 操作: Control lifecycle 操作会拆开处理,确保每个关键 transition 只有一个权威入口: -- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。 +- Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会按 workerd 64 MiB 上限估算最终 WorkerCode,包含 runtime 注入的 wrapper/client modules 和 workflow import rewrite,并对候选 metadata 和当前 secret envelope 做一次 advisory pass。真正的 Redis WATCH commit 会在分配真实 version 并完成 metadata materialization(例如 D1 database id 解析)之后用 headroomed `workerLoader` env-budget 做权威检查,然后才写入 version。 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 -- Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。 +- Secret update/delete 会修改 secret store;如果 worker 有 active route,则通过 `bumpActiveAndPromote()` 生成新的 active version。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Secret DELETE 仍是 repair surface:估算删除后的 env 时,会跳过另一层 secret scope 里的坏 envelope,避免无关的 namespace 或 worker 坏 secret 阻塞删除;Secret PUT 仍会对 retained corrupt envelope fail closed。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前,会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 - Worker delete lock value 和 `s3cleanup:` task id 必须由服务端生成随机值。`x-request-id` 只用于诊断,客户端或重试可能复用它;不能把它用作 lock token 或 cleanup-task id。 - Auth 不是一层约定俗成的 middleware。`parseControlRoute()` 分配 action,control 把 action 和 namespace 发给 auth,auth 用存储的 token record 对照 `shared/auth-roles.js` 评估权限。Dispatcher 代码不应自己从 URL prefix 推断权限。 @@ -135,6 +135,7 @@ Auth 子合同: - Details 可以增加字段,但不能覆盖 `error`、`message` 或 legacy `reason`。Auth reject reason 是 `error` machine code;日志可以把 `reason` 作为诊断上下文。 - Delegated issue 的 409 reason 有不同 retry 语义:`delegated_issue_busy` 表示 issuer/template lock 清除后可重试;`active_quota_exceeded` 在已有 delegated credential 过期或 revoke 前不应重试;`namespace_collision` 表示 Auth 已耗尽 configured candidate retry budget。 - Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志;除非 endpoint 明确拥有某个 diagnostic response field,否则不进入客户端 body。 +- Deploy 和 secret mutation 在 tenant module bodies 加 runtime 注入模块后超过 workerd 64 MiB dynamic code limit 时返回 `worker_code_too_large`,在估算的 `workerLoader` env 超过 WDL headroomed 1 MiB budget 时返回 `worker_env_too_large`。`worker_env_too_large` details 包含 `namespace`、可选 `worker`、`env_bytes`、`max_env_bytes`、`upstream_max_env_bytes` 和 `headroom_bytes`。如果 blocker 是 secret mutation 重新估算到的既有 retained version,details 还会包含 `source_version` 和 `estimated_version`,message 会标出 `/@`,方便 operator 定位要删除或 redeploy 的 retained version。 - Control 不直接调用 gateway。它写 Redis 并 publish invalidation message。 - Control 在进入 Redis mutation loop 前加密 secret PUT value。加密/provider 失败会返回 control error,不写 plaintext fallback。 - Worker delete 先 commit Redis lifecycle state;异步 S3 cleanup enqueue 是 best-effort,失败时返回 warning。 diff --git a/docs/modules/d1.md b/docs/modules/d1.md index 29dceca..516efec 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -74,6 +74,11 @@ Runtime ownership: - Shared EFS/localDisk is safe only when the owner lease model preserves single-writer access. +workerd 2026-07-01 rejects SQLite object names under the reserved `_cf_` namespace +case-insensitively, including `ALTER TABLE ... RENAME TO _cf_*`. Existing databases +that somehow contain `_CF_*` or other case variants should treat those objects as +reserved upgrade debris rather than application-owned tables. + Key families: | Key | Type | Owner | Authority | Cleanup/delete semantics | diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index fc82b76..4909186 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -46,6 +46,8 @@ Runtime ownership: - Redis owner record 包含 task identity 和 monotonic generation。 - Shared EFS/localDisk 只有在 owner lease 模型保证 single-writer 时才安全。 +workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespace 下的 object name,包括 `ALTER TABLE ... RENAME TO _cf_*`。如果既有 database 中已经有 `_CF_*` 等大小写变体,应把它们视为 reserved upgrade debris,而不是应用拥有的表。 + Key families: | Key | Type | Owner | Authority | Cleanup/delete 语义 | diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 5542006..942ddfb 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -93,6 +93,12 @@ do-runtime and stores one internal job per pending row. Row tokens fence user-dr delete against stale backend delivery; Workflows run tokens fence dispatch retry and completion inside DB 2. +workerd 2026-07-01 rejects SQLite object names under the reserved `_cf_` namespace +case-insensitively. `ctx.storage.deleteAll()` skips those names case-insensitively as +well, so old storage created before the stricter check with variants such as `_CF_*` +does not make cleanup fail. Those legacy reserved-name objects remain inaccessible to +tenant SQL and should be treated as upgrade debris, not application tables. + `getAlarm()` performs alarm-scoped read repair: if SQLite has a pending alarm row but the Workflows DB 2 due index is missing, it idempotently rewrites the backend due index without adding Redis IO to ordinary DO fetches. Active and retained alarms keep their diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 6e2b04f..fec25a9 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -59,6 +59,8 @@ Ownership 按 shard 划分: Alarm state 存在 object SQLite。Workflows 接收 do-runtime 的 set/delete 请求,并为每个 pending row 保存一个 internal job。Row token 用于 fence 用户驱动 delete 和 stale backend delivery;Workflows run token 在 DB 2 内 fence dispatch retry 和 completion。 +workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespace 下的 object name。`ctx.storage.deleteAll()` 也会大小写不敏感地跳过这些名字,因此 0617 以前可能创建出的 `_CF_*` 这类大小写变体不会让 cleanup 失败。这些 legacy reserved-name object 对 tenant SQL 仍不可访问,应视为升级遗留物,而不是应用表。 + `getAlarm()` 会做 alarm-scoped read repair:如果 SQLite 中有 pending alarm row,但 Workflows DB 2 due index 缺失,它会幂等重写 backend due index,而不会给普通 DO fetch 增加 Redis IO。Active/retained alarm 保留调度时的 worker version;旧 version 删除后,只有 `doStorageId` 仍匹配时 alarm dispatch 才 retarget 到当前 active version。逻辑 worker 已消失或指向不同 `doStorageId` 时,alarm 会自清理。 ## Ownership / 并发 / 失败语义 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 2fb2ef0..86484e1 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -84,6 +84,17 @@ pipe, not durable log storage. - Active tail sessions are time-bounded authorization leases and must reconnect through normal auth. `LOG_TAIL_MAX_SESSION_MS` sets the control-side maximum; invalid or empty values fall back to 15 minutes. +- Current stock workerd behavior, tracked upstream as + [#6832](https://github.com/cloudflare/workerd/issues/6832), does not reliably call + async response-body `ReadableStream.cancel()` on client disconnect. WDL treats this + as a permanent compatibility boundary: Control has independent watchdogs, not a + temporary workaround waiting on upstream. The max-session watchdog bounds + reauthorization, and the idle-pull watchdog closes a session when the SSE body has not + been pulled for three keepalive intervals. Active clients naturally pull at the + keepalive cadence because each heartbeat frees queue space; abandoned clients stop + pulling and are cleaned up without waiting for the full session lifetime. A TCP + connection that stays open but whose application stops reading for that window may be + closed and should reconnect. - Tail events racing activation can be dropped. - High QPS or slow SSE readers can miss middle events due to stream caps. diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index eb618bc..1892098 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -64,6 +64,7 @@ Tail streams 使用有界 `MAXLEN ~ 500`,并在写入时刷新 TTL。它们是 - 结构化 stdout 是持久平台日志的事实来源。 - 没有 active tailer 时,runtime 仍输出 stdout,但在本地 active-set miss cache 后跳过 per-event stream append work。 - Active tail session 是有时限的授权租约,必须通过正常 auth reconnect。`LOG_TAIL_MAX_SESSION_MS` 设置 control-side 最大时长;非法值或空值会回退到 15 分钟。 +- 当前 stock workerd 的行为(上游 issue [#6832](https://github.com/cloudflare/workerd/issues/6832) 跟踪)不会可靠地在 client disconnect 时触发 async response-body `ReadableStream.cancel()`。WDL 把它当作永久兼容边界处理:Control 的独立 watchdog 不是等待上游修复的临时 workaround。max-session watchdog 负责 reauthorization 上界,idle-pull watchdog 在 SSE body 连续三个 keepalive 周期没有被 pull 时关闭 session。活跃客户端会因为每次 heartbeat 腾出 queue 空间而自然按 keepalive 粒度继续 pull;遗弃客户端会停止 pull,因此不需要等完整 session lifetime 才清理。TCP 连接还在但应用层长时间不读的客户端可能被关闭,应自行 reconnect。 - 与 activation race 的 tail event 可以丢失。 - 高 QPS 或慢 SSE reader 可能因为 stream cap 丢中间事件。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 0059c69..5b5f834 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -82,7 +82,11 @@ services. Runtime therefore treats bindings as adapters: decrypts `WDL-ENC:` values; tenant-facing `env` shape stays unchanged. Env materialization merges in fixed precedence: bundle vars, then namespace secrets, then worker secrets. A worker-level secret with the same name wins over a namespace-level - secret, which wins over a var. + secret, which wins over a var. Control enforces a headroomed estimate of workerd's + `workerLoader` serialized env budget during deploys and secret mutations. That + estimate includes user vars/secrets plus runtime-injected binding/workflow env values, + including required caller secret copies in platform/service binding props, so an + over-large env fails in the control plane instead of during runtime cold-load. - Stateful bindings such as D1, Durable Objects, and Workflows call dedicated backend services. The hidden backend Fetchers stay in runtime and are removed before tenant code observes `env`. @@ -179,6 +183,9 @@ Data-plane bindings use their own storage: Runtime must treat Redis bundle metadata as control-authored, but still revalidates reserved runtime entrypoint and binding names when materializing older stored metadata. +It also fails closed if older metadata contains Python module entries or upstream +experimental compatibility flags, because those would otherwise reach workerd as +opaque cold-load failures under the current stock binary. ## Ownership / Concurrency / Failure Semantics @@ -230,11 +237,49 @@ when a matching active tail session exists. binding shape changes. - Runtime must roll before scheduler/workflows if they depend on a new `:8088` internal path or dispatch body. -- workerd upgrades can change experimental surfaces; runtime currently enables - `experimental` because `abortIsolate()` is required for isolate eviction. -- `experimental` is broader than `abortIsolate()`. Tenants inherit every experimental - workerd surface enabled by that flag, so workerd upgrades require review of the - exposed surface, not only the loader/abort path. +- Runtime does not enable workerd's broad `experimental` flag for loaded workers. + Historical-version eviction still injects `__WdlAbort__`, but `abortIsolate()` is + available without that flag in the bundled workerd baseline. +- Removing the broad loaded-worker `experimental` flag intentionally removes access to + non-GA experimental-only tenant surfaces, such as irrevocable long-term stub storage. + Do not re-enable it as a compatibility workaround without an explicit feature design. +- Control rejects upstream `$experimental` compatibility enable flags at deploy, and + runtime rejects retained metadata that still contains them. Disable-style flags such + as `no_*` are not part of that mirror unless upstream marks the enable flag itself + experimental. +- Python Workers modules are not supported. Control rejects new `py` module manifests, + and runtime/do-runtime reject retained metadata that contains them instead of letting + workerd fail later with a mixed JS/Python bundle error. +- Before rolling the 2026-07-01 workerd adaptation over an existing Redis DB 0, install + `redis-cli` and run `node scripts/scan-workerd-0701-metadata.mjs` against the control + Redis database to find retained versions that are missing metadata, contain Python + modules or upstream experimental compatibility flags, or exceed the headroomed + `workerLoader` env budget under the 2026-07-01 estimator. The scanner is read-only and + does not decrypt secrets; it uses encrypted envelope length as a conservative + two-byte string upper bound. Treat `worker_env_too_large` findings involving secrets + as rollout blockers to review precisely, for example by redeploying or using the + normal secret mutation API path that runs the exact decrypting estimator. Set + `ASSETS_CDN_BASE` when scanning deployments whose assets CDN base is longer than the + default placeholder. +- The runtime workerd processes still run with process-level `--experimental` because + upstream workerd 2026-07-01 continues to gate `workerLoader` bindings on that switch. + Do not re-add the `experimental` compatibility flag or `allowExperimental` to loaded + WorkerCode unless another upstream API explicitly requires it. +- Upstream workerd 2026-07-01 caps dynamic worker code at 64 MiB and serialized dynamic + env at 1 MiB. Control estimates final WorkerCode before version allocation, including + runtime-injected wrapper/client modules and workflow import rewrites. Vars, + namespace/worker secrets, and runtime-injected binding/workflow env values get + an advisory deploy pass and an authoritative watched-commit check against a headroomed + `workerLoader` env budget because workerd's final enforcement includes the full `env` + estimate after version allocation and metadata materialization. The estimate starts + from JSON bytes and adds V8 two-byte string overhead for non-Latin-1 strings, so mixed + ASCII plus CJK or emoji secrets do not slip past control and fail later at cold-load. +- In current stock workerd, a client disconnect during an async `ReadableStream` + response body may not call the stream source's `cancel()` callback. Tenant streaming + and SSE workers should use their own heartbeat, timeout, or application close path + instead of relying on disconnect-driven `cancel()` as the only resource cleanup hook. +- workerd upgrades can still change default or compatibility-flagged runtime + surfaces; review the exposed surface, not only the loader/abort path. ## Tests That Protect This Module diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 14be694..0098fa6 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -40,7 +40,7 @@ Tenant 可见 binding 包括 KV、R2、D1、Durable Objects、Queues、ASSETS、 workerd 提供 isolate、module evaluation、named entrypoint 和 JSRPC 机制。Cloudflare 生产平台通常在外部服务里实现的 binding 后端,则由 WDL 自己补齐。因此 runtime 把 binding 当作 adapter: - KV、queue producer 这类纯数据 binding 调 colocated redis-proxy sidecar。Loaded worker 看到 Cloudflare-shaped object,但 method call 会先通过 workerd JSRPC 回到 runtime,再经 HTTP 调 redis-proxy。 -- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。 +- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 过程中用留有 headroom 的预算估算完整 workerLoader env,包括用户 vars/secrets、runtime 注入的 binding/workflow env value,以及 platform/service binding props 中复制的 required caller secret,避免超限配置拖到 runtime cold-load 才失败。 - D1、Durable Objects、Workflows 这类 stateful binding 调专门 backend service。Hidden backend Fetcher 留在 runtime 内部,并在 tenant code 观察 `env` 前被删除。 - R2 是 S3-compatible object-storage adapter:runtime 使用平台 credential 签名请求,并发送到配置的 endpoint。 - ASSETS 是 deploy artifact URL helper:control 在 deploy 时把 assets 上传到 S3-compatible storage;runtime 读取 `__meta__.assets` 和 `ASSETS_CDN_BASE`,只暴露 `env.ASSETS.url(path)` 用来生成 tokenized CDN URL。 @@ -74,7 +74,7 @@ Runtime 通过 `redis-proxy` 从 DB 0 读取不可变 bundle 和 metadata。Data - D1 和 DO binding 调用专门 runtime service。 - R2/ASSETS 使用 S3-compatible object storage。 -Runtime 可以把 Redis bundle metadata 视为 control-authored,但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。 +Runtime 可以把 Redis bundle metadata 视为 control-authored,但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含 Python module entry 或上游 experimental compatibility flag,也会 fail closed;否则这些形态会在当前 stock workerd binary 下变成 opaque cold-load failure。 ## Ownership / 并发 / 失败语义 @@ -104,8 +104,15 @@ Runtime 为 loading、binding operation、`redis-proxy` call、workflow replay c - 修改 bundle metadata、wrapper generation 或 binding shape 时,runtime 和 control 应一起滚。 - 如果 scheduler/workflows 依赖新的 `:8088` internal path 或 dispatch body,runtime 必须先滚。 -- workerd 升级可能改变 experimental surface;runtime 当前需要开启 `experimental`,因为 isolate eviction 依赖 `abortIsolate()`。 -- `experimental` 覆盖的范围大于 `abortIsolate()`。Tenant 会继承该 flag 打开的所有 experimental workerd surface,因此 workerd 升级时要审 exposed surface,而不只审 loader/abort path。 +- Runtime 不再为 loaded worker 开启 workerd 的宽泛 `experimental` flag。Historical-version eviction 仍然注入 `__WdlAbort__`,但当前 bundled workerd baseline 中 `abortIsolate()` 已经不需要该 flag。 +- 移除 loaded worker 的宽泛 `experimental` flag 会有意收紧 tenant 对非 GA experimental-only surface 的访问,例如不可撤销的长期 stub storage。不要为了兼容绕过而重新打开它,除非先完成明确的功能设计。 +- Control 会在 deploy 时拒绝上游 `$experimental` compatibility enable flags,runtime 也会拒绝仍带有这些 flags 的 retained metadata。`no_*` 这类 disable-style flag 不属于这个 mirror,除非上游把对应 enable flag 本身标为 experimental。 +- Python Workers modules 不受支持。Control 会拒绝新的 `py` module manifest,runtime/do-runtime 会拒绝 retained metadata 中的 `py` module,而不是让 workerd 之后抛 mixed JS/Python bundle error。 +- 在已有 Redis DB 0 上 rollout 2026-07-01 workerd 适配前,先安装 `redis-cli`,再对 control Redis database 运行 `node scripts/scan-workerd-0701-metadata.mjs`,找出缺少 metadata、仍包含 Python modules 或上游 experimental compatibility flags,或按 2026-07-01 估算器会超过 headroomed `workerLoader` env budget 的 retained versions。这个 scanner 只读且不解密 secret;它用 encrypted envelope length 作为 conservative two-byte string 上界。涉及 secret 的 `worker_env_too_large` finding 应作为 rollout blocker 精确复核,例如 redeploy,或走正常 secret mutation API path 触发会解密的精确估算。扫描 assets CDN base 比默认 placeholder 更长的部署时设置 `ASSETS_CDN_BASE`。 +- Runtime workerd 进程仍需要进程级 `--experimental`,因为上游 workerd 2026-07-01 仍用它 gate `workerLoader` binding。不要重新给 loaded WorkerCode 加 `experimental` compatibility flag 或 `allowExperimental`,除非新的上游 API 明确需要。 +- 上游 workerd 2026-07-01 把 dynamic worker code 限制为 64 MiB、serialized dynamic env 限制为 1 MiB。Control 会在分配 version 前估算最终 WorkerCode,包含 runtime 注入的 wrapper/client modules 和 workflow import rewrite;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 会先做 advisory deploy pass,并在 Redis WATCH commit 内用真实 version 和 materialization 后的 metadata 做权威检查,因为 workerd 的最终检查看的是完整 `env` estimate。估算以 JSON bytes 为基底,并对非 Latin-1 字符串补上 V8 two-byte string overhead,因此 ASCII 混 CJK 或 emoji 的 secret 不会绕过 control 后在 cold-load 才失败。 +- 当前 stock workerd 中,客户端在 async `ReadableStream` response body 中途断开时,不一定调用 stream source 的 `cancel()`。Tenant streaming/SSE worker 应使用自己的 heartbeat、timeout 或应用层 close path,不要把 disconnect-driven `cancel()` 当成唯一资源清理信号。 +- workerd 升级仍可能改变默认或 compatibility-flagged runtime surface;升级时要审 exposed surface,而不只审 loader/abort path。 ## 保护该模块的测试 diff --git a/docs/source-map.md b/docs/source-map.md index 158c005..585386e 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -43,6 +43,8 @@ are outside this map unless they own runtime or deployable service behavior. | `control/topology.js` | Route, pattern, cron, queue consumer, and workflow declaration parsing for deploy metadata. | | `control/routing.js`, `control/routing/route-plan.js` | Promote, secret bump/promote, host reconcile WATCH/MULTI loops, and pure route/pattern planning helpers. | | `control/lifecycle-indexes.js` | Redis mutation helpers for worker lifecycle, cron, queue consumer, and referrer indexes. | +| `control/env-budget.js` | Control-plane estimate of workerd `workerLoader` env size for deploy and secret mutation guards. | +| `control/worker-code-budget.js` | Control-plane final WorkerCode size estimate for deploy guards, sharing runtime wrapper/module injection rules. | | `control/d1-*` | D1 control metadata, store, lifecycle, migration, and d1-runtime client modules. | | `control/r2.js` | Control-plane R2 bucket/object API client for the configured S3-compatible store. | | `control/s3.js` | S3-compatible ASSETS upload helper. | @@ -65,6 +67,7 @@ are outside this map unless they own runtime or deployable service behavior. | `shared/bounded-body.js` | Shared bounded request body byte/text readers; each tier maps limit errors to its own HTTP error contract. | | `shared/ns-pattern.js` | Namespace, worker, binding, queue, KV id, module path, reserved object-key, and reserved namespace grammars. | | `shared/version.js` | Worker version formatting and bundle key helpers. | +| `shared/workerd-compat-flags.js` | Pinned mirror of upstream workerd experimental compatibility enable flags used to reject tenant metadata before cold-load. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | | `shared/d1-*.js`, `shared/sql-splitter.js` | D1 parameter, data-field, transport, timeout, query-wire, and SQL splitting utilities shared by runtime, d1-runtime, control, and tests. | @@ -102,6 +105,8 @@ are outside this map unless they own runtime or deployable service behavior. | `examples/` | Manual demos and reference projects. Tests should not silently depend on them unless the fixture graduates to `test-workers/`. | | `scripts/run-integration-tests.js` | Integration worker-pool runner. | | `scripts/compile-workerd-configs.js` | Compiles workerd Cap'n Proto configs into `dist/workerd-configs/*.bin`. | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | Pin-bump flag extractor. | +| `scripts/scan-workerd-0701-metadata.mjs` | Read-only Redis metadata scanner for rollout checks before the workerd 2026-07-01 adaptation: reports retained versions with missing metadata, experimental flags, Python modules, or oversized estimated env. | ## Infrastructure diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index d8ae6b6..52b7819 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -40,6 +40,8 @@ | `control/topology.js` | Deploy metadata 中 routes、patterns、cron、queue consumer 和 workflow declaration parsing。 | | `control/routing.js`、`control/routing/route-plan.js` | Promote、secret bump/promote、host reconcile WATCH/MULTI loops,以及纯 route/pattern planning helpers。 | | `control/lifecycle-indexes.js` | Worker lifecycle、cron、queue consumer 和 referrer indexes 的 Redis mutation helpers。 | +| `control/env-budget.js` | Deploy 和 secret mutation guard 使用的 workerd `workerLoader` env size 控制面估算。 | +| `control/worker-code-budget.js` | Deploy guard 使用的最终 WorkerCode size 控制面估算,复用 runtime wrapper/module injection 规则。 | | `control/d1-*` | D1 control metadata、store、lifecycle、migration 和 d1-runtime client modules。 | | `control/r2.js` | 面向配置的 S3-compatible store 的 control-plane R2 bucket/object API client。 | | `control/s3.js` | S3-compatible ASSETS upload helper。 | @@ -62,6 +64,7 @@ | `shared/bounded-body.js` | 共享 bounded request body byte/text readers;各 tier 自己把 limit error 映射为对应 HTTP error contract。 | | `shared/ns-pattern.js` | Namespace、worker、binding、queue、KV id、module path、reserved object-key 和 reserved namespace grammars。 | | `shared/version.js` | Worker version formatting 和 bundle key helpers。 | +| `shared/workerd-compat-flags.js` | 上游 workerd experimental compatibility enable flags 的 pinned mirror,用于在 cold-load 前拒绝 tenant metadata。 | | `shared/queue-keys.js` | JavaScript queue key helpers,供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | | `shared/d1-*.js`、`shared/sql-splitter.js` | Runtime、d1-runtime、control 和 tests 共用的 D1 parameter、data-field、transport、timeout、query-wire 和 SQL splitting utilities。 | @@ -99,6 +102,8 @@ | `examples/` | 手工 demo 和 reference projects。测试不应悄悄依赖它们,除非 fixture 明确迁入 `test-workers/`。 | | `scripts/run-integration-tests.js` | Integration worker-pool runner。 | | `scripts/compile-workerd-configs.js` | 把 workerd Cap'n Proto configs 编译成 `dist/workerd-configs/*.bin`。 | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | pin bump flag 提取脚本。 | +| `scripts/scan-workerd-0701-metadata.mjs` | workerd 2026-07-01 适配 rollout 前使用的只读 Redis metadata scanner,报告缺失 metadata、使用 experimental flags、Python modules 或估算 env 超限的 retained versions。 | ## Infrastructure diff --git a/package-lock.json b/package-lock.json index 8c1ee32..cd81236 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,10 +11,10 @@ "control" ], "dependencies": { - "workerd": "1.20260617.1" + "workerd": "1.20260701.1" }, "devDependencies": { - "@cloudflare/workers-types": "4.20260617.1", + "@cloudflare/workers-types": "4.20260701.1", "@eslint/js": "^10.0.1", "@types/node": "^24.13.2", "esbuild": "^0.28.0", @@ -36,9 +36,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260617.1.tgz", - "integrity": "sha512-jWwmgEVVWbsHNrLSNXzwjJaH90VzRxq1cWkQFUidxyeUPnMxemeNE8I9qFAfrpzGgE11e9sKDcE3ettJW08swQ==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260701.1.tgz", + "integrity": "sha512-Zd9Y1bah6DwwBN2RW8vJohffQrIUazb8UXnqSNecOxM+jJLhUuvv5IOG8dbHcV83TyZAubea6gsQXo2yH1lDdw==", "cpu": [ "x64" ], @@ -52,9 +52,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260617.1.tgz", - "integrity": "sha512-LHH7b565g9znfCUOkwbec6FG2rmRbsgCy6aJiU9KN662mNheWl5sw/iKleiFSiljPKQQP3HkjnC/NSkdgi/aSA==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260701.1.tgz", + "integrity": "sha512-yBLsjS1qCWqFyCY37qRUrYfzHHvMGvjh8zRKJ6MvUivYDhkZTzqduppK38FoqYvayLJ5KbcxH7zo5rkxGqbsaA==", "cpu": [ "arm64" ], @@ -68,9 +68,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260617.1.tgz", - "integrity": "sha512-FMnaAKXe4Cfd8TQurCVd9fs2XQVBFRCsP+Id/SRdUv89MlwYu9zXfoyx6BxM+brPTIUK38SHbo8iaxiwzLi9JQ==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260701.1.tgz", + "integrity": "sha512-vMfqSIMfoo4xmZXEuUVqLpSFS921YKjiR9q7kDXPi6Vld1PK74UHg9LZuBavT2KSyemHUCTpj9y/4JSYOEyQbQ==", "cpu": [ "x64" ], @@ -84,9 +84,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260617.1.tgz", - "integrity": "sha512-MRoifFYcqbxxIIQy7PqO5tFY/qPFSnjXzakWl0sO93l+HLyG35jRAgOi6jfqa4kBxc7gKKtH861DcewjxUfkjA==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260701.1.tgz", + "integrity": "sha512-HRfwbKU2pK44V2NhoM0+iH0JJSj7nQ9Wv13ifIiGYCmTtDL8/zKtEhX7kQ3D4Vy/Cpjhttl0FkfqXj1aqLDPPg==", "cpu": [ "arm64" ], @@ -100,9 +100,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260617.1.tgz", - "integrity": "sha512-rgBV9wQrv0OSKgCTTbhFUFY3sLGNANZ88aqaLvtmEn2gmbFVb1J4PDGochVUdB7NSEp4D/ghHva6/8SZmbONpw==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260701.1.tgz", + "integrity": "sha512-ngxCiIN9s/fM2o1IBMD0o1/mcXrv2NJVdyznh51UH8sQuvrTrXvV2nM0Uj/qU2wMwF6prgNBcdcd7AZeZGiBQA==", "cpu": [ "x64" ], @@ -116,9 +116,9 @@ } }, "node_modules/@cloudflare/workers-types": { - "version": "4.20260617.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260617.1.tgz", - "integrity": "sha512-HdbP3CNcdMZBwegitFDjWvzv+6wPkFXvV9gBXMnf6RjV2Cy3W8TJL3IhSEGul0S6F1DHjnucP7lrpIsvkzNEjA==", + "version": "4.20260701.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20260701.1.tgz", + "integrity": "sha512-eGZ+PWPlu/yUxH+BJoplV45oSA709sjYFYr2kjrqSo+601qE15X4tsuZcPz1KE6Fb3ObGA9d9ZXn53n34NtyiA==", "dev": true, "license": "MIT OR Apache-2.0" }, @@ -2595,9 +2595,9 @@ } }, "node_modules/workerd": { - "version": "1.20260617.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260617.1.tgz", - "integrity": "sha512-Re5pl6pdowt3ZmWUzGlOuB7jbRIIPetgKalmo4cYmucQnVhpo7/3e4MfpekbhLi2EhZZz5EY9NWRu8zFzuEZew==", + "version": "1.20260701.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260701.1.tgz", + "integrity": "sha512-uF813NG09JwNRRUfJ0zBomyTslSPM810dMj9LVvkQ7RAkLrQLzAlPU8Xh/3dIqZDo2bfd7tChbf2PtqLRARRJQ==", "hasInstallScript": true, "license": "Apache-2.0", "bin": { @@ -2607,11 +2607,11 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260617.1", - "@cloudflare/workerd-darwin-arm64": "1.20260617.1", - "@cloudflare/workerd-linux-64": "1.20260617.1", - "@cloudflare/workerd-linux-arm64": "1.20260617.1", - "@cloudflare/workerd-windows-64": "1.20260617.1" + "@cloudflare/workerd-darwin-64": "1.20260701.1", + "@cloudflare/workerd-darwin-arm64": "1.20260701.1", + "@cloudflare/workerd-linux-64": "1.20260701.1", + "@cloudflare/workerd-linux-arm64": "1.20260701.1", + "@cloudflare/workerd-windows-64": "1.20260701.1" } }, "node_modules/yaml": { diff --git a/package.json b/package.json index 18318ab..83d47b8 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,7 @@ "build:third-party-licenses": "node scripts/generate-third-party-license-bundles.mjs" }, "devDependencies": { - "@cloudflare/workers-types": "4.20260617.1", + "@cloudflare/workers-types": "4.20260701.1", "@eslint/js": "^10.0.1", "@types/node": "^24.13.2", "esbuild": "^0.28.0", @@ -45,6 +45,6 @@ "typescript": "^6.0.3" }, "dependencies": { - "workerd": "1.20260617.1" + "workerd": "1.20260701.1" } } diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 438f0ec..911e060 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -48,6 +48,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-load", esModule = embed "load.js"), (name = "runtime-load-env-build", esModule = embed "load/env-build.js"), + (name = "runtime-load-code-budget", esModule = embed "load/code-budget.js"), (name = "runtime-load-module-rewrite", esModule = embed "load/module-rewrite.js"), (name = "runtime-load-wrapper-generate", esModule = embed "load/wrapper-generate.js"), (name = "runtime-metrics", esModule = embed "metrics.js"), @@ -92,6 +93,7 @@ const loaderWorker :Workerd.Worker = ( (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), @@ -104,9 +106,7 @@ const loaderWorker :Workerd.Worker = ( (name = "@wdl-dev/aws-sigv4", esModule = embed "../shared/vendor/aws-sigv4.js"), ], compatibilityDate = "2026-04-24", - # `experimental` lets this worker pass `allowExperimental: true` in WorkerCode - # so loaded workers can import `abortIsolate` for historical-version eviction. - compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "experimental"], + compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers"], globalOutbound = "network", bindings = [ (name = "SERVICE_NAME", text = "system-runtime"), @@ -160,6 +160,8 @@ const controlWorker :Workerd.Worker = ( (name = "control-bundle", esModule = embed "../control/bundle.js"), (name = "control-bindings", esModule = embed "../control/bindings.js"), (name = "control-topology", esModule = embed "../control/topology.js"), + (name = "control-env-budget", esModule = embed "../control/env-budget.js"), + (name = "control-worker-code-budget", esModule = embed "../control/worker-code-budget.js"), (name = "control-lifecycle-indexes", esModule = embed "../control/lifecycle-indexes.js"), (name = "control-routing", esModule = embed "../control/routing.js"), (name = "control-routing-route-plan", esModule = embed "../control/routing/route-plan.js"), @@ -200,6 +202,7 @@ const controlWorker :Workerd.Worker = ( (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), (name = "shared-version", esModule = embed "../shared/version.js"), @@ -217,6 +220,22 @@ const controlWorker :Workerd.Worker = ( (name = "shared-env", esModule = embed "../shared/env.js"), (name = "shared-d1-timeout", esModule = embed "../shared/d1-timeout.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), + (name = "runtime-load-code-budget", esModule = embed "load/code-budget.js"), + (name = "runtime-load-module-rewrite", esModule = embed "load/module-rewrite.js"), + (name = "runtime-load-wrapper-generate", esModule = embed "load/wrapper-generate.js"), + (name = "runtime-d1-client-source", text = embed "d1-client.js"), + (name = "runtime-d1-data-field-source", text = embed "../shared/d1-data-field.js"), + (name = "runtime-d1-params-source", text = embed "../shared/d1-params.js"), + (name = "runtime-sql-splitter-source", text = embed "../shared/sql-splitter.js"), + (name = "runtime-d1-transport-source", text = embed "../shared/d1-transport.js"), + (name = "runtime-r2-client-source", text = embed "r2-client.js"), + (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), + (name = "runtime-do-client-source", text = embed "do-client.js"), + (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), + (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), + (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), + (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), (name = "wdl-package-json-source", text = embed "../package.json"), (name = "runtime-r2-utils", esModule = embed "r2-utils.js"), # Pre-bundled via `npm run build:vendor`; workerd embed can't walk node_modules. @@ -244,6 +263,7 @@ const controlWorker :Workerd.Worker = ( (name = "ASSETS_CDN_BASE", fromEnvironment = "ASSETS_CDN_BASE"), (name = "PLATFORM_DOMAIN", fromEnvironment = "PLATFORM_DOMAIN"), (name = "LOG_LEVEL", fromEnvironment = "LOG_LEVEL"), + (name = "LOG_TAIL_MAX_SESSION_MS", fromEnvironment = "LOG_TAIL_MAX_SESSION_MS"), (name = "SECRET_ENVELOPE_LOCAL_KEY_B64", fromEnvironment = "SECRET_ENVELOPE_LOCAL_KEY_B64"), (name = "SECRET_ENVELOPE_KID", fromEnvironment = "SECRET_ENVELOPE_KID"), (name = "TAIL_WORKER", service = "tail-worker"), diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index fa62afa..a34494a 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -47,6 +47,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-load", esModule = embed "load.js"), (name = "runtime-load-env-build", esModule = embed "load/env-build.js"), + (name = "runtime-load-code-budget", esModule = embed "load/code-budget.js"), (name = "runtime-load-module-rewrite", esModule = embed "load/module-rewrite.js"), (name = "runtime-load-wrapper-generate", esModule = embed "load/wrapper-generate.js"), (name = "runtime-metrics", esModule = embed "metrics.js"), @@ -91,6 +92,7 @@ const loaderWorker :Workerd.Worker = ( (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), @@ -105,10 +107,7 @@ const loaderWorker :Workerd.Worker = ( compatibilityDate = "2026-04-24", # service_binding_extra_handlers exposes stub.queue()/scheduled() on # Fetcher stubs returned by workerLoader.get(). Runtime-only flag. - # `experimental` is required so this worker may pass `allowExperimental: true` - # in its WorkerCode — loaded workers need that to import `abortIsolate` from - # cloudflare:workers for the historical-version eviction shim. - compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers", "experimental"], + compatibilityFlags = ["nodejs_compat", "service_binding_extra_handlers"], globalOutbound = "internal-network", bindings = [ (name = "SERVICE_NAME", text = "user-runtime"), diff --git a/runtime/lib.js b/runtime/lib.js index 3ebc0ab..8e3e0f8 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -1,5 +1,6 @@ -// Pure helpers for the runtime worker. No bare-specifier imports so this -// file can be loaded both as a workerd embedded module and by node --test. +// Pure helpers for the runtime worker. + +import { isWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; const BASE64_CHUNK_SIZE = 0x8000; const utf8Encoder = new TextEncoder(); @@ -122,9 +123,6 @@ function deepFreeze(obj) { const DEFAULT_COMPATIBILITY_DATE = "2026-04-24"; const ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE = "2026-04-21"; const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; -// Gate for the __WdlAbort__ shim's abortIsolate import. Paired with -// `allowExperimental: true` on the WorkerCode. -const EXPERIMENTAL_FLAG = "experimental"; const ROUTE_NS_RE = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?|__system__)$/; const RESERVED_TENANT_NS = new Set(["admin"]); const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; @@ -133,10 +131,9 @@ const VERSION_RE = /^v[1-9][0-9]*$/; // Loaded workers may declare an older compatibilityDate than the platform // workers. Keep enhanced error serialization as a floor only before the date // where workerd made it the default; newer dates reject the explicit flag. -// `experimental` is unconditional — it is the gate for our eviction shim. /** @param {string} compatibilityDate */ function platformFloorCompatFlags(compatibilityDate) { - const out = [EXPERIMENTAL_FLAG]; + const out = []; if (compatibilityDate < ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE) { out.push(ENHANCED_ERROR_SERIALIZATION_FLAG); } @@ -164,6 +161,11 @@ function mergeCompatFlags(userFlags, compatibilityDate) { `meta.compatibilityFlags entries must be non-empty strings, got ${JSON.stringify(f)}` ); } + if (isWorkerdExperimentalCompatFlag(f)) { + throw new Error( + `meta.compatibilityFlags contains experimental workerd flag ${JSON.stringify(f)}, which WDL does not support for tenant workers` + ); + } out.push(f); } for (const f of platformFloorCompatFlags(compatibilityDate)) { @@ -187,14 +189,13 @@ function mergeCompatFlags(userFlags, compatibilityDate) { * exports?: { entrypoint?: unknown }[] | null, * [key: string]: unknown, * }} WorkerBundleMeta - * @typedef {string | { cjs: string } | { py: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue */ /** @type {[string, (bytes: Uint8Array) => WorkerModuleValue][]} */ const moduleDecoderEntries = [ ["module", (bytes) => utf8Decoder.decode(bytes)], ["cjs", (bytes) => ({ cjs: utf8Decoder.decode(bytes) })], - ["py", (bytes) => ({ py: utf8Decoder.decode(bytes) })], ["text", (bytes) => ({ text: utf8Decoder.decode(bytes) })], ["json", (bytes) => ({ json: JSON.parse(utf8Decoder.decode(bytes)) })], ["wasm", (bytes) => ({ wasm: bytes })], @@ -227,6 +228,9 @@ export function bundleToWorkerCode(hash) { for (const [path, info] of Object.entries(meta.modules)) { const bytes = hash[path]; if (!bytes) throw new Error(`Bundle missing module "${path}"`); + if (info.type === "py") { + throw new Error(`Module "${path}": Python Workers modules are not supported by WDL`); + } const decoder = typeof info.type === "string" ? moduleDecoders.get(info.type) : undefined; if (!decoder) throw new Error(`Module "${path}": unknown type "${info.type}"`); modules[path] = decoder(bytes); diff --git a/runtime/load.js b/runtime/load.js index 525135a..91fe8d6 100644 --- a/runtime/load.js +++ b/runtime/load.js @@ -5,10 +5,6 @@ import { bundleToWorkerCode } from "runtime-lib"; import { formatError } from "shared-observability"; import { withInternalAuth } from "shared-internal-auth"; import { discardResponseBody } from "shared-respond"; -import { - WDL_RESERVED_ENTRYPOINT_RE, - isValidJsClassDeclarationName, -} from "shared-ns-pattern"; import { parseRuntimeLoadWorkerId } from "shared-worker-id"; import D1_CLIENT_SOURCE from "runtime-d1-client-source"; import D1_DATA_FIELD_SOURCE from "runtime-d1-data-field-source"; @@ -24,16 +20,9 @@ import OWNER_HINT_CACHE_SOURCE from "runtime-owner-hint-cache-source"; import REQUEST_ID_SOURCE from "runtime-request-id-source"; import WORKFLOWS_CLIENT_SOURCE from "runtime-workflows-client-source"; import { - HOST_BINDING_RESERVED_MODULES, - HOST_BINDING_RESERVED_MODULE_NAMES, - WORKFLOWS_MODULE_NAME, - WORKFLOWS_MODULE_SOURCE, - rewriteCloudflareWorkflowsImports, -} from "runtime-load-module-rewrite"; -import { - generateAbortShimWrapperModule, - generateHostBindingWrapperModule, -} from "runtime-load-wrapper-generate"; + analyzeRuntimeMeta, + injectRuntimeModulesForHostBindings, +} from "runtime-load-code-budget"; import { buildWorkerEnv } from "runtime-load-env-build"; export { buildWorkerEnv } from "runtime-load-env-build"; @@ -46,7 +35,7 @@ const utf8Decoder = new TextDecoder(); /** * @typedef {{ bundle: Record, ns_secrets: Record, worker_secrets: Record }} RuntimeLoadPayload - * @typedef {string | { cjs: string } | { py: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue * @typedef {{ modules: Record, mainModule: string, [key: string]: unknown }} WorkerCodeShape * @typedef {Record & { type?: string, className?: unknown }} RuntimeBindingSpec * @typedef {{ binding?: unknown, className?: unknown }} RuntimeWorkflowSpec @@ -81,95 +70,7 @@ const utf8Decoder = new TextDecoder(); * }} RuntimeLoaderMetrics * @typedef {(options: { props: Record }) => unknown} RuntimeEntrypointFactory * @typedef {{ exports: Record & { KV: RuntimeEntrypointFactory, Assets: RuntimeEntrypointFactory, QueueProducer: RuntimeEntrypointFactory, D1Database: RuntimeEntrypointFactory, R2Bucket: RuntimeEntrypointFactory, ServiceBinding: RuntimeEntrypointFactory, DurableObjectNamespace: RuntimeEntrypointFactory, InternalAuthBackend: RuntimeEntrypointFactory } }} RuntimeContext - * @typedef {[name: string, source: string]} RuntimeModuleInjection - */ - -/** @type {RuntimeModuleInjection} */ -const REQUEST_ID_MODULE_INJECTION = ["_wdl-request-id.js", REQUEST_ID_SOURCE]; -const D1_DATA_FIELD_MODULE_NAME = "_wdl-d1-data-field.js"; -const D1_TRANSPORT_INJECTED_SOURCE = D1_TRANSPORT_SOURCE.replace( - /from "shared-d1-data-field";/, - `from "./${D1_DATA_FIELD_MODULE_NAME}";` -); -/** @type {RuntimeModuleInjection[]} */ -const D1_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - [D1_DATA_FIELD_MODULE_NAME, D1_DATA_FIELD_SOURCE], - ["_wdl-d1-params.js", D1_PARAMS_SOURCE], - ["_wdl-sql-splitter.js", SQL_SPLITTER_SOURCE], - ["_wdl-d1-transport.js", D1_TRANSPORT_INJECTED_SOURCE], - ["_wdl-d1-client.js", D1_CLIENT_SOURCE], -]; -/** @type {RuntimeModuleInjection[]} */ -const R2_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - ["_wdl-r2-utils.js", R2_UTILS_SOURCE], - ["_wdl-r2-client.js", R2_CLIENT_SOURCE], -]; -/** @type {RuntimeModuleInjection[]} */ -const DO_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - ["_wdl-do-transport.js", DO_TRANSPORT_SOURCE], - ["_wdl-owner-endpoint.js", OWNER_ENDPOINT_SOURCE], - ["_wdl-owner-hint-cache.js", OWNER_HINT_CACHE_SOURCE], - ["_wdl-do-client.js", DO_CLIENT_SOURCE], -]; -/** @type {RuntimeModuleInjection[]} */ -const WORKFLOWS_MODULE_INJECTIONS = [ - REQUEST_ID_MODULE_INJECTION, - ["_wdl-workflows-client.js", WORKFLOWS_CLIENT_SOURCE], -]; - -/** - * @typedef {{ - * type: string, - * modules: RuntimeModuleInjection[], - * addBinding(plan: Pick, name: string): void, - * bindingNames(plan: Pick): string[], - * }} HostFacadeBindingDefinition - */ - -/** @type {HostFacadeBindingDefinition[]} */ -const HOST_FACADE_BINDING_DEFINITIONS = [ - { - type: "d1", - modules: D1_MODULE_INJECTIONS, - addBinding(plan, name) { plan.d1Bindings.push(name); }, - bindingNames(plan) { return plan.d1Bindings; }, - }, - { - type: "r2", - modules: R2_MODULE_INJECTIONS, - addBinding(plan, name) { plan.r2Bindings.push(name); }, - bindingNames(plan) { return plan.r2Bindings; }, - }, - { - type: "do", - modules: DO_MODULE_INJECTIONS, - addBinding(plan, name) { plan.doBindings.push(name); }, - bindingNames(plan) { return plan.doBindings; }, - }, -]; - -/** @param {WorkerCodeShape} workerCode @param {RuntimeModuleInjection[]} modules */ -function injectRuntimeModules(workerCode, modules) { - for (const [name, source] of modules) workerCode.modules[name] = source; -} - -/** - * @param {RuntimeMetaPlan} plan - * @param {RuntimeBindingSpec} spec - * @param {string} name */ -function addHostFacadeBinding(plan, spec, name) { - const definition = HOST_FACADE_BINDING_DEFINITIONS.find((entry) => entry.type === spec?.type); - if (definition) definition.addBinding(plan, name); -} - -/** @param {RuntimeMetaPlan} plan */ -function hasHostFacadeBindings(plan) { - return HOST_FACADE_BINDING_DEFINITIONS.some((entry) => entry.bindingNames(plan).length > 0); -} /** * Keep the internal auth token in the host loader realm. Generated tenant @@ -280,128 +181,25 @@ function redisProxyUrl(env) { return String(env.REDIS_PROXY_URL).replace(/\/+$/, ""); } -/** @param {RuntimeBundleMeta} meta */ -function d1ExportedEntrypointNames(meta) { - /** @type {string[]} */ - const out = []; - for (const entry of meta.exports || []) { - const name = entry?.entrypoint; - if (!name || name === "default") continue; - if (typeof name !== "string") { - throw new Error(`Host binding wrapper requires exported entrypoint names to be strings, got ${JSON.stringify(name)}`); - } - if (!isValidJsClassDeclarationName(name)) { - throw new Error(`Host binding wrapper requires exported entrypoint names to be valid JS class declaration names, got ${JSON.stringify(name)}`); - } - if (WDL_RESERVED_ENTRYPOINT_RE.test(name)) { - throw new Error(`Exported entrypoint targets reserved runtime entrypoint "${name}" (redeploy worker)`); - } - out.push(name); - } - return out; -} - -/** @param {RuntimeBundleMeta} meta @param {Array<[string, RuntimeBindingSpec]>} bindingEntries @param {RuntimeWorkflowSpec[]} workflows */ -function hostWrappedClassNames(meta, bindingEntries, workflows) { - const out = new Set(d1ExportedEntrypointNames(meta)); - for (const [, spec] of bindingEntries) { - if (spec?.type === "do" && typeof spec.className === "string" && spec.className) { - if (!isValidJsClassDeclarationName(spec.className)) { - throw new Error(`Host binding wrapper requires Durable Object class names to be valid JS class declaration names, got ${JSON.stringify(spec.className)}`); - } - if (WDL_RESERVED_ENTRYPOINT_RE.test(spec.className)) { - throw new Error(`Durable Object binding targets reserved runtime entrypoint "${spec.className}" (redeploy worker)`); - } - out.add(spec.className); - } - } - for (const workflow of workflows) { - const className = workflow?.className; - if (typeof className === "string" && className) { - if (!isValidJsClassDeclarationName(className)) { - throw new Error(`Host binding wrapper requires Workflow class names to be valid JS class declaration names, got ${JSON.stringify(className)}`); - } - if (WDL_RESERVED_ENTRYPOINT_RE.test(className)) { - throw new Error(`Workflow binding targets reserved runtime entrypoint "${className}" (redeploy worker)`); - } - out.add(className); - } - } - return [...out]; -} - -/** @param {RuntimeBundleMeta} meta @returns {RuntimeMetaPlan} */ -export function analyzeRuntimeMeta(meta) { - const bindingEntries = Object.entries(meta.bindings || {}); - const workflows = Array.isArray(meta.workflows) ? meta.workflows : []; - /** @type {RuntimeMetaPlan} */ - const plan = { - bindingEntries, - workflows, - d1Bindings: [], - r2Bindings: [], - doBindings: [], - workflowBindings: Object.create(null), - hostWrappedClassNames: [], - needsDoBackend: false, - needsWorkflowsBackend: false, - needsHostBindingWrapper: false, - }; - for (const [name, spec] of bindingEntries) { - addHostFacadeBinding(plan, spec, name); - } - for (const workflow of workflows) { - if (typeof workflow?.binding === "string" && workflow.binding) plan.workflowBindings[workflow.binding] = workflow; - } - plan.needsDoBackend = plan.doBindings.length > 0; - plan.needsWorkflowsBackend = Object.keys(plan.workflowBindings).length > 0; - plan.needsHostBindingWrapper = hasHostFacadeBindings(plan) || plan.needsWorkflowsBackend; - if (plan.needsHostBindingWrapper) { - plan.hostWrappedClassNames = hostWrappedClassNames(meta, bindingEntries, workflows); - } - return plan; -} +const RUNTIME_INJECTION_SOURCES = Object.freeze({ + d1ClientSource: D1_CLIENT_SOURCE, + d1DataFieldSource: D1_DATA_FIELD_SOURCE, + d1ParamsSource: D1_PARAMS_SOURCE, + sqlSplitterSource: SQL_SPLITTER_SOURCE, + d1TransportSource: D1_TRANSPORT_SOURCE, + r2ClientSource: R2_CLIENT_SOURCE, + r2UtilsSource: R2_UTILS_SOURCE, + doClientSource: DO_CLIENT_SOURCE, + doTransportSource: DO_TRANSPORT_SOURCE, + ownerEndpointSource: OWNER_ENDPOINT_SOURCE, + ownerHintCacheSource: OWNER_HINT_CACHE_SOURCE, + requestIdSource: REQUEST_ID_SOURCE, + workflowsClientSource: WORKFLOWS_CLIENT_SOURCE, +}); /** @param {WorkerCodeShape} workerCode @param {RuntimeBundleMeta} meta @param {RuntimeMetaPlan} [plan] */ export function wrapWorkerCodeForHostBindings(workerCode, meta, plan = analyzeRuntimeMeta(meta)) { - const { d1Bindings, r2Bindings, doBindings, workflowBindings } = plan; - const originalMain = workerCode.mainModule; - if (typeof originalMain !== "string" || !originalMain) { - throw new Error("Host binding wrapper requires a string mainModule"); - } - rewriteCloudflareWorkflowsImports(workerCode); - if ( - HOST_BINDING_RESERVED_MODULES.has(originalMain) || - [...HOST_BINDING_RESERVED_MODULES].some((name) => workerCode.modules[name]) - ) { - throw new Error( - `Host binding wrapper requires reserved module names ${HOST_BINDING_RESERVED_MODULE_NAMES.join(", ")}` - ); - } - // Every loaded worker needs the __WdlAbort__ entrypoint for historical - // version eviction. Host facade bindings need the heavier request-context - // wrapper that swaps loaded-isolate facades into env. - for (const definition of HOST_FACADE_BINDING_DEFINITIONS) { - if (definition.bindingNames(plan).length > 0) { - injectRuntimeModules(workerCode, definition.modules); - } - } - if (plan.needsWorkflowsBackend) { - injectRuntimeModules(workerCode, WORKFLOWS_MODULE_INJECTIONS); - } - workerCode.modules[WORKFLOWS_MODULE_NAME] = WORKFLOWS_MODULE_SOURCE; - workerCode.modules["_wdl-wrapper.js"] = plan.needsHostBindingWrapper - ? generateHostBindingWrapperModule( - originalMain, - d1Bindings, - r2Bindings, - doBindings, - workflowBindings, - plan.hostWrappedClassNames - ) - : generateAbortShimWrapperModule(originalMain); - workerCode.mainModule = "_wdl-wrapper.js"; - return workerCode; + return injectRuntimeModulesForHostBindings(workerCode, meta, RUNTIME_INJECTION_SOURCES, plan); } /** @param {Record} env @param {string} ns @param {string} worker @param {string} version */ @@ -565,12 +363,9 @@ export function createLoaderCallback({ requestId, env, ctx, ns, worker, version, }, Date.now() - envStartedAt)); // Loaded workers must not inherit runtime's private-reaching - // outbound (pinned to PUBLIC_NETWORK). `allowExperimental: true` is - // gated by the host carrying `experimental` itself; the loaded - // worker needs it for the __WdlAbort__ shim's abortIsolate import. + // outbound; pin them to PUBLIC_NETWORK. const workerCode = { ...codeBase, - allowExperimental: true, env: workerEnv, globalOutbound: env.PUBLIC_NETWORK, ...(env.TAIL_WORKER ? { tails: [env.TAIL_WORKER] } : {}), diff --git a/runtime/load/code-budget.js b/runtime/load/code-budget.js new file mode 100644 index 0000000..39361bf --- /dev/null +++ b/runtime/load/code-budget.js @@ -0,0 +1,370 @@ +import { + WDL_RESERVED_ENTRYPOINT_RE, + isValidJsClassDeclarationName, +} from "shared-ns-pattern"; +import { + HOST_BINDING_RESERVED_MODULES, + HOST_BINDING_RESERVED_MODULE_NAMES, + WORKFLOWS_MODULE_NAME, + WORKFLOWS_MODULE_SOURCE, + rewriteCloudflareWorkflowsImports, +} from "runtime-load-module-rewrite"; +import { + generateAbortShimWrapperModule, + generateHostBindingWrapperModule, +} from "runtime-load-wrapper-generate"; + +export const WORKER_LOADER_CODE_MAX_BYTES = 64 * 1024 * 1024; + +const D1_DATA_FIELD_MODULE_NAME = "_wdl-d1-data-field.js"; + +/** + * @typedef {string | Uint8Array} NormalizedModuleBody + * @typedef {[name: string, body: NormalizedModuleBody]} NormalizedModule + * @typedef {string | { cjs: string } | { text: string } | { json: unknown } | { wasm: Uint8Array } | { data: Uint8Array }} WorkerModuleValue + * @typedef {{ modules: Record, mainModule: string, [key: string]: unknown }} WorkerCodeShape + * @typedef {Record & { type?: string, className?: unknown }} RuntimeBindingSpec + * @typedef {{ binding?: unknown, className?: unknown }} RuntimeWorkflowSpec + * @typedef {{ entrypoint?: unknown }} RuntimeExportSpec + * @typedef {{ bindings?: Record | null, workflows?: RuntimeWorkflowSpec[] | null, exports?: RuntimeExportSpec[] | null, modules?: Record | null }} RuntimeBundleMeta + * @typedef {{ + * bindingEntries: Array<[string, RuntimeBindingSpec]>, + * workflows: RuntimeWorkflowSpec[], + * d1Bindings: string[], + * r2Bindings: string[], + * doBindings: string[], + * workflowBindings: Record, + * hostWrappedClassNames: string[], + * needsDoBackend: boolean, + * needsWorkflowsBackend: boolean, + * needsHostBindingWrapper: boolean, + * }} RuntimeMetaPlan + * @typedef {{ + * d1ClientSource: string, + * d1DataFieldSource: string, + * d1ParamsSource: string, + * sqlSplitterSource: string, + * d1TransportSource: string, + * r2ClientSource: string, + * r2UtilsSource: string, + * doClientSource: string, + * doTransportSource: string, + * ownerEndpointSource: string, + * ownerHintCacheSource: string, + * requestIdSource: string, + * workflowsClientSource: string, + * }} RuntimeInjectionSources + * @typedef {[name: string, source: string]} RuntimeModuleInjection + */ + +/** @param {string | Uint8Array} body */ +export function moduleBodyByteLength(body) { + return typeof body === "string" ? Buffer.byteLength(body, "utf8") : body.byteLength; +} + +/** @param {RuntimeInjectionSources} sources */ +function runtimeModuleInjections(sources) { + /** @type {RuntimeModuleInjection} */ + const requestIdModuleInjection = ["_wdl-request-id.js", sources.requestIdSource]; + const d1TransportInjectedSource = sources.d1TransportSource.replace( + /from "shared-d1-data-field";/, + `from "./${D1_DATA_FIELD_MODULE_NAME}";` + ); + /** @type {RuntimeModuleInjection[]} */ + const d1ModuleInjections = [ + requestIdModuleInjection, + [D1_DATA_FIELD_MODULE_NAME, sources.d1DataFieldSource], + ["_wdl-d1-params.js", sources.d1ParamsSource], + ["_wdl-sql-splitter.js", sources.sqlSplitterSource], + ["_wdl-d1-transport.js", d1TransportInjectedSource], + ["_wdl-d1-client.js", sources.d1ClientSource], + ]; + /** @type {RuntimeModuleInjection[]} */ + const r2ModuleInjections = [ + requestIdModuleInjection, + ["_wdl-r2-utils.js", sources.r2UtilsSource], + ["_wdl-r2-client.js", sources.r2ClientSource], + ]; + /** @type {RuntimeModuleInjection[]} */ + const doModuleInjections = [ + requestIdModuleInjection, + ["_wdl-do-transport.js", sources.doTransportSource], + ["_wdl-owner-endpoint.js", sources.ownerEndpointSource], + ["_wdl-owner-hint-cache.js", sources.ownerHintCacheSource], + ["_wdl-do-client.js", sources.doClientSource], + ]; + /** @type {RuntimeModuleInjection[]} */ + const workflowsModuleInjections = [ + requestIdModuleInjection, + ["_wdl-workflows-client.js", sources.workflowsClientSource], + ]; + return { + d1ModuleInjections, + r2ModuleInjections, + doModuleInjections, + workflowsModuleInjections, + }; +} + +/** + * @typedef {{ + * type: string, + * modules: RuntimeModuleInjection[], + * addBinding(plan: Pick, name: string): void, + * bindingNames(plan: Pick): string[], + * }} HostFacadeBindingDefinition + */ + +/** @param {RuntimeInjectionSources} sources @returns {HostFacadeBindingDefinition[]} */ +function hostFacadeBindingDefinitions(sources) { + const { + d1ModuleInjections, + r2ModuleInjections, + doModuleInjections, + } = runtimeModuleInjections(sources); + return [ + { + type: "d1", + modules: d1ModuleInjections, + addBinding(plan, name) { plan.d1Bindings.push(name); }, + bindingNames(plan) { return plan.d1Bindings; }, + }, + { + type: "r2", + modules: r2ModuleInjections, + addBinding(plan, name) { plan.r2Bindings.push(name); }, + bindingNames(plan) { return plan.r2Bindings; }, + }, + { + type: "do", + modules: doModuleInjections, + addBinding(plan, name) { plan.doBindings.push(name); }, + bindingNames(plan) { return plan.doBindings; }, + }, + ]; +} + +/** + * @param {RuntimeMetaPlan} plan + * @param {RuntimeBindingSpec} spec + * @param {string} name + */ +function addHostFacadeBinding(plan, spec, name) { + switch (spec?.type) { + case "d1": + plan.d1Bindings.push(name); + return; + case "r2": + plan.r2Bindings.push(name); + return; + case "do": + plan.doBindings.push(name); + return; + } +} + +/** @param {RuntimeMetaPlan} plan */ +function hasHostFacadeBindings(plan) { + return plan.d1Bindings.length > 0 || plan.r2Bindings.length > 0 || plan.doBindings.length > 0; +} + +/** @param {RuntimeBundleMeta} meta */ +function d1ExportedEntrypointNames(meta) { + /** @type {string[]} */ + const out = []; + for (const entry of meta.exports || []) { + const name = entry?.entrypoint; + if (!name || name === "default") continue; + if (typeof name !== "string") { + throw new Error( + `Host binding wrapper requires exported entrypoint names to be strings, got ${JSON.stringify(name)}` + ); + } + if (!isValidJsClassDeclarationName(name)) { + throw new Error( + `Host binding wrapper requires exported entrypoint names to be valid JS class declaration names, got ${JSON.stringify(name)}` + ); + } + if (WDL_RESERVED_ENTRYPOINT_RE.test(name)) { + throw new Error(`Exported entrypoint targets reserved runtime entrypoint "${name}" (redeploy worker)`); + } + out.push(name); + } + return out; +} + +/** + * @param {RuntimeBundleMeta} meta + * @param {Array<[string, RuntimeBindingSpec]>} bindingEntries + * @param {RuntimeWorkflowSpec[]} workflows + */ +function hostWrappedClassNames(meta, bindingEntries, workflows) { + const out = new Set(d1ExportedEntrypointNames(meta)); + for (const [, spec] of bindingEntries) { + if (spec?.type === "do" && typeof spec.className === "string" && spec.className) { + if (!isValidJsClassDeclarationName(spec.className)) { + throw new Error( + `Host binding wrapper requires Durable Object class names to be valid JS class declaration names, got ${JSON.stringify(spec.className)}` + ); + } + if (WDL_RESERVED_ENTRYPOINT_RE.test(spec.className)) { + throw new Error(`Durable Object binding targets reserved runtime entrypoint "${spec.className}" (redeploy worker)`); + } + out.add(spec.className); + } + } + for (const workflow of workflows) { + const className = workflow?.className; + if (typeof className === "string" && className) { + if (!isValidJsClassDeclarationName(className)) { + throw new Error( + `Host binding wrapper requires Workflow class names to be valid JS class declaration names, got ${JSON.stringify(className)}` + ); + } + if (WDL_RESERVED_ENTRYPOINT_RE.test(className)) { + throw new Error(`Workflow binding targets reserved runtime entrypoint "${className}" (redeploy worker)`); + } + out.add(className); + } + } + return [...out]; +} + +/** @param {RuntimeBundleMeta} meta @returns {RuntimeMetaPlan} */ +export function analyzeRuntimeMeta(meta) { + const bindingEntries = Object.entries(meta.bindings || {}); + const workflows = Array.isArray(meta.workflows) ? meta.workflows : []; + /** @type {RuntimeMetaPlan} */ + const plan = { + bindingEntries, + workflows, + d1Bindings: [], + r2Bindings: [], + doBindings: [], + workflowBindings: Object.create(null), + hostWrappedClassNames: [], + needsDoBackend: false, + needsWorkflowsBackend: false, + needsHostBindingWrapper: false, + }; + for (const [name, spec] of bindingEntries) { + addHostFacadeBinding(plan, spec, name); + } + for (const workflow of workflows) { + if (typeof workflow?.binding === "string" && workflow.binding) plan.workflowBindings[workflow.binding] = workflow; + } + plan.needsDoBackend = plan.doBindings.length > 0; + plan.needsWorkflowsBackend = Object.keys(plan.workflowBindings).length > 0; + plan.needsHostBindingWrapper = hasHostFacadeBindings(plan) || plan.needsWorkflowsBackend; + if (plan.needsHostBindingWrapper) { + plan.hostWrappedClassNames = hostWrappedClassNames(meta, bindingEntries, workflows); + } + return plan; +} + +/** + * @param {string} mainModule + * @param {RuntimeBundleMeta} meta + * @param {RuntimeInjectionSources} runtimeSources + * @param {RuntimeMetaPlan} [plan] + */ +export function runtimeInjectedModuleSources(mainModule, meta, runtimeSources, plan = analyzeRuntimeMeta(meta)) { + /** @type {Map} */ + const out = new Map(); + /** @param {RuntimeModuleInjection[]} modules */ + const addModules = (modules) => { + for (const [name, source] of modules) out.set(name, source); + }; + for (const definition of hostFacadeBindingDefinitions(runtimeSources)) { + if (definition.bindingNames(plan).length > 0) addModules(definition.modules); + } + if (plan.needsWorkflowsBackend) { + addModules(runtimeModuleInjections(runtimeSources).workflowsModuleInjections); + } + out.set(WORKFLOWS_MODULE_NAME, WORKFLOWS_MODULE_SOURCE); + out.set( + "_wdl-wrapper.js", + plan.needsHostBindingWrapper + ? generateHostBindingWrapperModule( + mainModule, + plan.d1Bindings, + plan.r2Bindings, + plan.doBindings, + plan.workflowBindings, + plan.hostWrappedClassNames + ) + : generateAbortShimWrapperModule(mainModule) + ); + return [...out]; +} + +/** + * @param {WorkerCodeShape} workerCode + * @param {RuntimeBundleMeta} meta + * @param {RuntimeInjectionSources} runtimeSources + * @param {RuntimeMetaPlan} [plan] + */ +export function injectRuntimeModulesForHostBindings( + workerCode, + meta, + runtimeSources, + plan = analyzeRuntimeMeta(meta) +) { + const originalMain = workerCode.mainModule; + if (typeof originalMain !== "string" || !originalMain) { + throw new Error("Host binding wrapper requires a string mainModule"); + } + rewriteCloudflareWorkflowsImports(workerCode); + if ( + HOST_BINDING_RESERVED_MODULES.has(originalMain) || + [...HOST_BINDING_RESERVED_MODULES].some((name) => workerCode.modules[name]) + ) { + throw new Error( + `Host binding wrapper requires reserved module names ${HOST_BINDING_RESERVED_MODULE_NAMES.join(", ")}` + ); + } + for (const [name, source] of runtimeInjectedModuleSources(originalMain, meta, runtimeSources, plan)) { + workerCode.modules[name] = source; + } + workerCode.mainModule = "_wdl-wrapper.js"; + return workerCode; +} + +/** @param {Record | null | undefined} modules @param {string} name */ +function moduleType(modules, name) { + const type = modules?.[name]?.type; + return typeof type === "string" ? type : ""; +} + +/** + * @param {{ + * mainModule: string, + * normalized: NormalizedModule[], + * meta: RuntimeBundleMeta, + * runtimeSources: RuntimeInjectionSources, + * }} args + */ +export function estimateFinalWorkerLoaderCodeBytes({ mainModule, normalized, meta, runtimeSources }) { + /** @type {Record} */ + const jsModules = Object.create(null); + /** @type {Map} */ + const userModuleBytes = new Map(); + for (const [name, body] of normalized) { + const type = moduleType(meta.modules, name); + if (type === "module" || type === "cjs") { + jsModules[name] = typeof body === "string" ? body : new TextDecoder().decode(body); + } else { + userModuleBytes.set(name, moduleBodyByteLength(body)); + } + } + rewriteCloudflareWorkflowsImports({ modules: jsModules }); + for (const [name, source] of Object.entries(jsModules)) { + userModuleBytes.set(name, Buffer.byteLength(source, "utf8")); + } + let total = 0; + for (const bytes of userModuleBytes.values()) total += bytes; + for (const [, source] of runtimeInjectedModuleSources(mainModule, meta, runtimeSources)) { + total += Buffer.byteLength(source, "utf8"); + } + return total; +} diff --git a/rust/supervisor/src/config.rs b/rust/supervisor/src/config.rs index 9a17c51..a7e2b8e 100644 --- a/rust/supervisor/src/config.rs +++ b/rust/supervisor/src/config.rs @@ -240,13 +240,12 @@ pub(crate) fn validate_shutdown_timing(config: &SupervisorConfig) { } } -pub(crate) fn workerd_args(compiled_config: &str) -> Vec { - vec![ - "serve".into(), - "-b".into(), - compiled_config.into(), - "--experimental".into(), - ] +pub(crate) fn workerd_args(compiled_config: &str, experimental: bool) -> Vec { + let mut args = vec!["serve".into(), "-b".into(), compiled_config.into()]; + if experimental { + args.push("--experimental".into()); + } + args } pub(crate) fn pick_do_compiled_config() -> &'static str { @@ -343,6 +342,23 @@ mod tests { assert_eq!(signal_exit_code(libc::SIGHUP), 1); } + #[test] + fn workerd_args_adds_experimental_only_when_requested() { + assert_eq!( + workerd_args("/app/dist/workerd-configs/d1-runtime.bin", false), + vec!["serve", "-b", "/app/dist/workerd-configs/d1-runtime.bin"] + ); + assert_eq!( + workerd_args("/app/dist/workerd-configs/do-runtime.bin", true), + vec![ + "serve", + "-b", + "/app/dist/workerd-configs/do-runtime.bin", + "--experimental" + ] + ); + } + #[test] fn evaluate_shutdown_timing_default_no_warnings() { assert_eq!(evaluate_shutdown_timing(10_000, 15_000, 120_000, 0), vec![]); diff --git a/rust/supervisor/src/lib.rs b/rust/supervisor/src/lib.rs index fdb1a98..2051ec3 100644 --- a/rust/supervisor/src/lib.rs +++ b/rust/supervisor/src/lib.rs @@ -8,9 +8,14 @@ pub(crate) use config::*; pub(crate) use wdl_rust_common::text::truncate_chars; pub async fn run_d1() -> ! { - process::run(&D1_CONFIG, WORKERD, workerd_args(D1_COMPILED_CONFIG)).await + process::run(&D1_CONFIG, WORKERD, workerd_args(D1_COMPILED_CONFIG, false)).await } pub async fn run_do() -> ! { - process::run(&DO_CONFIG, WORKERD, workerd_args(pick_do_compiled_config())).await + process::run( + &DO_CONFIG, + WORKERD, + workerd_args(pick_do_compiled_config(), true), + ) + .await } diff --git a/scripts/extract-workerd-experimental-compat-flags.mjs b/scripts/extract-workerd-experimental-compat-flags.mjs new file mode 100644 index 0000000..ae93822 --- /dev/null +++ b/scripts/extract-workerd-experimental-compat-flags.mjs @@ -0,0 +1,50 @@ +#!/usr/bin/env node + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const DEFAULT_COMPATIBILITY_DATE_CAPNP = "src/workerd/io/compatibility-date.capnp"; + +/** @param {string} source */ +export function extractExperimentalCompatFlags(source) { + /** @type {string[]} */ + const blocks = []; + /** @type {string[]} */ + let currentBlock = []; + for (const line of source.split(/\n/)) { + if (/^\s*[A-Za-z][A-Za-z0-9_]*\s+@\d+\s*:\s*Bool/.test(line)) { + if (currentBlock.length) blocks.push(currentBlock.join("\n")); + currentBlock = [line]; + } else if (currentBlock.length) { + currentBlock.push(line); + } + } + if (currentBlock.length) blocks.push(currentBlock.join("\n")); + + const flags = new Set(); + for (const block of blocks) { + if (!block.includes("$experimental")) continue; + for (const match of block.matchAll(/\$compatEnableFlag\("([^"]+)"\)/g)) { + flags.add(match[1]); + } + } + return [...flags].sort(); +} + +/** @param {string[]} [argv] */ +export function runCli(argv = process.argv.slice(2)) { + const input = argv[0] || DEFAULT_COMPATIBILITY_DATE_CAPNP; + const source = readFileSync(input, "utf8"); + for (const flag of extractExperimentalCompatFlags(source)) { + console.log(flag); + } +} + +function isMainModule() { + return process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); +} + +if (isMainModule()) { + runCli(); +} diff --git a/scripts/scan-workerd-0701-metadata.mjs b/scripts/scan-workerd-0701-metadata.mjs new file mode 100644 index 0000000..901a96f --- /dev/null +++ b/scripts/scan-workerd-0701-metadata.mjs @@ -0,0 +1,375 @@ +#!/usr/bin/env node + +import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { firstWorkerdExperimentalCompatFlag } from "../shared/workerd-compat-flags.js"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const REPO_ROOT = path.resolve(__dirname, ".."); +const REDIS_PATTERN = "worker:*:*:v:*"; +const TWO_BYTE_SECRET_CHAR = "\u0100"; + +const { + estimatedWorkerLoaderEnv, + estimatedWorkerLoaderEnvBytes, + WORKER_LOADER_ENV_MAX_BYTES, + UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_HEADROOM_BYTES, +} = await importControlEnvBudget(); + +/** @param {string} relativePath */ +function repoFileUrl(relativePath) { + return pathToFileURL(path.resolve(REPO_ROOT, relativePath)).href; +} + +/** + * @param {string} relativePath + * @param {Array<[RegExp | string, string]>} [replacements] + */ +function repoModuleDataUrl(relativePath, replacements = []) { + let source = readFileSync(path.resolve(REPO_ROOT, relativePath), "utf8"); + for (const [pattern, replacement] of replacements) { + source = source.replace(pattern, replacement); + } + return `data:text/javascript,${encodeURIComponent(source)}`; +} + +async function importControlEnvBudget() { + return await import(repoModuleDataUrl("control/env-budget.js", [ + [/from "shared-secret-envelope";/, `from ${JSON.stringify(repoFileUrl("shared/secret-envelope.js"))};`], + [/from "shared-errors";/, `from ${JSON.stringify(repoFileUrl("shared/errors.js"))};`], + [/from "shared-version";/, `from ${JSON.stringify(repoFileUrl("shared/version.js"))};`], + ])); +} + +export function redisUrlFromEnv(env = process.env) { + const raw = env.REDIS_URL || env.REDIS_ADDR || "redis://127.0.0.1:6379/0"; + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(raw)) return raw; + return `redis://${raw}`; +} + +/** + * @param {string[]} args + * @param {{ redisUrl?: string, spawn?: typeof spawnSync }} [options] + */ +export function redisCli(args, { redisUrl = redisUrlFromEnv(), spawn = spawnSync } = {}) { + const result = spawn("redis-cli", ["-u", redisUrl, "--raw", ...args], { + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + if (result.error) { + if (/** @type {NodeJS.ErrnoException} */ (result.error).code === "ENOENT") { + throw new Error( + "redis-cli not found. Install redis-cli/redis-tools before running " + + "scripts/scan-workerd-0701-metadata.mjs; local compose users can inspect " + + "Redis with `docker compose exec -T redis redis-cli`." + ); + } + throw result.error; + } + if (result.status !== 0) { + throw new Error(result.stderr.trim() || `redis-cli exited with status ${result.status}`); + } + return result.stdout; +} + +/** @param {string} raw */ +export function parseRedisHash(raw) { + const lines = raw.split("\n"); + if (lines.at(-1) === "") lines.pop(); + if (lines.length % 2 !== 0) { + throw new Error("redis-cli HGETALL returned an odd number of raw lines"); + } + /** @type {Record} */ + const out = Object.create(null); + for (let i = 0; i < lines.length; i += 2) { + out[lines[i]] = lines[i + 1]; + } + return out; +} + +/** @param {string} key */ +export function parseBundleKey(key) { + const match = /^worker:([^:]+):([^:]+):v:([1-9][0-9]*)$/.exec(key); + if (!match) return { namespace: null, worker: null, version: null }; + return { namespace: match[1], worker: match[2], version: `v${match[3]}` }; +} + +/** @param {unknown} meta */ +export function pythonModules(meta) { + if (!meta || typeof meta !== "object" || Array.isArray(meta)) return []; + const modules = /** @type {{ modules?: unknown }} */ (meta).modules; + if (!modules || typeof modules !== "object" || Array.isArray(modules)) return []; + return Object.entries(/** @type {Record} */ (modules)) + .filter(([, value]) => + value && + typeof value === "object" && + !Array.isArray(value) && + /** @type {{ type?: unknown }} */ (value).type === "py" + ) + .map(([name]) => name); +} + +/** @param {unknown} meta */ +export function experimentalFlag(meta) { + if (!meta || typeof meta !== "object" || Array.isArray(meta)) return null; + return firstWorkerdExperimentalCompatFlag(/** @type {{ compatibilityFlags?: unknown }} */ (meta).compatibilityFlags); +} + +/** + * @param {string} key + * @param {string} rawMeta + */ +function parseMetadataRecord(key, rawMeta) { + const identity = parseBundleKey(key); + if (!rawMeta) { + return { + identity, + meta: null, + findings: [{ kind: "missing_meta", key, ...identity }], + }; + } + + /** @type {unknown} */ + let meta; + try { + meta = JSON.parse(rawMeta); + } catch (err) { + return { + identity, + meta: null, + findings: [{ + kind: "corrupt_meta", + key, + ...identity, + error: err instanceof Error ? err.message : String(err), + }], + }; + } + if (!meta || typeof meta !== "object" || Array.isArray(meta)) { + return { + identity, + meta: null, + findings: [{ kind: "corrupt_meta", key, ...identity, error: "__meta__ must be a JSON object" }], + }; + } + return { + identity, + meta: /** @type {Record} */ (meta), + findings: [], + }; +} + +/** @param {Record} encrypted */ +export function secretUpperBoundStrings(encrypted) { + /** @type {Record} */ + const out = Object.create(null); + for (const [key, value] of Object.entries(encrypted || {})) { + if (typeof value !== "string") continue; + out[key] = TWO_BYTE_SECRET_CHAR.repeat(Math.min(value.length, UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES)); + } + return out; +} + +/** + * @param {{ + * identity: { namespace: string | null, worker: string | null, version: string | null }, + * meta: Record, + * nsSecretsEncrypted?: Record, + * workerSecretsEncrypted?: Record, + * assetsCdnBase?: string | null, + * }} args + */ +export function estimateBundleEnvBytes({ + identity, + meta, + nsSecretsEncrypted = {}, + workerSecretsEncrypted = {}, + assetsCdnBase = null, +}) { + if (!identity.namespace || !identity.worker || !identity.version) return null; + const env = estimatedWorkerLoaderEnv({ + ns: identity.namespace, + worker: identity.worker, + version: identity.version, + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) + ? /** @type {Record} */ (meta.vars) + : null, + nsSecrets: secretUpperBoundStrings(nsSecretsEncrypted), + workerSecrets: secretUpperBoundStrings(workerSecretsEncrypted), + meta, + assetsCdnBase, + }); + return estimatedWorkerLoaderEnvBytes(env); +} + +/** + * @param {{ + * key: string, + * rawMeta: string, + * nsSecretsEncrypted?: Record, + * workerSecretsEncrypted?: Record, + * assetsCdnBase?: string | null, + * }} args + */ +export function findingsForBundleMetadata({ + key, + rawMeta, + nsSecretsEncrypted = {}, + workerSecretsEncrypted = {}, + assetsCdnBase = null, +}) { + const { identity, meta, findings: parseFindings } = parseMetadataRecord(key, rawMeta); + if (!meta) return parseFindings; + return findingsForParsedBundleMetadata({ + key, + identity, + meta, + nsSecretsEncrypted, + workerSecretsEncrypted, + assetsCdnBase, + }); +} + +/** + * @param {{ + * key: string, + * identity: { namespace: string | null, worker: string | null, version: string | null }, + * meta: Record, + * nsSecretsEncrypted?: Record, + * workerSecretsEncrypted?: Record, + * assetsCdnBase?: string | null, + * }} args + */ +function findingsForParsedBundleMetadata({ + key, + identity, + meta, + nsSecretsEncrypted = {}, + workerSecretsEncrypted = {}, + assetsCdnBase = null, +}) { + /** @type {Array>} */ + const findings = []; + const flag = experimentalFlag(meta); + if (flag) { + findings.push({ + kind: "experimental_compat_flag", + key, + ...identity, + flag, + }); + } + for (const module of pythonModules(meta)) { + findings.push({ + kind: "python_worker_module", + key, + ...identity, + module, + }); + } + + const envBytes = estimateBundleEnvBytes({ + identity, + meta, + nsSecretsEncrypted, + workerSecretsEncrypted, + assetsCdnBase, + }); + if (envBytes !== null && envBytes > WORKER_LOADER_ENV_MAX_BYTES) { + findings.push({ + kind: "worker_env_too_large", + key, + ...identity, + env_bytes: envBytes, + max_env_bytes: WORKER_LOADER_ENV_MAX_BYTES, + upstream_max_env_bytes: UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + headroom_bytes: WORKER_LOADER_ENV_HEADROOM_BYTES, + secret_value_estimate: "encrypted_envelope_length_as_two_byte_string", + }); + } + return findings; +} + +/** + * @param {{ redis: (args: string[]) => string, assetsCdnBase?: string | null }} args + */ +export function scanWorkerd0701Metadata({ redis, assetsCdnBase = null }) { + const keys = redis(["--scan", "--pattern", REDIS_PATTERN]) + .split("\n") + .map((line) => line.trim()) + .filter(Boolean); + /** @type {Map>} */ + const secretHashCache = new Map(); + /** @param {string} key */ + const readSecretHash = (key) => { + let cached = secretHashCache.get(key); + if (!cached) { + cached = parseRedisHash(redis(["HGETALL", key])); + secretHashCache.set(key, cached); + } + return cached; + }; + + /** @type {Array>} */ + const findings = []; + for (const key of keys) { + const rawMeta = redis(["HGET", key, "__meta__"]).replace(/\n$/, ""); + const { identity, meta, findings: parseFindings } = parseMetadataRecord(key, rawMeta); + if (!meta) { + findings.push(...parseFindings); + continue; + } + const nsSecretsEncrypted = identity.namespace + ? readSecretHash(`secrets:${identity.namespace}`) + : {}; + const workerSecretsEncrypted = identity.namespace && identity.worker + ? readSecretHash(`secrets:${identity.namespace}:${identity.worker}`) + : {}; + findings.push(...findingsForParsedBundleMetadata({ + key, + identity, + meta, + nsSecretsEncrypted, + workerSecretsEncrypted, + assetsCdnBase, + })); + } + return { keysScanned: keys.length, findings }; +} + +export async function runCli() { + const redisUrl = redisUrlFromEnv(); + const { keysScanned, findings } = scanWorkerd0701Metadata({ + redis: (args) => redisCli(args, { redisUrl }), + assetsCdnBase: process.env.ASSETS_CDN_BASE || null, + }); + for (const finding of findings) { + console.log(JSON.stringify(finding)); + } + if (findings.length === 0) { + console.error( + `Scanned ${keysScanned} worker bundle metadata keys; ` + + "no workerd 0701 blockers found." + ); + } else { + console.error( + `Scanned ${keysScanned} worker bundle metadata keys; ` + + `found ${findings.length} workerd 0701 blocker(s).` + ); + process.exitCode = 1; + } +} + +function isMainModule() { + return process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]); +} + +if (isMainModule()) { + runCli().catch((err) => { + console.error(err instanceof Error ? err.message : String(err)); + process.exitCode = 1; + }); +} diff --git a/shared/redis-session.js b/shared/redis-session.js index 3a198bc..3d5de2f 100644 --- a/shared/redis-session.js +++ b/shared/redis-session.js @@ -76,12 +76,23 @@ export class RedisSession { return this; } + hasOpenResources() { + return !this._closed && Boolean(this.socket || this.writer || this.reader || this.parser); + } + async close() { if (this._closed) return; this._closed = true; - try { this.writer?.close(); } catch { /* already closed */ } - try { this.reader?.releaseLock(); } catch { /* already released */ } - try { this.socket?.close?.(); } catch { /* already closed */ } + const writer = this.writer; + const reader = this.reader; + const socket = this.socket; + this.writer = null; + this.reader = null; + this.parser = null; + this.socket = null; + try { writer?.close(); } catch { /* already closed */ } + try { reader?.releaseLock(); } catch { /* already released */ } + try { socket?.close?.(); } catch { /* already closed */ } } /** @param {RedisCommandEvent} event */ diff --git a/shared/workerd-compat-flags.js b/shared/workerd-compat-flags.js new file mode 100644 index 0000000..33e72b9 --- /dev/null +++ b/shared/workerd-compat-flags.js @@ -0,0 +1,61 @@ +/* + * Mirrors workerd v1.20260701.1 src/workerd/io/compatibility-date.capnp. + * Regenerate on every workerd pin bump from an upstream workerd source checkout: + * + * node scripts/extract-workerd-experimental-compat-flags.mjs \ + * /path/to/workerd/src/workerd/io/compatibility-date.capnp + */ + +export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION = "1.20260701.1"; + +export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS = Object.freeze([ + "allow_insecure_inefficient_logged_eval", + "allow_irrevocable_stub_storage", + "auto_grpc_convert", + "cache_reload_enabled", + "connect_pass_through", + "durable_object_get_existing", + "durable_object_rename", + "enable_abortsignal_rpc", + "enable_ctx_version_metadata", + "enable_d1_with_sessions_api", + "enable_version_api", + "enable_web_file_system", + "experimental", + "increase_websocket_message_size", + "js_rpc", + "kv_direct_binding", + "memory_cache_delete", + "new_module_registry", + "precise_timers", + "python_workers_development", + "python_workers_durable_objects", + "replica_routing", + "rtti_api", + "service_binding_extra_handlers", + "spec_compliant_property_attributes", + "streaming_tail_worker", + "streams_no_default_auto_allocate_chunk_size", + "tail_worker_user_spans", + "typescript_strip_types", + "unsafe_module", + "unsupported_process_actual_platform", + "webgpu", + "workflows_step_rollback", +]); + +const WORKERD_EXPERIMENTAL_COMPAT_FLAG_SET = new Set(WORKERD_EXPERIMENTAL_COMPAT_FLAGS); + +/** @param {unknown} flag */ +export function isWorkerdExperimentalCompatFlag(flag) { + return typeof flag === "string" && WORKERD_EXPERIMENTAL_COMPAT_FLAG_SET.has(flag); +} + +/** @param {unknown} flags */ +export function firstWorkerdExperimentalCompatFlag(flags) { + if (!Array.isArray(flags)) return null; + for (const flag of flags) { + if (isWorkerdExperimentalCompatFlag(flag)) return flag; + } + return null; +} diff --git a/terraform/README.md b/terraform/README.md index 5869071..229423e 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -225,9 +225,17 @@ The EC2 capacity provider uses a fixed Auto Scaling Group: - spread placement by AZ, then by instance ID - ENI trunking enabled with `awsvpcTrunking` -Task `cpu` and `memory` values are placement reservations on EC2 launch type, not -hard per-container caps. CPU is a cgroup share weight. Memory is an ECS placement -reservation because the containers do not set hard container memory limits. +Task-level `memory` is a task cgroup ceiling on this EC2-only stack, but it does not +separate one container's memory from another container in the same task. CPU remains a +cgroup share weight. D1 and Durable Object stateful runtime containers also set +explicit container `memory` hard limits. D1 defaults to `runtime_memory`; DO defaults +to `runtime_memory - 128 MiB` so the colocated redis-proxy sidecar keeps task-level +headroom. Both are overrideable with `d1_runtime_container_memory` / +`do_runtime_container_memory`; D1 must be no greater than `runtime_memory`, and DO must +still stay below `runtime_memory` to leave sidecar headroom. This matters because newer +workerd releases no longer cap SQLite's process hard heap at 512 MiB; the container cap +keeps a runaway SQLite query inside the stateful runtime container budget. The +supervisor is PID 1 in that same container. The launch template keeps IMDSv2 enabled for the ECS host agent, sets metadata hop limit to 1, and sets `ECS_AWSVPC_BLOCK_IMDS=true` so awsvpc tasks cannot read diff --git a/terraform/main.tf b/terraform/main.tf index 017248b..023ac44 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -40,24 +40,26 @@ module "compute" { secret_envelope_secret_arn = module.data.secret_envelope_secret_arn internal_auth_previous_token_secret_arn = var.internal_auth_previous_token_secret_arn - gateway_desired_count = var.gateway_desired_count - runtime_desired_count = var.runtime_desired_count - d1_runtime_desired_count = var.d1_runtime_desired_count - do_runtime_desired_count = var.do_runtime_desired_count - d1_test_hooks_enabled = var.d1_test_hooks_enabled - do_test_hooks_enabled = var.do_test_hooks_enabled - scheduler_desired_count = var.scheduler_desired_count - workflows_desired_count = var.workflows_desired_count - gateway_cpu = var.gateway_cpu - gateway_memory = var.gateway_memory - system_runtime_cpu = var.system_runtime_cpu - system_runtime_memory = var.system_runtime_memory - runtime_cpu = var.runtime_cpu - runtime_memory = var.runtime_memory - scheduler_cpu = var.scheduler_cpu - scheduler_memory = var.scheduler_memory - workflows_cpu = var.workflows_cpu - workflows_memory = var.workflows_memory + gateway_desired_count = var.gateway_desired_count + runtime_desired_count = var.runtime_desired_count + d1_runtime_desired_count = var.d1_runtime_desired_count + do_runtime_desired_count = var.do_runtime_desired_count + d1_test_hooks_enabled = var.d1_test_hooks_enabled + do_test_hooks_enabled = var.do_test_hooks_enabled + scheduler_desired_count = var.scheduler_desired_count + workflows_desired_count = var.workflows_desired_count + gateway_cpu = var.gateway_cpu + gateway_memory = var.gateway_memory + system_runtime_cpu = var.system_runtime_cpu + system_runtime_memory = var.system_runtime_memory + runtime_cpu = var.runtime_cpu + runtime_memory = var.runtime_memory + d1_runtime_container_memory = var.d1_runtime_container_memory + do_runtime_container_memory = var.do_runtime_container_memory + scheduler_cpu = var.scheduler_cpu + scheduler_memory = var.scheduler_memory + workflows_cpu = var.workflows_cpu + workflows_memory = var.workflows_memory log_level = var.log_level log_retention_days = var.log_retention_days diff --git a/terraform/modules/compute/d1_runtime_service.tf b/terraform/modules/compute/d1_runtime_service.tf index b819e46..37a2df2 100644 --- a/terraform/modules/compute/d1_runtime_service.tf +++ b/terraform/modules/compute/d1_runtime_service.tf @@ -27,6 +27,7 @@ resource "aws_ecs_task_definition" "d1_runtime" { image = var.workerd_image essential = true entryPoint = ["d1-supervisor"] + memory = local.d1_runtime_container_memory portMappings = [{ name = "d1-http" @@ -74,6 +75,14 @@ resource "aws_ecs_task_definition" "d1_runtime" { }]) lifecycle { + precondition { + condition = ( + local.d1_runtime_container_memory > 0 && + local.d1_runtime_container_memory <= var.runtime_memory + ) + error_message = "d1_runtime_container_memory must be positive and no greater than runtime_memory." + } + precondition { condition = !var.d1_test_hooks_enabled || can(regex("(^|-)test($|-)", var.name)) error_message = "d1_test_hooks_enabled may only be enabled for test-named compute stacks." diff --git a/terraform/modules/compute/do_runtime_service.tf b/terraform/modules/compute/do_runtime_service.tf index c11b2ad..ed96e3a 100644 --- a/terraform/modules/compute/do_runtime_service.tf +++ b/terraform/modules/compute/do_runtime_service.tf @@ -50,6 +50,7 @@ resource "aws_ecs_task_definition" "do_runtime" { image = var.workerd_image essential = true entryPoint = ["do-supervisor"] + memory = local.do_runtime_container_memory dependsOn = [{ containerName = "redis-proxy" @@ -104,6 +105,11 @@ resource "aws_ecs_task_definition" "do_runtime" { ]) lifecycle { + precondition { + condition = local.do_runtime_container_memory > 0 && local.do_runtime_container_memory < var.runtime_memory + error_message = "do_runtime_container_memory must leave positive task-level memory headroom for the redis-proxy sidecar." + } + precondition { condition = !var.do_test_hooks_enabled || can(regex("(^|-)test($|-)", var.name)) error_message = "do_test_hooks_enabled may only be enabled for test-named compute stacks." diff --git a/terraform/modules/compute/gateway_service.tf b/terraform/modules/compute/gateway_service.tf index f2a866b..3892905 100644 --- a/terraform/modules/compute/gateway_service.tf +++ b/terraform/modules/compute/gateway_service.tf @@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "gateway" { image = var.workerd_image essential = true entryPoint = ["workerd"] - command = ["serve", "-b", "/app/dist/workerd-configs/gateway.bin", "--experimental"] + command = ["serve", "-b", "/app/dist/workerd-configs/gateway.bin"] stopTimeout = 20 portMappings = [{ diff --git a/terraform/modules/compute/locals.tf b/terraform/modules/compute/locals.tf index 7af0153..44802a6 100644 --- a/terraform/modules/compute/locals.tf +++ b/terraform/modules/compute/locals.tf @@ -1,6 +1,15 @@ locals { - redis_addr = "${var.valkey_host}:${var.valkey_port}" - data_redis_url = "redis://${local.redis_addr}/1" + redis_addr = "${var.valkey_host}:${var.valkey_port}" + data_redis_url = "redis://${local.redis_addr}/1" + do_redis_proxy_memory_headroom = 128 + d1_runtime_container_memory = coalesce( + var.d1_runtime_container_memory, + var.runtime_memory, + ) + do_runtime_container_memory = coalesce( + var.do_runtime_container_memory, + var.runtime_memory - local.do_redis_proxy_memory_headroom, + ) # PLATFORM_DOMAIN is the base hostname gateway uses to split ns from host # (regex builds "."). platform_domain carries a leading diff --git a/terraform/modules/compute/variables.tf b/terraform/modules/compute/variables.tf index f2f3b9a..1866f2a 100644 --- a/terraform/modules/compute/variables.tf +++ b/terraform/modules/compute/variables.tf @@ -53,6 +53,14 @@ variable "system_runtime_cpu" { type = number } variable "system_runtime_memory" { type = number } variable "runtime_cpu" { type = number } variable "runtime_memory" { type = number } +variable "d1_runtime_container_memory" { + type = number + default = null +} +variable "do_runtime_container_memory" { + type = number + default = null +} variable "scheduler_cpu" { type = number } variable "scheduler_memory" { type = number } variable "workflows_cpu" { type = number } diff --git a/terraform/variables.tf b/terraform/variables.tf index b7d47dd..be21b76 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -201,6 +201,26 @@ variable "runtime_memory" { default = 768 } +variable "d1_runtime_container_memory" { + type = number + default = null + description = "Optional hard memory limit, in MiB, for the d1-runtime container. Defaults to runtime_memory." + validation { + condition = var.d1_runtime_container_memory == null || var.d1_runtime_container_memory > 0 + error_message = "d1_runtime_container_memory must be null or a positive number of MiB." + } +} + +variable "do_runtime_container_memory" { + type = number + default = null + description = "Optional hard memory limit, in MiB, for the do-runtime container. Defaults to runtime_memory minus 128 MiB of redis-proxy sidecar headroom." + validation { + condition = var.do_runtime_container_memory == null || var.do_runtime_container_memory > 0 + error_message = "do_runtime_container_memory must be null or a positive number of MiB." + } +} + variable "scheduler_desired_count" { type = number default = 1 diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index 290fe06..9effdbd 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -3,6 +3,7 @@ import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { freshRepositoryModuleDataUrl, + importSpecifierReplacements, moduleDataUrl, readRepositoryFile, repositoryFileUrl, @@ -17,6 +18,7 @@ const SHARED_NS_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_QUEUE_KEYS_URL = repositoryFileUrl("shared/queue-keys.js"); const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_WORKERD_COMPAT_FLAGS_URL = repositoryFileUrl("shared/workerd-compat-flags.js"); /** * Compile the control/* module graph against shared/* deps. Returns URLs so @@ -64,7 +66,10 @@ export async function compileControlGraph(opts = {}) { `export default ${JSON.stringify(readRepositoryFile("package.json"))};` ); const bundleUrl = freshRepositoryModuleDataUrl("control/bundle.js", [ - [/from "shared-ns-pattern"/g, `from ${JSON.stringify(SHARED_NS_URL)}`], + ...importSpecifierReplacements({ + "shared-ns-pattern": SHARED_NS_URL, + "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + }), [/from "control-bindings"/g, `from ${JSON.stringify(bindingsUrl)}`], [/from "wdl-package-json-source"/g, `from ${JSON.stringify(packageJsonSourceUrl)}`], ]); diff --git a/tests/helpers/load-do-protocol.js b/tests/helpers/load-do-protocol.js index fd81f52..1ef16ee 100644 --- a/tests/helpers/load-do-protocol.js +++ b/tests/helpers/load-do-protocol.js @@ -1,10 +1,15 @@ -import { repositoryFileUrl, repositoryModuleDataUrl } from "./load-shared-module.js"; +import { + importSpecifierReplacements, + repositoryFileUrl, + repositoryModuleDataUrl, +} from "./load-shared-module.js"; const SHARED_FNV_URL = repositoryFileUrl("shared/fnv1a32.js"); const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_WORKERD_COMPAT_FLAGS_URL = repositoryFileUrl("shared/workerd-compat-flags.js"); const DO_WIRE_GRAMMAR_URL = repositoryFileUrl("do-runtime/protocol/wire-grammar.js"); const DO_ERRORS_URL = repositoryModuleDataUrl("do-runtime/protocol/errors.js", [ [/from "shared-respond";/g, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], @@ -17,12 +22,15 @@ const DO_IDENTITY_URL = repositoryModuleDataUrl("do-runtime/protocol/identity.js export function doProtocolDataUrl() { return repositoryModuleDataUrl("do-runtime/protocol.js", [ - [/from "do-runtime-protocol-wire-grammar";/g, `from ${JSON.stringify(DO_WIRE_GRAMMAR_URL)};`], - [/from "do-runtime-protocol-errors";/g, `from ${JSON.stringify(DO_ERRORS_URL)};`], - [/from "do-runtime-protocol-identity";/g, `from ${JSON.stringify(DO_IDENTITY_URL)};`], - [/from "shared-worker-id";/g, `from ${JSON.stringify(SHARED_WORKER_ID_URL)};`], - [/from "shared-bounded-body";/g, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)};`], - [/from "shared-internal-auth";/g, `from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)};`], + ...importSpecifierReplacements({ + "do-runtime-protocol-wire-grammar": DO_WIRE_GRAMMAR_URL, + "do-runtime-protocol-errors": DO_ERRORS_URL, + "do-runtime-protocol-identity": DO_IDENTITY_URL, + "shared-worker-id": SHARED_WORKER_ID_URL, + "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + "shared-bounded-body": SHARED_BOUNDED_BODY_URL, + "shared-internal-auth": SHARED_INTERNAL_AUTH_URL, + }), ]); } diff --git a/tests/helpers/load-runtime-dispatch.js b/tests/helpers/load-runtime-dispatch.js index 4cc9275..adf5154 100644 --- a/tests/helpers/load-runtime-dispatch.js +++ b/tests/helpers/load-runtime-dispatch.js @@ -7,6 +7,7 @@ import { moduleDataUrl, repositoryFileUrl, repositoryModuleDataUrl, + runtimeLibModuleDataUrl, } from "./load-shared-module.js"; import { runtimeProxyBindingStubUrl, sharedInternalAuthUrl } from "./runtime-proxy-stub.js"; @@ -15,7 +16,7 @@ const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); const RESPOND_URL = repositoryFileUrl("shared/respond.js"); const BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); -const RUNTIME_LIB_URL = repositoryFileUrl("runtime/lib.js"); +const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); const METRICS_MOCK_URL = moduleDataUrl(` export const metrics = { increment() {}, diff --git a/tests/helpers/load-shared-module.js b/tests/helpers/load-shared-module.js index 2b530e3..949dc48 100644 --- a/tests/helpers/load-shared-module.js +++ b/tests/helpers/load-shared-module.js @@ -11,6 +11,9 @@ const REPO_ROOT = path.resolve(__dirname, "../.."); const SHARED_ENV_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/env.js")).href; const SHARED_ERRORS_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/errors.js")).href; const SHARED_HEX_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/hex.js")).href; +const SHARED_WORKERD_COMPAT_FLAGS_URL = pathToFileURL( + path.resolve(REPO_ROOT, "shared/workerd-compat-flags.js") +).href; /** @typedef {Array<[RegExp | string, string]>} ModuleReplacements */ @@ -82,6 +85,12 @@ export function repositoryModuleDataUrl(relativePath, replacements = []) { return moduleDataUrl(readRepositoryModuleSource(relativePath, replacements)); } +export function runtimeLibModuleDataUrl() { + return repositoryModuleDataUrl("runtime/lib.js", importSpecifierReplacements({ + "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + })); +} + /** * Like repositoryModuleDataUrl but cache-busted per call. Returns the URL (not * the imported module) so loaders can chain it into a sibling's replacements. diff --git a/tests/integration/admin-api.test.js b/tests/integration/admin-api.test.js index 97c5463..a191dd9 100644 --- a/tests/integration/admin-api.test.js +++ b/tests/integration/admin-api.test.js @@ -68,6 +68,25 @@ test("deploy rejects malformed optional metadata instead of silently dropping it } }); +test("deploy rejects workerd 0701 unsupported bundle metadata before cold-load", async () => { + const experimental = await adminPost("/ns/test3/worker/unsupported/deploy", { + code: "export default { fetch() { return new Response('ok'); } };", + compatibilityFlags: ["unsafe_module"], + }); + assert.equal(experimental.status, 400); + assert.equal(experimental.json.error, "experimental_compat_flag_unsupported"); + + const python = await adminPost("/ns/test3/worker/unsupported/deploy", { + mainModule: "worker.js", + modules: { + "worker.js": "export default {};", + "mod.py": { py: "print(1)" }, + }, + }); + assert.equal(python.status, 400); + assert.equal(python.json.error, "python_workers_unsupported"); +}); + test("deploy rejects invalid namespace", async () => { const d = await adminPost("/ns/Bad_NS/worker/x/deploy", { code: "x" }); assert.equal(d.status, 400); diff --git a/tests/integration/http-features.test.js b/tests/integration/http-features.test.js index 96832f6..6722099 100644 --- a/tests/integration/http-features.test.js +++ b/tests/integration/http-features.test.js @@ -24,7 +24,7 @@ const ABORT_WORKER = readFileSync( "utf8" ); -test("client disconnect cancels the response stream in the loaded worker", async () => { +test("client disconnect response stream behavior is bounded on current workerd", async () => { const ns = uniqueNs("abort"); await deployAndPromote(ns, "w", { mainModule: "worker.js", @@ -57,12 +57,17 @@ test("client disconnect cancels the response stream in the loaded worker", async await waitUntil("disconnect marker written", async () => { const r = await gatewayFetch(ns, `/w/poll?key=${encodeURIComponent(key)}`); const text = await r.text(); - // cancel / enqueue-threw prove disconnect reached the worker; - // ended-normally or __null__ mean the signal didn't propagate. - if (text === "cancel" || text === "enqueue-threw") return true; + // workerd #6832 means 2026-06-19+ no longer calls ReadableStream.cancel() + // for this async response-body pattern on client disconnect. If "cancel" + // appears again, upstream likely fixed the bug and WDL should restore the + // strict cancel assertion and re-evaluate log-tail watchdog/documentation. + if (text === "cancel") { + throw new Error("upstream workerd #6832 appears fixed; restore strict cancel assertion and re-evaluate WDL stream watchdogs"); + } + if (text === "enqueue-threw" || text === "ended-normally") return true; if (text === "__null__") return false; throw new Error(`unexpected marker value ${JSON.stringify(text)}`); - }, { timeoutMs: 15000, intervalMs: 250 }); + }, { timeoutMs: 30000, intervalMs: 250 }); }); const STREAM_WORKER = ` diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 11042a7..0b3f5cc 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -6,7 +6,11 @@ import { } from "../helpers/control-handler-harness.js"; import { compileControlGraph } from "../helpers/load-control-lib.js"; import { + importSpecifierReplacements, moduleDataUrl, + readRepositoryFile, + readRepositoryModuleSource, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { readJsonResponse } from "../helpers/response-json.js"; @@ -28,6 +32,9 @@ const CONTROL_DEPLOY_TEST_STATE = { parsedCrons: null, parsedQueueConsumers: null, watchedKeys: null, + envBudgetError: false, + envBudgetFailuresRemaining: 0, + envBudgetCalls: [], redis: null, logs: [], metrics: { increment() {}, observe() {} }, @@ -51,6 +58,9 @@ function resetControlDeployTestState() { CONTROL_DEPLOY_TEST_STATE.parsedCrons = null; CONTROL_DEPLOY_TEST_STATE.parsedQueueConsumers = null; CONTROL_DEPLOY_TEST_STATE.watchedKeys = null; + CONTROL_DEPLOY_TEST_STATE.envBudgetError = false; + CONTROL_DEPLOY_TEST_STATE.envBudgetFailuresRemaining = 0; + CONTROL_DEPLOY_TEST_STATE.envBudgetCalls = []; CONTROL_DEPLOY_TEST_STATE.redis = null; CONTROL_DEPLOY_TEST_STATE.logs = []; CONTROL_DEPLOY_TEST_STATE.metrics = { increment() {}, observe() {} }; @@ -77,14 +87,19 @@ export async function recordS3CleanupIntent(intent) { const controlBundleUrl = moduleDataUrl(` export function deepFreeze(value) { return value; } export function prepareBundle(mainModule, modules, options = {}) { - return /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle || { - meta: { - mainModule, - modules, - bindings: options.bindings, - exports: options.exports, - workflows: options.workflows, - }, + if (/** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle) { + return /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle; + } + const meta = { + mainModule, + modules, + bindings: options.bindings, + exports: options.exports, + workflows: options.workflows, + }; + if (options.vars !== undefined) meta.vars = options.vars; + return { + meta, normalized: Object.entries(modules || {}), }; } @@ -197,6 +212,70 @@ export async function resolveDatabaseRefFrom(session, ns, databaseRef) { } `); +const controlEnvBudgetUrl = moduleDataUrl(` +export class WorkerEnvBudgetError extends Error { + constructor(message) { + super(message); + this.status = 400; + this.code = "worker_env_too_large"; + } +} +export function assertWorkerLoaderUserEnvBudget() { + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.push(Array.from(arguments)[0] || {}); + if (/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining > 0) { + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining -= 1; + throw new WorkerEnvBudgetError("env too large"); + } + if (/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetError) { + throw new WorkerEnvBudgetError("env too large"); + } + return 0; +} +export async function decryptSecretHash() { return {}; } +`); + +const secretEnvelopeUrl = moduleDataUrl(` +export class SecretEnvelopeError extends Error { + constructor(code, message) { + super(message); + this.code = code; + } +} +`); + +/** @param {string} path */ +function textModuleUrl(path) { + return moduleDataUrl(`export default ${JSON.stringify(readRepositoryFile(path))};`); +} + +const runtimeLoadCodeBudgetUrl = moduleDataUrl(readRepositoryModuleSource( + "runtime/load/code-budget.js", + importSpecifierReplacements({ + "shared-ns-pattern": repositoryFileUrl("shared/ns-pattern.js"), + "runtime-load-module-rewrite": repositoryFileUrl("runtime/load/module-rewrite.js"), + "runtime-load-wrapper-generate": repositoryFileUrl("runtime/load/wrapper-generate.js"), + }) +)); +const controlWorkerCodeBudgetUrl = moduleDataUrl(readRepositoryModuleSource( + "control/worker-code-budget.js", + importSpecifierReplacements({ + "runtime-load-code-budget": runtimeLoadCodeBudgetUrl, + "runtime-d1-client-source": textModuleUrl("runtime/d1-client.js"), + "runtime-d1-data-field-source": textModuleUrl("shared/d1-data-field.js"), + "runtime-d1-params-source": textModuleUrl("shared/d1-params.js"), + "runtime-sql-splitter-source": textModuleUrl("shared/sql-splitter.js"), + "runtime-d1-transport-source": textModuleUrl("shared/d1-transport.js"), + "runtime-r2-client-source": textModuleUrl("runtime/r2-client.js"), + "runtime-r2-utils-source": textModuleUrl("runtime/r2-utils.js"), + "runtime-do-client-source": textModuleUrl("runtime/do-client.js"), + "runtime-do-transport-source": textModuleUrl("runtime/_wdl-do-transport.js"), + "runtime-owner-endpoint-source": textModuleUrl("runtime/_wdl-owner-endpoint.js"), + "runtime-owner-hint-cache-source": textModuleUrl("runtime/_wdl-owner-hint-cache.js"), + "runtime-request-id-source": textModuleUrl("runtime/_wdl-request-id.js"), + "runtime-workflows-client-source": textModuleUrl("runtime/workflows-client.js"), + }) +)); + const { commitWithWatch, handle } = await importControlHandler("control/handlers/deploy.js", { globalName: "__controlDeployTestState", extraSharedSource: controlSharedExtraSource, @@ -213,6 +292,9 @@ const { commitWithWatch, handle } = await importControlHandler("control/handlers "control-s3": controlS3Url, "shared-assets-token": sharedAssetsUrl, "control-d1-store": d1StoreUrl, + "control-env-budget": controlEnvBudgetUrl, + "control-worker-code-budget": controlWorkerCodeBudgetUrl, + "shared-secret-envelope": secretEnvelopeUrl, }, }); const { WatchError } = await import(sharedRedisUrl); @@ -333,6 +415,94 @@ test("commitWithWatch re-resolves a D1 alias after a watched recreate race", asy assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("d1:database:tenant-a:d1_new")); }); +test("commitWithWatch validates deploy env budget under watched secret hashes", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; + /** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys = []; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls = []; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + await commitWithWatch({ + redis, + ns: "tenant-a", + name: "demo", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + vars: { TOKEN: "from-vars" }, + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [], + d1Refs: [], + controlEnv: { ASSETS_CDN_BASE: "https://assets.example/cdn" }, + }); + + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("secrets:tenant-a")); + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("secrets:tenant-a:demo")); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.length, 1); + assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].vars, { TOKEN: "from-vars" }); + assert.equal( + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].assetsCdnBase, + "https://assets.example/cdn" + ); +}); + +test("commitWithWatch validates env budget against materialized D1 metadata", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map([ + ["d1:database-name:tenant-a:main", "d1_0123456789abcdef0123456789abcdef"], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ + ["d1:database:tenant-a:d1_0123456789abcdef0123456789abcdef", { + databaseId: "d1_0123456789abcdef0123456789abcdef", + databaseName: "main", + }], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls = []; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + await commitWithWatch({ + redis, + ns: "tenant-a", + name: "demo", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + bindings: { + DB: { type: "d1", databaseId: "main" }, + }, + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [], + d1Refs: [{ binding: "DB", databaseId: "main" }], + controlEnv: {}, + }); + + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.length, 1); + assert.equal( + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].meta.bindings.DB.databaseId, + "d1_0123456789abcdef0123456789abcdef" + ); +}); + test("deploy handler resolves cross-namespace service-binding meta from the target namespace", async () => { /** @type {string[]} */ const metaReads = []; @@ -524,6 +694,108 @@ test("deploy handler rejects assets without S3 before allocating a version", asy } }); +test("deploy handler treats pre-allocation workerLoader env budget failures as advisory", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining = 1; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls = []; + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; + + const session = makeSession(); + let incrCalled = false; + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async incr() { + incrCalled = true; + return 1; + }, + /** @param {string} key */ + async hGetAll(key) { + return await session.hGetAll(key); + }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(session); + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/env-heavy/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + vars: { BIG: "x" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "env-heavy", + requestId: "rid-env-budget", + }); + + const body = await readJsonResponse(response, 201); + assert.equal(body.version, "v1"); + assert.equal(incrCalled, true); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls.length, 2); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[0].version, undefined); + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.envBudgetCalls[1].version, "v1"); + assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta.vars, { BIG: "x" }); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + /** @type {any} */ (globalThis).__controlDeployTestState.envBudgetFailuresRemaining = 0; + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + } +}); + +test("deploy handler counts runtime-generated wrapper code before allocating a version", async () => { + const oversizedEntrypoint = `Entrypoint${"A".repeat(600 * 1024)}`; + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + bindings: { DB: { type: "d1", databaseId: "db" } }, + exports: [{ entrypoint: oversizedEntrypoint }], + }, + normalized: [["worker.js", new Uint8Array(64 * 1024 * 1024 - 512 * 1024)]], + }; + + let incrCalled = false; + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async incr() { + incrCalled = true; + return 1; + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/code-heavy/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "code-heavy", + requestId: "rid-code-budget", + }); + + const body = await readJsonResponse(response, 413); + assert.equal(body.error, "worker_code_too_large"); + assert.match(body.message, /final WorkerCode/); + assert.equal(incrCalled, false); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + } +}); + test("deploy handler skips cleanup for empty assets when commit fails", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js new file mode 100644 index 0000000..a042f28 --- /dev/null +++ b/tests/unit/control-env-budget.test.js @@ -0,0 +1,478 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + importRepositoryModule, + importSpecifierReplacements, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { encryptSecretValue } from "../../shared/secret-envelope.js"; + +const secretEnvelopeUrl = repositoryFileUrl("shared/secret-envelope.js"); +const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const { + UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_HEADROOM_BYTES, + WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + WorkerEnvBudgetError, + assertWorkerLoaderUserEnvBudget, + assertWorkerVersionsUserEnvBudget, + decryptSecretHash, + estimatedWorkerLoaderEnv, + estimatedWorkerLoaderEnvBytes, +} = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ + "shared-secret-envelope": secretEnvelopeUrl, + "shared-errors": sharedErrorsUrl, + "shared-version": sharedVersionUrl, +})); + +const envelopeEnv = { + SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", + SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", +}; + +test("worker env budget counts merged vars and secrets with worker-secret precedence", () => { + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", + vars: { TOKEN: "var", ONLY_VAR: "v" }, + nsSecrets: { TOKEN: "ns", ONLY_NS: "n" }, + workerSecrets: { TOKEN: "worker" }, + }); + + assert.deepEqual(estimated, { + TOKEN: "worker", + ONLY_VAR: "v", + ONLY_NS: "n", + }); + assert.equal( + assertWorkerLoaderUserEnvBudget({ + ns: "demo", + vars: { TOKEN: "var", ONLY_VAR: "v" }, + nsSecrets: { TOKEN: "ns", ONLY_NS: "n" }, + workerSecrets: { TOKEN: "worker" }, + }), + estimatedWorkerLoaderEnvBytes(estimated) + ); +}); + +test("worker env budget rejects user-controlled env above workerd workerLoader limit", () => { + assert.equal(WORKER_LOADER_ENV_MAX_BYTES, UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES - WORKER_LOADER_ENV_HEADROOM_BYTES); + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "api", + vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES) }, + }), + (err) => { + if (!(err instanceof WorkerEnvBudgetError)) return false; + const budgetErr = /** @type {InstanceType} */ (err); + assert.equal(budgetErr.code, "worker_env_too_large"); + assert.equal(budgetErr.status, 400); + assert.equal(budgetErr.details.namespace, "demo"); + assert.equal(budgetErr.details.worker, "api"); + assert.equal(budgetErr.details.upstream_max_env_bytes, UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES); + assert.equal(budgetErr.details.headroom_bytes, WORKER_LOADER_ENV_HEADROOM_BYTES); + return true; + } + ); +}); + +test("worker env budget counts required caller secret copies in service binding props", () => { + const secret = "x".repeat(Math.floor(WORKER_LOADER_ENV_MAX_BYTES * 0.6)); + + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "caller", + vars: { SMALL: "ok" }, + nsSecrets: { API_TOKEN: secret }, + meta: { + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + entrypoint: "Api", + requiredCallerSecrets: ["API_TOKEN"], + }, + }, + }, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + +test("worker env budget accounts for V8 two-byte strings on mixed non-Latin-1 env", () => { + const mixed = `${"x".repeat(Math.floor(WORKER_LOADER_ENV_MAX_BYTES / 2) + 1)}中`; + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + vars: { BIG: mixed }, + }); + const jsonBytes = Buffer.byteLength(JSON.stringify(estimated), "utf8"); + const estimatedBytes = estimatedWorkerLoaderEnvBytes(estimated); + + assert.ok(jsonBytes <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(estimatedBytes > WORKER_LOADER_ENV_MAX_BYTES); + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "api", + vars: { BIG: mixed }, + }), + (err) => { + if (!(err instanceof WorkerEnvBudgetError)) return false; + const budgetErr = /** @type {WorkerEnvBudgetError} */ (err); + assert.equal(budgetErr.code, "worker_env_too_large"); + assert.equal(budgetErr.details.env_bytes, estimatedBytes); + return true; + } + ); +}); + +test("worker env budget stores required caller secrets in a null-prototype map", () => { + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "caller", + nsSecrets: JSON.parse('{"__proto__":"secret"}'), + meta: { + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + requiredCallerSecrets: ["__proto__"], + }, + }, + }, + }); + const callerSecrets = /** @type {any} */ (estimated.PLATFORM).props.callerSecrets; + + assert.equal(Object.getPrototypeOf(callerSecrets), null); + assert.equal(Object.hasOwn(callerSecrets, "__proto__"), true); + assert.equal(callerSecrets.__proto__, "secret"); +}); + +test("worker env budget counts configured assets CDN base", () => { + const meta = { + vars: { PAD: "" }, + assets: { prefix: "assets/demo/api/token/" }, + bindings: { + ASSETS: { type: "assets" }, + }, + }; + const assetsCdnBase = `https://${"assets-subdomain-".repeat(600)}example.test`; + /** @param {number} padLength @param {string | null | undefined} cdnBase */ + const bytesWithPad = (padLength, cdnBase) => estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + vars: { PAD: "x".repeat(padLength) }, + meta, + assetsCdnBase: cdnBase, + })); + const padLength = WORKER_LOADER_ENV_MAX_BYTES - bytesWithPad(0, assetsCdnBase) + 1; + + assert.ok(bytesWithPad(padLength, null) <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(bytesWithPad(padLength, assetsCdnBase) > WORKER_LOADER_ENV_MAX_BYTES); + assert.throws( + () => assertWorkerLoaderUserEnvBudget({ + ns: "demo", + worker: "api", + vars: { PAD: "x".repeat(padLength) }, + meta, + assetsCdnBase, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + +test("worker env budget includes do-runtime alarm binding for Durable Object workers", () => { + const estimated = estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + version: "v12", + meta: { + bindings: { + ROOM: { + type: "do", + className: "Room", + doStorageId: "do_0123456789abcdef0123456789abcdef", + }, + }, + }, + }); + + assert.deepEqual(estimated.ROOM, { + __wdlBinding: "do", + ns: "demo", + worker: "api", + version: "v12", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + hostProxy: { + __wdlBinding: "do-host-proxy", + props: { + ns: "demo", + worker: "api", + version: "v12", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, + }, + }); + assert.deepEqual(estimated.__WDL_DO_ALARMS__, { + __wdlBinding: "do-alarms", + props: { + ns: "demo", + worker: "api", + version: "v12", + doStorageId: "do_0123456789abcdef0123456789abcdef", + }, + }); +}); + +test("decryptSecretHash returns plaintext secret values for budget checks", async () => { + const hashKey = "secrets:demo"; + const encrypted = await encryptSecretValue("plain", { + env: envelopeEnv, + hashKey, + fieldName: "TOKEN", + }); + + assert.deepEqual( + { + ...(await decryptSecretHash({ + encrypted: { TOKEN: encrypted, MISSING: null }, + env: envelopeEnv, + hashKey, + })), + }, + { TOKEN: "plain" } + ); +}); + +test("decryptSecretHash can ignore corrupt envelopes for DELETE repair budgets", async () => { + const hashKey = "secrets:demo"; + const encrypted = await encryptSecretValue("plain", { + env: envelopeEnv, + hashKey, + fieldName: "TOKEN", + }); + + assert.deepEqual( + { + ...(await decryptSecretHash({ + encrypted: { TOKEN: encrypted, BAD: "WDL-ENC:not-json" }, + env: envelopeEnv, + hashKey, + ignoreSecretEnvelopeErrors: true, + })), + }, + { TOKEN: "plain" } + ); +}); + +test("worker env budget checks every retained worker version", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(field, "__meta__"); + if (key === "worker:demo:api:v:1") return JSON.stringify({ vars: { SMALL: "ok" } }); + if (key === "worker:demo:api:v:2") { + return JSON.stringify({ vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES) } }); + } + return null; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1", "v2"], + nsSecrets: { TOKEN: "secret" }, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + +test("worker env budget checks retained-version binding env injections", async () => { + const secret = "x".repeat(Math.floor(WORKER_LOADER_ENV_MAX_BYTES * 0.6)); + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:caller:v:1"); + assert.equal(field, "__meta__"); + return JSON.stringify({ + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + requiredCallerSecrets: ["API_TOKEN"], + }, + }, + }); + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "caller", + versions: ["v1"], + nsSecrets: { API_TOKEN: secret }, + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + assert.equal(/** @type {WorkerEnvBudgetError} */ (err).code, "worker_env_too_large"); + return true; + } + ); +}); + +test("worker env budget can estimate a source bundle under a future version string", async () => { + const baseMeta = { + vars: { PAD: "" }, + workflows: [{ + binding: "FLOW", + name: "flow", + className: "Flow", + workflowKey: "wf_0123456789abcdef0123456789abcdef", + }], + }; + /** @param {number} padLength @param {string} version */ + const bytesWithPad = (padLength, version) => estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + version, + vars: { PAD: "x".repeat(padLength) }, + meta: baseMeta, + })); + const padLength = WORKER_LOADER_ENV_MAX_BYTES - bytesWithPad(0, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) + 1; + assert.ok(bytesWithPad(padLength, "v1") <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(bytesWithPad(padLength, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) > WORKER_LOADER_ENV_MAX_BYTES); + + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return JSON.stringify({ + ...baseMeta, + vars: { PAD: "x".repeat(padLength) }, + }); + }, + }; + + await assert.doesNotReject(() => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + })); + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: [], + versionEstimates: [{ + sourceVersion: "v1", + estimatedVersion: WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + }], + }), + (err) => { + assert.equal(err instanceof WorkerEnvBudgetError, true); + const budgetErr = /** @type {WorkerEnvBudgetError} */ (err); + assert.equal(budgetErr.code, "worker_env_too_large"); + assert.match(budgetErr.message, /demo\/api@v1/); + assert.equal(budgetErr.details.source_version, "v1"); + assert.equal(budgetErr.details.estimated_version, WORKER_LOADER_ENV_VERSION_PLACEHOLDER); + return true; + } + ); +}); + +test("worker env budget reports bundle metadata parse context", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return "{not-json"; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + /invalid bundle metadata for demo\/api@v1/ + ); +}); + +test("worker env budget fails closed when retained bundle metadata is missing", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return null; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + /bundle metadata missing for demo\/api@v1/ + ); +}); + +test("worker env budget fails closed when retained bundle metadata is not an object", async () => { + const redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + assert.equal(key, "worker:demo:api:v:1"); + assert.equal(field, "__meta__"); + return "[]"; + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + /__meta__ must be a JSON object/ + ); +}); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 2ad2352..c48b63a 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -3,6 +3,10 @@ import assert from "node:assert/strict"; import { loadControlLib } from "../helpers/load-control-lib.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { RESERVED_NS } from "../../shared/ns-pattern.js"; +import { + WORKERD_EXPERIMENTAL_COMPAT_FLAGS, + WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, +} from "../../shared/workerd-compat-flags.js"; const packageJson = /** @type {any} */ (readRepositoryJson("package.json")); const packageWorkerdDep = packageJson?.dependencies?.workerd; @@ -239,9 +243,21 @@ test("normalizeModule: json null is allowed", () => { assert.equal(r.bytes.toString(), "null"); }); -test("normalizeModule: cjs / py", () => { +test("normalizeModule: cjs", () => { assert.equal(normalizeModule({ cjs: "module.exports = {}" }).type, "cjs"); - assert.equal(normalizeModule({ py: "print(1)" }).type, "py"); +}); + +test("normalizeModule: py is rejected before workerd cold-load", () => { + assert.throws( + () => normalizeModule({ py: "print(1)" }), + (err) => { + if (!(err instanceof Error)) return false; + const coded = /** @type {Error & { code?: unknown, status?: unknown }} */ (err); + return coded.code === "python_workers_unsupported" && + coded.status === 400 && + /Python Workers modules are not supported by WDL/.test(coded.message); + } + ); }); test("normalizeModule: unknown shape throws", () => { @@ -689,6 +705,30 @@ test("prepareBundle: compatibilityFlags preserved", () => { assert.deepEqual(meta.compatibilityFlags, ["nodejs_compat"]); }); +test("prepareBundle: experimental workerd compatibility flags are rejected", () => { + assert.throws( + () => prepareBundle( + "w.js", + { "w.js": "x" }, + { compatibilityFlags: ["nodejs_compat", "unsafe_module"] } + ), + (err) => { + if (!(err instanceof Error)) return false; + const coded = /** @type {Error & { code?: unknown, status?: unknown }} */ (err); + return coded.code === "experimental_compat_flag_unsupported" && + coded.status === 400 && + /"unsafe_module"/.test(coded.message); + } + ); + assert.doesNotThrow(() => + prepareBundle( + "w.js", + { "w.js": "x" }, + { compatibilityFlags: ["nodejs_compat", "no_nodejs_compat"] } + ) + ); +}); + test("prepareBundle: compatibilityDate validates shape before commit", () => { assert.equal( prepareBundle("w.js", { "w.js": "x" }, { compatibilityDate: "2026-04-24" }).meta.compatibilityDate, @@ -710,6 +750,8 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( String(unsupported.getUTCMonth() + 1).padStart(2, "0"), String(unsupported.getUTCDate()).padStart(2, "0"), ].join("-"); + const afterUnsupported = new Date(unsupported); + afterUnsupported.setUTCDate(afterUnsupported.getUTCDate() + 1); assert.equal( validateCompatibilityDate("2026-06-20", new Date("2026-06-30T00:00:00Z")), @@ -720,7 +762,7 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( /must not be later than today UTC/ ); assert.throws( - () => validateCompatibilityDate(unsupportedDate, new Date("2026-06-30T00:00:00Z")), + () => validateCompatibilityDate(unsupportedDate, afterUnsupported), /newer than bundled workerd supports/ ); }); @@ -748,6 +790,28 @@ test("MAX_WORKER_COMPATIBILITY_DATE matches pinned workerd release plus seven da assert.equal(maxWorkerCompatibilityDateFromPackageJson(JSON.stringify({ dependencies: {} })), null); }); +test("workerd experimental compat flag mirror matches pinned workerd source version", () => { + const regenerate = [ + "Regenerate shared/workerd-compat-flags.js from an upstream workerd checkout:", + "node scripts/extract-workerd-experimental-compat-flags.mjs", + "/path/to/workerd/src/workerd/io/compatibility-date.capnp", + ].join(" "); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, WORKERD_VERSION, regenerate); + assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("experimental")); + assert.ok(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unsafe_module")); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.length, 33); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unique_ctx_per_invocation"), false); + assert.equal( + WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("nonclass_entrypoint_reuses_ctx_across_invocations"), + false + ); + assert.equal( + WORKERD_EXPERIMENTAL_COMPAT_FLAGS.some((flag) => flag.startsWith("no_")), + false, + "only enable flags from $experimental compatibility entries should be mirrored" + ); +}); + test("prepareBundle: compatibilityFlags rejected when not an array (would be silently dropped at runtime floor merge)", () => { assert.throws( () => prepareBundle("w.js", { "w.js": "x" }, { compatibilityFlags: "nodejs_compat" }), diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 3def174..d39eb4e 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -1,10 +1,16 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { importRepositoryModuleFresh } from "../helpers/load-shared-module.js"; -import { delay } from "../helpers/timing.js"; +import { delay, waitUntil } from "../helpers/timing.js"; -function loadLogsTailHandler() { - return importRepositoryModuleFresh("control/handlers/logs-tail.js", [ +/** @param {{ keepaliveMs?: number }} [options] */ +function loadLogsTailHandler(options = {}) { + /** @type {Array<[RegExp | string, string]>} */ + const replacements = []; + if (options.keepaliveMs) { + replacements.push([/const SSE_KEEPALIVE_MS = 5_000;/, `const SSE_KEEPALIVE_MS = ${options.keepaliveMs};`]); + } + replacements.push( [ /import \{ envValueOr \} from "shared-env";/, "const envValueOr = (value, fallback) => value == null || value === '' ? fallback : value;", @@ -20,9 +26,19 @@ function loadLogsTailHandler() { this.publishCalls = []; this.closed = false; } - async open() { this.opened = true; } + async open() { + this.openStarted = true; + this.socket = {}; + this.openPromise = (async () => { + const state = /** @type {any} */ (globalThis).__tailState; + if (state.openBlocker) await state.openBlocker; + this.opened = true; + })(); + await this.openPromise; + } async publish(channel, payload) { this.publishCalls.push([channel, payload]); } async xRead(...args) { this.xReadCalls.push(args); return null; } + hasOpenResources() { return Boolean(this.socket); } async close() { this.closed = true; } } const redisDbFromEnv = (env, name) => Number(env?.[name] || 0);`, @@ -51,7 +67,8 @@ function loadLogsTailHandler() { const requireControlLog = () => state.log; const controlTailRedis = () => state.dataRedis || state.redis;`, ], - ]); + ); + return importRepositoryModuleFresh("control/handlers/logs-tail.js", replacements); } function resetTailState() { @@ -85,6 +102,7 @@ function resetTailState() { }, }, logs: [], + openBlocker: null, log: (/** @type {string} */ level, /** @type {string} */ event, /** @type {any} */ data) => { /** @type {any} */ (globalThis).__tailState.logs.push({ level, event, data }); }, @@ -192,6 +210,134 @@ test("logs tail closes after the max session lifetime so reconnect reauthorizes" assert.ok(/** @type {any} */ (globalThis).__tailState.logs.some((/** @type {any} */ entry) => entry.event === "tail_session_expired")); }); +test("logs tail max-session watchdog closes even without stream cancel", async () => { + resetTailState(); + const { handle } = await loadLogsTailHandler(); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "50" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-watchdog", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + assert.match((await readText(reader)).text, /tail-open/); + await delay(80); + await Promise.all(waitUntilPromises); + + assert.equal(/** @type {any} */ (globalThis).__tailSessions.length, 1); + assert.equal(/** @type {any} */ (globalThis).__tailSessions[0].closed, true); + assert.ok(/** @type {any} */ (globalThis).__tailState.logs.some((/** @type {any} */ entry) => entry.event === "tail_session_expired")); + await reader.cancel().catch(() => {}); +}); + +test("logs tail idle-pull watchdog closes abandoned streams before max-session", async () => { + resetTailState(); + const { handle } = await loadLogsTailHandler({ keepaliveMs: 5 }); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "1000" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-idle", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + assert.match((await readText(reader)).text, /tail-open/); + await waitUntil("tail idle watchdog to close session", () => + /** @type {any} */ (globalThis).__tailSessions[0]?.closed === true, { + timeoutMs: 500, intervalMs: 5, + }); + await Promise.all(waitUntilPromises); + + assert.equal(/** @type {any} */ (globalThis).__tailSessions.length, 1); + assert.equal(/** @type {any} */ (globalThis).__tailSessions[0].closed, true); + assert.ok(/** @type {any} */ (globalThis).__tailState.logs.some((/** @type {any} */ entry) => entry.event === "tail_session_idle")); + await reader.cancel().catch(() => {}); +}); + +test("logs tail closes session if watchdog fires while Redis open is pending", async () => { + resetTailState(); + const state = /** @type {any} */ (globalThis).__tailState; + let releaseOpen = () => {}; + state.openBlocker = new Promise((resolve) => { + releaseOpen = () => resolve(undefined); + }); + + const { handle } = await loadLogsTailHandler(); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "30" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-open-race", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + const pendingRead = readText(reader); + const session = /** @type {any} */ (globalThis).__tailSessions[0]; + await waitUntil("tail Redis open to start", () => session.openStarted === true, { + timeoutMs: 500, intervalMs: 5, + }); + await delay(50); + releaseOpen(); + await session.openPromise; + await Promise.all(waitUntilPromises); + await waitUntil("tail Redis session to close", () => session.closed === true, { + timeoutMs: 500, intervalMs: 5, + }); + + assert.equal(session.opened, true); + assert.equal(session.closed, true); + assert.match((await pendingRead).text, /session_expired/); + await reader.cancel().catch(() => {}); +}); + +test("logs tail max-session watchdog closes a socket while Redis open is pending", async () => { + resetTailState(); + const state = /** @type {any} */ (globalThis).__tailState; + let releaseOpen = () => {}; + state.openBlocker = new Promise((resolve) => { + releaseOpen = () => resolve(undefined); + }); + + const { handle } = await loadLogsTailHandler(); + /** @type {Promise[]} */ + const waitUntilPromises = []; + const response = await handle({ + request: new Request("http://control.test/ns/demo/logs/tail?worker=foo"), + env: { REDIS_ADDR: "redis://unit", LOG_TAIL_MAX_SESSION_MS: "30" }, + ctx: { waitUntil(/** @type {Promise} */ promise) { waitUntilPromises.push(promise); } }, + ns: "demo", + requestId: "rid-tail-open-socket-race", + }); + + assert.equal(response.status, 200); + const reader = response.body.getReader(); + const pendingRead = readText(reader); + const session = /** @type {any} */ (globalThis).__tailSessions[0]; + await waitUntil("tail Redis open to assign socket", () => session.socket, { + timeoutMs: 500, intervalMs: 5, + }); + await Promise.all(waitUntilPromises); + + assert.equal(session.closed, true); + releaseOpen(); + await session.openPromise; + assert.match((await pendingRead).text, /session_expired/); + await reader.cancel().catch(() => {}); +}); + test("logs tail emits idle keepalives below common proxy idle timeouts", async () => { resetTailState(); const { handle } = await loadLogsTailHandler(); diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index bff423b..e84ed43 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -414,6 +414,62 @@ test("bumpActiveAndPromote also rewrites full queue consumer projection", async }); }); +test("bumpActiveAndPromote runs beforeStageCopy inside the watched copy transaction", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { queueConsumers: [] }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + /** @type {unknown} */ + let seen = null; + + const result = await bumpActiveAndPromote(redis, "demo", "worker", { + /** @param {{ iso: any, currentVersion: string, newVersion: string, sourceMeta: any }} context */ + async beforeStageCopy(context) { + const { iso, currentVersion, newVersion, sourceMeta } = context; + await iso.watch("secrets:demo", "secrets:demo:worker"); + seen = { + currentVersion, + newVersion, + route: await iso.hGet("routes:demo", "worker"), + queueConsumers: sourceMeta.queueConsumers, + }; + }, + }); + + assert.equal(result.version, "v2"); + assert.deepEqual(seen, { + currentVersion: "v1", + newVersion: "v2", + route: "v1", + queueConsumers: [], + }); + assert.ok(redis.state.watchBatches.some((batch) => + batch.length === 2 && + batch.includes("secrets:demo") && + batch.includes("secrets:demo:worker") + )); + assert.ok(redis.state.hashes.has("bundle:demo:worker:v2")); +}); + +test("bumpActiveAndPromote does not copy or flip routes when beforeStageCopy rejects", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { queueConsumers: [] }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await assert.rejects( + bumpActiveAndPromote(redis, "demo", "worker", { + async beforeStageCopy() { + throw new Error("budget failed"); + }, + }), + /budget failed/ + ); + + assert.equal(redis.state.hashes.has("bundle:demo:worker:v2"), false); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); +}); + test("bumpActiveAndPromote rejects active routes that no longer declare their hosts", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 2b3e630..b4681e0 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -1,16 +1,103 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; -import { decryptSecretValue, isSecretEnvelope } from "../../shared/secret-envelope.js"; -import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { decryptSecretValue, encryptSecretValue, isSecretEnvelope } from "../../shared/secret-envelope.js"; +import { + applyModuleReplacements, + moduleDataUrl, + readRepositoryFile, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { installMockProperty } from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; const SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const env = { SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", }; +/** @returns {Record} */ +function emptySecretHash() { + return {}; +} + +/** + * @param {string} key + * @param {string} field + */ +function defaultWorkerSecretBundleMeta(key, field) { + return /^worker:[^:]+:[^:]+:v:[1-9][0-9]*$/.test(key) && field === "__meta__" + ? "{}" + : null; +} + +/** + * @param {{ + * hKeys?: string[], + * hGet?: (key: string, field: string) => string | null | Promise, + * hGetAll?: (key: string) => Record | Promise>, + * zCard?: number, + * zRange?: string[], + * onHSet?: (key: string, field: string, value: string) => void, + * onHDel?: (key: string, field: string) => void, + * onExec?: () => void | Promise, + * }} options + */ +function makeWorkerSecretSession({ + hKeys = [], + hGet = defaultWorkerSecretBundleMeta, + hGetAll = emptySecretHash, + zCard = 0, + zRange = [], + onHSet = () => {}, + onHDel = () => {}, + onExec = () => {}, +} = {}) { + return { + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return hKeys; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { return await hGet(key, field); }, + /** @param {string} key */ + async hGetAll(key) { return await hGetAll(key); }, + async zCard() { return zCard; }, + async zRange() { return zRange; }, + multi() { + return { + /** @param {string} key @param {string} field @param {string} value */ + hSet(key, field, value) { onHSet(key, field, value); }, + /** @param {string} key @param {string} field */ + hDel(key, field) { onHDel(key, field); }, + sAdd() {}, + sRem() {}, + async exec() { await onExec(); }, + }; + }, + }; +} + +/** + * @param {{ redis: Record }} state + * @param {ReturnType} session + * @param {() => unknown | Promise} callback + */ +async function withWorkerSecretSession(state, session, callback) { + const restore = installMockProperty(state.redis, "session", + async (/** @type {(session: ReturnType) => unknown | Promise} */ fn) => + await fn(session) + ); + try { + return await callback(); + } finally { + restore(); + } +} + const validateSecretKeyStubSource = ` const SECRET_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; const WDL_RESERVED_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; @@ -34,25 +121,91 @@ function secretPutUrl(controlSharedUrl, controlLibUrl) { return moduleDataUrl(source); } +function envBudgetUrl() { + const source = applyModuleReplacements(readRepositoryFile("control/env-budget.js"), [ + [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], + [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + ]); + return moduleDataUrl(source); +} + const controlSharedUrl = controlSharedStubUrl(` +class WatchError extends Error {} export const state = { log() {}, redis: { + execCalls: 0, + execFailures: 0, writes: [], + deletes: [], + watchedKeys: [], async hKeys() { return []; }, + async hGet(key, field) { + return /^worker:[^:]+:[^:]+:v:[1-9][0-9]*$/.test(key) && field === "__meta__" + ? "{}" + : null; + }, + async hGetAll() { return {}; }, + async sMembers() { return []; }, + async zRange() { return []; }, async hSet(key, field, value) { this.writes.push({ key, field, value }); return 1; }, async hDel() { return 0; }, + async session(fn) { + return await fn({ + async watch(...keys) { + state.redis.watchedKeys.push(...keys); + }, + async hGet(key, field) { + return await state.redis.hGet(key, field); + }, + async hGetAll(key) { + return await state.redis.hGetAll(key); + }, + async sMembers(key) { + return await state.redis.sMembers(key); + }, + async zRange(key, start, stop) { + return await state.redis.zRange(key, start, stop); + }, + multi() { + return { + hSet(key, field, value) { + state.redis.writes.push({ key, field, value }); + return this; + }, + hDel(key, field) { + state.redis.deletes.push({ key, field }); + return this; + }, + async exec() { + state.redis.execCalls += 1; + if (state.redis.execFailures > 0) { + state.redis.execFailures -= 1; + throw new WatchError("simulated namespace secret contention"); + } + }, + }; + }, + }); + }, }, }; `); -const controlLibStubUrl = moduleDataUrl(validateSecretKeyStubSource); +const controlLibStubUrl = moduleDataUrl(` +${validateSecretKeyStubSource} +export const workersIndexKey = (ns) => \`workers:\${ns}\`; +`); const src = applyModuleReplacements(readRepositoryFile("control/handlers/ns-secrets.js"), [ [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(controlLibStubUrl)};`], [/from "control-handlers-secret-put";/, `from ${JSON.stringify(secretPutUrl(controlSharedUrl, controlLibStubUrl))};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "control-env-budget";/, `from ${JSON.stringify(envBudgetUrl())};`], + [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], ]); const { handle } = await import(moduleDataUrl(src)); @@ -125,6 +278,295 @@ test("namespace secret PUT accepts lowercase secret keys like production", async assert.equal(state.redis.writes.at(-1).field, "lowercase"); }); +test("namespace secret PUT runs as a WATCH/MULTI mutation and retries contention", async () => { + const { state } = await import(controlSharedUrl); + const writesBefore = state.redis.writes.length; + const execBefore = state.redis.execCalls; + state.redis.execFailures = 1; + state.redis.watchedKeys = []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/RETRY_TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "RETRY_TOKEN", + requestId: "rid-secret-retry", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.execCalls - execBefore, 2); + assert.equal(state.redis.writes.length - writesBefore, 2); + assert.equal(state.redis.writes.at(-1).field, "RETRY_TOKEN"); + assert.ok(state.redis.watchedKeys.includes("secrets:demo")); + assert.ok(state.redis.watchedKeys.includes("routes:demo")); + assert.ok(state.redis.watchedKeys.includes("workers:demo")); + } finally { + state.redis.execFailures = 0; + } +}); + +test("namespace secret PUT checks retained worker versions before storing", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGet: state.redis.hGet, + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + zRange: state.redis.zRange, + }; + const writesBefore = state.redis.writes.length; + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? ["api"] : []; + /** @param {string} key */ + state.redis.zRange = async (key) => key === "worker-versions:demo:api" ? ["v1"] : []; + /** @param {string} key @param {string} field */ + state.redis.hGet = async (key, field) => { + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { BIG: "x".repeat(1024 * 1024) } }); + } + return null; + }; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-retained", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(state.redis.writes.length, writesBefore); + } finally { + Object.assign(state.redis, original); + } +}); + +test("namespace secret DELETE checks env revealed by removing a namespace secret", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGet: state.redis.hGet, + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + zRange: state.redis.zRange, + }; + const deletesBefore = state.redis.deletes.length; + const encrypted = await encryptSecretValue("small", { + env, + hashKey: "secrets:demo", + fieldName: "TOKEN", + }); + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: encrypted }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? ["api"] : []; + /** @param {string} key */ + state.redis.zRange = async (key) => key === "worker-versions:demo:api" ? ["v1"] : []; + /** @param {string} key @param {string} field */ + state.redis.hGet = async (key, field) => { + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { TOKEN: "x".repeat(1024 * 1024) } }); + } + return null; + }; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-budget", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(state.redis.deletes.length, deletesBefore); + } finally { + Object.assign(state.redis, original); + } +}); + +test("namespace secret DELETE skips decrypting the removed corrupt envelope", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + }; + const deletesBefore = state.redis.deletes.length; + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: "WDL-ENC:not-json" }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? [] : []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.deletes.length, deletesBefore + 1); + assert.deepEqual(state.redis.deletes.at(-1), { key: "secrets:demo", field: "TOKEN" }); + } finally { + Object.assign(state.redis, original); + } +}); + +test("namespace secret DELETE skips other corrupt namespace envelopes for repair", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + }; + const deletesBefore = state.redis.deletes.length; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo", + fieldName: "TOKEN", + }); + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: encrypted, BAD: "WDL-ENC:not-json" }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? [] : []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-other-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.deletes.length, deletesBefore + 1); + assert.deepEqual(state.redis.deletes.at(-1), { key: "secrets:demo", field: "TOKEN" }); + } finally { + Object.assign(state.redis, original); + } +}); + +test("namespace secret DELETE skips corrupt worker envelopes for repair", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + zRange: state.redis.zRange, + }; + const deletesBefore = state.redis.deletes.length; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo", + fieldName: "TOKEN", + }); + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { TOKEN: encrypted }; + if (key === "routes:demo") return {}; + if (key === "secrets:demo:api") return { BAD: "WDL-ENC:not-json" }; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? ["api"] : []; + state.redis.zRange = async () => []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-delete-worker-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(state.redis.deletes.length, deletesBefore + 1); + assert.deepEqual(state.redis.deletes.at(-1), { key: "secrets:demo", field: "TOKEN" }); + } finally { + Object.assign(state.redis, original); + } +}); + +test("namespace secret PUT still fails closed on other corrupt namespace envelopes", async () => { + const { state } = await import(controlSharedUrl); + const original = { + hGetAll: state.redis.hGetAll, + sMembers: state.redis.sMembers, + }; + const writesBefore = state.redis.writes.length; + /** @param {string} key */ + state.redis.hGetAll = async (key) => { + if (key === "secrets:demo") return { BAD: "WDL-ENC:not-json" }; + if (key === "routes:demo") return {}; + return {}; + }; + /** @param {string} key */ + state.redis.sMembers = async (key) => key === "workers:demo" ? [] : []; + + try { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-put-other-corrupt", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "invalid_envelope"); + assert.equal(state.redis.writes.length, writesBefore); + } finally { + Object.assign(state.redis, original); + } +}); + const workerControlSharedUrl = controlSharedStubUrl(` class WatchError extends Error {} export function formatError(err) { @@ -134,15 +576,24 @@ export const state = { log() {}, redis: { execCalls: 0, + watchedKeys: [], writes: [], async session(fn) { return await fn({ - async watch() {}, + async watch(...keys) { + state.redis.watchedKeys.push(...keys); + }, async unwatch() {}, async get() { return null; }, async hKeys() { return []; }, - async hGet() { return null; }, + async hGet(key, field) { + return /^worker:[^:]+:[^:]+:v:[1-9][0-9]*$/.test(key) && field === "__meta__" + ? "{}" + : null; + }, + async hGetAll() { return {}; }, async zCard() { return 0; }, + async zRange() { return []; }, multi() { return { hSet(key, field, value) { @@ -167,6 +618,7 @@ ${validateSecretKeyStubSource} export const deleteLockKey = (ns, worker) => \`worker-delete-lock:\${ns}:\${worker}\`; export const workerVersionsKey = (ns, worker) => \`worker-versions:\${ns}:\${worker}\`; export const routesKey = (ns) => \`routes:\${ns}\`; +export const workersIndexKey = (ns) => \`workers:\${ns}\`; `); const lifecycleStubUrl = moduleDataUrl(` export function stageWorkerHidden() {} @@ -176,18 +628,39 @@ export function stageWorkerVisible(multi, ns, name) { `); const routingStubUrl = moduleDataUrl(` export class RoutingError extends Error {} -export async function bumpActiveAndPromote() { - return { previousVersion: "v1", version: "v2" }; +export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) { + return await redis.session(async (iso) => { + const currentVersion = await iso.hGet(\`routes:\${ns}\`, workerName) || "v1"; + const currentNumber = Number(/^v(\\d+)$/.exec(currentVersion)?.[1] || 1); + const newVersion = \`v\${currentNumber + 1}\`; + await options.beforeStageCopy?.({ + iso, + currentVersion, + newVersion, + sourceMeta: {}, + }); + return { previousVersion: currentVersion, version: newVersion }; + }); } `); +const workerSecretPutUrl = secretPutUrl(workerControlSharedUrl, workerLibStubUrl); const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/worker-secrets.js"), [ [/from "control-shared";/, `from ${JSON.stringify(workerControlSharedUrl)};`], [/from "control-lib";/, `from ${JSON.stringify(workerLibStubUrl)};`], - [/from "control-handlers-secret-put";/, `from ${JSON.stringify(secretPutUrl(workerControlSharedUrl, workerLibStubUrl))};`], + [/from "control-handlers-secret-put";/, `from ${JSON.stringify(workerSecretPutUrl)};`], [/from "control-lifecycle-indexes";/, `from ${JSON.stringify(lifecycleStubUrl)};`], [/from "control-routing";/, `from ${JSON.stringify(routingStubUrl)};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], + [/from "control-env-budget";/, `from ${JSON.stringify(envBudgetUrl())};`], + [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], ]); const { handle: workerHandle } = await import(moduleDataUrl(workerSrc)); +const { + WORKER_LOADER_ENV_MAX_BYTES, + WORKER_LOADER_ENV_VERSION_PLACEHOLDER, + estimatedWorkerLoaderEnv, + estimatedWorkerLoaderEnvBytes, +} = await import(envBudgetUrl()); test("worker secret PUT encrypts before WATCH retries and reuses the envelope", async () => { const response = await workerHandle({ @@ -206,6 +679,7 @@ test("worker secret PUT encrypts before WATCH retries and reuses the envelope", assert.equal(response.status, 200); const { state } = await import(workerControlSharedUrl); assert.equal(state.redis.execCalls, 2); + assert.ok(state.redis.watchedKeys.includes("secrets:demo")); assert.equal(state.redis.writes.length, 2); assert.equal(state.redis.writes[0].key, "secrets:demo:api"); assert.equal(state.redis.writes[0].field, "TOKEN"); @@ -242,3 +716,432 @@ test("worker secret mutation rejects invalid keys through shared validator", asy assert.equal(body.error, "invalid_request"); assert.equal(state.redis.writes.length, writesBefore); }); + +test("worker secret DELETE checks env revealed by removing a higher-precedence secret", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const encrypted = await encryptSecretValue("small", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let execCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["TOKEN"]; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") return "v1"; + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { TOKEN: "x".repeat(1024 * 1024) } }); + } + return null; + }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo:api") return { TOKEN: encrypted }; + return {}; + }, + async zCard() { return 1; }, + async zRange() { return ["v1"]; }, + multi() { + return { + hSet() {}, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-budget", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(execCalled, false); + } finally { + state.redis.session = originalSession; + } +}); + +test("worker secret PUT budgets the copied active bundle under a future version string", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const baseMeta = { + vars: { PAD: "" }, + workflows: [{ + binding: "FLOW", + name: "flow", + className: "Flow", + workflowKey: "wf_0123456789abcdef0123456789abcdef", + }], + }; + /** @param {number} padLength @param {string} version */ + const bytesWithPad = (padLength, version) => estimatedWorkerLoaderEnvBytes(estimatedWorkerLoaderEnv({ + ns: "demo", + worker: "api", + version, + vars: { PAD: "x".repeat(padLength) }, + workerSecrets: { TOKEN: "plain-secret" }, + meta: baseMeta, + })); + const padLength = WORKER_LOADER_ENV_MAX_BYTES - + bytesWithPad(0, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) + + 1; + assert.ok(bytesWithPad(padLength, "v1") <= WORKER_LOADER_ENV_MAX_BYTES); + assert.ok(bytesWithPad(padLength, WORKER_LOADER_ENV_VERSION_PLACEHOLDER) > WORKER_LOADER_ENV_MAX_BYTES); + let execCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return []; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") return "v1"; + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ + ...baseMeta, + vars: { PAD: "x".repeat(padLength) }, + }); + } + return null; + }, + async hGetAll() { return {}; }, + async zCard() { return 1; }, + async zRange() { return []; }, + multi() { + return { + hSet() {}, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-future-version-budget", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "worker_env_too_large"); + assert.equal(execCalled, false); + } finally { + state.redis.session = originalSession; + } +}); + +test("worker secret PUT rechecks the active bundle copied by bump after the secret write", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const writesBefore = state.redis.writes.length; + let sessionCalls = 0; + let mutateExecCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => { + sessionCalls += 1; + return await fn({ + /** @param {...string} keys */ + async watch(...keys) { + state.redis.watchedKeys.push(...keys); + }, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return []; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") { + return sessionCalls === 1 ? "v1" : "v2"; + } + if (key === "worker:demo:api:v:1" && field === "__meta__") { + return JSON.stringify({ vars: { SAFE: "ok" } }); + } + if (key === "worker:demo:api:v:2" && field === "__meta__") { + return JSON.stringify({ vars: { BIG: "x".repeat(WORKER_LOADER_ENV_MAX_BYTES + 1) } }); + } + return null; + }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo:api") { + const written = state.redis.writes.at(-1); + return written?.key === key ? { [written.field]: written.value } : {}; + } + return {}; + }, + async zCard() { return 1; }, + async zRange() { return []; }, + multi() { + return { + /** @param {string} key @param {string} field @param {string} value */ + hSet(key, field, value) { + state.redis.writes.push({ key, field, value }); + }, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { mutateExecCalled = true; }, + }; + }, + }); + }; + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-bump-recheck", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(mutateExecCalled, true); + assert.equal(state.redis.writes.length, writesBefore + 1); + assert.equal(body.secretWritten, true); + assert.equal(body.reloadForced, false); + assert.equal(body.warnings[0].kind, "promote_failed"); + assert.match(body.warnings[0].reason, /workerLoader env/); + assert.ok(state.redis.watchedKeys.includes("secrets:demo")); + assert.ok(state.redis.watchedKeys.includes("secrets:demo:api")); + } finally { + state.redis.session = originalSession; + } +}); + +test("worker secret DELETE skips decrypting the removed corrupt envelope", async () => { + const { state } = await import(workerControlSharedUrl); + let execCalled = false; + /** @type {string | null} */ + let deletedField = null; + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["TOKEN"], + hGetAll(key) { + if (key === "secrets:demo:api") return { TOKEN: "WDL-ENC:not-json" }; + return emptySecretHash(); + }, + onHDel(_key, field) { deletedField = field; }, + onExec() { execCalled = true; }, + }), async () => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(execCalled, true); + assert.equal(deletedField, "TOKEN"); + }); +}); + +test("worker secret DELETE skips other corrupt worker envelopes for repair", async () => { + const { state } = await import(workerControlSharedUrl); + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let execCalled = false; + /** @type {string | null} */ + let deletedField = null; + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["TOKEN", "BAD"], + hGetAll(key) { + if (key === "secrets:demo:api") return { TOKEN: encrypted, BAD: "WDL-ENC:not-json" }; + return emptySecretHash(); + }, + onHDel(_key, field) { deletedField = field; }, + onExec() { execCalled = true; }, + }), async () => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-other-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(execCalled, true); + assert.equal(deletedField, "TOKEN"); + }); +}); + +test("worker secret DELETE skips corrupt namespace envelopes for repair", async () => { + const { state } = await import(workerControlSharedUrl); + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let execCalled = false; + /** @type {string | null} */ + let deletedField = null; + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["TOKEN"], + hGetAll(key) { + if (key === "secrets:demo") return { BAD: "WDL-ENC:not-json" }; + if (key === "secrets:demo:api") return { TOKEN: encrypted }; + return emptySecretHash(); + }, + onHDel(_key, field) { deletedField = field; }, + onExec() { execCalled = true; }, + }), async () => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-ns-corrupt", + }); + + assert.equal(response.status, 200); + assert.equal(execCalled, true); + assert.equal(deletedField, "TOKEN"); + }); +}); + +test("worker secret DELETE skips corrupt namespace envelopes during bump repair", async () => { + const { state } = await import(workerControlSharedUrl); + const originalSession = state.redis.session; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + let sessionCalls = 0; + let execCalled = false; + /** @param {(session: unknown) => Promise} fn */ + state.redis.session = async (fn) => { + sessionCalls += 1; + return await fn({ + async watch() {}, + async unwatch() {}, + async get() { return null; }, + async hKeys() { return ["TOKEN"]; }, + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:demo" && field === "api") return "v1"; + if (key === "worker:demo:api:v:1" && field === "__meta__") return JSON.stringify({ vars: { SAFE: "ok" } }); + return null; + }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "secrets:demo") return { BAD: "WDL-ENC:not-json" }; + if (key === "secrets:demo:api" && sessionCalls === 1) return { TOKEN: encrypted }; + return {}; + }, + async zCard() { return 1; }, + async zRange() { return []; }, + multi() { + return { + hSet() {}, + hDel() {}, + sAdd() {}, + sRem() {}, + async exec() { execCalled = true; }, + }; + }, + }); + }; + + try { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-delete-bump-ns-corrupt", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(execCalled, true); + assert.equal(body.deleted, true); + assert.equal(body.previousVersion, "v1"); + assert.equal(body.version, "v2"); + } finally { + state.redis.session = originalSession; + } +}); + +test("worker secret PUT still fails closed on other corrupt worker envelopes", async () => { + const { state } = await import(workerControlSharedUrl); + let hSetCalled = false; + await withWorkerSecretSession(state, makeWorkerSecretSession({ + hKeys: ["BAD"], + hGetAll(key) { + if (key === "secrets:demo:api") return { BAD: "WDL-ENC:not-json" }; + return emptySecretHash(); + }, + onHSet() { hSetCalled = true; }, + }), async () => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-put-other-corrupt", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "invalid_envelope"); + assert.equal(hSetCalled, false); + }); +}); diff --git a/tests/unit/do-alarm-shim.test.js b/tests/unit/do-alarm-shim.test.js index 00692b9..8f43e0a 100644 --- a/tests/unit/do-alarm-shim.test.js +++ b/tests/unit/do-alarm-shim.test.js @@ -446,6 +446,42 @@ test("DO alarm shim: deleteAll clears alarm row and cancels backend schedule by assert.equal(kv.size, 0); }); +test("DO alarm shim: deleteAll skips _cf_ reserved SQL objects case-insensitively", async () => { + /** @type {string[]} */ + const dropped = []; + const storage = { + sql: { + /** @param {string} statement */ + exec(statement) { + if (statement.startsWith("CREATE TABLE")) return []; + if (statement.startsWith("SELECT scheduled_time")) return []; + if (statement.startsWith("SELECT type, name FROM sqlite_master")) { + return [ + { type: "table", name: "_CF_legacy" }, + { type: "index", name: "_Cf_legacy_idx" }, + { type: "table", name: "tenant_table" }, + ]; + } + if (statement.startsWith("PRAGMA foreign_keys")) return []; + if (statement.startsWith("DROP ")) { + dropped.push(statement); + return []; + } + throw new Error(`unexpected SQL: ${statement}`); + }, + }, + async list() { + return new Map(); + }, + async delete() {}, + }; + const wrapped = wrapStorage(storage, makeDoAlarmBinding([]), "Room", "alice"); + + await wrapped.deleteAll(); + + assert.deepEqual(dropped, ['DROP TABLE IF EXISTS "tenant_table"']); +}); + test("DO alarm shim: deleteAll deleteAlarm false preserves alarm row without backend cancel", async () => { /** @type {unknown[][]} */ const calls = []; diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 196c9bf..60790cf 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -236,7 +236,23 @@ test("normalizes inline workerCode only for test hooks", () => { ); const invoke = normalizeDoInvokeRequest(INLINE_BODY, { allowInlineWorkerCode: true }); assert.equal(invoke.hostId, CHAT_ROOM_HOST_ID); - assert.equal("workerCode" in invoke ? invoke.workerCode.allowExperimental : undefined, true); + assert.equal("workerCode" in invoke ? "allowExperimental" in invoke.workerCode : undefined, false); +}); + +test("rejects experimental workerd compatibility flags in inline workerCode", () => { + assert.throws( + () => normalizeDoInvokeRequest({ + ...INLINE_BODY, + workerCode: { + ...INLINE_BODY.workerCode, + compatibilityFlags: ["nodejs_compat", "unsafe_module"], + }, + }, { allowInlineWorkerCode: true }), + (err) => err instanceof DoRuntimeError && + err.status === 400 && + err.code === "experimental_compat_flag_unsupported" && + /"unsafe_module"/.test(err.message) + ); }); test("builds forwarded Request for user durable object fetch", async () => { diff --git a/tests/unit/redis-session.test.js b/tests/unit/redis-session.test.js index 32322b8..113a46f 100644 --- a/tests/unit/redis-session.test.js +++ b/tests/unit/redis-session.test.js @@ -532,6 +532,54 @@ test("RedisSession selects configured DB on its held socket", async () => { assert.equal(decode(socket._writes[1]), "*3\r\n$4\r\nHGET\r\n$1\r\nk\r\n$1\r\nf\r\n"); }); +test("RedisSession reports resources while SELECT is still pending", async () => { + let releaseSelect = () => {}; + const writer = { + /** @param {Uint8Array} _buf */ + async write(_buf) {}, + close() { writer.closed = true; }, + /** @type {boolean} */ + closed: false, + }; + const reader = { + read() { + return new Promise((resolve, reject) => { + releaseSelect = () => { + if (reader.released) { + reject(new Error("reader released")); + } else { + resolve({ done: false, value: bytes("+OK\r\n") }); + } + }; + }); + }, + releaseLock() { reader.released = true; }, + /** @type {boolean} */ + released: false, + }; + const socket = { + writable: { getWriter: () => writer }, + readable: { getReader: () => reader }, + close() { socket.closed = true; }, + /** @type {boolean} */ + closed: false, + }; + const { connect } = scriptedConnect(socket); + const session = new RedisSession("x", { connect, db: 1 }); + + assert.equal(session.hasOpenResources(), false); + const openPromise = session.open(); + assert.equal(session.hasOpenResources(), true); + await Promise.resolve(); + await session.close(); + assert.equal(session.hasOpenResources(), false); + assert.equal(writer.closed, true); + assert.equal(reader.released, true); + assert.equal(socket.closed, true); + releaseSelect(); + await assert.rejects(openPromise, /reader released/); +}); + test("RedisClient.set supports Valkey IFEQ and delIfEq", async () => { const socketA = makeFakeSocket([bytes("+OK\r\n")]); const socketB = makeFakeSocket([bytes(":1\r\n")]); @@ -616,7 +664,9 @@ test("RedisSession.close is idempotent and blocks further commands", async () => // close() twice and assert the post-close command throws. const s = new RedisSession("x", { connect }); await s.open(); + assert.equal(s.hasOpenResources(), true); await s.close(); + assert.equal(s.hasOpenResources(), false); await s.close(); // idempotent — no throw await assert.rejects(() => s.get("k"), /session closed/); }); diff --git a/tests/unit/runtime-kv-binding.test.js b/tests/unit/runtime-kv-binding.test.js index 53c1c7a..3d1bc68 100644 --- a/tests/unit/runtime-kv-binding.test.js +++ b/tests/unit/runtime-kv-binding.test.js @@ -1,6 +1,10 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { + importRepositoryModule, + repositoryFileUrl, + runtimeLibModuleDataUrl, +} from "../helpers/load-shared-module.js"; import { installMockFetch, makeRecordingFetch } from "../helpers/mock-fetch.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { RUNTIME_METRICS_NOOP_URL } from "../helpers/mocks/runtime-metrics.js"; @@ -8,7 +12,7 @@ import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); -const RUNTIME_LIB_URL = repositoryFileUrl("runtime/lib.js"); +const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); /** @param {Array<[RegExp | string, string]>} [replacements] */ diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 6d2d4be..76b96bb 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -1,7 +1,9 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseBase64Json } from "../helpers/json-payload.js"; -import { +import { runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; + +const { toBytes, bundleToWorkerCode, buildAssetUrl, @@ -13,7 +15,7 @@ import { normalizeQueueDelaySeconds, normalizeQueuedDispatchBody, normalizeScheduledDispatchBody, -} from "../../runtime/lib.js"; +} = await import(runtimeLibModuleDataUrl()); const enc = new TextEncoder(); @@ -241,7 +243,7 @@ function mkBundle(meta, files) { return out; } -test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => { +test("bundleToWorkerCode: module + data + wasm + json + text + cjs", () => { const meta = { mainModule: "worker.js", modules: { @@ -251,7 +253,6 @@ test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => "config.json": { type: "json" }, "readme.txt": { type: "text" }, "commonjs.cjs": { type: "cjs" }, - "mod.py": { type: "py" }, }, }; const code = bundleToWorkerCode( @@ -262,7 +263,6 @@ test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => "config.json": enc.encode('{"k":1}'), "readme.txt": enc.encode("hello"), "commonjs.cjs": enc.encode("module.exports = 1"), - "mod.py": enc.encode("x = 1"), }) ); assert.equal(code.mainModule, "worker.js"); @@ -273,13 +273,23 @@ test("bundleToWorkerCode: module + data + wasm + json + text + cjs + py", () => const jsonModule = /** @type {{ json: unknown }} */ (code.modules["config.json"]); const textModule = /** @type {{ text: string }} */ (code.modules["readme.txt"]); const cjsModule = /** @type {{ cjs: string }} */ (code.modules["commonjs.cjs"]); - const pyModule = /** @type {{ py: string }} */ (code.modules["mod.py"]); assert.deepEqual(Array.from(dataModule.data), [1, 2]); assert.deepEqual(Array.from(wasmModule.wasm), [0, 97, 115, 109]); assert.deepEqual(jsonModule.json, { k: 1 }); assert.equal(textModule.text, "hello"); assert.equal(cjsModule.cjs, "module.exports = 1"); - assert.equal(pyModule.py, "x = 1"); +}); + +test("bundleToWorkerCode: py modules fail closed with WDL error", () => { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { mainModule: "worker.js", modules: { "worker.js": { type: "module" }, "mod.py": { type: "py" } } }, + { "worker.js": enc.encode("export default {}"), "mod.py": enc.encode("x = 1") } + ) + ), + /Module "mod\.py": Python Workers modules are not supported by WDL/ + ); }); test("bundleToWorkerCode: uses meta.compatibilityDate when set", () => { @@ -310,7 +320,7 @@ test("bundleToWorkerCode: compatibilityFlags merge user-declared with old-date p { "w.js": enc.encode("x") } ) ); - assert.deepEqual(code.compatibilityFlags, ["nodejs_compat", "experimental", "enhanced_error_serialization"]); + assert.deepEqual(code.compatibilityFlags, ["nodejs_compat", "enhanced_error_serialization"]); assert.deepEqual(/** @type {any} */ (code.meta.bindings), { KV: { type: "kv", id: "x" } }); assert.deepEqual(code.meta.vars, { G: "hi" }); assert.ok(Object.isFrozen(code.meta), "meta must be frozen"); @@ -327,7 +337,7 @@ test("bundleToWorkerCode: default compatibilityDate does not inject defaulted pl ) ); assert.equal(code.compatibilityDate, "2026-04-24"); - assert.deepEqual(code.compatibilityFlags, ["experimental"]); + assert.deepEqual(code.compatibilityFlags, []); }); test("bundleToWorkerCode: old compatibilityDate absent flags → floor only", () => { @@ -341,7 +351,7 @@ test("bundleToWorkerCode: old compatibilityDate absent flags → floor only", () { "w.js": enc.encode("x") } ) ); - assert.deepEqual(code.compatibilityFlags, ["experimental", "enhanced_error_serialization"]); + assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization"]); }); test("bundleToWorkerCode: compatibilityFlags already includes floor → no dup", () => { @@ -356,7 +366,7 @@ test("bundleToWorkerCode: compatibilityFlags already includes floor → no dup", { "w.js": enc.encode("x") } ) ); - assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization", "nodejs_compat", "experimental"]); + assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization", "nodejs_compat"]); }); test("bundleToWorkerCode: throws (doesn't silently drop) on malformed compatibilityFlags in bundle bytes", () => { @@ -388,6 +398,22 @@ test("bundleToWorkerCode: throws (doesn't silently drop) on malformed compatibil ); }); +test("bundleToWorkerCode: experimental workerd compatibility flags fail closed", () => { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { + mainModule: "w.js", + compatibilityFlags: ["nodejs_compat", "unsafe_module"], + modules: { "w.js": { type: "module" } }, + }, + { "w.js": enc.encode("x") } + ) + ), + /meta\.compatibilityFlags contains experimental workerd flag "unsafe_module"/ + ); +}); + test("bundleToWorkerCode: missing __meta__ throws", () => { assert.throws( () => bundleToWorkerCode({ "worker.js": enc.encode("x") }), diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index 99dd168..1281199 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -96,6 +96,10 @@ const src = readRepositoryModuleSource("runtime/load.js", [ /import WORKFLOWS_CLIENT_SOURCE from "runtime-workflows-client-source";/, 'const WORKFLOWS_CLIENT_SOURCE = "export class Workflow { constructor(metadata) { this.metadata = metadata; } }";' ], + [ + /from "runtime-load-code-budget";/, + 'from "./load/code-budget.js";' + ], [ /from "runtime-load-module-rewrite";/, 'from "./load/module-rewrite.js";' @@ -118,10 +122,12 @@ const ENV_BUILD_SOURCE = readRepositoryModuleSource("runtime/load/env-build.js", })); mkdirSync(LOAD_TEST_SUBMODULE_DIR, { recursive: true }); writeFileSync(path.join(LOAD_TEST_RUNTIME_DIR, "load.js"), src); -for (const name of ["env-build.js", "module-rewrite.js", "wrapper-generate.js"]) { +for (const name of ["env-build.js", "code-budget.js", "module-rewrite.js", "wrapper-generate.js"]) { const moduleSource = name === "env-build.js" ? ENV_BUILD_SOURCE : readRepositoryModuleSource(`runtime/load/${name}`, importSpecifierReplacements({ + "runtime-load-module-rewrite": pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "module-rewrite.js")).href, + "runtime-load-wrapper-generate": pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "wrapper-generate.js")).href, "shared-ns-pattern": SHARED_NS_PATTERN_URL, })); writeFileSync( @@ -132,6 +138,7 @@ for (const name of ["env-build.js", "module-rewrite.js", "wrapper-generate.js"]) after(() => removeTempDir(LOAD_TEST_DIR)); const mod = await import(pathToFileURL(path.join(LOAD_TEST_RUNTIME_DIR, "load.js")).href); +const codeBudgetMod = await import(pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "code-budget.js")).href); const { buildWorkerEnv, createLoaderCallback, @@ -139,6 +146,10 @@ const { runtimeLoadContentTypeMatches, wrapWorkerCodeForHostBindings, } = mod; +const { + estimateFinalWorkerLoaderCodeBytes, + injectRuntimeModulesForHostBindings, +} = codeBudgetMod; const RUNTIME_LOAD_MAGIC = "WDLLOAD!"; const RUNTIME_LOAD_CONTENT_TYPE = "application/vnd.wdl.runtime-load"; @@ -1062,7 +1073,7 @@ test("createLoaderCallback: attaches configured tail worker and always wraps mai assert.equal(/** @type {any} */ (workerCode.modules)["worker.js"], "export default {};"); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class __WdlAbort__/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \{ WorkerEntrypoint, abortIsolate \} from "cloudflare:workers"/); - assert.equal(workerCode.allowExperimental, true); + assert.equal("allowExperimental" in workerCode, false); assert.deepEqual(workerCode.tails, [{ kind: "tail-fetcher" }]); assert.deepEqual(workerCode.globalOutbound, { kind: "public-network" }); assert.ok(!("meta" in workerCode), "loader callback should not propagate meta into the workerLoader object"); @@ -1375,6 +1386,66 @@ test("createLoaderCallback: aborts hung redis proxy runtime load requests", asyn ); }); +const TEST_RUNTIME_INJECTION_SOURCES = Object.freeze({ + d1ClientSource: "export class D1Database {}", + d1DataFieldSource: "export function setDataField() {}", + d1ParamsSource: "export function normalizeD1Param(value) { return value; }", + sqlSplitterSource: "export function splitSqlStatements(sql) { return [{ sql }]; }", + d1TransportSource: 'import { setDataField } from "shared-d1-data-field"; export function decode() { setDataField(); }', + r2ClientSource: "export class R2Bucket {}", + r2UtilsSource: "export const R2_OBJECT_MAX_BUFFER_BYTES = 1;", + doClientSource: "export class DurableObjectNamespace {}", + doTransportSource: "export function requestSpec() {}", + ownerEndpointSource: "export function validOwnerEndpointForService() { return true; }", + ownerHintCacheSource: "export function createOwnerHintCache() { return {}; }", + requestIdSource: "export function requestIdFromOptions() { return null; }", + workflowsClientSource: "export class Workflow {}", +}); + +/** @param {{ modules: Record }} workerCode */ +function workerCodeModuleBytes(workerCode) { + let total = 0; + for (const value of Object.values(workerCode.modules)) { + assert.equal(typeof value, "string"); + total += Buffer.byteLength(/** @type {string} */ (value), "utf8"); + } + return total; +} + +test("workerLoader code estimator matches runtime wrapper injection exactly", () => { + const entrypoint = `Api${"A".repeat(1024)}`; + const source = 'import { WorkflowEntrypoint } from "cloudflare:workflows"; export default {};'; + const meta = { + mainModule: "src/worker.js", + modules: { "src/worker.js": { type: "module" } }, + bindings: { DB: { type: "d1", databaseId: "main" } }, + exports: [{ entrypoint }], + workflows: [{ binding: "FLOW", name: "flow", className: "FlowHandler" }], + }; + /** @type {{ mainModule: string, modules: Record }} */ + const workerCode = { + mainModule: meta.mainModule, + modules: { "src/worker.js": source }, + }; + + injectRuntimeModulesForHostBindings(workerCode, meta, TEST_RUNTIME_INJECTION_SOURCES); + + assert.equal( + estimateFinalWorkerLoaderCodeBytes({ + mainModule: meta.mainModule, + normalized: [["src/worker.js", Buffer.from(source)]], + meta, + runtimeSources: TEST_RUNTIME_INJECTION_SOURCES, + }), + workerCodeModuleBytes(workerCode) + ); + assert.match( + /** @type {string} */ (workerCode.modules["src/worker.js"]), + /"\.\.\/_wdl-cloudflare-workflows\.js"/ + ); + assert.match(/** @type {string} */ (workerCode.modules["_wdl-wrapper.js"]), new RegExp(entrypoint)); +}); + test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserves original main module", () => { const workerCode = { mainModule: "src/worker.js", diff --git a/tests/unit/runtime-queue-producer.test.js b/tests/unit/runtime-queue-producer.test.js index cbbc29e..36aadf3 100644 --- a/tests/unit/runtime-queue-producer.test.js +++ b/tests/unit/runtime-queue-producer.test.js @@ -1,6 +1,9 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { + importRepositoryModule, + runtimeLibModuleDataUrl, +} from "../helpers/load-shared-module.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { RUNTIME_METRICS_NOOP_URL } from "../helpers/mocks/runtime-metrics.js"; import { withRecordingFetch } from "../helpers/mock-fetch.js"; @@ -10,7 +13,7 @@ import { parseJsonArrayRequestBody } from "../helpers/request-body.js"; import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); -const RUNTIME_LIB_URL = repositoryFileUrl("runtime/lib.js"); +const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); const IMMEDIATE_VISIBLE_AT = 0; // Arbitrary fixed Unix epoch ms for deterministic time-based queue assertions. const MOCK_TIMESTAMP_BASE_MS = 1_700_000_000_000; diff --git a/tests/unit/scan-workerd-0701-metadata.test.js b/tests/unit/scan-workerd-0701-metadata.test.js new file mode 100644 index 0000000..fc70583 --- /dev/null +++ b/tests/unit/scan-workerd-0701-metadata.test.js @@ -0,0 +1,150 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + estimateBundleEnvBytes, + findingsForBundleMetadata, + parseRedisHash, + redisCli, + scanWorkerd0701Metadata, + secretUpperBoundStrings, +} from "../../scripts/scan-workerd-0701-metadata.mjs"; +import { WORKERD_EXPERIMENTAL_COMPAT_FLAGS } from "../../shared/workerd-compat-flags.js"; + +test("workerd experimental compat flag mirror excludes GA context reuse flags", () => { + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("experimental"), true); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unsafe_module"), true); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.length, 33); + assert.equal(WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("unique_ctx_per_invocation"), false); + assert.equal( + WORKERD_EXPERIMENTAL_COMPAT_FLAGS.includes("nonclass_entrypoint_reuses_ctx_across_invocations"), + false + ); +}); + +test("workerd 0701 scanner reports missing metadata, py modules, and experimental flags", () => { + assert.deepEqual( + findingsForBundleMetadata({ key: "worker:demo:api:v:1", rawMeta: "" }), + [{ + kind: "missing_meta", + key: "worker:demo:api:v:1", + namespace: "demo", + worker: "api", + version: "v1", + }] + ); + + const findings = findingsForBundleMetadata({ + key: "worker:demo:api:v:1", + rawMeta: JSON.stringify({ + compatibilityFlags: ["unsafe_module"], + modules: { "worker.py": { type: "py" } }, + }), + }); + assert.equal(findings.some((finding) => finding.kind === "experimental_compat_flag"), true); + assert.equal(findings.some((finding) => finding.kind === "python_worker_module"), true); +}); + +test("workerd 0701 scanner reports retained env that would exceed workerLoader budget", () => { + const encryptedEnvelope = `WDL-ENC:${"a".repeat(600_000)}`; + const { findings } = scanWorkerd0701Metadata({ + redis(args) { + if (args[0] === "--scan") return "worker:demo:api:v:1\n"; + if (args[0] === "HGET") return `${JSON.stringify({ vars: { SMALL: "ok" } })}\n`; + if (args[0] === "HGETALL" && args[1] === "secrets:demo") return `BIG\n${encryptedEnvelope}\n`; + if (args[0] === "HGETALL" && args[1] === "secrets:demo:api") return ""; + throw new Error(`unexpected redis args ${JSON.stringify(args)}`); + }, + }); + + const finding = findings.find((entry) => entry.kind === "worker_env_too_large"); + assert.ok(finding); + assert.equal(finding.namespace, "demo"); + assert.equal(finding.worker, "api"); + assert.equal(finding.version, "v1"); + assert.equal(finding.secret_value_estimate, "encrypted_envelope_length_as_two_byte_string"); +}); + +test("workerd 0701 scanner parses raw redis hashes and rejects odd replies", () => { + assert.deepEqual(Object.fromEntries(Object.entries(parseRedisHash("A\n1\nB\n2\n"))), { A: "1", B: "2" }); + assert.deepEqual(Object.fromEntries(Object.entries(parseRedisHash(""))), {}); + assert.throws(() => parseRedisHash("A\n1\nB\n"), /odd number/); +}); + +test("workerd 0701 scanner uses two-byte secret upper bounds capped at upstream max", () => { + const out = secretUpperBoundStrings({ + ASCII: "WDL-ENC:abc", + HUGE: "x".repeat(2 * 1024 * 1024), + }); + assert.equal(out.ASCII.length, "WDL-ENC:abc".length); + assert.equal(out.ASCII.charCodeAt(0), 0x100); + assert.equal(out.HUGE.length, 1024 * 1024); +}); + +test("workerd 0701 scanner estimates assets env with the supplied CDN base", () => { + const meta = { + assets: { prefix: "assets/demo/api/v1/" }, + bindings: { ASSETS: { type: "assets" } }, + }; + const identity = { namespace: "demo", worker: "api", version: "v1" }; + const shortBytes = estimateBundleEnvBytes({ identity, meta, assetsCdnBase: "https://a.invalid" }); + const longBytes = estimateBundleEnvBytes({ + identity, + meta, + assetsCdnBase: "https://very-long-assets-hostname.example.invalid", + }); + assert.ok(longBytes > shortBytes); +}); + +test("workerd 0701 scanner reports missing metadata before reading secret hashes", () => { + let secretReads = 0; + const { findings } = scanWorkerd0701Metadata({ + redis(args) { + if (args[0] === "--scan") return "worker:demo:api:v:1\n"; + if (args[0] === "HGET") return ""; + if (args[0] === "HGETALL") { + secretReads += 1; + throw new Error("secret hash should not be read for a bundle missing __meta__"); + } + throw new Error(`unexpected redis args ${JSON.stringify(args)}`); + }, + }); + + assert.equal(secretReads, 0); + assert.deepEqual(findings.map((finding) => finding.kind), ["missing_meta"]); +}); + +test("workerd 0701 scanner reports corrupt metadata before reading secret hashes", () => { + let secretReads = 0; + const { findings } = scanWorkerd0701Metadata({ + redis(args) { + if (args[0] === "--scan") return "worker:demo:api:v:1\n"; + if (args[0] === "HGET") return "not-json\n"; + if (args[0] === "HGETALL") { + secretReads += 1; + throw new Error("secret hash should not be read for corrupt __meta__"); + } + throw new Error(`unexpected redis args ${JSON.stringify(args)}`); + }, + }); + + assert.equal(secretReads, 0); + assert.deepEqual(findings.map((finding) => finding.kind), ["corrupt_meta"]); +}); + +test("workerd 0701 scanner fails clearly when redis-cli is missing", () => { + const error = Object.assign(new Error("spawn redis-cli ENOENT"), { code: "ENOENT" }); + assert.throws( + () => redisCli(["PING"], { + redisUrl: "redis://unit", + spawn() { + return /** @type {any} */ ({ + error, + status: null, + stdout: "", + stderr: "", + }); + }, + }), + /redis-cli not found/ + ); +}); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index cb1a4b1..da89433 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -1443,7 +1443,14 @@ test("local compose routes private HTTP hops through Envoy only", () => { const integrationCompose = withoutLineComments(readRepoFile("tests/integration/helpers/compose.js")); const compileConfigs = withoutLineComments(readRepoFile("scripts/compile-workerd-configs.js")); const supervisorConfig = readRepoFile("rust/supervisor/src/config.rs"); + const supervisorLib = readRepoFile("rust/supervisor/src/lib.rs"); const dockerfileWorkerd = withoutLineComments(readRepoFile("Dockerfile.workerd")); + const kubeGateway = withoutLineComments(readRepoFile("deploy/kubernetes/base/gateway.yaml")); + const kubeSystemRuntime = withoutLineComments(readRepoFile("deploy/kubernetes/base/system-runtime.yaml")); + const kubeUserRuntime = withoutLineComments(readRepoFile("deploy/kubernetes/base/user-runtime.yaml")); + const terraformGatewayService = withoutLineComments(readRepoFile("terraform/modules/compute/gateway_service.tf")); + const terraformRuntimeService = withoutLineComments(readRepoFile("terraform/modules/compute/runtime_service.tf")); + const terraformSystemRuntimeService = withoutLineComments(readRepoFile("terraform/modules/compute/system_runtime_service.tf")); const packageJson = withoutLineComments(readRepoFile("package.json")); // Local/integration should mirror production Service Connect for HTTP @@ -1501,9 +1508,28 @@ test("local compose routes private HTTP hops through Envoy only", () => { for (const tier of ["gateway-local", "user-runtime-local", "system-runtime-local"]) { assert.match( compose, - new RegExp(`serve.*workerd-configs/${RegExp.escape(tier)}\\.bin.*--experimental`), + new RegExp(`serve.*workerd-configs/${RegExp.escape(tier)}\\.bin`), ); } + // workerd 2026-07-01 still gates workerLoader bindings on the process-level + // --experimental switch. Keep it only on workerLoader-owning processes, not + // on gateway or D1 and not as a Worker compatibility flag. + for (const tier of ["user-runtime-local", "system-runtime-local"]) { + assert.match( + compose, + new RegExp(`workerd-configs/${RegExp.escape(tier)}\\.bin", "--experimental"`), + ); + } + assert.doesNotMatch(compose, /gateway-local\.bin", "--experimental"/); + assert.match(kubeUserRuntime, /user-runtime\.bin[\s\S]*?- --experimental/); + assert.match(kubeSystemRuntime, /system-runtime\.bin[\s\S]*?- --experimental/); + assert.doesNotMatch(kubeGateway, /--experimental/); + assert.match(terraformRuntimeService, /user-runtime\.bin", "--experimental"/); + assert.match(terraformSystemRuntimeService, /system-runtime\.bin", "--experimental"/); + assert.doesNotMatch(terraformGatewayService, /--experimental/); + assert.match(supervisorLib, /workerd_args\(D1_COMPILED_CONFIG, false\)/); + assert.match(supervisorLib, /workerd_args\(pick_do_compiled_config\(\), true\)/); + assert.match(supervisorConfig, /args\.push\("--experimental"\.into\(\)\)/); // The supervisor config owns the local/production .bin choice; only it // references do-runtime-local.bin since compose forwards via the // supervisor binary, not via raw workerd serve. @@ -1859,6 +1885,39 @@ test("Terraform gates DO test hooks like D1 test hooks", () => { assert.match(doService, /!var\.do_test_hooks_enabled \|\| can\(regex\("\(\^\|-\)test\(\$\|-\)", var\.name\)\)/); }); +test("D1 and DO workerd containers keep explicit memory ceilings", () => { + const rootVars = readRepoFile("terraform/variables.tf"); + const moduleVars = readRepoFile("terraform/modules/compute/variables.tf"); + const main = readRepoFile("terraform/main.tf"); + const locals = readRepoFile("terraform/modules/compute/locals.tf"); + const d1Service = readRepoFile("terraform/modules/compute/d1_runtime_service.tf"); + const doService = readRepoFile("terraform/modules/compute/do_runtime_service.tf"); + const d1Kube = readRepoFile("deploy/kubernetes/base/d1-runtime.yaml"); + const doKube = readRepoFile("deploy/kubernetes/base/do-runtime.yaml"); + + for (const name of ["d1_runtime_container_memory", "do_runtime_container_memory"]) { + assert.match(rootVars, new RegExp(`variable "${name}"`)); + assert.match(moduleVars, new RegExp(`variable "${name}"`)); + assert.match(main, new RegExp(`${name}\\s+=\\s+var\\.${name}`)); + } + assert.match( + locals, + /d1_runtime_container_memory\s+=\s+coalesce\(\s*var\.d1_runtime_container_memory,\s*var\.runtime_memory,\s*\)/ + ); + assert.match(d1Service, /memory\s+=\s+local\.d1_runtime_container_memory/); + assert.match( + d1Service, + /local\.d1_runtime_container_memory > 0 &&\s*local\.d1_runtime_container_memory <= var\.runtime_memory/ + ); + assert.match(locals, /do_redis_proxy_memory_headroom\s+=\s+128/); + assert.match(locals, /do_runtime_container_memory\s+=\s+coalesce\(\s*var\.do_runtime_container_memory,\s*var\.runtime_memory - local\.do_redis_proxy_memory_headroom,\s*\)/); + assert.match(doService, /memory\s+=\s+local\.do_runtime_container_memory/); + assert.match(doService, /local\.do_runtime_container_memory > 0 && local\.do_runtime_container_memory < var\.runtime_memory/); + assert.match(doService, /redis-proxy sidecar/); + assert.match(d1Kube, /name: d1-runtime[\s\S]*?limits:\n\s+memory: 1Gi/); + assert.match(doKube, /name: do-runtime[\s\S]*?limits:\n\s+memory: 1Gi/); +}); + test("EC2 capacity hosts block awsvpc task access to host IMDS", () => { const terraformCapacity = readRepoFile("terraform/modules/compute/ec2_capacity.tf"); diff --git a/tests/unit/workerd-compat-flags-extract.test.js b/tests/unit/workerd-compat-flags-extract.test.js new file mode 100644 index 0000000..9c42bba --- /dev/null +++ b/tests/unit/workerd-compat-flags-extract.test.js @@ -0,0 +1,20 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { extractExperimentalCompatFlags } from "../../scripts/extract-workerd-experimental-compat-flags.mjs"; + +test("workerd experimental compat flag extractor mirrors only experimental enable flags", () => { + const flags = extractExperimentalCompatFlags(` +struct CompatibilityFlags { + gaFlag @1 :Bool $compatEnableFlag("unique_ctx_per_invocation"); + experimentalFlag @2 :Bool + $experimental + $compatEnableFlag("experimental_one") + $compatDisableFlag("no_experimental_one"); + spacedExperimentalFlag @3 : Bool + $compatEnableFlag("experimental_two") + $experimental; +} +`); + assert.deepEqual(flags, ["experimental_one", "experimental_two"]); +});