From c814d48a94bd4e067729cd3187a1bb19dd36f2c9 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sun, 3 May 2026 20:46:31 +1000 Subject: [PATCH 1/5] fix(baseline): add .ts extension to update.worker import Matches the pattern used by xref and caniuse update routes. Co-Authored-By: Claude Opus 4.6 (1M context) --- routes/api/baseline/update.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/api/baseline/update.ts b/routes/api/baseline/update.ts index 887ba97a..f2029987 100644 --- a/routes/api/baseline/update.ts +++ b/routes/api/baseline/update.ts @@ -6,7 +6,7 @@ import { BackgroundTaskQueue } from "../../../utils/background-task-queue.js"; import { store } from "./lib/store-init.js"; const workerFile = path.join(import.meta.dirname, "update.worker.js"); -const taskQueue = new BackgroundTaskQueue( +const taskQueue = new BackgroundTaskQueue( workerFile, "baseline_update", ); From 74ee2412ba762537cb73eacc2744808f949987a7 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sun, 3 May 2026 20:46:51 +1000 Subject: [PATCH 2/5] feat(xref): move term autocomplete from client to server Add GET /xref/meta/terms/search?q=&limit= endpoint using binary search for prefix matches with infix fallback on a pre-sorted terms array. Case-insensitive matching, original-case results. Client no longer downloads 483KB of terms on page load. Instead, the autocomplete debounces at 150ms and fetches 15 suggestions per keystroke. The existing /xref/meta/terms endpoint is unchanged. Closes #226 Co-Authored-By: Claude Opus 4.6 (1M context) --- routes/xref/index.ts | 2 + routes/xref/terms.get.ts | 79 ++++++++++++++++++ static/xref/script.js | 14 ++-- tests/routes/xref/terms.test.js | 143 ++++++++++++++++++++++++++++++++ 4 files changed, 232 insertions(+), 6 deletions(-) create mode 100644 routes/xref/terms.get.ts create mode 100644 tests/routes/xref/terms.test.js diff --git a/routes/xref/index.ts b/routes/xref/index.ts index 9bc774a2..e605f35e 100644 --- a/routes/xref/index.ts +++ b/routes/xref/index.ts @@ -12,6 +12,7 @@ import searchRouteGet from "./search.get.js"; import searchRoutePost from "./search.post.js"; import headingsRoutePost from "./headings.post.js"; import metaRoute from "./meta.js"; +import termsRoute from "./terms.get.js"; import updateRoute from "./update.js"; import { search, Options, Query } from "./lib/search.js"; @@ -26,6 +27,7 @@ xref .options("/search", cors({ methods: ["POST", "GET"], maxAge: ms("1day") })) .get("/search", cors(), searchRouteGet) .post("/search", express.json({ limit: "2mb" }), cors(), searchRoutePost); +xref.get("/meta/terms/search", cors(), termsRoute); xref.get("/meta{/:field}", cors(), metaRoute); xref .options("/search/headings", cors({ methods: ["POST"], maxAge: ms("1day") })) diff --git a/routes/xref/terms.get.ts b/routes/xref/terms.get.ts new file mode 100644 index 00000000..5a96e6a5 --- /dev/null +++ b/routes/xref/terms.get.ts @@ -0,0 +1,79 @@ +import { Request, Response } from "express"; + +import { seconds } from "../../utils/misc.js"; +import { store } from "./lib/store-init.js"; + +interface TermEntry { + lower: string; + original: string; +} + +let termsIndex: TermEntry[] = []; +let termsVersion = -1; + +function getTermsIndex(): TermEntry[] { + if (termsVersion < store.version) { + const keys = Object.keys(store.byTerm).filter(k => k !== ""); + termsIndex = keys + .map(k => ({ lower: k.toLowerCase(), original: k })) + .sort((a, b) => (a.lower < b.lower ? -1 : a.lower > b.lower ? 1 : 0)); + termsVersion = store.version; + } + return termsIndex; +} + +function bisectLeft(arr: TermEntry[], target: string): number { + let lo = 0; + let hi = arr.length; + while (lo < hi) { + const mid = (lo + hi) >>> 1; + if (arr[mid].lower < target) lo = mid + 1; + else hi = mid; + } + return lo; +} + +function searchTerms(query: string, limit: number): string[] { + const index = getTermsIndex(); + const q = query.toLowerCase(); + const results: string[] = []; + const seen = new Set(); + + const start = bisectLeft(index, q); + for (let i = start; i < index.length && results.length < limit; i++) { + if (!index[i].lower.startsWith(q)) break; + seen.add(index[i].lower); + results.push(index[i].original); + } + + if (results.length < limit) { + for (let i = 0; i < index.length && results.length < limit; i++) { + if (seen.has(index[i].lower)) continue; + if (index[i].lower.includes(q)) { + results.push(index[i].original); + } + } + } + + return results; +} + +interface QueryParams { + q?: string; + limit?: string; +} +type IRequest = Request; + +export default function route(req: IRequest, res: Response) { + const { q } = req.query; + if (!q || q.length < 2) { + res.status(400).json({ error: "query must be at least 2 characters" }); + return; + } + + const limit = Math.min(Math.max(parseInt(req.query.limit || "15", 10) || 15, 1), 50); + const results = searchTerms(q, limit); + + res.set("Cache-Control", `max-age=${seconds("24h")}`); + res.json(results); +} diff --git a/static/xref/script.js b/static/xref/script.js index df8ee2d2..2e656f23 100644 --- a/static/xref/script.js +++ b/static/xref/script.js @@ -235,9 +235,9 @@ async function ready() { }; const metaURL = new URL( - `${form.action}/meta?fields=types,specs,terms,version`, + `${form.action}/meta?fields=types,specs,version`, ).href; - const { specs, types, terms, version } = await fetch(metaURL).then(res => + const { specs, types, version } = await fetch(metaURL).then(res => res.json(), ); @@ -260,13 +260,15 @@ async function ready() { const allTypes = [].concat(...Object.values(types)).sort(); updateInput(form.types, allTypes); - const fuse = new Fuse(terms); + const termsBaseURL = new URL(`${form.action}/meta/terms/search`, location.href).href; autocomplete({ input: form.term, + debounceWaitMs: 150, fetch(text, update) { - const searchResults = fuse.search(text).slice(0, 15); - const suggestions = searchResults.map(r => r.item); - update(suggestions); + if (text.length < 2) return update([]); + fetch(`${termsBaseURL}?q=${encodeURIComponent(text)}&limit=15`) + .then(r => r.json()) + .then(update); }, onSelect(suggestion) { this.input.value = suggestion; diff --git a/tests/routes/xref/terms.test.js b/tests/routes/xref/terms.test.js new file mode 100644 index 00000000..1f22574e --- /dev/null +++ b/tests/routes/xref/terms.test.js @@ -0,0 +1,143 @@ +import { mkdtemp, writeFile, mkdir, rm } from "fs/promises"; +import path from "path"; +import { tmpdir } from "os"; + +let tmpDir; +let route; +let origDataDir; + +const FIXTURE_XREF = { + EventTarget: [{ type: "interface", spec: "dom", uri: "#eventtarget" }], + "event target": [{ type: "dfn", spec: "dom", uri: "#concept-event-target" }], + event: [{ type: "dfn", spec: "dom", uri: "#concept-event" }], + "event handler": [{ type: "dfn", spec: "html", uri: "#event-handler" }], + "event loop": [{ type: "dfn", spec: "html", uri: "#event-loop" }], + element: [{ type: "dfn", spec: "dom", uri: "#concept-element" }], + "foreignObject": [{ type: "element", spec: "svg", uri: "#foreignObject" }], + fetch: [{ type: "dfn", spec: "fetch", uri: "#concept-fetch" }], + "fire an event": [{ type: "dfn", spec: "dom", uri: "#concept-event-fire" }], + URL: [{ type: "interface", spec: "url", uri: "#url" }], + "url": [{ type: "dfn", spec: "url", uri: "#concept-url" }], + AbortController: [{ type: "interface", spec: "dom", uri: "#abortcontroller" }], + AbortSignal: [{ type: "interface", spec: "dom", uri: "#abortsignal" }], + "abort signal": [{ type: "dfn", spec: "dom", uri: "#concept-abort-signal" }], + Node: [{ type: "interface", spec: "dom", uri: "#node" }], + navigator: [{ type: "attribute", spec: "html", uri: "#navigator" }], +}; + +beforeAll(async () => { + tmpDir = await mkdtemp(path.join(tmpdir(), "xref-terms-test-")); + const xrefDir = path.join(tmpDir, "xref"); + await mkdir(xrefDir, { recursive: true }); + await writeFile(path.join(xrefDir, "xref.json"), JSON.stringify(FIXTURE_XREF)); + await writeFile(path.join(xrefDir, "specs.json"), "{}"); + await writeFile(path.join(xrefDir, "specmap.json"), '{"current":{},"snapshot":{}}'); + await writeFile(path.join(xrefDir, "headings.json"), "{}"); + + origDataDir = process.env.DATA_DIR; + process.env.DATA_DIR = tmpDir; + + const mod = await import("../../../build/routes/xref/terms.get.js"); + route = mod.default; +}); + +afterAll(async () => { + if (origDataDir !== undefined) { + process.env.DATA_DIR = origDataDir; + } + if (tmpDir) { + await rm(tmpDir, { recursive: true, force: true }); + } +}); + +function mockReq(query = {}) { + return { query }; +} + +function mockRes() { + const res = { + _status: 200, + _body: undefined, + _headers: {}, + status(code) { res._status = code; return res; }, + json(body) { res._body = body; return res; }, + set(header, value) { res._headers[header] = value; return res; }, + send(body) { res._body = body; return res; }, + }; + return res; +} + +describe("xref/terms - server autocomplete", () => { + it("returns 400 when q is missing", () => { + const res = mockRes(); + route(mockReq(), res); + expect(res._status).toBe(400); + }); + + it("returns 400 when q is too short", () => { + const res = mockRes(); + route(mockReq({ q: "a" }), res); + expect(res._status).toBe(400); + }); + + it("returns prefix matches first", () => { + const res = mockRes(); + route(mockReq({ q: "ev" }), res); + expect(res._status).toBe(200); + expect(Array.isArray(res._body)).toBeTrue(); + expect(res._body.length).toBeGreaterThan(0); + for (const term of res._body) { + expect(term.toLowerCase()).toMatch(/ev/); + } + }); + + it("matches case-insensitively but preserves original case", () => { + const res = mockRes(); + route(mockReq({ q: "eventtarget" }), res); + expect(res._body).toContain("EventTarget"); + }); + + it("returns infix matches when prefix results are insufficient", () => { + const res = mockRes(); + route(mockReq({ q: "signal" }), res); + expect(res._body).toContain("AbortSignal"); + expect(res._body).toContain("abort signal"); + }); + + it("respects the limit parameter", () => { + const res = mockRes(); + route(mockReq({ q: "ev", limit: "3" }), res); + expect(res._body.length).toBeLessThanOrEqual(3); + }); + + it("caps limit at 50", () => { + const res = mockRes(); + route(mockReq({ q: "ev", limit: "100" }), res); + expect(res._body.length).toBeLessThanOrEqual(50); + }); + + it("defaults limit to 15", () => { + const res = mockRes(); + route(mockReq({ q: "ab" }), res); + expect(res._body.length).toBeLessThanOrEqual(15); + }); + + it("sets Cache-Control header", () => { + const res = mockRes(); + route(mockReq({ q: "ev" }), res); + expect(res._headers["Cache-Control"]).toBeDefined(); + expect(res._headers["Cache-Control"]).toMatch(/max-age=/); + }); + + it("returns empty array for no matches", () => { + const res = mockRes(); + route(mockReq({ q: "zzzzzzz" }), res); + expect(res._body).toEqual([]); + }); + + it("handles mixed case query preserving original", () => { + const res = mockRes(); + route(mockReq({ q: "ForeignObj" }), res); + expect(res._body).toContain("foreignObject"); + }); +}); From 43a5052a4da33a7727f0609ea3428ea8ecbcb6f6 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sun, 3 May 2026 20:52:14 +1000 Subject: [PATCH 3/5] fix(xref): handle fetch errors in autocomplete, test ordering Add .catch() and r.ok check to the client fetch so network errors and non-JSON responses degrade gracefully to an empty suggestion list. Add test verifying prefix matches precede infix-only matches. Co-Authored-By: Claude Opus 4.6 (1M context) --- static/xref/script.js | 5 +++-- tests/routes/xref/terms.test.js | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/static/xref/script.js b/static/xref/script.js index 2e656f23..6fe3cc94 100644 --- a/static/xref/script.js +++ b/static/xref/script.js @@ -267,8 +267,9 @@ async function ready() { fetch(text, update) { if (text.length < 2) return update([]); fetch(`${termsBaseURL}?q=${encodeURIComponent(text)}&limit=15`) - .then(r => r.json()) - .then(update); + .then(r => r.ok ? r.json() : []) + .then(update) + .catch(() => update([])); }, onSelect(suggestion) { this.input.value = suggestion; diff --git a/tests/routes/xref/terms.test.js b/tests/routes/xref/terms.test.js index 1f22574e..73696c11 100644 --- a/tests/routes/xref/terms.test.js +++ b/tests/routes/xref/terms.test.js @@ -16,6 +16,7 @@ const FIXTURE_XREF = { "foreignObject": [{ type: "element", spec: "svg", uri: "#foreignObject" }], fetch: [{ type: "dfn", spec: "fetch", uri: "#concept-fetch" }], "fire an event": [{ type: "dfn", spec: "dom", uri: "#concept-event-fire" }], + "live event": [{ type: "dfn", spec: "html", uri: "#concept-live-event" }], URL: [{ type: "interface", spec: "url", uri: "#url" }], "url": [{ type: "dfn", spec: "url", uri: "#concept-url" }], AbortController: [{ type: "interface", spec: "dom", uri: "#abortcontroller" }], @@ -80,6 +81,17 @@ describe("xref/terms - server autocomplete", () => { expect(res._status).toBe(400); }); + it("returns prefix matches before infix-only matches", () => { + const res = mockRes(); + route(mockReq({ q: "ev" }), res); + const results = res._body; + const firstInfix = results.findIndex(t => !t.toLowerCase().startsWith("ev")); + const lastPrefix = results.findLastIndex(t => t.toLowerCase().startsWith("ev")); + if (firstInfix !== -1) { + expect(lastPrefix).toBeLessThan(firstInfix); + } + }); + it("returns prefix matches first", () => { const res = mockRes(); route(mockReq({ q: "ev" }), res); From 06120d7388e70b2415c288c0bd1597d6adc360a9 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Sun, 3 May 2026 21:08:47 +1000 Subject: [PATCH 4/5] fix(xref): handle array query params, remove stale terms reference Normalize req.query.q to a string when Express parses repeated ?q= params as an array. Remove the unused terms reference from the metadata object that would throw a ReferenceError. Co-Authored-By: Claude Opus 4.6 (1M context) --- routes/xref/terms.get.ts | 4 ++-- static/xref/script.js | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/routes/xref/terms.get.ts b/routes/xref/terms.get.ts index 5a96e6a5..96859aa3 100644 --- a/routes/xref/terms.get.ts +++ b/routes/xref/terms.get.ts @@ -59,13 +59,13 @@ function searchTerms(query: string, limit: number): string[] { } interface QueryParams { - q?: string; + q?: string | string[]; limit?: string; } type IRequest = Request; export default function route(req: IRequest, res: Response) { - const { q } = req.query; + const q = Array.isArray(req.query.q) ? req.query.q[0] : req.query.q; if (!q || q.length < 2) { res.status(400).json({ error: "query must be at least 2 characters" }); return; diff --git a/static/xref/script.js b/static/xref/script.js index 6fe3cc94..2acee928 100644 --- a/static/xref/script.js +++ b/static/xref/script.js @@ -333,7 +333,6 @@ async function ready() { http: new Set(types.http), }, specs, - terms, }; } From dd311c40f48d101d7e70453e4bdd00c204c121d7 Mon Sep 17 00:00:00 2001 From: Marcos Caceres Date: Mon, 4 May 2026 21:40:10 +1000 Subject: [PATCH 5/5] test(xref): add tests for array query param normalization Express parses repeated ?q= query parameters as arrays at runtime, which would crash searchTerms() since arrays lack toLowerCase(). Add tests verifying array params are normalized to the first element. --- tests/routes/xref/terms.test.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/routes/xref/terms.test.js b/tests/routes/xref/terms.test.js index 73696c11..4e192411 100644 --- a/tests/routes/xref/terms.test.js +++ b/tests/routes/xref/terms.test.js @@ -152,4 +152,18 @@ describe("xref/terms - server autocomplete", () => { route(mockReq({ q: "ForeignObj" }), res); expect(res._body).toContain("foreignObject"); }); + + it("normalizes array query params to first element", () => { + const res = mockRes(); + route(mockReq({ q: ["event", "ignored"] }), res); + expect(res._status).toBe(200); + expect(Array.isArray(res._body)).toBeTrue(); + expect(res._body.length).toBeGreaterThan(0); + }); + + it("returns 400 when array query param has short first element", () => { + const res = mockRes(); + route(mockReq({ q: ["a", "event"] }), res); + expect(res._status).toBe(400); + }); });