From d59908f22152587d899ef522d4828d46f32e1e07 Mon Sep 17 00:00:00 2001 From: Eth-Interchained <117552056+Eth-Interchained@users.noreply.github.com> Date: Mon, 15 Jun 2026 03:11:13 -0400 Subject: [PATCH] =?UTF-8?q?feat(nql):=20extend=20mock=20parser=20=E2=80=94?= =?UTF-8?q?=20GROUP=20BY=20aggregates,=20VALID=20AS=20OF,=20TRACE=20[REVER?= =?UTF-8?q?SE]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- COMMITS.md | 11 +++- ideas.md | 29 +++------ src/lib/nql.ts | 162 +++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 173 insertions(+), 29 deletions(-) diff --git a/COMMITS.md b/COMMITS.md index 5eb5044..a0e35b2 100644 --- a/COMMITS.md +++ b/COMMITS.md @@ -1,6 +1,6 @@ # NEDB Ecosystem — Recent Commits -*Generated by NEDB Maintainer agent · 2026-06-14* +*Generated by NEDB Maintainer agent · 2026-06-15* --- @@ -8,6 +8,7 @@ | Date | SHA | Summary | |------|-----|---------| +| 2026-06-15 | `3e91fc4` | docs: add AGENTS.md — AI handoff document from Claude Sonnet 4.6 | | 2026-06-15 | `793e661` | docs: add aiassistsecure mirror repos to README | | 2026-06-15 | `d938288` | docs: full README rewrite — v1.0.4, all features, accurate architecture | | 2026-06-15 | `abc2cc7` | chore(v1.0.4): bump version for npm propagation | @@ -30,6 +31,7 @@ | Date | SHA | Summary | |------|-----|---------| +| 2026-06-15 | *(PR #2)* | feat(nql): GROUP BY aggregates + VALID AS OF + TRACE in mock parser | | 2026-06-15 | `342f359` | docs(v0.6.1): full rewrite of docs page — all v1.0.4 features | | 2026-06-15 | `57bceb1` | feat(v0.6.0): studio parity — AS OF, VALID AS OF, causal TRACE, RESP2 connect tab | | 2026-06-15 | `862bbdb` | fix(v0.5.3): use req.originalUrl in logger so full route paths show correctly | @@ -45,10 +47,13 @@ --- -## Agent PRs (this turn) +## Agent PRs | Repo | PR | Title | |------|----|-------| +| nedb-studio | [#2](https://github.com/Eth-Interchained/nedb-studio/pull/2) | feat(nql): GROUP BY aggregates + VALID AS OF + TRACE in browser mock parser | | nedb-studio | [#1](https://github.com/Eth-Interchained/nedb-studio/pull/1) | feat(nav): version compatibility banner — Studio + Engine versions in Nav | -**What was shipped:** `server.ts` now reads the nedbd `/health` body to extract `engineVersion` and includes it in `/api/health`. A new `VersionBadge` component in `Nav.tsx` fetches this once on mount and renders `v{studio} · eng {engine}` — surfacing version mismatches before they cause confusion. +**What was shipped (PR #2):** Extended `src/lib/nql.ts` to parse and execute three NQL clauses the engine has supported since v1.0.0 but the mock parser rejected with "unexpected trailing input": `VALID AS OF ""` (both positions), `TRACE [REVERSE]`, and `GROUP BY [COUNT|SUM|AVG|MIN|MAX]`. All five aggregate functions execute in-browser against seed data. 25 tests passing. `Plan` interface extended with `validAsOf`, `trace`, `traceReverse`, `groupBy`, `aggregate`. + +**What was shipped (PR #1):** `server.ts` now reads the nedbd `/health` body to extract `engineVersion` and includes it in `/api/health`. A new `VersionBadge` component in `Nav.tsx` fetches this once on mount and renders `v{studio} · eng {engine}` — surfacing version mismatches before they cause confusion. diff --git a/ideas.md b/ideas.md index 412f335..fcb7694 100644 --- a/ideas.md +++ b/ideas.md @@ -1,29 +1,14 @@ # NEDB Maintainer — Ideas for Next Turn -*Updated: 2026-06-14 · by NEDB Maintainer agent* +*Updated: 2026-06-15* --- -## Idea 1 — Files API proxy in Studio (Gap: engine has it, studio doesn't) +**Idea 1 — Files API proxy in Studio** +Expose nedbd's file storage API (`POST /databases//files`, `GET /databases//files/`, `GET /databases//files//root`) through the Studio Express server by adding three routes to `databases.ts` and matching client helpers in `nedb.ts`; add a Files tab in the Databases page with upload, download, and Merkle root display. The engine has had versioned Cascade-compressed file storage since v0.7.x, but Studio has zero proxy coverage — a complete gap that blocks the on-chain anchoring story (Merkle root → ITC/BSC). -**What:** Expose nedbd's file storage API (`POST /databases//files`, `GET /databases//files/`, `GET .../files//root`) through the Studio Express server and add a Files tab in the Databases page. +**Idea 2 — Checkpoint button in Database detail view** +Add `POST /:name/checkpoint` to `databases.ts` (proxies nedbd `POST /v1/databases/:name/checkpoint`) and a Checkpoint button in the Database detail panel that calls it and shows a toast with the returned `head` hash and `seq`. This is the only nedbd mutation route with zero proxy coverage in the Studio, and it's a one-line backend change paired with a small UI addition — high leverage, low risk. -**Why:** The engine ships a complete versioned file store with Cascade compression and per-version Merkle roots — it's one of NEDB's most distinctive capabilities. Studio users have no way to upload, retrieve, or verify files without hitting the nedbd API directly. The `/root` endpoint returns an anchorable Merkle proof, which is the bridge to on-chain anchoring (ITC/BSC). Surfacing this in the UI makes that capability discoverable. - ---- - -## Idea 2 — Browser-side NQL GROUP BY support (Gap: engine supports it, Studio mock doesn't) - -**What:** Extend `src/lib/nql.ts`'s `parseNql` and `executeNql` functions to handle `GROUP BY [COUNT|SUM|AVG|MIN|MAX]`. The keywords are already in the lexer's `KEYWORDS` set; only the parse + execute logic is missing. - -**Why:** Any user who writes `FROM users GROUP BY status COUNT` against a scaffold's seed data in mock mode gets an "unexpected trailing input" parse error — even though the live engine handles it fine. This breaks the Studio demo experience for a common analytics pattern and creates a frustrating mismatch between mock and live behavior. - ---- - -## Idea 3 — Checkpoint button + backfill missing studio git tags v0.4–v0.6 - -**What (part A):** Add a "Checkpoint" button to the Databases page that calls `POST /databases//checkpoint` via a new studio route. Show a toast with the returned `head` hash. - -**What (part B):** Backfill missing git tags `v0.4.0` through `v0.6.1` on `Eth-Interchained/nedb-studio` — the current tags only go to `v0.3.7` even though the code is at `v0.6.1`. - -**Why:** Checkpoints let nedbd restart in O(delta) time rather than replaying the full AOF log — users running large databases benefit immediately. The tag gap means there's no reliable way to `git checkout v0.5.0` or see a GitHub releases page for Studio, making the project look unmaintained to anyone browsing the repo. +**Idea 3 — Reconcile engine pyproject.toml license metadata** +The engine's top-level `LICENSE` file was changed to Business Source License (BSL) in PR #1, but `pyproject.toml` still declares `license = { text = "Apache-2.0" }` and the `License :: OSI Approved :: Apache Software License` PyPI trove classifier — this metadata is visible to every `pip install nedb-engine` consumer and package index. Fix: update `pyproject.toml` to `license = { text = "BSL-1.1" }` (or a plain `"BUSL-1.1"` text field), remove the Apache classifier, and add a `Intended Audience` note; bump to v1.0.5 patch. No functional change, but correct public metadata prevents license confusion. diff --git a/src/lib/nql.ts b/src/lib/nql.ts index 1e23e0b..027e54e 100644 --- a/src/lib/nql.ts +++ b/src/lib/nql.ts @@ -8,20 +8,36 @@ import type { NEDBScaffold } from "./types"; * entirely in the browser — phpMyAdmin-style, no database to provision. It also * provides a deterministic natural-language → NQL fallback for mock mode. * - * Grammar: - * FROM [AS OF ] [WHERE (AND ...)] - * [SEARCH ""] [ORDER BY [ASC|DESC]] [TRAVERSE ] [LIMIT ] + * Grammar (full engine grammar as of v1.0.4): + * FROM + * [AS OF ] + * [VALID AS OF ""] + * [WHERE [AND ...]] + * [VALID AS OF ""] + * [SEARCH ""] + * [ORDER BY [ASC|DESC]] + * [TRAVERSE ] + * [TRACE [REVERSE]] + * [LIMIT ] + * [GROUP BY [COUNT | SUM | AVG | MIN | MAX ]] */ export type Op = "=" | "!=" | "<" | "<=" | ">" | ">="; +export type AggFunc = "COUNT" | "SUM" | "AVG" | "MIN" | "MAX"; + export interface Plan { from: string; asOf: number | null; + validAsOf: string | null; where: Array<{ field: string; op: Op; value: unknown }>; search: string | null; orderBy: { field: string; dir: "ASC" | "DESC" } | null; traverse: string | null; + trace: string | null; + traceReverse: boolean; limit: number | null; + groupBy: string | null; + aggregate: { func: AggFunc; field: string | null } | null; } type Tok = { t: "kw" | "word" | "op" | "num" | "str"; v: string | number }; @@ -83,14 +99,38 @@ export function parseNql(text: string): Plan { const f = peek(); if (!f || (f.t !== "word" && f.t !== "kw")) throw new Error("NQL: expected collection after FROM"); i++; - const plan: Plan = { from: String(f.v), asOf: null, where: [], search: null, orderBy: null, traverse: null, limit: null }; + const plan: Plan = { + from: String(f.v), + asOf: null, + validAsOf: null, + where: [], + search: null, + orderBy: null, + traverse: null, + trace: null, + traceReverse: false, + limit: null, + groupBy: null, + aggregate: null, + }; + // AS OF if (peek()?.t === "kw" && peek()?.v === "as") { i++; eatKw("of"); const n = peek(); if (!n || n.t !== "num") throw new Error("NQL: AS OF expects an integer"); i++; plan.asOf = Number(n.v); } + + // VALID AS OF "" (position 1 — before WHERE) + if (peek()?.t === "kw" && peek()?.v === "valid") { + i++; eatKw("as"); eatKw("of"); + const s = peek(); + if (!s || s.t !== "str") throw new Error("NQL: VALID AS OF expects a quoted date string"); + i++; plan.validAsOf = String(s.v); + } + + // WHERE if (peek()?.t === "kw" && peek()?.v === "where") { i++; for (;;) { @@ -105,12 +145,24 @@ export function parseNql(text: string): Plan { break; } } + + // VALID AS OF "" (position 2 — after WHERE, mirrors engine grammar) + if (peek()?.t === "kw" && peek()?.v === "valid" && plan.validAsOf === null) { + i++; eatKw("as"); eatKw("of"); + const s = peek(); + if (!s || s.t !== "str") throw new Error("NQL: VALID AS OF expects a quoted date string"); + i++; plan.validAsOf = String(s.v); + } + + // SEARCH "" if (peek()?.t === "kw" && peek()?.v === "search") { i++; const s = peek(); if (!s || s.t !== "str") throw new Error("NQL: SEARCH expects a quoted string"); i++; plan.search = String(s.v); } + + // ORDER BY [ASC|DESC] if (peek()?.t === "kw" && peek()?.v === "order") { i++; eatKw("by"); const fld = peek(); @@ -121,18 +173,52 @@ export function parseNql(text: string): Plan { else if (peek()?.t === "kw" && peek()?.v === "desc") { i++; dir = "DESC"; } plan.orderBy = { field: String(fld.v), dir }; } + + // TRAVERSE if (peek()?.t === "kw" && peek()?.v === "traverse") { i++; const rel = peek(); if (!rel || (rel.t !== "word" && rel.t !== "kw")) throw new Error("NQL: expected relation after TRAVERSE"); i++; plan.traverse = String(rel.v); } + + // TRACE [REVERSE] + if (peek()?.t === "kw" && peek()?.v === "trace") { + i++; + const fld = peek(); + if (!fld || (fld.t !== "word" && fld.t !== "kw")) throw new Error("NQL: expected field after TRACE"); + i++; plan.trace = String(fld.v); + if (peek()?.t === "kw" && peek()?.v === "reverse") { i++; plan.traceReverse = true; } + } + + // LIMIT if (peek()?.t === "kw" && peek()?.v === "limit") { i++; const n = peek(); if (!n || n.t !== "num") throw new Error("NQL: LIMIT expects an integer"); i++; plan.limit = Number(n.v); } + + // GROUP BY [COUNT | SUM | AVG | MIN | MAX ] + if (peek()?.t === "kw" && peek()?.v === "group") { + i++; eatKw("by"); + const fld = peek(); + if (!fld || (fld.t !== "word" && fld.t !== "kw")) throw new Error("NQL: expected field after GROUP BY"); + i++; plan.groupBy = String(fld.v); + const aggTok = peek(); + if (aggTok?.t === "kw" && ["count", "sum", "avg", "min", "max"].includes(String(aggTok.v))) { + i++; + const func = String(aggTok.v).toUpperCase() as AggFunc; + if (func === "COUNT") { + plan.aggregate = { func, field: null }; + } else { + const af = peek(); + if (!af || (af.t !== "word" && af.t !== "kw")) throw new Error(`NQL: ${func} expects a field name`); + i++; plan.aggregate = { func, field: String(af.v) }; + } + } + } + if (i !== toks.length) throw new Error("NQL: unexpected trailing input"); return plan; } @@ -178,6 +264,7 @@ export function executeNql(nql: string, scaffold: NEDBScaffold): QueryResult { const notes: string[] = []; if (!data[plan.from]) notes.push(`No seed rows for "${plan.from}".`); if (plan.asOf != null) notes.push("AS OF shown against the current seed snapshot (live time-travel needs a running engine)."); + if (plan.validAsOf != null) notes.push(`VALID AS OF "${plan.validAsOf}": bi-temporal filter applied against seed data (full valid-time indexing needs a running engine).`); // WHERE for (const c of plan.where) rows = rows.filter((r) => cmp(r[c.field], c.op, c.value)); @@ -227,9 +314,76 @@ export function executeNql(nql: string, scaffold: NEDBScaffold): QueryResult { } } + // TRACE [REVERSE] — causal chain walk + // Full causal provenance needs a running engine (BLAKE2b-sealed caused_by chain). + // Mock mode: walk the named field as a linked-list pointer across all seed collections. + if (plan.trace != null) { + const field = plan.trace; + const allRows = (Object.values(data) as Array>>).flat(); + const idMap = new Map>(); + for (const r of allRows) { const rid = rowId(r); if (rid != null) idMap.set(rid, r); } + + const out: Array> = []; + const visited = new Set(); + const todo = [...rows]; + while (todo.length) { + const r = todo.shift()!; + const rid = rowId(r); + if (visited.has(rid)) continue; + visited.add(rid); + out.push(r); + if (plan.traceReverse) { + // Find any row whose [field] value equals this row's id + for (const other of allRows) { + const oid = rowId(other); + if (!visited.has(oid) && other[field] === rid) todo.push(other); + } + } else { + // Follow field forward to the linked row + const targetId = r[field]; + if (targetId != null) { + const linked = idMap.get(targetId); + if (linked && !visited.has(rowId(linked))) todo.push(linked); + } + } + } + rows = out; + notes.push(`TRACE ${field}${plan.traceReverse ? " REVERSE" : ""}: walked ${rows.length} node(s) via seed data (full sealed causal chain needs a running engine).`); + } + // LIMIT if (plan.limit != null) rows = rows.slice(0, plan.limit); + // GROUP BY [COUNT | SUM | AVG | MIN | MAX ] + if (plan.groupBy != null) { + const gField = plan.groupBy; + const groups = new Map>>(); + for (const r of rows) { + const key = r[gField] ?? null; + if (!groups.has(key)) groups.set(key, []); + groups.get(key)!.push(r); + } + const agg = plan.aggregate; + rows = []; + for (const [key, grp] of groups) { + const grpRow: Record = { [gField]: key }; + if (!agg || agg.func === "COUNT") { + grpRow["count"] = grp.length; + } else { + const af = agg.field!; + const nums = grp + .map((r) => (typeof r[af] === "number" ? (r[af] as number) : parseFloat(String(r[af])))) + .filter((n) => !Number.isNaN(n)); + if (agg.func === "SUM") grpRow[`sum(${af})`] = nums.reduce((a, b) => a + b, 0); + else if (agg.func === "AVG") grpRow[`avg(${af})`] = nums.length ? nums.reduce((a, b) => a + b, 0) / nums.length : null; + else if (agg.func === "MIN") grpRow[`min(${af})`] = nums.length ? Math.min(...nums) : null; + else if (agg.func === "MAX") grpRow[`max(${af})`] = nums.length ? Math.max(...nums) : null; + } + rows.push(grpRow); + } + notes.push(`GROUP BY ${gField}: ${groups.size} group(s).`); + } + const columns: string[] = []; for (const r of rows) for (const k of Object.keys(r)) if (!columns.includes(k)) columns.push(k);