From ac75c06d1faebc1a0d03d49c2ec810c08c14662d Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Mon, 2 Mar 2026 20:48:12 +0100 Subject: [PATCH 1/6] feat(psl-parser): add lossless, fault-tolerant PSL tokenizer Add a lazy tokenizer for Prisma Schema Language with cursor API (next/peek), lossless round-tripping, and single-char Invalid fallback for fault tolerance. Supports unicode identifiers and dashed idents for pack namespaces. Co-Authored-By: Claude Opus 4.6 --- .../2-authoring/psl-parser/package.json | 1 + .../psl-parser/src/exports/tokenizer.ts | 2 + .../2-authoring/psl-parser/src/tokenizer.ts | 210 +++++++++++ .../psl-parser/test/tokenizer.test.ts | 330 ++++++++++++++++++ .../2-authoring/psl-parser/tsdown.config.ts | 7 +- 5 files changed, 549 insertions(+), 1 deletion(-) create mode 100644 packages/1-framework/2-authoring/psl-parser/src/exports/tokenizer.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts diff --git a/packages/1-framework/2-authoring/psl-parser/package.json b/packages/1-framework/2-authoring/psl-parser/package.json index e4d9846abe..f4b06fca0d 100644 --- a/packages/1-framework/2-authoring/psl-parser/package.json +++ b/packages/1-framework/2-authoring/psl-parser/package.json @@ -31,6 +31,7 @@ "exports": { ".": "./dist/index.mjs", "./parser": "./dist/parser.mjs", + "./tokenizer": "./dist/tokenizer.mjs", "./types": "./dist/types.mjs", "./package.json": "./package.json" }, diff --git a/packages/1-framework/2-authoring/psl-parser/src/exports/tokenizer.ts b/packages/1-framework/2-authoring/psl-parser/src/exports/tokenizer.ts new file mode 100644 index 0000000000..57881da276 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/exports/tokenizer.ts @@ -0,0 +1,2 @@ +export type { Token, TokenKind } from '../tokenizer'; +export { Tokenizer } from '../tokenizer'; diff --git a/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts new file mode 100644 index 0000000000..847cdda9bb --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts @@ -0,0 +1,210 @@ +export type TokenKind = + | 'Ident' + | 'StringLiteral' + | 'NumberLiteral' + | 'At' + | 'DoubleAt' + | 'LBrace' + | 'RBrace' + | 'LParen' + | 'RParen' + | 'LBracket' + | 'RBracket' + | 'Equals' + | 'Question' + | 'Dot' + | 'Comma' + | 'Colon' + | 'Whitespace' + | 'Newline' + | 'Comment' + | 'Invalid' + | 'Eof'; + +export interface Token { + readonly kind: TokenKind; + readonly text: string; + readonly offset: number; +} + +export class Tokenizer { + readonly #source: string; + #pos: number; + readonly #buffer: Token[]; + + constructor(source: string) { + this.#source = source; + this.#pos = 0; + this.#buffer = []; + } + + next(): Token { + const token = this.#buffer.shift() ?? scan(this.#source, this.#pos); + this.#pos = token.offset + token.text.length; + return token; + } + + peek(offset = 0): Token { + while (this.#buffer.length <= offset) { + const last = this.#buffer.at(-1); + if (last?.kind === 'Eof') { + break; + } + const scanPos = last !== undefined ? last.offset + last.text.length : this.#pos; + this.#buffer.push(scan(this.#source, scanPos)); + } + return ( + this.#buffer[offset] ?? this.#buffer[this.#buffer.length - 1] ?? scan(this.#source, this.#pos) + ); + } +} + +function scan(source: string, pos: number): Token { + if (pos >= source.length) { + return { kind: 'Eof', text: '', offset: source.length }; + } + + return ( + scanNewline(source, pos) ?? + scanWhitespace(source, pos) ?? + scanComment(source, pos) ?? + scanAt(source, pos) ?? + scanIdent(source, pos) ?? + scanNumber(source, pos) ?? + scanString(source, pos) ?? + scanPunctuation(source, pos) ?? { + kind: 'Invalid' as const, + text: source.charAt(pos), + offset: pos, + } + ); +} + +function scanNewline(source: string, pos: number): Token | undefined { + const ch = source.charAt(pos); + if (ch !== '\r' && ch !== '\n') return undefined; + if (ch === '\r' && source.charAt(pos + 1) === '\n') { + return { kind: 'Newline', text: '\r\n', offset: pos }; + } + return { kind: 'Newline', text: ch, offset: pos }; +} + +function scanWhitespace(source: string, pos: number): Token | undefined { + const ch = source.charAt(pos); + if (ch !== ' ' && ch !== '\t') return undefined; + let end = pos + 1; + while (end < source.length) { + const c = source.charAt(end); + if (c !== ' ' && c !== '\t') break; + end++; + } + return { kind: 'Whitespace', text: source.slice(pos, end), offset: pos }; +} + +function scanComment(source: string, pos: number): Token | undefined { + if (source.charAt(pos) !== '/' || source.charAt(pos + 1) !== '/') return undefined; + let end = pos + 2; + while (end < source.length) { + const c = source.charAt(end); + if (c === '\n' || c === '\r') break; + end++; + } + return { kind: 'Comment', text: source.slice(pos, end), offset: pos }; +} + +function scanAt(source: string, pos: number): Token | undefined { + if (source.charAt(pos) !== '@') return undefined; + if (source.charAt(pos + 1) === '@') { + return { kind: 'DoubleAt', text: '@@', offset: pos }; + } + return { kind: 'At', text: '@', offset: pos }; +} + +function scanIdent(source: string, pos: number): Token | undefined { + if (!isIdentStart(source.charAt(pos))) return undefined; + let end = pos + 1; + while (end < source.length) { + if (isIdentPart(source.charAt(end))) { + end++; + } else if ( + source.charAt(end) === '-' && + end + 1 < source.length && + isIdentPart(source.charAt(end + 1)) + ) { + end++; + } else { + break; + } + } + return { kind: 'Ident', text: source.slice(pos, end), offset: pos }; +} + +function scanNumber(source: string, pos: number): Token | undefined { + if (!isDigit(source.charAt(pos))) return undefined; + let end = pos + 1; + while (end < source.length && isDigit(source.charAt(end))) { + end++; + } + if (source.charAt(end) === '.' && end + 1 < source.length && isDigit(source.charAt(end + 1))) { + end++; // consume the dot + while (end < source.length && isDigit(source.charAt(end))) { + end++; + } + } + return { kind: 'NumberLiteral', text: source.slice(pos, end), offset: pos }; +} + +function scanString(source: string, pos: number): Token | undefined { + if (source.charAt(pos) !== '"') return undefined; + let end = pos + 1; + while (end < source.length) { + const c = source.charAt(end); + if (c === '\\' && end + 1 < source.length) { + end += 2; // skip escape sequence + continue; + } + if (c === '"') { + end++; // include closing quote + return { kind: 'StringLiteral', text: source.slice(pos, end), offset: pos }; + } + if (c === '\n' || c === '\r') { + // Unterminated: stop before newline + return { kind: 'StringLiteral', text: source.slice(pos, end), offset: pos }; + } + end++; + } + // Unterminated at EOF + return { kind: 'StringLiteral', text: source.slice(pos, end), offset: pos }; +} + +function scanPunctuation(source: string, pos: number): Token | undefined { + const kind = PUNCTUATION[source.charAt(pos)]; + if (kind === undefined) return undefined; + return { kind, text: source.charAt(pos), offset: pos }; +} + +function isIdentStart(ch: string): boolean { + return /\p{L}/u.test(ch) || ch === '_'; +} + +function isIdentPart(ch: string): boolean { + return isIdentStart(ch) || isDigit(ch); +} + +function isDigit(ch: string): boolean { + return ch >= '0' && ch <= '9'; +} + +const PUNCTUATION: Record = { + '{': 'LBrace', + '}': 'RBrace', + '(': 'LParen', + ')': 'RParen', + '[': 'LBracket', + ']': 'RBracket', + '=': 'Equals', + '?': 'Question', + '.': 'Dot', + ',': 'Comma', + ':': 'Colon', +}; diff --git a/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts new file mode 100644 index 0000000000..77f0e9b66a --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts @@ -0,0 +1,330 @@ +import { describe, expect, it } from 'vitest'; +import type { Token } from '../src/tokenizer'; +import { Tokenizer } from '../src/tokenizer'; + +const KIND_COLUMN_WIDTH = 15; + +function escapeForDebug(text: string): string { + return text + .replaceAll('\\', '\\\\') + .replaceAll('\n', '\\n') + .replaceAll('\r', '\\r') + .replaceAll('\t', '\\t') + .replaceAll('"', '\\"'); +} + +function debugTokens(tokens: Iterable): string { + const lines: string[] = []; + for (const token of tokens) { + lines.push(`${token.kind.padEnd(KIND_COLUMN_WIDTH)}"${escapeForDebug(token.text)}"`); + } + return lines.join('\n'); +} + +function collectAll(source: string): Token[] { + const t = new Tokenizer(source); + const tokens: Token[] = []; + let tok: Token; + do { + tok = t.next(); + tokens.push(tok); + } while (tok.kind !== 'Eof'); + return tokens; +} + +function tokenize(source: string): string { + return debugTokens(collectAll(source)); +} + +function assertLossless(source: string): void { + const tokens = collectAll(source); + expect(tokens.map((t) => t.text).join('')).toBe(source); +} + +describe('Tokenizer', () => { + describe('PSL fragments', () => { + it('tokenizes a model with fields and attributes', () => { + assertLossless('model User {\n id Int @id\n}'); + expect(tokenize('model User {\n id Int @id\n}')).toMatchInlineSnapshot(` + "Ident "model" + Whitespace " " + Ident "User" + Whitespace " " + LBrace "{" + Newline "\\n" + Whitespace " " + Ident "id" + Whitespace " " + Ident "Int" + Whitespace " " + At "@" + Ident "id" + Newline "\\n" + RBrace "}" + Eof """ + `); + }); + + it('tokenizes optional and array types', () => { + expect(tokenize('role Role?\nposts Post[]')).toMatchInlineSnapshot(` + "Ident "role" + Whitespace " " + Ident "Role" + Question "?" + Newline "\\n" + Ident "posts" + Whitespace " " + Ident "Post" + LBracket "[" + RBracket "]" + Eof """ + `); + }); + + it('tokenizes @relation with named arguments', () => { + expect(tokenize('@relation(fields: [userId], references: [id])')).toMatchInlineSnapshot(` + "At "@" + Ident "relation" + LParen "(" + Ident "fields" + Colon ":" + Whitespace " " + LBracket "[" + Ident "userId" + RBracket "]" + Comma "," + Whitespace " " + Ident "references" + Colon ":" + Whitespace " " + LBracket "[" + Ident "id" + RBracket "]" + RParen ")" + Eof """ + `); + }); + + it('tokenizes block attribute @@index', () => { + expect(tokenize('@@index([userId])')).toMatchInlineSnapshot(` + "DoubleAt "@@" + Ident "index" + LParen "(" + LBracket "[" + Ident "userId" + RBracket "]" + RParen ")" + Eof """ + `); + }); + + it('tokenizes comment followed by model', () => { + expect(tokenize('// config\nmodel C {}')).toMatchInlineSnapshot(` + "Comment "// config" + Newline "\\n" + Ident "model" + Whitespace " " + Ident "C" + Whitespace " " + LBrace "{" + RBrace "}" + Eof """ + `); + }); + + it('tokenizes string default value', () => { + expect(tokenize('@default("unknown")')).toMatchInlineSnapshot(` + "At "@" + Ident "default" + LParen "(" + StringLiteral "\\"unknown\\"" + RParen ")" + Eof """ + `); + }); + + it('tokenizes namespaced attribute with dot', () => { + expect(tokenize('@db.VarChar(191)')).toMatchInlineSnapshot(` + "At "@" + Ident "db" + Dot "." + Ident "VarChar" + LParen "(" + NumberLiteral "191" + RParen ")" + Eof """ + `); + }); + + it('tokenizes types block with equals', () => { + expect(tokenize('Email = String')).toMatchInlineSnapshot(` + "Ident "Email" + Whitespace " " + Equals "=" + Whitespace " " + Ident "String" + Eof """ + `); + }); + + it('tokenizes hyphenated attribute namespace', () => { + expect(tokenize('@my-pack.column')).toMatchInlineSnapshot(` + "At "@" + Ident "my-pack" + Dot "." + Ident "column" + Eof """ + `); + }); + + it('tokenizes unicode identifiers', () => { + expect(tokenize('café Ñame 名前')).toMatchInlineSnapshot(` + "Ident "café" + Whitespace " " + Ident "Ñame" + Whitespace " " + Ident "名前" + Eof """ + `); + }); + }); + + describe('edge cases', () => { + it('handles \\r\\n line endings (lossless)', () => { + const schema = 'model User {\r\n id Int\r\n}'; + assertLossless(schema); + expect(tokenize(schema)).toMatchInlineSnapshot(` + "Ident "model" + Whitespace " " + Ident "User" + Whitespace " " + LBrace "{" + Newline "\\r\\n" + Whitespace " " + Ident "id" + Whitespace " " + Ident "Int" + Newline "\\r\\n" + RBrace "}" + Eof """ + `); + }); + + it('handles number literals and trailing dots', () => { + expect(tokenize('1.5')).toMatchInlineSnapshot(` + "NumberLiteral "1.5" + Eof """ + `); + expect(tokenize('1.')).toMatchInlineSnapshot(` + "NumberLiteral "1" + Dot "." + Eof """ + `); + }); + + it('handles string escapes and unterminated strings', () => { + expect(tokenize('"hello \\"world\\""')).toMatchInlineSnapshot(` + "StringLiteral "\\"hello \\\\\\"world\\\\\\"\\"" + Eof """ + `); + expect(tokenize('"hello\nworld')).toMatchInlineSnapshot(` + "StringLiteral "\\"hello" + Newline "\\n" + Ident "world" + Eof """ + `); + expect(tokenize('"hello')).toMatchInlineSnapshot(` + "StringLiteral "\\"hello" + Eof """ + `); + }); + + it('emits single-char Invalid tokens for unknown characters', () => { + assertLossless('#$%^&'); + expect(tokenize('#$%^&')).toMatchInlineSnapshot(` + "Invalid "#" + Invalid "$" + Invalid "%" + Invalid "^" + Invalid "&" + Eof """ + `); + }); + + it('resumes known tokens after Invalid', () => { + assertLossless('#$model'); + expect(tokenize('#$model')).toMatchInlineSnapshot(` + "Invalid "#" + Invalid "$" + Ident "model" + Eof """ + `); + }); + + it('handles lone / and ! as Invalid', () => { + expect(tokenize('!/')).toMatchInlineSnapshot(` + "Invalid "!" + Invalid "/" + Eof """ + `); + }); + + it('never throws on pathological input', () => { + const nasty = '!@#$%^&*(){}[]<>~`|\\;\'"/?.,\x00\x01\x02'; + assertLossless(nasty); + expect(() => tokenize(nasty)).not.toThrow(); + }); + }); + + describe('offsets', () => { + it('each token offset equals sum of preceding text lengths', () => { + const source = 'model User {\n id Int @id\n}\n'; + const tokens = collectAll(source); + let expectedOffset = 0; + for (const token of tokens) { + expect(token.offset).toBe(expectedOffset); + expectedOffset += token.text.length; + } + }); + + it('Eof offset equals source length', () => { + const source = 'model User {}'; + const tokens = collectAll(source); + const eof = tokens[tokens.length - 1]!; + expect(eof.kind).toBe('Eof'); + expect(eof.offset).toBe(source.length); + }); + }); + + describe('cursor API', () => { + it('peek(0) returns the same token as a subsequent next()', () => { + const t = new Tokenizer('model User'); + const peeked = t.peek(0); + const consumed = t.next(); + expect(peeked).toEqual(consumed); + }); + + it('peek(1) returns the token after the next one', () => { + const t = new Tokenizer('model User'); + const peekOne = t.peek(1); + t.next(); // consume 'model' + const second = t.next(); // consume ' ' + expect(peekOne).toEqual(second); + }); + + it('returns Eof indefinitely after source is exhausted', () => { + const t = new Tokenizer('a'); + expect(t.next().kind).toBe('Ident'); + expect(t.next().kind).toBe('Eof'); + expect(t.next().kind).toBe('Eof'); + expect(t.next().kind).toBe('Eof'); + }); + + it('peek(0) returns Eof after Eof has been consumed', () => { + const t = new Tokenizer('a'); + t.next(); // 'a' + t.next(); // Eof + expect(t.peek(0).kind).toBe('Eof'); + }); + }); +}); diff --git a/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts b/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts index 524df841e4..6cb54562be 100644 --- a/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts +++ b/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts @@ -1,5 +1,10 @@ import { defineConfig } from '@prisma-next/tsdown'; export default defineConfig({ - entry: ['src/exports/index.ts', 'src/exports/parser.ts', 'src/exports/types.ts'], + entry: [ + 'src/exports/index.ts', + 'src/exports/parser.ts', + 'src/exports/tokenizer.ts', + 'src/exports/types.ts', + ], }); From ce6edf424aa6fa086ced15fe1793c1a17cb56e28 Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Mon, 2 Mar 2026 20:58:23 +0100 Subject: [PATCH 2/6] feat(psl-parser): add negative number literal tokenization scanNumber now accepts an optional leading minus when followed by a digit, producing tokens like NumberLiteral("-1") and NumberLiteral("-3.14"). Co-Authored-By: Claude Opus 4.6 --- .../2-authoring/psl-parser/src/tokenizer.ts | 10 ++++++++-- .../psl-parser/test/tokenizer.test.ts | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts index 847cdda9bb..b0cdb203e6 100644 --- a/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts +++ b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts @@ -140,8 +140,14 @@ function scanIdent(source: string, pos: number): Token | undefined { } function scanNumber(source: string, pos: number): Token | undefined { - if (!isDigit(source.charAt(pos))) return undefined; - let end = pos + 1; + let end = pos; + if (source.charAt(end) === '-') { + if (end + 1 >= source.length || !isDigit(source.charAt(end + 1))) return undefined; + end++; + } else if (!isDigit(source.charAt(end))) { + return undefined; + } + end++; while (end < source.length && isDigit(source.charAt(end))) { end++; } diff --git a/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts index 77f0e9b66a..906fd37451 100644 --- a/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts +++ b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts @@ -222,6 +222,25 @@ describe('Tokenizer', () => { `); }); + it('handles negative number literals', () => { + expect(tokenize('-1')).toMatchInlineSnapshot(` + "NumberLiteral "-1" + Eof """ + `); + expect(tokenize('-3.14')).toMatchInlineSnapshot(` + "NumberLiteral "-3.14" + Eof """ + `); + expect(tokenize('@default(-1)')).toMatchInlineSnapshot(` + "At "@" + Ident "default" + LParen "(" + NumberLiteral "-1" + RParen ")" + Eof """ + `); + }); + it('handles string escapes and unterminated strings', () => { expect(tokenize('"hello \\"world\\""')).toMatchInlineSnapshot(` "StringLiteral "\\"hello \\\\\\"world\\\\\\"\\"" From 53ee1d54301e734881f2e038031afc21e05ff503 Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Tue, 3 Mar 2026 12:30:35 +0100 Subject: [PATCH 3/6] fix(psl-parser): support astral unicode idents and simplify dash handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use codePointAt-based readChar() for code-point-aware iteration in scanIdent and the Invalid fallback, fixing broken tokenization of non-BMP letters (e.g. Deseret 𐐀). Move dash into isIdentPart to simplify scanIdent loop. Co-Authored-By: Claude Opus 4.6 --- .../2-authoring/psl-parser/src/tokenizer.ts | 25 ++++++++++--------- .../psl-parser/test/tokenizer.test.ts | 9 +++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts index b0cdb203e6..4da4f3e22e 100644 --- a/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts +++ b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts @@ -74,7 +74,7 @@ function scan(source: string, pos: number): Token { scanString(source, pos) ?? scanPunctuation(source, pos) ?? { kind: 'Invalid' as const, - text: source.charAt(pos), + text: readChar(source, pos), offset: pos, } ); @@ -121,17 +121,13 @@ function scanAt(source: string, pos: number): Token | undefined { } function scanIdent(source: string, pos: number): Token | undefined { - if (!isIdentStart(source.charAt(pos))) return undefined; - let end = pos + 1; + const ch = readChar(source, pos); + if (!isIdentStart(ch)) return undefined; + let end = pos + ch.length; while (end < source.length) { - if (isIdentPart(source.charAt(end))) { - end++; - } else if ( - source.charAt(end) === '-' && - end + 1 < source.length && - isIdentPart(source.charAt(end + 1)) - ) { - end++; + const c = readChar(source, end); + if (isIdentPart(c)) { + end += c.length; } else { break; } @@ -189,12 +185,17 @@ function scanPunctuation(source: string, pos: number): Token | undefined { return { kind, text: source.charAt(pos), offset: pos }; } +function readChar(source: string, pos: number): string { + const cp = source.codePointAt(pos); + return cp !== undefined ? String.fromCodePoint(cp) : ''; +} + function isIdentStart(ch: string): boolean { return /\p{L}/u.test(ch) || ch === '_'; } function isIdentPart(ch: string): boolean { - return isIdentStart(ch) || isDigit(ch); + return isIdentStart(ch) || isDigit(ch) || ch === '-'; } function isDigit(ch: string): boolean { diff --git a/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts index 906fd37451..24b0663d12 100644 --- a/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts +++ b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts @@ -187,6 +187,15 @@ describe('Tokenizer', () => { Eof """ `); }); + + it('tokenizes astral unicode identifiers', () => { + expect(tokenize('𐐀𐐁 test')).toMatchInlineSnapshot(` + "Ident "𐐀𐐁" + Whitespace " " + Ident "test" + Eof """ + `); + }); }); describe('edge cases', () => { From ba27de246c4df9efe65e00f9a89e7edb9e9c2c7a Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Thu, 5 Mar 2026 15:13:08 +0100 Subject: [PATCH 4/6] refactor(psl-parser): remove offset tracking from tokenizer Offsets on tokens are incompatible with the incremental parsing goal. The Tokenizer class now tracks position internally without exposing offsets on the Token interface. --- .../2-authoring/psl-parser/src/tokenizer.ts | 32 +++++++++---------- .../psl-parser/test/tokenizer.test.ts | 20 ------------ 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts index 4da4f3e22e..edaa29fc79 100644 --- a/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts +++ b/packages/1-framework/2-authoring/psl-parser/src/tokenizer.ts @@ -24,7 +24,6 @@ export type TokenKind = export interface Token { readonly kind: TokenKind; readonly text: string; - readonly offset: number; } export class Tokenizer { @@ -40,7 +39,7 @@ export class Tokenizer { next(): Token { const token = this.#buffer.shift() ?? scan(this.#source, this.#pos); - this.#pos = token.offset + token.text.length; + this.#pos += token.text.length; return token; } @@ -50,7 +49,7 @@ export class Tokenizer { if (last?.kind === 'Eof') { break; } - const scanPos = last !== undefined ? last.offset + last.text.length : this.#pos; + const scanPos = this.#buffer.reduce((pos, t) => pos + t.text.length, this.#pos); this.#buffer.push(scan(this.#source, scanPos)); } return ( @@ -61,7 +60,7 @@ export class Tokenizer { function scan(source: string, pos: number): Token { if (pos >= source.length) { - return { kind: 'Eof', text: '', offset: source.length }; + return { kind: 'Eof', text: '' }; } return ( @@ -75,7 +74,6 @@ function scan(source: string, pos: number): Token { scanPunctuation(source, pos) ?? { kind: 'Invalid' as const, text: readChar(source, pos), - offset: pos, } ); } @@ -84,9 +82,9 @@ function scanNewline(source: string, pos: number): Token | undefined { const ch = source.charAt(pos); if (ch !== '\r' && ch !== '\n') return undefined; if (ch === '\r' && source.charAt(pos + 1) === '\n') { - return { kind: 'Newline', text: '\r\n', offset: pos }; + return { kind: 'Newline', text: '\r\n' }; } - return { kind: 'Newline', text: ch, offset: pos }; + return { kind: 'Newline', text: ch }; } function scanWhitespace(source: string, pos: number): Token | undefined { @@ -98,7 +96,7 @@ function scanWhitespace(source: string, pos: number): Token | undefined { if (c !== ' ' && c !== '\t') break; end++; } - return { kind: 'Whitespace', text: source.slice(pos, end), offset: pos }; + return { kind: 'Whitespace', text: source.slice(pos, end) }; } function scanComment(source: string, pos: number): Token | undefined { @@ -109,15 +107,15 @@ function scanComment(source: string, pos: number): Token | undefined { if (c === '\n' || c === '\r') break; end++; } - return { kind: 'Comment', text: source.slice(pos, end), offset: pos }; + return { kind: 'Comment', text: source.slice(pos, end) }; } function scanAt(source: string, pos: number): Token | undefined { if (source.charAt(pos) !== '@') return undefined; if (source.charAt(pos + 1) === '@') { - return { kind: 'DoubleAt', text: '@@', offset: pos }; + return { kind: 'DoubleAt', text: '@@' }; } - return { kind: 'At', text: '@', offset: pos }; + return { kind: 'At', text: '@' }; } function scanIdent(source: string, pos: number): Token | undefined { @@ -132,7 +130,7 @@ function scanIdent(source: string, pos: number): Token | undefined { break; } } - return { kind: 'Ident', text: source.slice(pos, end), offset: pos }; + return { kind: 'Ident', text: source.slice(pos, end) }; } function scanNumber(source: string, pos: number): Token | undefined { @@ -153,7 +151,7 @@ function scanNumber(source: string, pos: number): Token | undefined { end++; } } - return { kind: 'NumberLiteral', text: source.slice(pos, end), offset: pos }; + return { kind: 'NumberLiteral', text: source.slice(pos, end) }; } function scanString(source: string, pos: number): Token | undefined { @@ -167,22 +165,22 @@ function scanString(source: string, pos: number): Token | undefined { } if (c === '"') { end++; // include closing quote - return { kind: 'StringLiteral', text: source.slice(pos, end), offset: pos }; + return { kind: 'StringLiteral', text: source.slice(pos, end) }; } if (c === '\n' || c === '\r') { // Unterminated: stop before newline - return { kind: 'StringLiteral', text: source.slice(pos, end), offset: pos }; + return { kind: 'StringLiteral', text: source.slice(pos, end) }; } end++; } // Unterminated at EOF - return { kind: 'StringLiteral', text: source.slice(pos, end), offset: pos }; + return { kind: 'StringLiteral', text: source.slice(pos, end) }; } function scanPunctuation(source: string, pos: number): Token | undefined { const kind = PUNCTUATION[source.charAt(pos)]; if (kind === undefined) return undefined; - return { kind, text: source.charAt(pos), offset: pos }; + return { kind, text: source.charAt(pos) }; } function readChar(source: string, pos: number): string { diff --git a/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts index 24b0663d12..e40bb3da15 100644 --- a/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts +++ b/packages/1-framework/2-authoring/psl-parser/test/tokenizer.test.ts @@ -304,26 +304,6 @@ describe('Tokenizer', () => { }); }); - describe('offsets', () => { - it('each token offset equals sum of preceding text lengths', () => { - const source = 'model User {\n id Int @id\n}\n'; - const tokens = collectAll(source); - let expectedOffset = 0; - for (const token of tokens) { - expect(token.offset).toBe(expectedOffset); - expectedOffset += token.text.length; - } - }); - - it('Eof offset equals source length', () => { - const source = 'model User {}'; - const tokens = collectAll(source); - const eof = tokens[tokens.length - 1]!; - expect(eof.kind).toBe('Eof'); - expect(eof.offset).toBe(source.length); - }); - }); - describe('cursor API', () => { it('peek(0) returns the same token as a subsequent next()', () => { const t = new Tokenizer('model User'); From 34c8ced03eebede5b47f7735fe8e0da742927c29 Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Tue, 3 Mar 2026 16:36:10 +0100 Subject: [PATCH 5/6] feat(psl-parser): add lossless red/green syntax tree and typed AST wrappers Implements a Roslyn/rust-analyzer style red/green tree infrastructure for PSL: - Green layer: immutable GreenToken/GreenNode with GreenNodeBuilder - Red layer: SyntaxNode with parent pointers, offsets, and navigation - Typed AST wrappers for all PSL constructs (documents, models, enums, fields, attributes, expressions) - Exported via new `@prisma-next/psl-parser/syntax` subpath Co-Authored-By: Claude Opus 4.6 --- .../2-authoring/psl-parser/package.json | 1 + .../psl-parser/src/exports/syntax.ts | 38 ++ .../psl-parser/src/syntax/ast-helpers.ts | 36 ++ .../psl-parser/src/syntax/ast/attributes.ts | 97 ++++ .../psl-parser/src/syntax/ast/declarations.ts | 265 +++++++++ .../psl-parser/src/syntax/ast/expressions.ts | 189 +++++++ .../psl-parser/src/syntax/ast/identifier.ts | 20 + .../src/syntax/ast/type-annotation.ts | 41 ++ .../psl-parser/src/syntax/green-builder.ts | 33 ++ .../psl-parser/src/syntax/green.ts | 29 + .../2-authoring/psl-parser/src/syntax/red.ts | 138 +++++ .../psl-parser/src/syntax/syntax-kind.ts | 21 + .../psl-parser/test/syntax/ast.test.ts | 533 ++++++++++++++++++ .../test/syntax/green-builder.test.ts | 101 ++++ .../psl-parser/test/syntax/green.test.ts | 68 +++ .../psl-parser/test/syntax/red.test.ts | 209 +++++++ .../2-authoring/psl-parser/tsdown.config.ts | 1 + 17 files changed, 1820 insertions(+) create mode 100644 packages/1-framework/2-authoring/psl-parser/src/exports/syntax.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/ast-helpers.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/ast/attributes.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/ast/declarations.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/ast/expressions.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/ast/identifier.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/ast/type-annotation.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/green-builder.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/green.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/red.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/src/syntax/syntax-kind.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/test/syntax/green-builder.test.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/test/syntax/green.test.ts create mode 100644 packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts diff --git a/packages/1-framework/2-authoring/psl-parser/package.json b/packages/1-framework/2-authoring/psl-parser/package.json index f4b06fca0d..261cf3503f 100644 --- a/packages/1-framework/2-authoring/psl-parser/package.json +++ b/packages/1-framework/2-authoring/psl-parser/package.json @@ -32,6 +32,7 @@ ".": "./dist/index.mjs", "./parser": "./dist/parser.mjs", "./tokenizer": "./dist/tokenizer.mjs", + "./syntax": "./dist/syntax.mjs", "./types": "./dist/types.mjs", "./package.json": "./package.json" }, diff --git a/packages/1-framework/2-authoring/psl-parser/src/exports/syntax.ts b/packages/1-framework/2-authoring/psl-parser/src/exports/syntax.ts new file mode 100644 index 0000000000..bea3dc74f5 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/exports/syntax.ts @@ -0,0 +1,38 @@ +export { + AttributeArgListAst, + FieldAttributeAst, + ModelAttributeAst, +} from '../syntax/ast/attributes'; +export { + BlockDeclarationAst, + DocumentAst, + EnumDeclarationAst, + EnumValueDeclarationAst, + FieldDeclarationAst, + KeyValuePairAst, + ModelDeclarationAst, + NamedTypeDeclarationAst, + TypesBlockAst, +} from '../syntax/ast/declarations'; +export type { ExpressionAst } from '../syntax/ast/expressions'; +export { + ArrayLiteralAst, + AttributeArgAst, + BooleanLiteralExprAst, + castExpression, + FunctionCallAst, + NumberLiteralExprAst, + StringLiteralExprAst, +} from '../syntax/ast/expressions'; +// AST wrappers +export { IdentifierAst } from '../syntax/ast/identifier'; +export { TypeAnnotationAst } from '../syntax/ast/type-annotation'; +export type { AstNode } from '../syntax/ast-helpers'; +export { filterChildren, findChildToken, findFirstChild } from '../syntax/ast-helpers'; +export type { GreenElement, GreenNode, GreenToken } from '../syntax/green'; +export { greenNode, greenToken } from '../syntax/green'; +export { GreenNodeBuilder } from '../syntax/green-builder'; +// Red layer +export type { SyntaxElement } from '../syntax/red'; +export { createSyntaxTree, SyntaxNode } from '../syntax/red'; +export type { SyntaxKind } from '../syntax/syntax-kind'; diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/ast-helpers.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast-helpers.ts new file mode 100644 index 0000000000..05daa42647 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast-helpers.ts @@ -0,0 +1,36 @@ +import type { Token, TokenKind } from '../tokenizer'; +import { SyntaxNode } from './red'; + +export interface AstNode { + readonly syntax: SyntaxNode; +} + +export function findChildToken(node: SyntaxNode, kind: TokenKind): Token | undefined { + for (const child of node.children()) { + if (!(child instanceof SyntaxNode) && child.kind === kind) { + return child; + } + } + return undefined; +} + +export function findFirstChild( + node: SyntaxNode, + cast: (node: SyntaxNode) => T | undefined, +): T | undefined { + for (const child of node.childNodes()) { + const result = cast(child); + if (result !== undefined) return result; + } + return undefined; +} + +export function* filterChildren( + node: SyntaxNode, + cast: (node: SyntaxNode) => T | undefined, +): Iterable { + for (const child of node.childNodes()) { + const result = cast(child); + if (result !== undefined) yield result; + } +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/attributes.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/attributes.ts new file mode 100644 index 0000000000..281b62414e --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/attributes.ts @@ -0,0 +1,97 @@ +import type { Token } from '../../tokenizer'; +import type { AstNode } from '../ast-helpers'; +import { filterChildren, findChildToken, findFirstChild } from '../ast-helpers'; +import type { SyntaxNode } from '../red'; +import { AttributeArgAst } from './expressions'; +import { IdentifierAst } from './identifier'; + +export class AttributeArgListAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + lparen(): Token | undefined { + return findChildToken(this.syntax, 'LParen'); + } + + rparen(): Token | undefined { + return findChildToken(this.syntax, 'RParen'); + } + + *args(): Iterable { + yield* filterChildren(this.syntax, AttributeArgAst.cast); + } + + static cast(node: SyntaxNode): AttributeArgListAst | undefined { + return node.kind === 'AttributeArgList' ? new AttributeArgListAst(node) : undefined; + } +} + +export class FieldAttributeAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + at(): Token | undefined { + return findChildToken(this.syntax, 'At'); + } + + name(): IdentifierAst | undefined { + if (this.dot()) { + let count = 0; + for (const child of this.syntax.childNodes()) { + if (child.kind === 'Identifier') { + count++; + if (count === 2) return new IdentifierAst(child); + } + } + return undefined; + } + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + dot(): Token | undefined { + return findChildToken(this.syntax, 'Dot'); + } + + namespaceName(): IdentifierAst | undefined { + if (!this.dot()) return undefined; + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + argList(): AttributeArgListAst | undefined { + return findFirstChild(this.syntax, AttributeArgListAst.cast); + } + + static cast(node: SyntaxNode): FieldAttributeAst | undefined { + return node.kind === 'FieldAttribute' ? new FieldAttributeAst(node) : undefined; + } +} + +export class ModelAttributeAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + doubleAt(): Token | undefined { + return findChildToken(this.syntax, 'DoubleAt'); + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + argList(): AttributeArgListAst | undefined { + return findFirstChild(this.syntax, AttributeArgListAst.cast); + } + + static cast(node: SyntaxNode): ModelAttributeAst | undefined { + return node.kind === 'ModelAttribute' ? new ModelAttributeAst(node) : undefined; + } +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/declarations.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/declarations.ts new file mode 100644 index 0000000000..a68c9eac57 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/declarations.ts @@ -0,0 +1,265 @@ +import type { Token } from '../../tokenizer'; +import type { AstNode } from '../ast-helpers'; +import { filterChildren, findChildToken, findFirstChild } from '../ast-helpers'; +import { SyntaxNode } from '../red'; +import { FieldAttributeAst, ModelAttributeAst } from './attributes'; +import type { ExpressionAst } from './expressions'; +import { castExpression } from './expressions'; +import { IdentifierAst } from './identifier'; +import { TypeAnnotationAst } from './type-annotation'; + +export class DocumentAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + *declarations(): Iterable< + ModelDeclarationAst | EnumDeclarationAst | TypesBlockAst | BlockDeclarationAst + > { + yield* filterChildren( + this.syntax, + (node) => + ModelDeclarationAst.cast(node) ?? + EnumDeclarationAst.cast(node) ?? + TypesBlockAst.cast(node) ?? + BlockDeclarationAst.cast(node), + ); + } + + static cast(node: SyntaxNode): DocumentAst | undefined { + return node.kind === 'Document' ? new DocumentAst(node) : undefined; + } +} + +export class ModelDeclarationAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + keyword(): Token | undefined { + return findChildToken(this.syntax, 'Ident'); + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + lbrace(): Token | undefined { + return findChildToken(this.syntax, 'LBrace'); + } + + rbrace(): Token | undefined { + return findChildToken(this.syntax, 'RBrace'); + } + + *fields(): Iterable { + yield* filterChildren(this.syntax, FieldDeclarationAst.cast); + } + + *attributes(): Iterable { + yield* filterChildren(this.syntax, ModelAttributeAst.cast); + } + + static cast(node: SyntaxNode): ModelDeclarationAst | undefined { + return node.kind === 'ModelDeclaration' ? new ModelDeclarationAst(node) : undefined; + } +} + +export class EnumDeclarationAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + keyword(): Token | undefined { + return findChildToken(this.syntax, 'Ident'); + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + lbrace(): Token | undefined { + return findChildToken(this.syntax, 'LBrace'); + } + + rbrace(): Token | undefined { + return findChildToken(this.syntax, 'RBrace'); + } + + *values(): Iterable { + yield* filterChildren(this.syntax, EnumValueDeclarationAst.cast); + } + + static cast(node: SyntaxNode): EnumDeclarationAst | undefined { + return node.kind === 'EnumDeclaration' ? new EnumDeclarationAst(node) : undefined; + } +} + +export class TypesBlockAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + keyword(): Token | undefined { + return findChildToken(this.syntax, 'Ident'); + } + + lbrace(): Token | undefined { + return findChildToken(this.syntax, 'LBrace'); + } + + rbrace(): Token | undefined { + return findChildToken(this.syntax, 'RBrace'); + } + + *declarations(): Iterable { + yield* filterChildren(this.syntax, NamedTypeDeclarationAst.cast); + } + + static cast(node: SyntaxNode): TypesBlockAst | undefined { + return node.kind === 'TypesBlock' ? new TypesBlockAst(node) : undefined; + } +} + +export class BlockDeclarationAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + keyword(): Token | undefined { + return findChildToken(this.syntax, 'Ident'); + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + lbrace(): Token | undefined { + return findChildToken(this.syntax, 'LBrace'); + } + + rbrace(): Token | undefined { + return findChildToken(this.syntax, 'RBrace'); + } + + *entries(): Iterable { + yield* filterChildren(this.syntax, KeyValuePairAst.cast); + } + + static cast(node: SyntaxNode): BlockDeclarationAst | undefined { + return node.kind === 'BlockDeclaration' ? new BlockDeclarationAst(node) : undefined; + } +} + +export class KeyValuePairAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + key(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + equals(): Token | undefined { + return findChildToken(this.syntax, 'Equals'); + } + + value(): ExpressionAst | undefined { + let pastEquals = false; + for (const child of this.syntax.children()) { + if (!(child instanceof SyntaxNode)) { + if (child.kind === 'Equals') pastEquals = true; + continue; + } + if (pastEquals) { + const expr = castExpression(child); + if (expr) return expr; + } + } + return undefined; + } + + static cast(node: SyntaxNode): KeyValuePairAst | undefined { + return node.kind === 'KeyValuePair' ? new KeyValuePairAst(node) : undefined; + } +} + +export class FieldDeclarationAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + typeAnnotation(): TypeAnnotationAst | undefined { + return findFirstChild(this.syntax, TypeAnnotationAst.cast); + } + + *attributes(): Iterable { + yield* filterChildren(this.syntax, FieldAttributeAst.cast); + } + + static cast(node: SyntaxNode): FieldDeclarationAst | undefined { + return node.kind === 'FieldDeclaration' ? new FieldDeclarationAst(node) : undefined; + } +} + +export class EnumValueDeclarationAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + static cast(node: SyntaxNode): EnumValueDeclarationAst | undefined { + return node.kind === 'EnumValueDeclaration' ? new EnumValueDeclarationAst(node) : undefined; + } +} + +export class NamedTypeDeclarationAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + equals(): Token | undefined { + return findChildToken(this.syntax, 'Equals'); + } + + typeAnnotation(): TypeAnnotationAst | undefined { + return findFirstChild(this.syntax, TypeAnnotationAst.cast); + } + + *attributes(): Iterable { + yield* filterChildren(this.syntax, FieldAttributeAst.cast); + } + + static cast(node: SyntaxNode): NamedTypeDeclarationAst | undefined { + return node.kind === 'NamedTypeDeclaration' ? new NamedTypeDeclarationAst(node) : undefined; + } +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/expressions.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/expressions.ts new file mode 100644 index 0000000000..331c45177d --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/expressions.ts @@ -0,0 +1,189 @@ +import type { Token } from '../../tokenizer'; +import type { AstNode } from '../ast-helpers'; +import { filterChildren, findChildToken, findFirstChild } from '../ast-helpers'; +import { SyntaxNode } from '../red'; +import { IdentifierAst } from './identifier'; + +export class FunctionCallAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + lparen(): Token | undefined { + return findChildToken(this.syntax, 'LParen'); + } + + rparen(): Token | undefined { + return findChildToken(this.syntax, 'RParen'); + } + + *args(): Iterable { + yield* filterChildren(this.syntax, AttributeArgAst.cast); + } + + static cast(node: SyntaxNode): FunctionCallAst | undefined { + return node.kind === 'FunctionCall' ? new FunctionCallAst(node) : undefined; + } +} + +export class ArrayLiteralAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + lbracket(): Token | undefined { + return findChildToken(this.syntax, 'LBracket'); + } + + rbracket(): Token | undefined { + return findChildToken(this.syntax, 'RBracket'); + } + + *elements(): Iterable { + yield* filterChildren(this.syntax, castExpression); + } + + static cast(node: SyntaxNode): ArrayLiteralAst | undefined { + return node.kind === 'ArrayLiteral' ? new ArrayLiteralAst(node) : undefined; + } +} + +export class StringLiteralExprAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + token(): Token | undefined { + return findChildToken(this.syntax, 'StringLiteral'); + } + + value(): string | undefined { + const tok = this.token(); + if (!tok) return undefined; + const raw = tok.text.slice(1, -1); + return raw + .replace(/\\n/g, '\n') + .replace(/\\r/g, '\r') + .replace(/\\t/g, '\t') + .replace(/\\"/g, '"') + .replace(/\\\\/g, '\\'); + } + + static cast(node: SyntaxNode): StringLiteralExprAst | undefined { + return node.kind === 'StringLiteralExpr' ? new StringLiteralExprAst(node) : undefined; + } +} + +export class NumberLiteralExprAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + token(): Token | undefined { + return findChildToken(this.syntax, 'NumberLiteral'); + } + + value(): number | undefined { + const tok = this.token(); + if (!tok) return undefined; + return Number(tok.text); + } + + static cast(node: SyntaxNode): NumberLiteralExprAst | undefined { + return node.kind === 'NumberLiteralExpr' ? new NumberLiteralExprAst(node) : undefined; + } +} + +export class BooleanLiteralExprAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + token(): Token | undefined { + return findChildToken(this.syntax, 'Ident'); + } + + value(): boolean | undefined { + const tok = this.token(); + if (!tok) return undefined; + if (tok.text === 'true') return true; + if (tok.text === 'false') return false; + return undefined; + } + + static cast(node: SyntaxNode): BooleanLiteralExprAst | undefined { + return node.kind === 'BooleanLiteralExpr' ? new BooleanLiteralExprAst(node) : undefined; + } +} + +export type ExpressionAst = + | FunctionCallAst + | ArrayLiteralAst + | StringLiteralExprAst + | NumberLiteralExprAst + | BooleanLiteralExprAst + | IdentifierAst; + +export function castExpression(node: SyntaxNode): ExpressionAst | undefined { + return ( + FunctionCallAst.cast(node) ?? + ArrayLiteralAst.cast(node) ?? + StringLiteralExprAst.cast(node) ?? + NumberLiteralExprAst.cast(node) ?? + BooleanLiteralExprAst.cast(node) ?? + IdentifierAst.cast(node) + ); +} + +export class AttributeArgAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + name(): IdentifierAst | undefined { + if (!this.colon()) return undefined; + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + colon(): Token | undefined { + return findChildToken(this.syntax, 'Colon'); + } + + value(): ExpressionAst | undefined { + if (this.colon()) { + let pastColon = false; + for (const child of this.syntax.children()) { + if (!(child instanceof SyntaxNode)) { + if (child.kind === 'Colon') pastColon = true; + continue; + } + if (pastColon) { + const expr = castExpression(child); + if (expr) return expr; + } + } + return undefined; + } + return findFirstChild(this.syntax, castExpression); + } + + static cast(node: SyntaxNode): AttributeArgAst | undefined { + return node.kind === 'AttributeArg' ? new AttributeArgAst(node) : undefined; + } +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/identifier.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/identifier.ts new file mode 100644 index 0000000000..0e6b1819fb --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/identifier.ts @@ -0,0 +1,20 @@ +import type { Token } from '../../tokenizer'; +import type { AstNode } from '../ast-helpers'; +import { findChildToken } from '../ast-helpers'; +import type { SyntaxNode } from '../red'; + +export class IdentifierAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + token(): Token | undefined { + return findChildToken(this.syntax, 'Ident'); + } + + static cast(node: SyntaxNode): IdentifierAst | undefined { + return node.kind === 'Identifier' ? new IdentifierAst(node) : undefined; + } +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/type-annotation.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/type-annotation.ts new file mode 100644 index 0000000000..686cfa4d1c --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/ast/type-annotation.ts @@ -0,0 +1,41 @@ +import type { Token } from '../../tokenizer'; +import type { AstNode } from '../ast-helpers'; +import { findChildToken, findFirstChild } from '../ast-helpers'; +import type { SyntaxNode } from '../red'; +import { IdentifierAst } from './identifier'; + +export class TypeAnnotationAst implements AstNode { + readonly syntax: SyntaxNode; + + constructor(syntax: SyntaxNode) { + this.syntax = syntax; + } + + name(): IdentifierAst | undefined { + return findFirstChild(this.syntax, IdentifierAst.cast); + } + + lbracket(): Token | undefined { + return findChildToken(this.syntax, 'LBracket'); + } + + rbracket(): Token | undefined { + return findChildToken(this.syntax, 'RBracket'); + } + + questionMark(): Token | undefined { + return findChildToken(this.syntax, 'Question'); + } + + isList(): boolean { + return this.lbracket() !== undefined; + } + + isOptional(): boolean { + return this.questionMark() !== undefined; + } + + static cast(node: SyntaxNode): TypeAnnotationAst | undefined { + return node.kind === 'TypeAnnotation' ? new TypeAnnotationAst(node) : undefined; + } +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/green-builder.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/green-builder.ts new file mode 100644 index 0000000000..a93b495d30 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/green-builder.ts @@ -0,0 +1,33 @@ +import type { TokenKind } from '../tokenizer'; +import type { GreenElement, GreenNode } from './green'; +import { greenNode, greenToken } from './green'; +import type { SyntaxKind } from './syntax-kind'; + +export class GreenNodeBuilder { + readonly #stack: Array<{ kind: SyntaxKind; children: GreenElement[] }> = []; + + startNode(kind: SyntaxKind): void { + this.#stack.push({ kind, children: [] }); + } + + token(kind: TokenKind, text: string): void { + const current = this.#stack.at(-1); + if (!current) { + throw new Error('GreenNodeBuilder: token() called with no open node'); + } + current.children.push(greenToken(kind, text)); + } + + finishNode(): GreenNode { + const completed = this.#stack.pop(); + if (!completed) { + throw new Error('GreenNodeBuilder: finishNode() called with no open node'); + } + const node = greenNode(completed.kind, completed.children); + const parent = this.#stack.at(-1); + if (parent) { + parent.children.push(node); + } + return node; + } +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/green.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/green.ts new file mode 100644 index 0000000000..3fc5423918 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/green.ts @@ -0,0 +1,29 @@ +import type { TokenKind } from '../tokenizer'; +import type { SyntaxKind } from './syntax-kind'; + +export interface GreenToken { + readonly type: 'token'; + readonly kind: TokenKind; + readonly text: string; +} + +export interface GreenNode { + readonly type: 'node'; + readonly kind: SyntaxKind; + readonly children: ReadonlyArray; + readonly textLength: number; +} + +export type GreenElement = GreenNode | GreenToken; + +export function greenToken(kind: TokenKind, text: string): GreenToken { + return { type: 'token', kind, text }; +} + +export function greenNode(kind: SyntaxKind, children: ReadonlyArray): GreenNode { + let textLength = 0; + for (const child of children) { + textLength += child.type === 'token' ? child.text.length : child.textLength; + } + return { type: 'node', kind, children, textLength }; +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/red.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/red.ts new file mode 100644 index 0000000000..314df27dc2 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/red.ts @@ -0,0 +1,138 @@ +import type { Token } from '../tokenizer'; +import type { GreenElement, GreenNode } from './green'; +import type { SyntaxKind } from './syntax-kind'; + +export type SyntaxElement = SyntaxNode | Token; + +export class SyntaxNode { + readonly green: GreenNode; + readonly offset: number; + readonly parent: SyntaxNode | undefined; + + constructor(green: GreenNode, offset: number, parent: SyntaxNode | undefined) { + this.green = green; + this.offset = offset; + this.parent = parent; + } + + get kind(): SyntaxKind { + return this.green.kind; + } + + get textLength(): number { + return this.green.textLength; + } + + get firstChild(): SyntaxElement | undefined { + return childAt(this, 0); + } + + get lastChild(): SyntaxElement | undefined { + const len = this.green.children.length; + if (len === 0) return undefined; + return childAt(this, len - 1); + } + + get nextSibling(): SyntaxElement | undefined { + if (!this.parent) return undefined; + const siblings = this.parent.green.children; + let offset = this.parent.offset; + let found = false; + for (const child of siblings) { + if (found) { + return wrapElement(child, offset, this.parent); + } + const childLen = elementTextLength(child); + if (child.type === 'node' && offset === this.offset && child === this.green) { + found = true; + } + offset += childLen; + } + return undefined; + } + + get prevSibling(): SyntaxElement | undefined { + if (!this.parent) return undefined; + const siblings = this.parent.green.children; + let offset = this.parent.offset; + let prev: { green: GreenElement; offset: number } | undefined; + for (const child of siblings) { + if (child.type === 'node' && offset === this.offset && child === this.green) { + if (!prev) return undefined; + return wrapElement(prev.green, prev.offset, this.parent); + } + prev = { green: child, offset }; + offset += elementTextLength(child); + } + return undefined; + } + + *children(): Iterable { + let offset = this.offset; + for (const child of this.green.children) { + yield wrapElement(child, offset, this); + offset += elementTextLength(child); + } + } + + *childNodes(): Iterable { + for (const child of this.children()) { + if (child instanceof SyntaxNode) yield child; + } + } + + *ancestors(): Iterable { + let current: SyntaxNode | undefined = this.parent; + while (current) { + yield current; + current = current.parent; + } + } + + *descendants(): Iterable { + const stack: SyntaxElement[] = [this]; + while (stack.length > 0) { + const el = stack.pop() as SyntaxElement; + yield el; + if (el instanceof SyntaxNode) { + const children = Array.from(el.children()); + for (let i = children.length - 1; i >= 0; i--) { + stack.push(children[i] as SyntaxElement); + } + } + } + } + + *tokens(): Iterable { + for (const el of this.descendants()) { + if (!(el instanceof SyntaxNode)) { + yield el; + } + } + } +} + +function elementTextLength(el: GreenElement): number { + return el.type === 'token' ? el.text.length : el.textLength; +} + +function wrapElement(green: GreenElement, offset: number, parent: SyntaxNode): SyntaxElement { + if (green.type === 'token') { + return { kind: green.kind, text: green.text, offset }; + } + return new SyntaxNode(green, offset, parent); +} + +function childAt(node: SyntaxNode, index: number): SyntaxElement | undefined { + const children = node.green.children; + if (index < 0 || index >= children.length) return undefined; + let offset = node.offset; + for (let i = 0; i < index; i++) { + offset += elementTextLength(children[i] as GreenElement); + } + return wrapElement(children[index] as GreenElement, offset, node); +} + +export function createSyntaxTree(green: GreenNode): SyntaxNode { + return new SyntaxNode(green, 0, undefined); +} diff --git a/packages/1-framework/2-authoring/psl-parser/src/syntax/syntax-kind.ts b/packages/1-framework/2-authoring/psl-parser/src/syntax/syntax-kind.ts new file mode 100644 index 0000000000..860d88e74c --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/src/syntax/syntax-kind.ts @@ -0,0 +1,21 @@ +export type SyntaxKind = + | 'Document' + | 'ModelDeclaration' + | 'EnumDeclaration' + | 'TypesBlock' + | 'BlockDeclaration' + | 'FieldDeclaration' + | 'EnumValueDeclaration' + | 'NamedTypeDeclaration' + | 'KeyValuePair' + | 'FieldAttribute' + | 'ModelAttribute' + | 'AttributeArgList' + | 'AttributeArg' + | 'TypeAnnotation' + | 'Identifier' + | 'FunctionCall' + | 'ArrayLiteral' + | 'StringLiteralExpr' + | 'NumberLiteralExpr' + | 'BooleanLiteralExpr'; diff --git a/packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts b/packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts new file mode 100644 index 0000000000..87e1cb7704 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts @@ -0,0 +1,533 @@ +import { describe, expect, it } from 'vitest'; +import { + AttributeArgListAst, + FieldAttributeAst, + ModelAttributeAst, +} from '../../src/syntax/ast/attributes'; +import { + BlockDeclarationAst, + DocumentAst, + EnumDeclarationAst, + EnumValueDeclarationAst, + FieldDeclarationAst, + KeyValuePairAst, + ModelDeclarationAst, + NamedTypeDeclarationAst, + TypesBlockAst, +} from '../../src/syntax/ast/declarations'; +import { + ArrayLiteralAst, + AttributeArgAst, + BooleanLiteralExprAst, + FunctionCallAst, + NumberLiteralExprAst, + StringLiteralExprAst, +} from '../../src/syntax/ast/expressions'; +import { IdentifierAst } from '../../src/syntax/ast/identifier'; +import { TypeAnnotationAst } from '../../src/syntax/ast/type-annotation'; +import { GreenNodeBuilder } from '../../src/syntax/green-builder'; +import { createSyntaxTree, type SyntaxNode } from '../../src/syntax/red'; +import type { SyntaxKind } from '../../src/syntax/syntax-kind'; + +function buildIdentifier(name: string) { + const b = new GreenNodeBuilder(); + b.startNode('Identifier'); + b.token('Ident', name); + return b.finishNode(); +} + +describe('IdentifierAst', () => { + it('exposes token()', () => { + const root = createSyntaxTree(buildIdentifier('User')); + const id = IdentifierAst.cast(root); + expect(id).toBeDefined(); + expect(id!.token()?.text).toBe('User'); + }); + + it('returns syntax property', () => { + const root = createSyntaxTree(buildIdentifier('User')); + const id = IdentifierAst.cast(root); + expect(id!.syntax).toBe(root); + }); + + it('cast returns undefined for non-matching kind', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + const green = b.finishNode(); + const root = createSyntaxTree(green); + expect(IdentifierAst.cast(root)).toBeUndefined(); + }); +}); + +describe('static cast', () => { + it('DocumentAst.cast matches Document kind', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + const root = createSyntaxTree(b.finishNode()); + expect(DocumentAst.cast(root)).toBeDefined(); + }); + + it('ModelDeclarationAst.cast returns undefined for wrong kind', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + const root = createSyntaxTree(b.finishNode()); + expect(ModelDeclarationAst.cast(root)).toBeUndefined(); + }); + + const castTests: Array<[string, (node: SyntaxNode) => unknown, SyntaxKind]> = [ + ['EnumDeclarationAst', EnumDeclarationAst.cast, 'EnumDeclaration'], + ['TypesBlockAst', TypesBlockAst.cast, 'TypesBlock'], + ['BlockDeclarationAst', BlockDeclarationAst.cast, 'BlockDeclaration'], + ['KeyValuePairAst', KeyValuePairAst.cast, 'KeyValuePair'], + ['FieldDeclarationAst', FieldDeclarationAst.cast, 'FieldDeclaration'], + ['EnumValueDeclarationAst', EnumValueDeclarationAst.cast, 'EnumValueDeclaration'], + ['NamedTypeDeclarationAst', NamedTypeDeclarationAst.cast, 'NamedTypeDeclaration'], + ['TypeAnnotationAst', TypeAnnotationAst.cast, 'TypeAnnotation'], + ['FieldAttributeAst', FieldAttributeAst.cast, 'FieldAttribute'], + ['ModelAttributeAst', ModelAttributeAst.cast, 'ModelAttribute'], + ['AttributeArgListAst', AttributeArgListAst.cast, 'AttributeArgList'], + ['AttributeArgAst', AttributeArgAst.cast, 'AttributeArg'], + ['FunctionCallAst', FunctionCallAst.cast, 'FunctionCall'], + ['ArrayLiteralAst', ArrayLiteralAst.cast, 'ArrayLiteral'], + ['StringLiteralExprAst', StringLiteralExprAst.cast, 'StringLiteralExpr'], + ['NumberLiteralExprAst', NumberLiteralExprAst.cast, 'NumberLiteralExpr'], + ['BooleanLiteralExprAst', BooleanLiteralExprAst.cast, 'BooleanLiteralExpr'], + ]; + + for (const [name, castFn, kind] of castTests) { + it(`${name}.cast matches ${kind}`, () => { + const b = new GreenNodeBuilder(); + b.startNode(kind); + const root = createSyntaxTree(b.finishNode()); + expect(castFn(root)).toBeDefined(); + }); + + it(`${name}.cast returns undefined for wrong kind`, () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + const root = createSyntaxTree(b.finishNode()); + expect(castFn(root)).toBeUndefined(); + }); + } +}); + +describe('accessors return undefined on missing children', () => { + it('IdentifierAst.token() returns undefined when empty', () => { + const b = new GreenNodeBuilder(); + b.startNode('Identifier'); + const root = createSyntaxTree(b.finishNode()); + const id = IdentifierAst.cast(root)!; + expect(id.token()).toBeUndefined(); + }); + + it('ModelDeclarationAst.name() returns undefined when missing', () => { + const b = new GreenNodeBuilder(); + b.startNode('ModelDeclaration'); + const root = createSyntaxTree(b.finishNode()); + const model = ModelDeclarationAst.cast(root)!; + expect(model.name()).toBeUndefined(); + expect(model.keyword()).toBeUndefined(); + expect(model.lbrace()).toBeUndefined(); + expect(model.rbrace()).toBeUndefined(); + }); + + it('FieldDeclarationAst accessors return undefined when missing', () => { + const b = new GreenNodeBuilder(); + b.startNode('FieldDeclaration'); + const root = createSyntaxTree(b.finishNode()); + const field = FieldDeclarationAst.cast(root)!; + expect(field.name()).toBeUndefined(); + expect(field.typeAnnotation()).toBeUndefined(); + }); + + it('TypeAnnotationAst accessors return undefined when missing', () => { + const b = new GreenNodeBuilder(); + b.startNode('TypeAnnotation'); + const root = createSyntaxTree(b.finishNode()); + const ta = TypeAnnotationAst.cast(root)!; + expect(ta.name()).toBeUndefined(); + expect(ta.lbracket()).toBeUndefined(); + expect(ta.rbracket()).toBeUndefined(); + expect(ta.questionMark()).toBeUndefined(); + expect(ta.isList()).toBe(false); + expect(ta.isOptional()).toBe(false); + }); +}); + +describe('ModelDeclarationAst', () => { + function buildModel() { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('ModelDeclaration'); + b.token('Ident', 'model'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'User'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('Newline', '\n'); + b.token('Whitespace', ' '); + b.startNode('FieldDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('TypeAnnotation'); + b.token('Ident', 'Int'); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.startNode('ModelAttribute'); + b.token('DoubleAt', '@@'); + b.startNode('Identifier'); + b.token('Ident', 'map'); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.token('RBrace', '}'); + b.finishNode(); + return b.finishNode(); + } + + it('exposes keyword, name, braces', () => { + const root = createSyntaxTree(buildModel()); + const doc = DocumentAst.cast(root)!; + const model = Array.from(doc.declarations())[0] as ModelDeclarationAst; + expect(model.keyword()?.text).toBe('model'); + expect(model.name()?.token()?.text).toBe('User'); + expect(model.lbrace()?.text).toBe('{'); + expect(model.rbrace()?.text).toBe('}'); + }); + + it('iterates fields', () => { + const root = createSyntaxTree(buildModel()); + const doc = DocumentAst.cast(root)!; + const model = Array.from(doc.declarations())[0] as ModelDeclarationAst; + const fields = Array.from(model.fields()); + expect(fields).toHaveLength(1); + expect(fields[0]!.name()?.token()?.text).toBe('id'); + }); + + it('iterates model attributes', () => { + const root = createSyntaxTree(buildModel()); + const doc = DocumentAst.cast(root)!; + const model = Array.from(doc.declarations())[0] as ModelDeclarationAst; + const attrs = Array.from(model.attributes()); + expect(attrs).toHaveLength(1); + expect(attrs[0]!.doubleAt()?.text).toBe('@@'); + expect(attrs[0]!.name()?.token()?.text).toBe('map'); + }); +}); + +describe('TypeAnnotationAst', () => { + it('detects list type', () => { + const b = new GreenNodeBuilder(); + b.startNode('TypeAnnotation'); + b.startNode('Identifier'); + b.token('Ident', 'String'); + b.finishNode(); + b.token('LBracket', '['); + b.token('RBracket', ']'); + const root = createSyntaxTree(b.finishNode()); + const ta = TypeAnnotationAst.cast(root)!; + expect(ta.isList()).toBe(true); + expect(ta.isOptional()).toBe(false); + expect(ta.name()?.token()?.text).toBe('String'); + }); + + it('detects optional type', () => { + const b = new GreenNodeBuilder(); + b.startNode('TypeAnnotation'); + b.startNode('Identifier'); + b.token('Ident', 'Int'); + b.finishNode(); + b.token('Question', '?'); + const root = createSyntaxTree(b.finishNode()); + const ta = TypeAnnotationAst.cast(root)!; + expect(ta.isList()).toBe(false); + expect(ta.isOptional()).toBe(true); + }); +}); + +describe('KeyValuePairAst', () => { + it('exposes key, equals, and value', () => { + const b = new GreenNodeBuilder(); + b.startNode('KeyValuePair'); + b.startNode('Identifier'); + b.token('Ident', 'provider'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('Equals', '='); + b.token('Whitespace', ' '); + b.startNode('StringLiteralExpr'); + b.token('StringLiteral', '"postgresql"'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const kv = KeyValuePairAst.cast(root)!; + expect(kv.key()?.token()?.text).toBe('provider'); + expect(kv.equals()?.text).toBe('='); + const val = kv.value(); + expect(val).toBeInstanceOf(StringLiteralExprAst); + }); +}); + +describe('FieldAttributeAst', () => { + it('exposes at and name for simple attribute', () => { + const b = new GreenNodeBuilder(); + b.startNode('FieldAttribute'); + b.token('At', '@'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const attr = FieldAttributeAst.cast(root)!; + expect(attr.at()?.text).toBe('@'); + expect(attr.name()?.token()?.text).toBe('id'); + expect(attr.dot()).toBeUndefined(); + expect(attr.namespaceName()).toBeUndefined(); + expect(attr.argList()).toBeUndefined(); + }); + + it('exposes namespaced attribute parts', () => { + // @db.VarChar + const b = new GreenNodeBuilder(); + b.startNode('FieldAttribute'); + b.token('At', '@'); + b.startNode('Identifier'); + b.token('Ident', 'db'); + b.finishNode(); + b.token('Dot', '.'); + b.startNode('Identifier'); + b.token('Ident', 'VarChar'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const attr = FieldAttributeAst.cast(root)!; + expect(attr.dot()?.text).toBe('.'); + expect(attr.namespaceName()?.token()?.text).toBe('db'); + expect(attr.name()?.token()?.text).toBe('VarChar'); + }); + + it('exposes argList', () => { + // @default(autoincrement()) + const b = new GreenNodeBuilder(); + b.startNode('FieldAttribute'); + b.token('At', '@'); + b.startNode('Identifier'); + b.token('Ident', 'default'); + b.finishNode(); + b.startNode('AttributeArgList'); + b.token('LParen', '('); + b.startNode('AttributeArg'); + b.startNode('FunctionCall'); + b.startNode('Identifier'); + b.token('Ident', 'autoincrement'); + b.finishNode(); + b.token('LParen', '('); + b.token('RParen', ')'); + b.finishNode(); + b.finishNode(); + b.token('RParen', ')'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const attr = FieldAttributeAst.cast(root)!; + const argList = attr.argList(); + expect(argList).toBeDefined(); + expect(argList!.lparen()?.text).toBe('('); + expect(argList!.rparen()?.text).toBe(')'); + const args = Array.from(argList!.args()); + expect(args).toHaveLength(1); + const val = args[0]!.value(); + expect(val).toBeInstanceOf(FunctionCallAst); + }); +}); + +describe('StringLiteralExprAst', () => { + it('returns unquoted string value', () => { + const b = new GreenNodeBuilder(); + b.startNode('StringLiteralExpr'); + b.token('StringLiteral', '"hello world"'); + const root = createSyntaxTree(b.finishNode()); + const expr = StringLiteralExprAst.cast(root)!; + expect(expr.value()).toBe('hello world'); + }); + + it('unescapes escape sequences', () => { + const b = new GreenNodeBuilder(); + b.startNode('StringLiteralExpr'); + b.token('StringLiteral', '"line1\\nline2\\ttab"'); + const root = createSyntaxTree(b.finishNode()); + const expr = StringLiteralExprAst.cast(root)!; + expect(expr.value()).toBe('line1\nline2\ttab'); + }); + + it('returns undefined when token missing', () => { + const b = new GreenNodeBuilder(); + b.startNode('StringLiteralExpr'); + const root = createSyntaxTree(b.finishNode()); + const expr = StringLiteralExprAst.cast(root)!; + expect(expr.token()).toBeUndefined(); + expect(expr.value()).toBeUndefined(); + }); +}); + +describe('NumberLiteralExprAst', () => { + it('returns parsed integer', () => { + const b = new GreenNodeBuilder(); + b.startNode('NumberLiteralExpr'); + b.token('NumberLiteral', '42'); + const root = createSyntaxTree(b.finishNode()); + const expr = NumberLiteralExprAst.cast(root)!; + expect(expr.value()).toBe(42); + }); + + it('returns parsed float', () => { + const b = new GreenNodeBuilder(); + b.startNode('NumberLiteralExpr'); + b.token('NumberLiteral', '3.14'); + const root = createSyntaxTree(b.finishNode()); + const expr = NumberLiteralExprAst.cast(root)!; + expect(expr.value()).toBe(3.14); + }); + + it('returns undefined when token missing', () => { + const b = new GreenNodeBuilder(); + b.startNode('NumberLiteralExpr'); + const root = createSyntaxTree(b.finishNode()); + const expr = NumberLiteralExprAst.cast(root)!; + expect(expr.value()).toBeUndefined(); + }); +}); + +describe('BooleanLiteralExprAst', () => { + it('returns true', () => { + const b = new GreenNodeBuilder(); + b.startNode('BooleanLiteralExpr'); + b.token('Ident', 'true'); + const root = createSyntaxTree(b.finishNode()); + const expr = BooleanLiteralExprAst.cast(root)!; + expect(expr.value()).toBe(true); + }); + + it('returns false', () => { + const b = new GreenNodeBuilder(); + b.startNode('BooleanLiteralExpr'); + b.token('Ident', 'false'); + const root = createSyntaxTree(b.finishNode()); + const expr = BooleanLiteralExprAst.cast(root)!; + expect(expr.value()).toBe(false); + }); + + it('returns undefined for non-boolean ident', () => { + const b = new GreenNodeBuilder(); + b.startNode('BooleanLiteralExpr'); + b.token('Ident', 'maybe'); + const root = createSyntaxTree(b.finishNode()); + const expr = BooleanLiteralExprAst.cast(root)!; + expect(expr.value()).toBeUndefined(); + }); + + it('returns undefined when token missing', () => { + const b = new GreenNodeBuilder(); + b.startNode('BooleanLiteralExpr'); + const root = createSyntaxTree(b.finishNode()); + const expr = BooleanLiteralExprAst.cast(root)!; + expect(expr.value()).toBeUndefined(); + }); +}); + +describe('AttributeArgAst', () => { + it('exposes positional arg value', () => { + const b = new GreenNodeBuilder(); + b.startNode('AttributeArg'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const arg = AttributeArgAst.cast(root)!; + expect(arg.name()).toBeUndefined(); // positional - no colon + expect(arg.colon()).toBeUndefined(); + const val = arg.value(); + expect(val).toBeInstanceOf(IdentifierAst); + }); + + it('exposes named arg with colon', () => { + const b = new GreenNodeBuilder(); + b.startNode('AttributeArg'); + b.startNode('Identifier'); + b.token('Ident', 'fields'); + b.finishNode(); + b.token('Colon', ':'); + b.token('Whitespace', ' '); + b.startNode('ArrayLiteral'); + b.token('LBracket', '['); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.token('RBracket', ']'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const arg = AttributeArgAst.cast(root)!; + expect(arg.name()?.token()?.text).toBe('fields'); + expect(arg.colon()?.text).toBe(':'); + const val = arg.value(); + expect(val).toBeInstanceOf(ArrayLiteralAst); + }); +}); + +describe('ArrayLiteralAst', () => { + it('exposes brackets and elements', () => { + const b = new GreenNodeBuilder(); + b.startNode('ArrayLiteral'); + b.token('LBracket', '['); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.token('Comma', ','); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'name'); + b.finishNode(); + b.token('RBracket', ']'); + const root = createSyntaxTree(b.finishNode()); + const arr = ArrayLiteralAst.cast(root)!; + expect(arr.lbracket()?.text).toBe('['); + expect(arr.rbracket()?.text).toBe(']'); + const elements = Array.from(arr.elements()); + expect(elements).toHaveLength(2); + }); +}); + +describe('DocumentAst', () => { + it('iterates mixed declarations', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('ModelDeclaration'); + b.token('Ident', 'model'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'User'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('RBrace', '}'); + b.finishNode(); + b.token('Newline', '\n'); + b.startNode('EnumDeclaration'); + b.token('Ident', 'enum'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'Role'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('RBrace', '}'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const doc = DocumentAst.cast(root)!; + const decls = Array.from(doc.declarations()); + expect(decls).toHaveLength(2); + expect(decls[0]).toBeInstanceOf(ModelDeclarationAst); + expect(decls[1]).toBeInstanceOf(EnumDeclarationAst); + }); +}); diff --git a/packages/1-framework/2-authoring/psl-parser/test/syntax/green-builder.test.ts b/packages/1-framework/2-authoring/psl-parser/test/syntax/green-builder.test.ts new file mode 100644 index 0000000000..9ac5d13208 --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/test/syntax/green-builder.test.ts @@ -0,0 +1,101 @@ +import { describe, expect, it } from 'vitest'; +import type { GreenElement } from '../../src/syntax/green'; +import { GreenNodeBuilder } from '../../src/syntax/green-builder'; + +function collectTexts(el: GreenElement): string { + if (el.type === 'token') return el.text; + return el.children.map(collectTexts).join(''); +} + +describe('GreenNodeBuilder', () => { + it('builds a flat node with tokens', () => { + const b = new GreenNodeBuilder(); + b.startNode('Identifier'); + b.token('Ident', 'User'); + const root = b.finishNode(); + + expect(root.kind).toBe('Identifier'); + expect(root.children).toHaveLength(1); + expect(root.children[0]?.type).toBe('token'); + expect(root.textLength).toBe(4); + }); + + it('builds nested nodes', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('ModelDeclaration'); + b.token('Ident', 'model'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'User'); + b.finishNode(); + b.finishNode(); + const root = b.finishNode(); + + expect(root.kind).toBe('Document'); + expect(root.children).toHaveLength(1); + const model = root.children[0]; + expect(model?.type).toBe('node'); + if (model?.type === 'node') { + expect(model.kind).toBe('ModelDeclaration'); + expect(model.children).toHaveLength(3); + } + }); + + it('inner finishNode returns the completed node', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('Identifier'); + b.token('Ident', 'x'); + const inner = b.finishNode(); + expect(inner.kind).toBe('Identifier'); + }); + + it('throws on finishNode with no open node', () => { + const b = new GreenNodeBuilder(); + expect(() => b.finishNode()).toThrow('no open node'); + }); + + it('throws on token with no open node', () => { + const b = new GreenNodeBuilder(); + expect(() => b.token('Ident', 'x')).toThrow('no open node'); + }); + + it('produces lossless round-trip', () => { + const source = 'model User {\n id Int @id\n}'; + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('ModelDeclaration'); + b.token('Ident', 'model'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'User'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('Newline', '\n'); + b.token('Whitespace', ' '); + b.startNode('FieldDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('TypeAnnotation'); + b.token('Ident', 'Int'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('FieldAttribute'); + b.token('At', '@'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.token('RBrace', '}'); + b.finishNode(); + const root = b.finishNode(); + + expect(collectTexts(root)).toBe(source); + }); +}); diff --git a/packages/1-framework/2-authoring/psl-parser/test/syntax/green.test.ts b/packages/1-framework/2-authoring/psl-parser/test/syntax/green.test.ts new file mode 100644 index 0000000000..86ea97078d --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/test/syntax/green.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest'; +import type { GreenElement } from '../../src/syntax/green'; +import { greenNode, greenToken } from '../../src/syntax/green'; + +describe('GreenToken', () => { + it('stores kind and text', () => { + const t = greenToken('Ident', 'model'); + expect(t.type).toBe('token'); + expect(t.kind).toBe('Ident'); + expect(t.text).toBe('model'); + }); +}); + +describe('GreenNode', () => { + it('computes textLength from single token child', () => { + const node = greenNode('Identifier', [greenToken('Ident', 'User')]); + expect(node.textLength).toBe(4); + }); + + it('computes textLength from multiple token children', () => { + const node = greenNode('TypeAnnotation', [ + greenToken('Ident', 'Int'), + greenToken('LBracket', '['), + greenToken('RBracket', ']'), + ]); + expect(node.textLength).toBe(5); + }); + + it('computes textLength from nested nodes', () => { + const inner = greenNode('Identifier', [greenToken('Ident', 'User')]); + const outer = greenNode('FieldDeclaration', [ + inner, + greenToken('Whitespace', ' '), + greenNode('TypeAnnotation', [greenToken('Ident', 'String')]), + ]); + expect(outer.textLength).toBe(11); // "User" + " " + "String" + }); + + it('has textLength 0 for empty node', () => { + const node = greenNode('Document', []); + expect(node.textLength).toBe(0); + }); + + it('collects all tokens in document order', () => { + const tree = greenNode('Document', [ + greenToken('Ident', 'model'), + greenToken('Whitespace', ' '), + greenNode('Identifier', [greenToken('Ident', 'User')]), + greenToken('Whitespace', ' '), + greenToken('LBrace', '{'), + greenToken('RBrace', '}'), + ]); + + const texts: string[] = []; + function collect(el: GreenElement): void { + if (el.type === 'token') { + texts.push(el.text); + } else { + for (const child of el.children) { + collect(child); + } + } + } + collect(tree); + + expect(texts.join('')).toBe('model User {}'); + }); +}); diff --git a/packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts b/packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts new file mode 100644 index 0000000000..8e69740e1d --- /dev/null +++ b/packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from 'vitest'; +import { GreenNodeBuilder } from '../../src/syntax/green-builder'; +import { createSyntaxTree, SyntaxNode } from '../../src/syntax/red'; + +/** Builds a tree for: model User {\n id Int @id\n} */ +function buildSampleTree() { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('ModelDeclaration'); + b.token('Ident', 'model'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'User'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('Newline', '\n'); + b.token('Whitespace', ' '); + b.startNode('FieldDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('TypeAnnotation'); + b.token('Ident', 'Int'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('FieldAttribute'); + b.token('At', '@'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.token('RBrace', '}'); + b.finishNode(); + return b.finishNode(); +} + +describe('createSyntaxTree', () => { + it('wraps green root with offset 0 and no parent', () => { + const green = buildSampleTree(); + const root = createSyntaxTree(green); + expect(root.offset).toBe(0); + expect(root.parent).toBeUndefined(); + expect(root.kind).toBe('Document'); + }); +}); + +describe('SyntaxNode offset correctness', () => { + it('computes correct offsets for all tokens', () => { + const source = 'model User {\n id Int @id\n}'; + const green = buildSampleTree(); + const root = createSyntaxTree(green); + + const tokens = Array.from(root.tokens()); + let expectedOffset = 0; + for (const tok of tokens) { + expect(tok.offset).toBe(expectedOffset); + expectedOffset += tok.text.length; + } + expect(expectedOffset).toBe(source.length); + }); + + it('computes correct offset for nested nodes', () => { + const green = buildSampleTree(); + const root = createSyntaxTree(green); + + // Document at 0 + expect(root.offset).toBe(0); + + // ModelDeclaration at 0 + const model = root.firstChild; + expect(model).toBeInstanceOf(SyntaxNode); + if (model instanceof SyntaxNode) { + expect(model.offset).toBe(0); + expect(model.kind).toBe('ModelDeclaration'); + } + }); +}); + +describe('SyntaxNode.parent', () => { + it('root has undefined parent', () => { + const root = createSyntaxTree(buildSampleTree()); + expect(root.parent).toBeUndefined(); + }); + + it('child nodes point back to parent', () => { + const root = createSyntaxTree(buildSampleTree()); + const model = root.firstChild; + expect(model).toBeInstanceOf(SyntaxNode); + if (model instanceof SyntaxNode) { + expect(model.parent).toBe(root); + } + }); +}); + +describe('SyntaxNode.firstChild / lastChild', () => { + it('returns first and last children', () => { + const root = createSyntaxTree(buildSampleTree()); + const model = root.firstChild; + expect(model).toBeInstanceOf(SyntaxNode); + expect(root.lastChild).toBeInstanceOf(SyntaxNode); + // Document has only one child (ModelDeclaration), so first === last by green identity + if (model instanceof SyntaxNode) { + expect(model.kind).toBe('ModelDeclaration'); + } + }); + + it('returns undefined for empty node', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + const green = b.finishNode(); + const root = createSyntaxTree(green); + expect(root.firstChild).toBeUndefined(); + expect(root.lastChild).toBeUndefined(); + }); +}); + +describe('SyntaxNode.nextSibling / prevSibling', () => { + it('navigates between siblings', () => { + // Build a Document with two children: model identifier tokens + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('Identifier'); + b.token('Ident', 'A'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'B'); + b.finishNode(); + const green = b.finishNode(); + const root = createSyntaxTree(green); + + const first = root.firstChild; + expect(first).toBeInstanceOf(SyntaxNode); + if (first instanceof SyntaxNode) { + const next = first.nextSibling; + // next sibling should be a whitespace token + expect(next).toBeDefined(); + expect(next).not.toBeInstanceOf(SyntaxNode); + if (next && !(next instanceof SyntaxNode)) { + expect(next.kind).toBe('Whitespace'); + } + } + }); + + it('returns undefined for no sibling', () => { + const root = createSyntaxTree(buildSampleTree()); + // Root has no parent, so no siblings + expect(root.nextSibling).toBeUndefined(); + expect(root.prevSibling).toBeUndefined(); + }); +}); + +describe('SyntaxNode.ancestors', () => { + it('walks from node to root', () => { + const root = createSyntaxTree(buildSampleTree()); + // Navigate: Document > ModelDeclaration > first child node + const model = root.firstChild; + expect(model).toBeInstanceOf(SyntaxNode); + if (model instanceof SyntaxNode) { + const ancestors = Array.from(model.ancestors()); + expect(ancestors).toHaveLength(1); + expect(ancestors[0]).toBe(root); + } + }); + + it('yields nothing for root', () => { + const root = createSyntaxTree(buildSampleTree()); + const ancestors = Array.from(root.ancestors()); + expect(ancestors).toHaveLength(0); + }); +}); + +describe('SyntaxNode.descendants', () => { + it('yields elements in depth-first pre-order', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('Identifier'); + b.token('Ident', 'A'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'B'); + b.finishNode(); + const green = b.finishNode(); + const root = createSyntaxTree(green); + + const kinds: string[] = []; + for (const el of root.descendants()) { + if (el instanceof SyntaxNode) { + kinds.push(`node:${el.kind}`); + } else { + kinds.push(`token:${el.kind}`); + } + } + + expect(kinds).toEqual([ + 'node:Document', + 'node:Identifier', + 'token:Ident', // A + 'token:Whitespace', + 'node:Identifier', + 'token:Ident', // B + ]); + }); +}); diff --git a/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts b/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts index 6cb54562be..1709e9f0ce 100644 --- a/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts +++ b/packages/1-framework/2-authoring/psl-parser/tsdown.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ 'src/exports/index.ts', 'src/exports/parser.ts', 'src/exports/tokenizer.ts', + 'src/exports/syntax.ts', 'src/exports/types.ts', ], }); From a367b14d3fa6eb63bb559ca6cd5fccba2e801574 Mon Sep 17 00:00:00 2001 From: Serhii Tatarintsev Date: Wed, 4 Mar 2026 14:50:48 +0100 Subject: [PATCH 6/6] test(psl-parser): add AST accessor and red tree tests to satisfy coverage gate Cover all previously untested AST wrapper methods (EnumDeclarationAst, TypesBlockAst, BlockDeclarationAst, NamedTypeDeclarationAst, FieldDeclarationAst.attributes, FunctionCallAst, ModelAttributeAst.argList) and SyntaxNode.textLength/prevSibling to bring functions coverage to 100%. Co-Authored-By: Claude Opus 4.6 --- .../psl-parser/test/syntax/ast.test.ts | 251 ++++++++++++++++++ .../psl-parser/test/syntax/red.test.ts | 36 +++ 2 files changed, 287 insertions(+) diff --git a/packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts b/packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts index 87e1cb7704..ed00f9274a 100644 --- a/packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts +++ b/packages/1-framework/2-authoring/psl-parser/test/syntax/ast.test.ts @@ -531,3 +531,254 @@ describe('DocumentAst', () => { expect(decls[1]).toBeInstanceOf(EnumDeclarationAst); }); }); + +describe('EnumDeclarationAst', () => { + function buildEnum() { + const b = new GreenNodeBuilder(); + b.startNode('EnumDeclaration'); + b.token('Ident', 'enum'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'Role'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('Newline', '\n'); + b.token('Whitespace', ' '); + b.startNode('EnumValueDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'ADMIN'); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.token('Whitespace', ' '); + b.startNode('EnumValueDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'USER'); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.token('RBrace', '}'); + return b.finishNode(); + } + + it('exposes keyword, name, braces', () => { + const root = createSyntaxTree(buildEnum()); + const decl = EnumDeclarationAst.cast(root)!; + expect(decl.keyword()?.text).toBe('enum'); + expect(decl.name()?.token()?.text).toBe('Role'); + expect(decl.lbrace()?.text).toBe('{'); + expect(decl.rbrace()?.text).toBe('}'); + }); + + it('iterates values', () => { + const root = createSyntaxTree(buildEnum()); + const decl = EnumDeclarationAst.cast(root)!; + const values = Array.from(decl.values()); + expect(values).toHaveLength(2); + expect(values[0]!.name()?.token()?.text).toBe('ADMIN'); + expect(values[1]!.name()?.token()?.text).toBe('USER'); + }); +}); + +describe('TypesBlockAst', () => { + function buildTypesBlock() { + const b = new GreenNodeBuilder(); + b.startNode('TypesBlock'); + b.token('Ident', 'type'); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('Newline', '\n'); + b.token('Whitespace', ' '); + b.startNode('NamedTypeDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'UserId'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('Equals', '='); + b.token('Whitespace', ' '); + b.startNode('TypeAnnotation'); + b.token('Ident', 'Int'); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.token('RBrace', '}'); + return b.finishNode(); + } + + it('exposes keyword, braces', () => { + const root = createSyntaxTree(buildTypesBlock()); + const decl = TypesBlockAst.cast(root)!; + expect(decl.keyword()?.text).toBe('type'); + expect(decl.lbrace()?.text).toBe('{'); + expect(decl.rbrace()?.text).toBe('}'); + }); + + it('iterates declarations', () => { + const root = createSyntaxTree(buildTypesBlock()); + const decl = TypesBlockAst.cast(root)!; + const namedTypes = Array.from(decl.declarations()); + expect(namedTypes).toHaveLength(1); + expect(namedTypes[0]!.name()?.token()?.text).toBe('UserId'); + }); +}); + +describe('NamedTypeDeclarationAst', () => { + function buildNamedType() { + const b = new GreenNodeBuilder(); + b.startNode('NamedTypeDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'UserId'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('Equals', '='); + b.token('Whitespace', ' '); + b.startNode('TypeAnnotation'); + b.startNode('Identifier'); + b.token('Ident', 'Int'); + b.finishNode(); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('FieldAttribute'); + b.token('At', '@'); + b.startNode('Identifier'); + b.token('Ident', 'db'); + b.finishNode(); + b.finishNode(); + return b.finishNode(); + } + + it('exposes name, equals, typeAnnotation, attributes', () => { + const root = createSyntaxTree(buildNamedType()); + const decl = NamedTypeDeclarationAst.cast(root)!; + expect(decl.name()?.token()?.text).toBe('UserId'); + expect(decl.equals()?.text).toBe('='); + expect(decl.typeAnnotation()?.name()?.token()?.text).toBe('Int'); + const attrs = Array.from(decl.attributes()); + expect(attrs).toHaveLength(1); + expect(attrs[0]!.name()?.token()?.text).toBe('db'); + }); +}); + +describe('BlockDeclarationAst', () => { + function buildBlock() { + const b = new GreenNodeBuilder(); + b.startNode('BlockDeclaration'); + b.token('Ident', 'datasource'); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'db'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('LBrace', '{'); + b.token('Newline', '\n'); + b.token('Whitespace', ' '); + b.startNode('KeyValuePair'); + b.startNode('Identifier'); + b.token('Ident', 'provider'); + b.finishNode(); + b.token('Whitespace', ' '); + b.token('Equals', '='); + b.token('Whitespace', ' '); + b.startNode('StringLiteralExpr'); + b.token('StringLiteral', '"postgresql"'); + b.finishNode(); + b.finishNode(); + b.token('Newline', '\n'); + b.token('RBrace', '}'); + return b.finishNode(); + } + + it('exposes keyword, name, braces', () => { + const root = createSyntaxTree(buildBlock()); + const decl = BlockDeclarationAst.cast(root)!; + expect(decl.keyword()?.text).toBe('datasource'); + expect(decl.name()?.token()?.text).toBe('db'); + expect(decl.lbrace()?.text).toBe('{'); + expect(decl.rbrace()?.text).toBe('}'); + }); + + it('iterates entries', () => { + const root = createSyntaxTree(buildBlock()); + const decl = BlockDeclarationAst.cast(root)!; + const entries = Array.from(decl.entries()); + expect(entries).toHaveLength(1); + expect(entries[0]!.key()?.token()?.text).toBe('provider'); + }); +}); + +describe('FieldDeclarationAst.attributes', () => { + it('iterates field attributes', () => { + const b = new GreenNodeBuilder(); + b.startNode('FieldDeclaration'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('TypeAnnotation'); + b.token('Ident', 'Int'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('FieldAttribute'); + b.token('At', '@'); + b.startNode('Identifier'); + b.token('Ident', 'id'); + b.finishNode(); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const field = FieldDeclarationAst.cast(root)!; + const attrs = Array.from(field.attributes()); + expect(attrs).toHaveLength(1); + expect(attrs[0]!.name()?.token()?.text).toBe('id'); + }); +}); + +describe('FunctionCallAst', () => { + it('exposes name, parens, and args', () => { + const b = new GreenNodeBuilder(); + b.startNode('FunctionCall'); + b.startNode('Identifier'); + b.token('Ident', 'autoincrement'); + b.finishNode(); + b.token('LParen', '('); + b.token('RParen', ')'); + const root = createSyntaxTree(b.finishNode()); + const fn = FunctionCallAst.cast(root)!; + expect(fn.name()?.token()?.text).toBe('autoincrement'); + expect(fn.lparen()?.text).toBe('('); + expect(fn.rparen()?.text).toBe(')'); + expect(Array.from(fn.args())).toHaveLength(0); + }); +}); + +describe('ModelAttributeAst.argList', () => { + it('exposes argList with args', () => { + // @@unique([email]) + const b = new GreenNodeBuilder(); + b.startNode('ModelAttribute'); + b.token('DoubleAt', '@@'); + b.startNode('Identifier'); + b.token('Ident', 'unique'); + b.finishNode(); + b.startNode('AttributeArgList'); + b.token('LParen', '('); + b.startNode('AttributeArg'); + b.startNode('ArrayLiteral'); + b.token('LBracket', '['); + b.startNode('Identifier'); + b.token('Ident', 'email'); + b.finishNode(); + b.token('RBracket', ']'); + b.finishNode(); + b.finishNode(); + b.token('RParen', ')'); + b.finishNode(); + const root = createSyntaxTree(b.finishNode()); + const attr = ModelAttributeAst.cast(root)!; + const argList = attr.argList(); + expect(argList).toBeDefined(); + expect(argList!.lparen()?.text).toBe('('); + const args = Array.from(argList!.args()); + expect(args).toHaveLength(1); + }); +}); diff --git a/packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts b/packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts index 8e69740e1d..2d86e0e6a0 100644 --- a/packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts +++ b/packages/1-framework/2-authoring/psl-parser/test/syntax/red.test.ts @@ -152,6 +152,42 @@ describe('SyntaxNode.nextSibling / prevSibling', () => { expect(root.nextSibling).toBeUndefined(); expect(root.prevSibling).toBeUndefined(); }); + + it('navigates prevSibling from last child back', () => { + const b = new GreenNodeBuilder(); + b.startNode('Document'); + b.startNode('Identifier'); + b.token('Ident', 'A'); + b.finishNode(); + b.token('Whitespace', ' '); + b.startNode('Identifier'); + b.token('Ident', 'B'); + b.finishNode(); + const green = b.finishNode(); + const root = createSyntaxTree(green); + + // Get the last child node (Identifier "B") + const children = Array.from(root.children()); + const lastNode = children[2]; // Identifier "B" + expect(lastNode).toBeInstanceOf(SyntaxNode); + if (lastNode instanceof SyntaxNode) { + const prev = lastNode.prevSibling; + expect(prev).toBeDefined(); + expect(prev).not.toBeInstanceOf(SyntaxNode); + if (prev && !(prev instanceof SyntaxNode)) { + expect(prev.kind).toBe('Whitespace'); + } + } + }); +}); + +describe('SyntaxNode.textLength', () => { + it('returns total text length of the subtree', () => { + const source = 'model User {\n id Int @id\n}'; + const green = buildSampleTree(); + const root = createSyntaxTree(green); + expect(root.textLength).toBe(source.length); + }); }); describe('SyntaxNode.ancestors', () => {