From 2cbd3135ecb73568afc0ab2415b3c97f31e81ca7 Mon Sep 17 00:00:00 2001 From: LBan Date: Tue, 16 Dec 2025 15:55:52 +0800 Subject: [PATCH 01/21] feat(afs): implement i18n driver for multilingual support and enhance view processing --- afs/core/package.json | 2 + afs/core/src/afs.ts | 110 +- afs/core/src/index.ts | 3 + afs/core/src/metadata/index.ts | 4 + afs/core/src/metadata/migrate.ts | 49 + afs/core/src/metadata/migrations/001-init.ts | 40 + afs/core/src/metadata/models/index.ts | 2 + .../src/metadata/models/source-metadata.ts | 16 + afs/core/src/metadata/models/view-metadata.ts | 29 + afs/core/src/metadata/store.ts | 283 +++ afs/core/src/metadata/type.ts | 61 + afs/core/src/type.ts | 69 + afs/core/src/view-processor.ts | 277 +++ afs/core/src/view-schema.ts | 82 + afs/i18n-driver.md | 1837 +++++++++++++++++ pnpm-lock.yaml | 78 +- 16 files changed, 2885 insertions(+), 57 deletions(-) create mode 100644 afs/core/src/metadata/index.ts create mode 100644 afs/core/src/metadata/migrate.ts create mode 100644 afs/core/src/metadata/migrations/001-init.ts create mode 100644 afs/core/src/metadata/models/index.ts create mode 100644 afs/core/src/metadata/models/source-metadata.ts create mode 100644 afs/core/src/metadata/models/view-metadata.ts create mode 100644 afs/core/src/metadata/store.ts create mode 100644 afs/core/src/metadata/type.ts create mode 100644 afs/core/src/view-processor.ts create mode 100644 afs/core/src/view-schema.ts create mode 100644 afs/i18n-driver.md diff --git a/afs/core/package.json b/afs/core/package.json index a0433521f..bc4a584e9 100644 --- a/afs/core/package.json +++ b/afs/core/package.json @@ -56,6 +56,8 @@ "postbuild": "node ../../scripts/post-build-lib.mjs" }, "dependencies": { + "@aigne/sqlite": "workspace:^", + "p-limit": "^6.1.0", "strict-event-emitter": "^0.5.1", "ufo": "^1.6.1", "zod": "^3.25.67" diff --git a/afs/core/src/afs.ts b/afs/core/src/afs.ts index 1585c4cc3..f755f7e53 100644 --- a/afs/core/src/afs.ts +++ b/afs/core/src/afs.ts @@ -1,11 +1,13 @@ import { Emitter } from "strict-event-emitter"; import { joinURL } from "ufo"; import { z } from "zod"; +import { SQLiteMetadataStore } from "./metadata/index.js"; import { type AFSContext, type AFSContextPreset, type AFSDeleteOptions, type AFSDeleteResult, + type AFSDriver, type AFSEntry, type AFSExecOptions, type AFSExecResult, @@ -26,8 +28,10 @@ import { type AFSWriteEntryPayload, type AFSWriteOptions, type AFSWriteResult, + type View, afsEntrySchema, } from "./type.js"; +import { ViewProcessor } from "./view-processor.js"; const DEFAULT_MAX_DEPTH = 1; @@ -36,20 +40,45 @@ const MODULES_ROOT_DIR = "/modules"; export interface AFSOptions { modules?: AFSModule[]; context?: AFSContext; + drivers?: AFSDriver[]; + metadataPath?: string; // SQLite database path for metadata, default: ".afs/metadata.db" } export class AFS extends Emitter implements AFSRoot { name: string = "AFSRoot"; + private modules = new Map(); + private _drivers: AFSDriver[] = []; + private metadataStore?: SQLiteMetadataStore; + private viewProcessor?: ViewProcessor; + constructor(public options: AFSOptions = {}) { super(); + // Mount modules for (const module of options?.modules ?? []) { this.mount(module); } + + // Initialize drivers + this._drivers = options?.drivers ?? []; + + // Initialize metadata store and view processor if drivers are present + if (this._drivers.length > 0) { + const metadataPath = options?.metadataPath || "file:.afs/metadata.db"; + this.metadataStore = new SQLiteMetadataStore({ url: metadataPath }); + this.viewProcessor = new ViewProcessor(this.metadataStore, this._drivers); + + // Mount drivers + for (const driver of this._drivers) { + driver.onMount?.(this); + } + } } - private modules = new Map(); + get drivers(): AFSDriver[] { + return this._drivers; + } mount(module: AFSModule): this { let path = joinURL("/", module.name); @@ -136,20 +165,36 @@ export class AFS extends Emitter implements AFSRoot { return { data: results }; } - async read(path: string, _options?: AFSReadOptions): Promise { + async read(path: string, options?: AFSReadOptions): Promise { const modules = this.findModules(path, { exactMatch: true }); for (const { module, modulePath, subpath } of modules) { - const res = await module.read?.(subpath); - - if (res?.data) { - return { - ...res, - data: { - ...res.data, - path: joinURL(modulePath, res.data.path), - }, - }; + // If view is requested and we have a view processor, use it + if (options?.view && this.viewProcessor) { + const res = await this.viewProcessor.handleRead(module, subpath, options); + + if (res?.data) { + return { + ...res, + data: { + ...res.data, + path: joinURL(modulePath, res.data.path), + }, + }; + } + } else { + // No view requested, read normally + const res = await module.read?.(subpath, options); + + if (res?.data) { + return { + ...res, + data: { + ...res.data, + path: joinURL(modulePath, res.data.path), + }, + }; + } } } @@ -166,6 +211,11 @@ export class AFS extends Emitter implements AFSRoot { const res = await module.module.write(module.subpath, content, options); + // Update metadata if view processor is available + if (this.viewProcessor) { + await this.viewProcessor.handleWrite(module.subpath, res.data); + } + return { ...res, data: { @@ -179,7 +229,14 @@ export class AFS extends Emitter implements AFSRoot { const module = this.findModules(path, { exactMatch: true })[0]; if (!module?.module.delete) throw new Error(`No module found for path: ${path}`); - return await module.module.delete(module.subpath, options); + const result = await module.module.delete(module.subpath, options); + + // Clean up metadata if view processor is available + if (this.viewProcessor) { + await this.viewProcessor.handleDelete(module.subpath); + } + + return result; } async rename( @@ -422,4 +479,31 @@ export class AFS extends Emitter implements AFSRoot { return metadataSuffix; } + + /** + * Prefetch views for batch generation + * @param pathOrGlob - Single path or array of paths (glob support TBD) + * @param options - View options + */ + async prefetch( + pathOrGlob: string | string[], + options: { view: View; concurrency?: number; context?: any }, + ): Promise { + if (!this.viewProcessor) { + throw new Error("Prefetch requires drivers to be configured"); + } + + const paths = Array.isArray(pathOrGlob) ? pathOrGlob : [pathOrGlob]; + + // For each path, find the module and prefetch + for (const path of paths) { + const module = this.findModules(path, { exactMatch: true })[0]; + if (module) { + await this.viewProcessor.prefetch(module.module, [module.subpath], options.view, { + concurrency: options.concurrency, + context: options.context, + }); + } + } + } } diff --git a/afs/core/src/index.ts b/afs/core/src/index.ts index 4505d4b0b..fa84e5695 100644 --- a/afs/core/src/index.ts +++ b/afs/core/src/index.ts @@ -1,2 +1,5 @@ export * from "./afs.js"; +export * from "./metadata/index.js"; export * from "./type.js"; +export * from "./view-processor.js"; +export * from "./view-schema.js"; diff --git a/afs/core/src/metadata/index.ts b/afs/core/src/metadata/index.ts new file mode 100644 index 000000000..cedbef171 --- /dev/null +++ b/afs/core/src/metadata/index.ts @@ -0,0 +1,4 @@ +export * from "./migrate.js"; +export * from "./models/index.js"; +export * from "./store.js"; +export * from "./type.js"; diff --git a/afs/core/src/metadata/migrate.ts b/afs/core/src/metadata/migrate.ts new file mode 100644 index 000000000..d3e737376 --- /dev/null +++ b/afs/core/src/metadata/migrate.ts @@ -0,0 +1,49 @@ +import type { initDatabase } from "@aigne/sqlite"; +import { sql } from "@aigne/sqlite"; +import { init } from "./migrations/001-init.js"; + +const migrations = [init]; + +/** + * Run database migrations + */ +export async function migrate(db: ReturnType): Promise { + const dbInstance = await db; + + // Create migrations tracking table + await dbInstance + .run( + sql`CREATE TABLE IF NOT EXISTS __metadata_migrations ( + hash TEXT PRIMARY KEY, + applied_at INTEGER NOT NULL + )`, + ) + .execute(); + + // Run pending migrations + for (const migration of migrations) { + const rows = await dbInstance + .values<[string, number]>( + sql`SELECT hash, applied_at FROM __metadata_migrations WHERE hash = ${sql.param(migration.hash)}`, + ) + .execute(); + + const existing = rows[0]; + + if (!existing) { + console.log(`Running migration: ${migration.hash}`); + + // Execute all SQL statements in the migration + for (const statement of migration.sql()) { + await dbInstance.run(statement).execute(); + } + + // Record migration as applied + await dbInstance + .run( + sql`INSERT INTO __metadata_migrations (hash, applied_at) VALUES (${sql.param(migration.hash)}, ${sql.param(Date.now())})`, + ) + .execute(); + } + } +} diff --git a/afs/core/src/metadata/migrations/001-init.ts b/afs/core/src/metadata/migrations/001-init.ts new file mode 100644 index 000000000..7b91fb434 --- /dev/null +++ b/afs/core/src/metadata/migrations/001-init.ts @@ -0,0 +1,40 @@ +import type { SQL } from "@aigne/sqlite"; +import { sql } from "@aigne/sqlite"; + +/** + * Initial migration: Create source_metadata and view_metadata tables + */ +export const init = { + hash: "001-init", + sql: (): SQL[] => [ + // Source metadata table + sql` +CREATE TABLE IF NOT EXISTS source_metadata ( + path TEXT PRIMARY KEY, + source_revision TEXT NOT NULL, + updated_at INTEGER NOT NULL, + drivers_hint TEXT, + created_at INTEGER NOT NULL +)`, + // View metadata table + sql` +CREATE TABLE IF NOT EXISTS view_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + view TEXT NOT NULL, + state TEXT NOT NULL, + derived_from TEXT NOT NULL, + generated_at INTEGER, + error TEXT, + storage_path TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(path, view) +)`, + // Indexes + sql`CREATE INDEX IF NOT EXISTS idx_view_path ON view_metadata(path)`, + sql`CREATE INDEX IF NOT EXISTS idx_view_state ON view_metadata(state)`, + sql`CREATE INDEX IF NOT EXISTS idx_view_derived_from ON view_metadata(derived_from)`, + sql`CREATE UNIQUE INDEX IF NOT EXISTS unique_path_view ON view_metadata(path, view)`, + ], +}; diff --git a/afs/core/src/metadata/models/index.ts b/afs/core/src/metadata/models/index.ts new file mode 100644 index 000000000..8544b73c2 --- /dev/null +++ b/afs/core/src/metadata/models/index.ts @@ -0,0 +1,2 @@ +export * from "./source-metadata.js"; +export * from "./view-metadata.js"; diff --git a/afs/core/src/metadata/models/source-metadata.ts b/afs/core/src/metadata/models/source-metadata.ts new file mode 100644 index 000000000..29a8e267a --- /dev/null +++ b/afs/core/src/metadata/models/source-metadata.ts @@ -0,0 +1,16 @@ +import { integer, sqliteTable, text } from "@aigne/sqlite"; + +/** + * Source metadata table schema + */ +export const sourceMetadataTable = sqliteTable("source_metadata", { + id: integer("id").notNull().primaryKey({ autoIncrement: true }), + path: text("path").notNull().primaryKey(), + sourceRevision: text("source_revision").notNull(), + updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), + driversHint: text("drivers_hint"), // JSON array stored as text + createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(), +}); + +export type SourceMetadataRow = typeof sourceMetadataTable.$inferSelect; +export type SourceMetadataInsert = typeof sourceMetadataTable.$inferInsert; diff --git a/afs/core/src/metadata/models/view-metadata.ts b/afs/core/src/metadata/models/view-metadata.ts new file mode 100644 index 000000000..d67df84ab --- /dev/null +++ b/afs/core/src/metadata/models/view-metadata.ts @@ -0,0 +1,29 @@ +import { index, integer, sqliteTable, text } from "@aigne/sqlite"; + +/** + * View metadata table schema + */ +export const viewMetadataTable = sqliteTable( + "view_metadata", + { + id: integer("id").notNull().primaryKey({ autoIncrement: true }), + path: text("path").notNull(), + view: text("view").notNull(), // JSON stringified View object + state: text("state").notNull(), // 'ready' | 'stale' | 'generating' | 'failed' + derivedFrom: text("derived_from").notNull(), + generatedAt: integer("generated_at", { mode: "timestamp_ms" }), + error: text("error"), + storagePath: text("storage_path"), + createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(), + updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), + }, + (table) => ({ + pathIdx: index("idx_view_path").on(table.path), + stateIdx: index("idx_view_state").on(table.state), + derivedFromIdx: index("idx_view_derived_from").on(table.derivedFrom), + uniquePathView: index("unique_path_view").on(table.path, table.view), + }), +); + +export type ViewMetadataRow = typeof viewMetadataTable.$inferSelect; +export type ViewMetadataInsert = typeof viewMetadataTable.$inferInsert; diff --git a/afs/core/src/metadata/store.ts b/afs/core/src/metadata/store.ts new file mode 100644 index 000000000..89fc68b8d --- /dev/null +++ b/afs/core/src/metadata/store.ts @@ -0,0 +1,283 @@ +import { and, eq, initDatabase } from "@aigne/sqlite"; +import type { View } from "../type.js"; +import { migrate } from "./migrate.js"; +import { sourceMetadataTable, viewMetadataTable } from "./models/index.js"; +import type { MetadataStore, SourceMetadata, ViewMetadata, ViewState } from "./type.js"; + +export interface SQLiteMetadataStoreOptions { + url?: string; +} + +/** + * SQLite-based metadata store implementation + */ +export class SQLiteMetadataStore implements MetadataStore { + private db: ReturnType; + private sourceTable = sourceMetadataTable; + private viewTable = viewMetadataTable; + + constructor(options?: SQLiteMetadataStoreOptions) { + this.db = initDatabase({ url: options?.url }).then(async (db) => { + // Run migrations + await migrate(Promise.resolve(db)); + return db; + }); + } + + // Source metadata operations + async getSourceMetadata(path: string): Promise { + const db = await this.db; + const rows = await db + .select() + .from(this.sourceTable) + .where(eq(this.sourceTable.path, path)) + .limit(1) + .execute(); + + const row = rows[0]; + if (!row) return null; + + return { + path: row.path, + sourceRevision: row.sourceRevision, + updatedAt: row.updatedAt, + driversHint: row.driversHint ? JSON.parse(row.driversHint) : undefined, + }; + } + + async setSourceMetadata(path: string, metadata: Omit): Promise { + const db = await this.db; + const now = new Date(); + + // Try to update first + const updated = await db + .update(this.sourceTable) + .set({ + sourceRevision: metadata.sourceRevision, + updatedAt: metadata.updatedAt, + driversHint: metadata.driversHint ? JSON.stringify(metadata.driversHint) : null, + }) + .where(eq(this.sourceTable.path, path)) + .returning() + .execute(); + + // If no rows updated, insert new record + if (!updated.length) { + await db + .insert(this.sourceTable) + .values({ + path, + sourceRevision: metadata.sourceRevision, + updatedAt: metadata.updatedAt, + driversHint: metadata.driversHint ? JSON.stringify(metadata.driversHint) : null, + createdAt: now, + }) + .execute(); + } + } + + async deleteSourceMetadata(path: string): Promise { + const db = await this.db; + await db.delete(this.sourceTable).where(eq(this.sourceTable.path, path)).execute(); + } + + // View metadata operations + async getViewMetadata(path: string, view: View): Promise { + const db = await this.db; + const viewKey = JSON.stringify(view); + + const rows = await db + .select() + .from(this.viewTable) + .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) + .limit(1) + .execute(); + + const row = rows[0]; + if (!row) return null; + + return { + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derivedFrom, + generatedAt: row.generatedAt || undefined, + error: row.error || undefined, + storagePath: row.storagePath || undefined, + }; + } + + async setViewMetadata(path: string, view: View, metadata: Partial): Promise { + const db = await this.db; + const viewKey = JSON.stringify(view); + const now = new Date(); + + // Get existing record + const existing = await this.getViewMetadata(path, view); + + const merged = { + state: metadata.state || existing?.state || "stale", + derivedFrom: metadata.derivedFrom || existing?.derivedFrom || "", + generatedAt: metadata.generatedAt || existing?.generatedAt || null, + error: metadata.error !== undefined ? metadata.error : existing?.error || null, + storagePath: metadata.storagePath || existing?.storagePath || null, + }; + + // Try to update first + const updated = await db + .update(this.viewTable) + .set({ + state: merged.state, + derivedFrom: merged.derivedFrom, + generatedAt: merged.generatedAt, + error: merged.error, + storagePath: merged.storagePath, + updatedAt: now, + }) + .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) + .returning() + .execute(); + + // If no rows updated, insert new record + if (!updated.length) { + await db + .insert(this.viewTable) + .values({ + path, + view: viewKey, + state: merged.state, + derivedFrom: merged.derivedFrom, + generatedAt: merged.generatedAt, + error: merged.error, + storagePath: merged.storagePath, + createdAt: now, + updatedAt: now, + }) + .execute(); + } + } + + async listViewMetadata(path: string): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.viewTable) + .where(eq(this.viewTable.path, path)) + .execute(); + + return rows.map((row) => ({ + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derivedFrom, + generatedAt: row.generatedAt || undefined, + error: row.error || undefined, + storagePath: row.storagePath || undefined, + })); + } + + async deleteViewMetadata(path: string, view?: View): Promise { + const db = await this.db; + + if (view) { + const viewKey = JSON.stringify(view); + await db + .delete(this.viewTable) + .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) + .execute(); + } else { + await db.delete(this.viewTable).where(eq(this.viewTable.path, path)).execute(); + } + } + + // Batch operations + async markViewsAsStale(path: string): Promise { + const db = await this.db; + const now = new Date(); + + await db + .update(this.viewTable) + .set({ + state: "stale", + updatedAt: now, + }) + .where(eq(this.viewTable.path, path)) + .execute(); + } + + async listStaleViews(): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.viewTable) + .where(eq(this.viewTable.state, "stale")) + .execute(); + + return rows.map((row) => ({ + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derivedFrom, + generatedAt: row.generatedAt || undefined, + error: row.error || undefined, + storagePath: row.storagePath || undefined, + })); + } + + async listGeneratingViews(): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.viewTable) + .where(eq(this.viewTable.state, "generating")) + .execute(); + + return rows.map((row) => ({ + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derivedFrom, + generatedAt: row.generatedAt || undefined, + error: row.error || undefined, + storagePath: row.storagePath || undefined, + })); + } + + // Cleanup operations + async cleanupOrphanedViewMetadata(): Promise { + const db = await this.db; + + // Get all distinct paths from view_metadata + const rows = await db + .selectDistinct({ path: this.viewTable.path }) + .from(this.viewTable) + .execute(); + + for (const { path } of rows) { + const sourceMeta = await this.getSourceMetadata(path); + if (!sourceMeta) { + // Source doesn't exist, delete all view metadata for this path + await this.deleteViewMetadata(path); + } + } + } + + async cleanupFailedViews(_olderThan?: Date): Promise { + const db = await this.db; + // const cutoff = olderThan || new Date(Date.now() - 24 * 60 * 60 * 1000); // Default 24 hours + + await db + .delete(this.viewTable) + .where( + and( + // Note: drizzle doesn't have a direct < operator for dates, we need to use sql + // For now, we'll keep this simple and delete all failed views + eq(this.viewTable.state, "failed"), + ), + ) + .execute(); + } +} diff --git a/afs/core/src/metadata/type.ts b/afs/core/src/metadata/type.ts new file mode 100644 index 000000000..cf116a4fd --- /dev/null +++ b/afs/core/src/metadata/type.ts @@ -0,0 +1,61 @@ +import type { View } from "../type.js"; + +/** + * View state + * - ready: View is generated and up-to-date + * - stale: View exists but is outdated (source has changed) + * - generating: View is currently being generated + * - failed: View generation failed + */ +export type ViewState = "ready" | "stale" | "generating" | "failed"; + +/** + * Source-level metadata (per path) + * Tracks the main content version + */ +export interface SourceMetadata { + path: string; + sourceRevision: string; // Content hash or mtime:size identifier + updatedAt: Date; + driversHint?: string[]; // Optional: driver names that may process this file +} + +/** + * View-level metadata (per path + view combination) + * Tracks the state of each view projection + */ +export interface ViewMetadata { + path: string; + view: View; + state: ViewState; + derivedFrom: string; // sourceRevision at generation time + generatedAt?: Date; + error?: string; // Error message if state = 'failed' + storagePath?: string; // Physical storage path (e.g., '.i18n/en/...') +} + +/** + * Metadata Store interface + * Manages source and view metadata using SQLite + */ +export interface MetadataStore { + // Source metadata operations + getSourceMetadata(path: string): Promise; + setSourceMetadata(path: string, metadata: Omit): Promise; + deleteSourceMetadata(path: string): Promise; + + // View metadata operations + getViewMetadata(path: string, view: View): Promise; + setViewMetadata(path: string, view: View, metadata: Partial): Promise; + listViewMetadata(path: string): Promise; + deleteViewMetadata(path: string, view?: View): Promise; + + // Batch operations + markViewsAsStale(path: string): Promise; + listStaleViews(): Promise; + listGeneratingViews(): Promise; + + // Cleanup operations + cleanupOrphanedViewMetadata(): Promise; + cleanupFailedViews(olderThan?: Date): Promise; +} diff --git a/afs/core/src/type.ts b/afs/core/src/type.ts index 53a7936cb..5c2b0a7c2 100644 --- a/afs/core/src/type.ts +++ b/afs/core/src/type.ts @@ -41,13 +41,43 @@ export interface AFSSearchResult { message?: string; } +/** + * View represents different projections of the same file + * V1: Only language dimension is implemented + * Future: format, policy, variant dimensions + */ +export interface View { + language?: string; // Target language for translation (e.g., "en", "zh", "ja") + // format?: string; // Future: format conversion (e.g., "html", "pdf") + // policy?: string; // Future: content style policy (e.g., "technical", "marketing") + // variant?: string; // Future: content variant (e.g., "summary", "toc", "index") +} + +/** + * Wait strategy for view processing + * - strict: Wait for view generation to complete before returning + * - fallback: Return source immediately and trigger background generation + */ +export type WaitStrategy = "strict" | "fallback"; + +/** + * View status in read result + * Indicates whether the requested view was returned or fell back to source + */ +export interface ViewStatus { + fallback?: boolean; // true = returned source content, view is being generated in background +} + export interface AFSReadOptions { + view?: View; + wait?: WaitStrategy; context?: any; } export interface AFSReadResult { data?: AFSEntry; message?: string; + viewStatus?: ViewStatus; } export interface AFSDeleteOptions { @@ -213,3 +243,42 @@ export interface AFSContext { presets?: Record; }; } + +/** + * AFSDriver interface for view transformation + */ +export interface AFSDriver { + readonly name: string; + readonly description?: string; + + /** + * Declare which view dimensions this driver can handle + */ + readonly capabilities: { + dimensions: (keyof View)[]; + }; + + /** + * Check if this driver can handle the given view + */ + canHandle(view: View): boolean; + + /** + * Process and generate the view projection + */ + process( + module: AFSModule, + path: string, + view: View, + options: { + sourceEntry: AFSEntry; + metadata: any; + context: any; + }, + ): Promise<{ result: AFSEntry; message?: string }>; + + /** + * Optional: Called when driver is mounted to AFS + */ + onMount?(root: AFSRoot): void; +} diff --git a/afs/core/src/view-processor.ts b/afs/core/src/view-processor.ts new file mode 100644 index 000000000..376bc43ed --- /dev/null +++ b/afs/core/src/view-processor.ts @@ -0,0 +1,277 @@ +import { createHash } from "node:crypto"; +import pLimit from "p-limit"; +import type { MetadataStore, ViewMetadata } from "./metadata/index.js"; +import type { AFSDriver, AFSEntry, AFSModule, AFSReadOptions, AFSReadResult, View } from "./type.js"; + +/** + * View processor for handling view generation and caching + * V1 implementation: Simplified without job deduplication + */ +export class ViewProcessor { + constructor( + private metadataStore: MetadataStore, + private drivers: AFSDriver[], + ) {} + + /** + * Find a driver that can handle the given view + */ + findDriver(view: View): AFSDriver | null { + const capable = this.drivers.filter((d) => d.canHandle(view)); + + if (capable.length === 0) { + return null; + } + + if (capable.length > 1) { + throw new Error( + `Multiple drivers can handle view ${JSON.stringify(view)}: ${capable.map((d) => d.name).join(", ")}`, + ); + } + + return capable[0] || null; + } + + /** + * Compute source revision for an entry + * - For text content: SHA-256 hash + * - For binary/other: mtime + size + */ + computeRevision(entry: AFSEntry): string { + if (typeof entry.content === "string") { + const hash = createHash("sha256").update(entry.content).digest("hex").substring(0, 16); + return `hash:sha256:${hash}`; + } + + const mtime = entry.updatedAt?.getTime() || Date.now(); + const size = entry.metadata?.size || 0; + return `mtime:${mtime}:size:${size}`; + } + + /** + * Check if a view is stale (outdated) + */ + async isViewStale(path: string, view: View): Promise { + const viewMeta = await this.metadataStore.getViewMetadata(path, view); + if (!viewMeta) return true; // Missing view is stale + + if (viewMeta.state === "stale" || viewMeta.state === "failed") { + return true; + } + + if (viewMeta.state === "generating") { + return false; // Currently generating, not stale + } + + // Check if derivedFrom matches current sourceRevision + const sourceMeta = await this.metadataStore.getSourceMetadata(path); + if (!sourceMeta) return true; + + return viewMeta.derivedFrom !== sourceMeta.sourceRevision; + } + + /** + * Process a view (generate or regenerate) + * V1: Direct execution without job deduplication + */ + async processView(module: AFSModule, path: string, view: View, context: any): Promise { + try { + // 1. Get or create source metadata + let sourceMeta = await this.metadataStore.getSourceMetadata(path); + + if (!sourceMeta) { + // Read source to create metadata + const sourceResult = await module.read?.(path); + if (!sourceResult?.data) { + throw new Error(`Source file not found: ${path}`); + } + + const sourceRevision = this.computeRevision(sourceResult.data); + await this.metadataStore.setSourceMetadata(path, { + sourceRevision, + updatedAt: new Date(), + driversHint: this.drivers.map((d) => d.name), + }); + + sourceMeta = await this.metadataStore.getSourceMetadata(path); + } + + // 2. Mark as generating + await this.metadataStore.setViewMetadata(path, view, { + state: "generating", + derivedFrom: sourceMeta!.sourceRevision, + }); + + // 3. Read source + const sourceResult = await module.read?.(path); + if (!sourceResult?.data) { + throw new Error(`Source file not found: ${path}`); + } + + // 4. Find and call driver + const driver = this.findDriver(view); + if (!driver) { + throw new Error(`No driver found for view: ${JSON.stringify(view)}`); + } + + const result = await driver.process(module, path, view, { + sourceEntry: sourceResult.data, + metadata: { derivedFrom: sourceMeta!.sourceRevision }, + context, + }); + + // 5. Update to ready + await this.metadataStore.setViewMetadata(path, view, { + state: "ready", + generatedAt: new Date(), + storagePath: result.result.metadata?.storagePath, + error: undefined, + }); + + return result.result; + } catch (error: any) { + // 6. Mark as failed + await this.metadataStore.setViewMetadata(path, view, { + state: "failed", + error: error.message, + }); + throw error; + } + } + + /** + * Read view result from storage + */ + async readViewResult(module: AFSModule, path: string, viewMeta: ViewMetadata): Promise { + if (!viewMeta.storagePath) { + throw new Error(`View metadata missing storagePath for ${path}`); + } + + const result = await module.read?.(viewMeta.storagePath); + if (!result?.data) { + throw new Error(`View storage file not found: ${viewMeta.storagePath}`); + } + + return { + ...result.data, + path, // Return logical path, not storage path + metadata: { + ...result.data.metadata, + view: viewMeta.view, + }, + }; + } + + /** + * Handle read operation with view support + */ + async handleRead( + module: AFSModule, + path: string, + options: AFSReadOptions | undefined, + context: any, + ): Promise { + // No view, read source directly + if (!options?.view) { + return (await module.read?.(path)) || { data: undefined }; + } + + // 1. Query view metadata + const viewMeta = await this.metadataStore.getViewMetadata(path, options.view); + const isStale = await this.isViewStale(path, options.view); + + // 2. If view is ready and not stale, return it + if (viewMeta?.state === "ready" && !isStale) { + const data = await this.readViewResult(module, path, viewMeta); + return { data }; + } + + // 3. Need to generate view + const wait = options.wait || "strict"; // Default to strict + + if (wait === "strict") { + // Wait for generation to complete + const data = await this.processView(module, path, options.view, context); + return { data }; + } else { + // Fallback: trigger background generation, return source + this.processView(module, path, options.view, context).catch((error) => { + console.error(`Background view processing failed for ${path}:`, error); + }); + + const sourceResult = await module.read?.(path); + return { + data: sourceResult?.data, + message: `View (${JSON.stringify(options.view)}) is being processed in background`, + viewStatus: { fallback: true }, + }; + } + } + + /** + * Update source metadata after write + */ + async handleWrite(path: string, entry: AFSEntry): Promise { + const newRevision = this.computeRevision(entry); + + // Get old metadata + const oldMeta = await this.metadataStore.getSourceMetadata(path); + + // Update source metadata + await this.metadataStore.setSourceMetadata(path, { + sourceRevision: newRevision, + updatedAt: new Date(), + driversHint: this.drivers.map((d) => d.name), + }); + + // If revision changed, mark all views as stale + if (!oldMeta || oldMeta.sourceRevision !== newRevision) { + await this.metadataStore.markViewsAsStale(path); + } + } + + /** + * Clean up metadata after delete + */ + async handleDelete(path: string): Promise { + await this.metadataStore.deleteViewMetadata(path); + await this.metadataStore.deleteSourceMetadata(path); + } + + /** + * Prefetch views for batch generation + */ + async prefetch( + module: AFSModule, + paths: string[], + view: View, + context: any, + options?: { concurrency?: number }, + ): Promise { + const tasksToGenerate: string[] = []; + + // Check which paths need generation + for (const path of paths) { + const isStale = await this.isViewStale(path, view); + const viewMeta = await this.metadataStore.getViewMetadata(path, view); + + if (isStale || !viewMeta || viewMeta.state !== "ready") { + tasksToGenerate.push(path); + } + } + + // Generate with concurrency limit using p-limit + const concurrency = options?.concurrency || 5; + const limit = pLimit(concurrency); + + await Promise.all( + tasksToGenerate.map((path) => + limit(() => + this.processView(module, path, view, context).catch((error) => { + console.error(`Prefetch failed for ${path}:`, error); + }), + ), + ), + ); + } +} diff --git a/afs/core/src/view-schema.ts b/afs/core/src/view-schema.ts new file mode 100644 index 000000000..8848fa758 --- /dev/null +++ b/afs/core/src/view-schema.ts @@ -0,0 +1,82 @@ +import { z } from "zod"; +import type { AFSDriver } from "./type.js"; + +/** + * Build view schema dynamically based on registered drivers + * + * @param drivers - List of registered drivers + * @returns Zod schema object, or undefined if no drivers support view + */ +export function buildViewSchema(drivers: AFSDriver[]): z.ZodObject | undefined { + if (!drivers || drivers.length === 0) { + return undefined; + } + + // Collect all supported view dimensions + const supportedDimensions = new Set(); + + drivers.forEach((driver) => { + driver.capabilities.dimensions.forEach((dim) => { + supportedDimensions.add(dim as string); + }); + }); + + if (supportedDimensions.size === 0) { + return undefined; + } + + // Dynamically build view schema object + const viewSchemaFields: Record> = {}; + + if (supportedDimensions.has("language")) { + viewSchemaFields.language = z + .string() + .optional() + .describe("Target language for translation (e.g., 'en', 'zh', 'ja')"); + } + + if (supportedDimensions.has("format")) { + viewSchemaFields.format = z + .string() + .optional() + .describe("Target format conversion (e.g., 'md', 'html', 'pdf')"); + } + + if (supportedDimensions.has("policy")) { + viewSchemaFields.policy = z + .string() + .optional() + .describe("Content style policy (e.g., 'technical', 'marketing')"); + } + + if (supportedDimensions.has("variant")) { + viewSchemaFields.variant = z + .string() + .optional() + .describe("Content variant (e.g., 'summary', 'toc', 'index')"); + } + + return z.object(viewSchemaFields); +} + +/** + * Extend base schema with view field + * + * @param baseSchema - Base schema without view + * @param drivers - List of registered drivers + * @returns Extended schema (with view field if drivers support it) + */ +export function extendSchemaWithView( + baseSchema: z.ZodObject, + drivers: AFSDriver[], +): z.ZodObject { + const viewSchema = buildViewSchema(drivers); + + if (!viewSchema) { + return baseSchema; + } + + return baseSchema.extend({ + view: viewSchema.optional().describe("View projection options"), + }); +} diff --git a/afs/i18n-driver.md b/afs/i18n-driver.md new file mode 100644 index 000000000..6b4b54e28 --- /dev/null +++ b/afs/i18n-driver.md @@ -0,0 +1,1837 @@ +# AFS i18n Driver 实施方案 + +## 一、背景与目标 + +### 1.1 问题分析 + +当前 DocSmith 中的多语言翻译是通过 Agent 编排流程实现的 workaround： +- 生成文档 → 调用翻译 Agent → 等待翻译 → 保存多语言版本 +- 应用层承担了本应属于文件系统的职责 +- AFS 只支持"单语言文件"，缺少语义化视图能力 + +### 1.2 设计目标 + +将多语言建模为**同一文件的不同语言视图**，由 AFS 框架统一提供： +- Agent 只需声明"我要什么"（path + language） +- 格式转换/翻译由 driver 层完成 +- Agent 不参与调度、不感知转换细节 +- 支持异步 I/O，读不到就挂起，数据就绪后唤醒 + +--- + +## 二、架构设计 + +### 2.1 核心概念 + +#### View（视图投影键） + +`View` 是对同一文件的不同投影结果的统一抽象，path 是唯一 identity，view 只是读取/生成的投影键。 + +```typescript +type View = { + language?: string; // "en", "zh", "ja" - 目标语言 + // format?: string; // 未来扩展：格式转换，如 "md", "html", "pdf" + // policy?: string; // 未来扩展：不同风格策略，如 "technical", "marketing" + // variant?: string; // 未来扩展：变体，如 "summary", "toc", "index" +}; +``` + +**当前实现范围：** +- V1 版本只实现 `language` 维度，用于多语言翻译 +- 其他维度（format、policy、variant）作为未来扩展点预留 +- 设计上支持多维度组合，但实现上分阶段进行 + +**关键原则：** +- `path` 是文件的唯一标识 +- `view` 是多维投影键的组合（可扩展） +- 多语言只是 view 的一个维度（当前唯一实现的维度） + +#### Driver（视图转换器） + +Driver 负责将 source 文件转换为指定 view 的投影结果： +- 以"能力"注册：声明自己处理哪些 view 维度（子集匹配） +- 对同一次 read(path, view)：AFS 选择唯一 capable driver +- 避免多 driver 行为叠加和组合爆炸 + +**Driver 示例：** +- i18n driver：关心 `{ language }`（V1 实现） +- format driver：关心 `{ format }`（未来扩展） +- summary driver：关心 `{ variant: "summary" }`（未来扩展） + +### 2.2 架构分层 + +``` +┌─────────────────────────────────────────────┐ +│ Agent / Application Layer │ +│ afs.read("docs/intro.md", { │ +│ view: { language: "en" }, │ +│ wait: "strict" │ +│ }) │ +└─────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────┐ +│ AFS Core Layer │ +│ - View resolution │ +│ - Driver matching & dispatch │ +│ - Async I/O coordination │ +│ - Metadata management │ +└─────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────┐ +│ AFSModule Layer │ +│ - LocalFS (读取源文件) │ +│ - History │ +│ - UserProfileMemory │ +└─────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────┐ +│ Driver Layer │ +│ - i18n Driver (处理 language view) │ +│ - Format Driver (未来：处理 format) │ +│ - ... (extensible) │ +└─────────────────────────────────────────────┘ +``` + +**数据流说明：** +1. Agent 请求读取文件的特定 view（如英文版） +2. AFS Core 解析请求，匹配对应的 driver +3. **先通过 AFSModule 读取源文件内容**（如 LocalFS 读取 `docs/intro.md`） +4. 将源文件内容交给 Driver 处理（如 i18n Driver 进行翻译） +5. Driver 返回处理后的结果给 AFS Core +6. AFS Core 返回最终结果给 Agent + +--- + +## 三、API 设计 + +### 3.1 扩展 AFS API + +```typescript +type View = { + language?: string; // 目标语言（V1 实现） + // format?: string; // 未来扩展：格式转换 + // policy?: string; // 未来扩展：风格策略 + // variant?: string; // 未来扩展：变体 +}; + +type ReadOptions = { + view?: View; + wait?: "strict" | "fallback"; +}; + +interface AFS { + // Read with view support + read(path: string, opts?: ReadOptions): Promise<{ result?: AFSEntry; message?: string }>; + + // Write - 只支持写入 source，不支持直接写入 view + // View 是内容的投影，应该由 driver 自动生成，不允许直接修改 + write( + path: string, + content: AFSWriteEntryPayload, + opts?: AFSWriteOptions // 不包含 view + ): Promise<{ result: AFSEntry; message?: string }>; + + // Stat with view support + stat(path: string, opts?: { view?: View }): Promise<{ result?: AFSEntryStat }>; + + // List with view support + list( + path: string, + opts?: AFSListOptions & { view?: View } + ): Promise<{ list: AFSEntry[]; message?: string }>; + + // Prefetch views for batch generation + prefetch( + pathOrGlob: string | string[], + opts: { view: View } + ): Promise; +} +``` + +### 3.2 Wait 策略 + +```typescript +type WaitStrategy = "strict" | "fallback"; +``` + +- **`strict`**: 严格等待，view 不存在/过期则生成并等待完成 + - 适用场景：必须获取最新翻译才能继续 + - 行为：挂起当前请求，等待生成完成或失败 + +- **`fallback`**: 降级策略，view 不存在/过期则触发后台生成，立即返回 source + - 适用场景：允许先使用源语言，后台生成翻译 + - 行为：返回 source + message 提示后台生成中 + +### 3.3 使用示例 + +```typescript +// 示例 1: 严格等待翻译 +const result = await afs.read("docs/intro.md", { + view: { language: "en" }, + wait: "strict" +}); +// 如果英文版不存在，会触发翻译并等待完成 + +// 示例 2: 降级策略 +const result = await afs.read("docs/intro.md", { + view: { language: "en" }, + wait: "fallback" +}); +// 英文版不存在，返回中文源文件 + 后台生成英文版 + +// 示例 3: 批量预生成 +await afs.prefetch("docs/**/*.md", { + view: { language: "en" } +}); +// 后台批量生成所有 markdown 文件的英文版 + +// 未来扩展示例：组合 view（language + format） +// const result = await afs.read("docs/intro.md", { +// view: { language: "en", format: "html" }, +// wait: "strict" +// }); +// 注：V1 版本暂不支持组合 view，仅支持 language 维度 +``` + +--- + +## 四、Driver 设计 + +### 4.1 Driver 接口 + +```typescript +interface AFSDriver { + readonly name: string; + readonly description?: string; + + // 声明 driver 能处理的 view 维度 + readonly capabilities: { + dimensions: (keyof View)[]; // e.g., ["language"] + }; + + // 判断是否可以处理指定 view + canHandle(view: View): boolean; + + // 处理并生成 view 投影 + process( + module: AFSModule, + path: string, + view: View, + options: { + sourceEntry: AFSEntry; + metadata: ViewMetadata; + context: any; + } + ): Promise<{ result: AFSEntry; message?: string }>; + + // 可选：driver 初始化时调用 + onMount?(root: AFSRoot): void; +} +``` + +### 4.2 i18n Driver 实现 + +```typescript +export class I18nDriver implements AFSDriver { + readonly name = "i18n"; + readonly description = "Multilingual translation driver"; + readonly capabilities = { + dimensions: ["language" as const] + }; + + constructor(private options: { + defaultSourceLanguage?: string; // 默认源语言，如 "zh" + supportedLanguages?: string[]; // 支持的目标语言 + translationAgent?: Agent; // 自定义翻译 Agent（可选） + model?: ChatModel; // LLM 模型（用于默认翻译 Agent） + } = {}) { + // 如果没有提供自定义 translationAgent，创建默认的翻译 Agent + if (!this.translationAgent) { + this.translationAgent = this.createDefaultTranslationAgent(); + } + } + + private translationAgent: Agent; + + private createDefaultTranslationAgent(): Agent { + // 创建内置的默认翻译 Agent + return AIAgent.from({ + name: "i18n_translator", + description: "Built-in translation agent for i18n driver", + model: this.options.model, + instructions: `You are a professional translator. + +Translate the provided content from the source language to the target language. +Maintain the original formatting, structure, and technical terms. +Preserve markdown syntax, code blocks, and special formatting. + +Requirements: +- Translate naturally and fluently +- Keep technical terms and proper nouns when appropriate +- Maintain the tone and style of the original content +- Do not add explanations or extra content`, + inputSchema: z.object({ + content: z.string().describe("Content to translate"), + targetLanguage: z.string().describe("Target language code (e.g., 'en', 'ja')"), + sourceLanguage: z.string().optional().describe("Source language code") + }), + outputSchema: z.object({ + translatedContent: z.string().describe("Translated content") + }) + }); + } + + canHandle(view: View): boolean { + // V1 实现：只处理单纯的 language view + // 确保只有 language 字段，没有其他 view 维度 + return !!view.language && Object.keys(view).length === 1; + } + + async process( + module: AFSModule, + path: string, + view: View, + options: { + sourceEntry: AFSEntry; + metadata: ViewMetadata; + context: any; + } + ): Promise<{ result: AFSEntry; message?: string }> { + const { language } = view; + const { sourceEntry, context } = options; + + // 调用翻译 Agent + const translatedContent = await this.translate( + sourceEntry.content, + language!, + context + ); + + // 返回翻译后的 entry + return { + result: { + ...sourceEntry, + content: translatedContent, + metadata: { + ...sourceEntry.metadata, + view: { language } + } + } + }; + } + + private async translate( + content: string, + targetLanguage: string, + context: any + ): Promise { + // 使用 translationAgent 进行翻译（默认或自定义） + const result = await this.translationAgent.invoke({ + content, + targetLanguage, + sourceLanguage: this.options.defaultSourceLanguage + }, { context }); + + return result.translatedContent; + } +} +``` + +### 4.3 Driver 注册与匹配 + +**注册方式（在 AFS 或 AFSModule 配置中）：** + +```typescript +import { I18nDriver } from "@aigne/afs-i18n-driver"; +import { OpenAIChatModel } from "@aigne/openai"; + +// 方式 1: 使用默认内置翻译 Agent（推荐） +const afs = new AFS({ + drivers: [ + new I18nDriver({ + defaultSourceLanguage: "zh", + supportedLanguages: ["en", "ja", "ko"], + model: new OpenAIChatModel({ apiKey: process.env.OPENAI_API_KEY }) + // 不提供 translationAgent，会自动使用内置的默认翻译 Agent + }) + ] +}); + +// 方式 2: 使用自定义翻译 Agent +const customTranslationAgent = AIAgent.from({ + name: "custom_translator", + instructions: "Custom translation instructions...", + model: myModel, + inputSchema: z.object({ + content: z.string(), + targetLanguage: z.string(), + sourceLanguage: z.string().optional() + }), + outputSchema: z.object({ + translatedContent: z.string() + }) +}); + +const afs2 = new AFS({ + drivers: [ + new I18nDriver({ + defaultSourceLanguage: "zh", + supportedLanguages: ["en", "ja", "ko"], + translationAgent: customTranslationAgent // 使用自定义 Agent + }) + ] +}); + +// 方式 3: 在 module 级别配置 driver（未来支持，当前在 AFS 层配置） +// const localFS = new LocalFS({ +// localPath: "/path/to/docs", +// drivers: [i18nDriver] +// }); +``` + +**匹配原则：** +1. 对于 `read(path, { view })` 请求，AFS 查找 `canHandle(view)` 返回 true 的 driver +2. 如果多个 driver 都能处理，抛出错误（避免歧义） +3. 如果没有 driver 能处理，返回错误提示 + +--- + +## 五、Metadata 设计 + +### 5.1 Metadata 分层 + +AFS 维护统一的 metadata，用于 view 状态判断、任务去重、增量更新。 + +#### Source-level（按 path） + +```typescript +interface SourceMetadata { + path: string; + sourceRevision: string; // 主内容版本标识（hash / mtime） + updatedAt: Date; // 主内容更新时间 + driversHint?: string[]; // 可选：可能涉及的 driver 名称 +} +``` + +**用途：** +- 主内容变更后，AFS 可据此将相关 view 标记为 stale +- 触发后续生成策略（strict / fallback / prefetch） + +#### View-level（按 path + view） + +```typescript +type ViewState = "ready" | "stale" | "generating" | "failed"; + +interface ViewMetadata { + path: string; + view: View; // 视图键 + state: ViewState; // 状态 + derivedFrom: string; // 对应的 sourceRevision + generatedAt?: Date; // 生成时间 + error?: string; // 失败原因（便于诊断与重试） + storagePath?: string; // 物理存储路径（如 .i18n/en/...） +} +``` + +**用途：** +- 快速判断某个 view 是否可用（ready）、是否需要重建（stale / missing）、是否正在生成（generating） +- (path, view) 维度的 job 去重与等待队列管理 + +### 5.2 Metadata 存储方案 + +采用**独立 Metadata Store**（SQLite 数据库），与 AFSHistory 类似的存储机制。 + +#### 数据库 Schema + +```sql +-- Source metadata table +CREATE TABLE source_metadata ( + path TEXT PRIMARY KEY, + source_revision TEXT NOT NULL, + updated_at INTEGER NOT NULL, -- Unix timestamp (ms) + drivers_hint TEXT, -- JSON array, e.g., '["i18n"]' + created_at INTEGER NOT NULL -- Unix timestamp (ms) +); + +-- View metadata table +CREATE TABLE view_metadata ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + path TEXT NOT NULL, + view TEXT NOT NULL, -- JSON stringified View, e.g., '{"language":"en"}' + state TEXT NOT NULL, -- 'ready' | 'stale' | 'generating' | 'failed' + derived_from TEXT NOT NULL, -- sourceRevision at generation time + generated_at INTEGER, -- Unix timestamp (ms) + error TEXT, -- Error message if state = 'failed' + storage_path TEXT, -- Physical storage path, e.g., 'docs/.i18n/en/intro.md' + created_at INTEGER NOT NULL, -- Unix timestamp (ms) + updated_at INTEGER NOT NULL, -- Unix timestamp (ms) + UNIQUE(path, view) +); + +CREATE INDEX idx_view_path ON view_metadata(path); +CREATE INDEX idx_view_state ON view_metadata(state); +CREATE INDEX idx_view_derived_from ON view_metadata(derived_from); +``` + +#### TypeScript 接口 + +```typescript +// Metadata Store 接口 +interface MetadataStore { + // Source metadata operations + getSourceMetadata(path: string): Promise; + setSourceMetadata(path: string, metadata: Omit): Promise; + deleteSourceMetadata(path: string): Promise; + + // View metadata operations + getViewMetadata(path: string, view: View): Promise; + setViewMetadata(path: string, view: View, metadata: Partial): Promise; + listViewMetadata(path: string): Promise; + deleteViewMetadata(path: string, view?: View): Promise; + + // Batch operations + markViewsAsStale(path: string): Promise; + listStaleViews(): Promise; + listGeneratingViews(): Promise; +} + +// 实现类 +class SQLiteMetadataStore implements MetadataStore { + constructor(private db: Database) {} + + async getSourceMetadata(path: string): Promise { + const row = await this.db.get( + 'SELECT * FROM source_metadata WHERE path = ?', + path + ); + if (!row) return null; + + return { + path: row.path, + sourceRevision: row.source_revision, + updatedAt: new Date(row.updated_at), + driversHint: row.drivers_hint ? JSON.parse(row.drivers_hint) : undefined + }; + } + + async setSourceMetadata( + path: string, + metadata: Omit + ): Promise { + const now = Date.now(); + await this.db.run( + `INSERT OR REPLACE INTO source_metadata + (path, source_revision, updated_at, drivers_hint, created_at) + VALUES (?, ?, ?, ?, COALESCE( + (SELECT created_at FROM source_metadata WHERE path = ?), ? + ))`, + path, + metadata.sourceRevision, + metadata.updatedAt.getTime(), + metadata.driversHint ? JSON.stringify(metadata.driversHint) : null, + path, + now + ); + } + + async getViewMetadata(path: string, view: View): Promise { + const viewKey = JSON.stringify(view); + const row = await this.db.get( + 'SELECT * FROM view_metadata WHERE path = ? AND view = ?', + path, + viewKey + ); + if (!row) return null; + + return { + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derived_from, + generatedAt: row.generated_at ? new Date(row.generated_at) : undefined, + error: row.error || undefined, + storagePath: row.storage_path || undefined + }; + } + + async setViewMetadata( + path: string, + view: View, + metadata: Partial + ): Promise { + const viewKey = JSON.stringify(view); + const now = Date.now(); + + // 获取现有记录 + const existing = await this.getViewMetadata(path, view); + + const merged = { + state: metadata.state || existing?.state || 'stale', + derivedFrom: metadata.derivedFrom || existing?.derivedFrom || '', + generatedAt: metadata.generatedAt?.getTime() || existing?.generatedAt?.getTime(), + error: metadata.error || existing?.error, + storagePath: metadata.storagePath || existing?.storagePath + }; + + await this.db.run( + `INSERT OR REPLACE INTO view_metadata + (path, view, state, derived_from, generated_at, error, storage_path, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, COALESCE( + (SELECT created_at FROM view_metadata WHERE path = ? AND view = ?), ? + ), ?)`, + path, + viewKey, + merged.state, + merged.derivedFrom, + merged.generatedAt || null, + merged.error || null, + merged.storagePath || null, + path, + viewKey, + now, + now + ); + } + + async markViewsAsStale(path: string): Promise { + await this.db.run( + `UPDATE view_metadata + SET state = 'stale', updated_at = ? + WHERE path = ? AND state IN ('ready', 'generating')`, + Date.now(), + path + ); + } + + async listViewMetadata(path: string): Promise { + const rows = await this.db.all( + 'SELECT * FROM view_metadata WHERE path = ?', + path + ); + + return rows.map(row => ({ + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derived_from, + generatedAt: row.generated_at ? new Date(row.generated_at) : undefined, + error: row.error || undefined, + storagePath: row.storage_path || undefined + })); + } + + async deleteViewMetadata(path: string, view?: View): Promise { + if (view) { + const viewKey = JSON.stringify(view); + await this.db.run( + 'DELETE FROM view_metadata WHERE path = ? AND view = ?', + path, + viewKey + ); + } else { + // 删除该 path 的所有 view metadata + await this.db.run('DELETE FROM view_metadata WHERE path = ?', path); + } + } + + async listStaleViews(): Promise { + const rows = await this.db.all( + "SELECT * FROM view_metadata WHERE state = 'stale'" + ); + + return rows.map(row => ({ + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derived_from, + generatedAt: row.generated_at ? new Date(row.generated_at) : undefined, + error: row.error || undefined, + storagePath: row.storage_path || undefined + })); + } + + async listGeneratingViews(): Promise { + const rows = await this.db.all( + "SELECT * FROM view_metadata WHERE state = 'generating'" + ); + + return rows.map(row => ({ + path: row.path, + view: JSON.parse(row.view), + state: row.state as ViewState, + derivedFrom: row.derived_from, + generatedAt: row.generated_at ? new Date(row.generated_at) : undefined, + error: row.error || undefined, + storagePath: row.storage_path || undefined + })); + } +} +``` + +### 5.3 Metadata 生成机制 + +#### 5.3.1 Source Metadata 生成 + +**触发时机：** +1. `write()` 操作写入文件后 +2. 首次 `read()` 一个文件时（如果 source metadata 不存在） + +**生成流程：** + +```typescript +async writeSource(path: string, content: AFSWriteEntryPayload): Promise { + // 1. 写入文件到 module + const result = await this.module.write(path, content); + + // 2. 计算 sourceRevision + const sourceRevision = this.computeRevision(result); + + // 3. 保存 source metadata + await this.metadataStore.setSourceMetadata(path, { + sourceRevision, + updatedAt: new Date(), + driversHint: this.getApplicableDrivers(path) + }); + + // 4. 标记所有相关 view 为 stale + await this.metadataStore.markViewsAsStale(path); + + return result; +} + +private computeRevision(entry: AFSEntry): string { + if (typeof entry.content === 'string') { + // 对文本内容计算 SHA-256 hash + const hash = crypto.createHash('sha256') + .update(entry.content) + .digest('hex') + .substring(0, 16); + return `hash:sha256:${hash}`; + } + + // 对二进制文件使用 mtime + size + const mtime = entry.updatedAt?.getTime() || Date.now(); + const size = entry.metadata?.size || 0; + return `mtime:${mtime}:size:${size}`; +} + +private getApplicableDrivers(path: string): string[] { + // 返回可能处理该文件的 driver 名称列表 + // 用于优化：只需检查相关 driver 的 view + return this.drivers.map(d => d.name); +} +``` + +#### 5.3.2 View Metadata 生成（V1 简化版） + +**触发时机：** +1. `read(path, { view })` 时，如果 view metadata 不存在或过期 +2. Driver 处理完成后更新状态 + +**生成流程（V1 - 无 job 去重）：** + +```typescript +async processView(path: string, view: View): Promise { + try { + // 1. 获取或创建 source metadata + let sourceMeta = await this.metadataStore.getSourceMetadata(path); + if (!sourceMeta) { + const sourceEntry = await this.readSource(path); + const sourceRevision = this.computeRevision(sourceEntry); + await this.metadataStore.setSourceMetadata(path, { + sourceRevision, + updatedAt: new Date() + }); + sourceMeta = await this.metadataStore.getSourceMetadata(path); + } + + // 2. 标记为 generating + await this.metadataStore.setViewMetadata(path, view, { + state: 'generating', + derivedFrom: sourceMeta!.sourceRevision + }); + + // 3. 读取 source + const sourceEntry = await this.readSource(path); + + // 4. 查找并调用 driver + const driver = this.findDriver(view); + if (!driver) { + throw new Error(`No driver found for view: ${JSON.stringify(view)}`); + } + + const result = await driver.process(this.module, path, view, { + sourceEntry, + metadata: { derivedFrom: sourceMeta!.sourceRevision }, + context: this.context + }); + + // 5. 更新为 ready + await this.metadataStore.setViewMetadata(path, view, { + state: 'ready', + generatedAt: new Date(), + storagePath: result.result.metadata?.storagePath, + error: undefined + }); + + return result.result; + + } catch (error) { + // 6. 处理失败，标记为 failed + await this.metadataStore.setViewMetadata(path, view, { + state: 'failed', + error: error.message + }); + throw error; + } +} +``` + +**V1 说明：** +- 直接执行处理，不检查并发 +- 如果多个请求同时触发同一个 view，会重复处理 +- V2 将增加 job 去重机制避免重复处理 + +### 5.4 Metadata 更新机制 + +#### 5.4.1 Source 更新时 + +```typescript +async write(path: string, content: AFSWriteEntryPayload): Promise { + const oldSourceMeta = await this.metadataStore.getSourceMetadata(path); + + // 写入新内容 + const result = await this.writeSource(path, content); + const newRevision = this.computeRevision(result); + + // 检查 revision 是否变化 + if (oldSourceMeta?.sourceRevision !== newRevision) { + // 更新 source metadata + await this.metadataStore.setSourceMetadata(path, { + sourceRevision: newRevision, + updatedAt: new Date() + }); + + // 标记所有 view 为 stale + await this.metadataStore.markViewsAsStale(path); + } + + return result; +} +``` + +#### 5.4.2 View 过期检测 + +```typescript +async isViewStale(path: string, view: View): Promise { + const viewMeta = await this.metadataStore.getViewMetadata(path, view); + if (!viewMeta) return true; // 不存在视为过期 + + if (viewMeta.state === 'stale' || viewMeta.state === 'failed') { + return true; + } + + if (viewMeta.state === 'generating') { + return false; // 正在生成，不算过期 + } + + // 检查 derivedFrom 是否匹配当前 sourceRevision + const sourceMeta = await this.metadataStore.getSourceMetadata(path); + if (!sourceMeta) return true; + + return viewMeta.derivedFrom !== sourceMeta.sourceRevision; +} +``` + +### 5.5 Metadata 使用机制 + +#### 5.5.1 Read 时使用 Metadata + +```typescript +async read(path: string, opts?: ReadOptions): Promise<{ result?: AFSEntry; message?: string }> { + // 无 view，直接读取 source + if (!opts?.view) { + return this.readSource(path); + } + + // 1. 查询 view metadata + const viewMeta = await this.metadataStore.getViewMetadata(path, opts.view); + const isStale = await this.isViewStale(path, opts.view); + + // 2. 如果 view 是 ready 且未过期，直接返回 + if (viewMeta?.state === 'ready' && !isStale) { + return this.readViewResult(viewMeta); + } + + // 3. 如果正在生成 + if (viewMeta?.state === 'generating') { + if (opts.wait === 'strict') { + return this.waitForProcess(path, opts.view); + } else { + // fallback: 返回 source + return this.readSourceWithMessage(path, 'View is being processed'); + } + } + + // 4. 需要生成 view + if (opts.wait === 'strict') { + // 等待生成完成 + const result = await this.processView(path, opts.view); + return { result }; + } else { + // 后台生成，立即返回 source + this.processViewInBackground(path, opts.view); + return this.readSourceWithMessage(path, 'View is being processed in background'); + } +} +``` + +#### 5.5.2 Prefetch 使用 Metadata + +```typescript +async prefetch(pathOrGlob: string | string[], opts: { view: View }): Promise { + const paths = Array.isArray(pathOrGlob) ? pathOrGlob : await this.glob(pathOrGlob); + + // 批量检查哪些需要生成 + const tasksToGenerate: Array<{ path: string; view: View }> = []; + + for (const path of paths) { + const isStale = await this.isViewStale(path, opts.view); + const viewMeta = await this.metadataStore.getViewMetadata(path, opts.view); + + // 只有 stale/failed/missing 才需要生成 + if (isStale || !viewMeta || viewMeta.state !== 'ready') { + tasksToGenerate.push({ path, view: opts.view }); + } + } + + // 并发生成（限制并发数） + const concurrency = 5; + const batches = chunk(tasksToGenerate, concurrency); + + for (const batch of batches) { + await Promise.all( + batch.map(({ path, view }) => this.processView(path, view)) + ); + } +} +``` + +### 5.6 Metadata 清理机制 + +```typescript +// 清理孤立的 view metadata（source 已删除） +async cleanupOrphanedViewMetadata(): Promise { + const allViewMeta = await this.db.all('SELECT DISTINCT path FROM view_metadata'); + + for (const row of allViewMeta) { + const sourceMeta = await this.metadataStore.getSourceMetadata(row.path); + if (!sourceMeta) { + // Source 已删除，清理所有相关 view metadata + await this.metadataStore.deleteViewMetadata(row.path); + } + } +} + +// 清理失败的 view metadata（可选，允许重试） +async cleanupFailedViews(olderThan?: Date): Promise { + const cutoff = olderThan || new Date(Date.now() - 24 * 60 * 60 * 1000); // 默认 24 小时 + + await this.db.run( + `DELETE FROM view_metadata + WHERE state = 'failed' AND updated_at < ?`, + cutoff.getTime() + ); +} +``` + +--- + +## 六、View 生成流程 + +### 6.1 Read 流程（with view）- V1 简化版 + +**V1 实现说明：** +- 暂不实现 job 去重和等待队列机制 +- 直接触发 driver 处理，等待完成后返回 +- V2 将增加并发控制和任务队列优化 + +``` +┌─────────────────────────────────────────────────────────────┐ +│ 1. Agent 调用 afs.read(path, { view, wait }) │ +└──────────────────────┬──────────────────────────────────────┘ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ 2. AFS Core: 查询 view metadata │ +│ - 如果 state = "ready" && !stale → 返回 view 结果 │ +│ - 如果 state = "stale" / missing → 需要生成 │ +└──────────────────────┬──────────────────────────────────────┘ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ 3. Driver Matching: 查找 canHandle(view) 的 driver │ +│ - 如果找到唯一 driver → 进入处理流程 │ +│ - 如果找不到 / 多个 driver → 返回错误 │ +└──────────────────────┬──────────────────────────────────────┘ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ 4. 触发处理：标记 state = "generating" │ +└──────────────────────┬──────────────────────────────────────┘ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ 5. Driver.process(): 读取 source，调用处理逻辑 │ +│ - 成功 → 写入 view 结果，更新 metadata state = "ready" │ +│ - 失败 → 更新 metadata state = "failed", error = ... │ +└──────────────────────┬──────────────────────────────────────┘ + ▼ +┌─────────────────────────────────────────────────────────────┐ +│ 6. 返回处理结果或错误 │ +└─────────────────────────────────────────────────────────────┘ +``` + +**V2 优化方向（未来）：** +- 增加 job 去重：多个并发请求同一个 view 时，复用同一个处理任务 +- 增加等待队列：正在处理时新请求加入队列等待 +- 增加超时机制：处理超时后标记失败并清理 + +### 6.2 Wait 策略实现（V1 简化版） + +**strict 模式（默认）：** +- 等待 view 处理完成再返回 +- 适用场景：必须获取翻译结果才能继续 + +```typescript +async read(path: string, opts?: ReadOptions): Promise<{ result?: AFSEntry; message?: string }> { + // 无 view，直接读取 source + if (!opts?.view) { + return this.readSource(path); + } + + // 1. 查询 view metadata + const viewMeta = await this.metadataStore.getViewMetadata(path, opts.view); + const isStale = await this.isViewStale(path, opts.view); + + // 2. 如果 view 是 ready 且未过期，直接返回 + if (viewMeta?.state === 'ready' && !isStale) { + return this.readViewResult(viewMeta); + } + + // 3. 需要生成 view（V1: 直接等待，不检查 generating 状态） + if (opts.wait === 'strict' || !opts.wait) { + // 触发处理并等待完成 + const result = await this.processView(path, opts.view); + return { result }; + } else { + // fallback 模式 + return this.readSourceAndProcessInBackground(path, opts.view); + } +} +``` + +**fallback 模式：** +- 立即返回 source，后台处理 view +- 适用场景：允许先使用原文，翻译在后台进行 + +```typescript +async readSourceAndProcessInBackground( + path: string, + view: View +): Promise<{ result?: AFSEntry; message?: string }> { + // 触发后台处理（fire and forget） + this.processView(path, view).catch(error => { + console.error(`Background view processing failed for ${path}:`, error); + }); + + // 立即返回 source + const source = await this.readSource(path); + return { + result: source, + message: `View (${JSON.stringify(view)}) is being processed in background` + }; +} +``` + +**processView 简化实现（V1）：** + +```typescript +async processView(path: string, view: View): Promise { + try { + // 1. 获取 source metadata + let sourceMeta = await this.metadataStore.getSourceMetadata(path); + + // 如果不存在，创建 source metadata + if (!sourceMeta) { + const sourceEntry = await this.readSource(path); + const sourceRevision = this.computeRevision(sourceEntry); + await this.metadataStore.setSourceMetadata(path, { + sourceRevision, + updatedAt: new Date() + }); + sourceMeta = await this.metadataStore.getSourceMetadata(path); + } + + // 2. 标记为 generating + await this.metadataStore.setViewMetadata(path, view, { + state: 'generating', + derivedFrom: sourceMeta!.sourceRevision + }); + + // 3. 读取 source + const sourceEntry = await this.readSource(path); + + // 4. 查找 driver + const driver = this.findDriver(view); + if (!driver) { + throw new Error(`No driver found for view: ${JSON.stringify(view)}`); + } + + // 5. 调用 driver 处理 + const result = await driver.process(this.module, path, view, { + sourceEntry, + metadata: { derivedFrom: sourceMeta!.sourceRevision }, + context: this.context + }); + + // 6. 更新为 ready + await this.metadataStore.setViewMetadata(path, view, { + state: 'ready', + generatedAt: new Date(), + storagePath: result.result.metadata?.storagePath, + error: undefined + }); + + return result.result; + + } catch (error) { + // 7. 处理失败，标记为 failed + await this.metadataStore.setViewMetadata(path, view, { + state: 'failed', + error: error.message + }); + throw error; + } +} +``` + +**V1 vs V2 对比：** + +| 特性 | V1 实现 | V2 优化 | +|------|---------|---------| +| 并发请求同一 view | 每次都触发新的处理 | Job 去重，复用同一任务 | +| generating 状态处理 | 直接覆盖并重新处理 | 等待队列，复用正在处理的任务 | +| 超时控制 | 依赖 driver 本身超时 | 框架级超时控制 | +| 复杂度 | 简单，易于实现 | 需要任务队列和状态同步 | + +### 6.3 Source 更新触发 View 失效 + +```typescript +async write(path: string, content: AFSWriteEntryPayload, opts?: AFSWriteOptions) { + // 写入 source（write 不支持 view 参数） + const result = await this.writeSource(path, content, opts); + + // 计算新的 sourceRevision + const newRevision = this.computeRevision(result.result); + + // 更新 source metadata + await this.updateSourceMetadata(path, { + sourceRevision: newRevision, + updatedAt: new Date() + }); + + // 标记所有相关 view 为 stale + // 下次 read 时会检测到过期并触发重新生成 + await this.markViewsAsStale(path); + + return result; +} +``` + +**设计说明：** +- `write` 操作只能修改 source 文件，不支持直接写入 view +- View 是 source 的投影，应该由 driver 自动生成 +- 当 source 更新时，所有相关的 view 会被标记为 stale +- 后续 `read` 请求会检测到 view 过期，根据 wait 策略重新生成 + +--- + +## 七、物理存储方案 + +### 7.1 文件落盘约定（LocalFS + i18n driver） + +以 DocSmith workspace 为例： + +``` +.aigne/doc-smith/ + docs/ # 主语言内容（source of truth） + intro.md + guide.md + docs/.i18n/ # 多语言派生版本（进 Git） + en/ + intro.md + guide.md + ja/ + intro.md + .afs/ # AFS 元数据目录（进 Git） + metadata.db # SQLite 数据库存储 metadata + intent/ # 用户 intent / lexicon / rules +``` + +**说明：** +- `docs/` 是 source 目录，包含主语言（如中文）内容 +- `docs/.i18n/{lang}/` 存储翻译后的各语言版本 +- `.afs/metadata.db` 使用 SQLite 存储 source 和 view metadata +- 对上层 API 始终使用：`afs.read('docs/intro.md', { view: { language: 'en' } })` +- 应用层不需要知道 `.i18n` 目录和 metadata 存储的存在 + +**Metadata 存储位置：** +- 数据库路径：`.afs/metadata.db`（相对于 workspace 根目录） +- 与 AFSHistory 类似，使用 SQLite 持久化存储 +- 进 Git 管理，团队共享 metadata 状态 +- Schema 见第五章 5.2 节 + +### 7.2 StoragePath 映射 + +Driver 负责将逻辑 view 映射到物理存储路径： + +```typescript +class I18nDriver implements AFSDriver { + getStoragePath(path: string, view: View): string { + const { language } = view; + const dir = dirname(path); + const filename = basename(path); + return join(dir, ".i18n", language, filename); + } + + async process(module, path, view, options) { + const storagePath = this.getStoragePath(path, view); + + // 翻译内容 + const translatedContent = await this.translate(...); + + // 写入物理路径 + await module.write(storagePath, { content: translatedContent }); + + // 更新 metadata + await this.updateViewMetadata(path, view, { + state: "ready", + storagePath, + derivedFrom: options.metadata.sourceRevision, + generatedAt: new Date() + }); + + return { result: { path, content: translatedContent } }; + } +} +``` + +--- + +## 八、Skill 层集成 + +### 8.1 动态 Schema 组装 + +基于配置的 drivers 动态组装 view schema，各 skill 独立调用公共方法，易于扩展。 + +#### 8.1.1 公共方法：buildViewSchema + +在 `afs/core/src/view-schema.ts` 中提供统一的 schema 构建方法： + +```typescript +import { z } from 'zod'; +import type { AFSDriver } from './type'; + +/** + * 根据已注册的 drivers 动态构建 view schema + * + * @param drivers - 已注册的 driver 列表 + * @returns Zod schema object，如果没有 driver 支持 view 则返回 undefined + */ +export function buildViewSchema(drivers: AFSDriver[]): z.ZodObject | undefined { + if (!drivers || drivers.length === 0) { + return undefined; + } + + // 收集所有 driver 支持的 view dimensions + const supportedDimensions = new Set(); + + drivers.forEach(driver => { + driver.capabilities.dimensions.forEach(dim => { + supportedDimensions.add(dim); + }); + }); + + if (supportedDimensions.size === 0) { + return undefined; + } + + // 动态构建 view schema object + const viewSchemaFields: Record> = {}; + + if (supportedDimensions.has('language')) { + viewSchemaFields.language = z.string().optional() + .describe("Target language for translation (e.g., 'en', 'zh', 'ja')"); + } + + if (supportedDimensions.has('format')) { + viewSchemaFields.format = z.string().optional() + .describe("Target format conversion (e.g., 'md', 'html', 'pdf')"); + } + + if (supportedDimensions.has('policy')) { + viewSchemaFields.policy = z.string().optional() + .describe("Content style policy (e.g., 'technical', 'marketing')"); + } + + if (supportedDimensions.has('variant')) { + viewSchemaFields.variant = z.string().optional() + .describe("Content variant (e.g., 'summary', 'toc', 'index')"); + } + + return z.object(viewSchemaFields); +} + +/** + * 为支持 view 的操作构建完整的 input schema + * + * @param baseSchema - 基础 schema（不含 view） + * @param drivers - 已注册的 driver 列表 + * @returns 扩展后的 schema（如果有 driver 则包含 view 字段） + */ +export function extendSchemaWithView( + baseSchema: z.ZodObject, + drivers: AFSDriver[] +): z.ZodObject { + const viewSchema = buildViewSchema(drivers); + + if (!viewSchema) { + return baseSchema; + } + + return baseSchema.extend({ + view: viewSchema.optional().describe("View projection options") + }); +} +``` + +#### 8.1.2 Skill 中使用 + +各 skill 在构造时调用公共方法动态扩展 schema： + +```typescript +// packages/core/src/prompt/skills/afs/read.ts +import { extendSchemaWithView } from '@aigne/afs-core/view-schema'; + +export class AFSReadAgent extends Agent { + constructor(options: AFSReadAgentOptions) { + // 定义基础 schema（不含 view） + const baseSchema = z.object({ + path: z.string().describe("Absolute file path to read"), + withLineNumbers: z.boolean().optional() + }); + + // 根据 AFS 配置的 drivers 动态扩展 schema + let inputSchema = extendSchemaWithView(baseSchema, options.afs.drivers); + + // 如果有 driver 支持 view，则添加 wait 策略字段 + if (options.afs.drivers && options.afs.drivers.length > 0) { + inputSchema = inputSchema.extend({ + wait: z.enum(["strict", "fallback"]).optional() + .describe("Wait strategy: 'strict' (wait for view, default) or 'fallback' (return source immediately)") + }); + } + + super({ + name: "afs_read", + description: "Read complete file contents with optional view projection", + inputSchema, + outputSchema: z.object({ + content: z.string(), + metadata: z.record(z.any()).optional() + }), + ...options, + }); + } + + async run(input: AFSReadInput): Promise { + const result = await this.afs.read(input.path, { + view: input.view, // 如果没有 driver，这个字段不会出现在 schema 中 + wait: input.wait, // 如果没有 driver，这个字段不会出现在 schema 中 + withLineNumbers: input.withLineNumbers + }); + + return { + content: result.result?.content || '', + metadata: result.result?.metadata + }; + } +} +``` + +```typescript +// packages/core/src/prompt/skills/afs/list.ts +export class AFSListAgent extends Agent { + constructor(options: AFSListAgentOptions) { + const baseSchema = z.object({ + path: z.string().describe("Directory path to list"), + recursive: z.boolean().optional() + }); + + // 使用同样的公共方法扩展 schema + const inputSchema = extendSchemaWithView(baseSchema, options.afs.drivers); + + super({ + name: "afs_list", + description: "List files and directories with optional view filtering", + inputSchema, + // ... + }); + } +} +``` + +```typescript +// packages/core/src/prompt/skills/afs/stat.ts +export class AFSStatAgent extends Agent { + constructor(options: AFSStatAgentOptions) { + const baseSchema = z.object({ + path: z.string().describe("File or directory path to stat") + }); + + // stat 也支持 view 查询 + const inputSchema = extendSchemaWithView(baseSchema, options.afs.drivers); + + super({ + name: "afs_stat", + description: "Get file or directory metadata with optional view status", + inputSchema, + // ... + }); + } +} +``` + +```typescript +// packages/core/src/prompt/skills/afs/write.ts +export class AFSWriteAgent extends Agent { + constructor(options: AFSWriteAgentOptions) { + // write 不支持 view，使用基础 schema 即可，不调用 extendSchemaWithView + const inputSchema = z.object({ + path: z.string().describe("File path to write"), + content: z.string().describe("Content to write") + }); + + super({ + name: "afs_write", + description: "Write content to file (source only, views are auto-generated)", + inputSchema, + // ... + }); + } +} +``` + +#### 8.1.3 getAFSSkills 保持简洁 + +```typescript +// packages/core/src/prompt/skills/afs/index.ts +export async function getAFSSkills(afs: AFS): Promise { + // 不需要检查 driver，各 skill 内部会根据 afs.drivers 动态处理 + return [ + new AFSListAgent({ afs }), + new AFSSearchAgent({ afs }), + new AFSReadAgent({ afs }), + new AFSStatAgent({ afs }), + new AFSWriteAgent({ afs }), + // ... + ]; +} +``` + +**设计优势：** +1. **易于扩展**：新增 driver 只需在 `capabilities.dimensions` 中声明，schema 自动更新 +2. **统一管理**：所有 view schema 逻辑集中在 `buildViewSchema`，避免重复代码 +3. **按需支持**：各 skill 根据自身需求决定是否调用 `extendSchemaWithView` +4. **类型安全**：基于已注册的 driver 动态生成 schema，LLM 只会看到真正支持的字段 +5. **V1 → V2 平滑过渡**：当添加新的 view dimension 时，无需修改 skill 代码 + +--- + +## 九、实施计划 + +### Phase 1: 基础架构（AFS Core） + +**目标：** 在 `afs/core` 中建立 view 和 driver 机制 + +**任务：** +1. [ ] 扩展 `afs/core/src/type.ts`： + - 定义 `View` 类型（V1 只包含 `language`） + - 定义 `AFSDriver` 接口 + - 扩展 `ReadOptions` 支持 view（`WriteOptions` 不需要修改） + +2. [ ] 扩展 `afs/core/src/afs.ts`： + - 增加 `drivers` 字段和注册机制 + - 实现 `read/stat/list` 支持 view 参数（`write` 不支持 view） + - 实现 driver 匹配逻辑 + - 实现 `prefetch` API + +3. [ ] 创建 `afs/core/src/metadata/`： + - `metadata/type.ts`: 定义 `SourceMetadata`, `ViewMetadata`, `MetadataStore` 接口 + - `metadata/store.ts`: 实现 `SQLiteMetadataStore` 类 + - `metadata/migrations/001-init.ts`: 创建 source_metadata 和 view_metadata 表 + - `metadata/index.ts`: 导出所有接口和实现 + +4. [ ] Metadata Store 核心功能： + - `getSourceMetadata()`: 查询 source metadata + - `setSourceMetadata()`: 更新/创建 source metadata + - `getViewMetadata()`: 查询 view metadata + - `setViewMetadata()`: 更新/创建 view metadata（支持部分更新） + - `markViewsAsStale()`: 批量标记 view 为 stale + - `listViewMetadata()`: 列出某个 path 的所有 view + - `cleanupOrphanedViewMetadata()`: 清理孤立的 view metadata + +5. [ ] 创建 `afs/core/src/view-processor.ts`（V1 简化版）： + - 实现 `processView()`: 异步处理 view 生成（无 job 去重） + - 实现 `isViewStale()`: 检查 view 是否过期 + - 实现 `computeRevision()`: 计算 sourceRevision + - 实现 `readViewResult()`: 从 storagePath 读取 view 结果 + - 实现 strict / fallback 策略 + - **V1 暂不实现：** job 去重、等待队列（留待 V2） + +6. [ ] 创建 `afs/core/src/view-schema.ts`： + - 实现 `buildViewSchema()`: 根据 drivers 动态构建 view schema + - 实现 `extendSchemaWithView()`: 为基础 schema 扩展 view 字段 + - 支持所有 view dimensions（language, format, policy, variant） + - **注意：** 这是 AFS Core 的基础功能，供 Skill 层使用 + +7. [ ] 扩展 `afs/core/src/afs.ts` 集成 Metadata： + - 在 constructor 中初始化 MetadataStore（数据库路径：`.afs/metadata.db`） + - 配置项：`metadataPath`（可选，默认 `.afs/metadata.db`） + - `read()` 中使用 metadata 判断 view 状态 + - `write()` 中更新 source metadata 并标记 view 为 stale + - `delete()` 中清理相关 metadata + +8. [ ] AFS 配置扩展： + - 增加 `AFSOptions.metadataPath` 配置项 + - 默认值：`.afs/metadata.db`（相对于 workspace 根目录） + - 支持自定义路径（绝对或相对路径） + +**输出：** +- `@aigne/afs` v2.0.0（breaking change） +- 向后兼容：不传 view 时行为与当前一致 + +--- + +### Phase 2: i18n Driver 实现 + +**目标：** 实现第一个 driver：i18n driver + +**任务：** +1. [ ] 创建 `afs/i18n-driver/` 包： + - `src/index.ts`: 实现 `I18nDriver` 类 + - `src/default-translation-agent.ts`: 实现内置默认翻译 Agent + - `src/storage.ts`: 实现 `.i18n/{lang}/` 物理路径映射 + +2. [ ] 配置选项设计： + - `defaultSourceLanguage`: 默认源语言（可选，如 "zh"） + - `supportedLanguages`: 支持的目标语言列表（可选） + - `model`: LLM 模型实例（用于默认翻译 Agent） + - `translationAgent`: 自定义翻译 Agent（可选，不提供则使用内置默认） + - `storagePath`: 物理存储路径模板（可选，默认 `.i18n/{language}/`） + +3. [ ] 默认翻译 Agent 实现： + - 使用 AIAgent.from 创建 + - 支持多语言翻译，保持格式和技术术语 + - 输入：content, targetLanguage, sourceLanguage + - 输出：translatedContent + +4. [ ] 测试： + - 单元测试：driver 匹配、生成逻辑 + - 集成测试：与 LocalFS 集成 + - 测试默认翻译 Agent 和自定义 Agent 两种场景 + +**输出：** +- `@aigne/afs-i18n-driver` v0.0.1 + +--- + +### Phase 3: Skill 层集成 + +**目标：** 在 `packages/core` 中集成 view 支持 + +**任务：** +1. [ ] 扩展 `packages/core/src/prompt/skills/afs/read.ts`： + - 使用 `extendSchemaWithView()` 动态扩展 schema + - 增加 `view` 字段到 `AFSReadInput`（根据 drivers 自动添加） + - 增加 `wait` 字段（可选，默认使用 AFS Core 的默认值 "strict"） + - 更新 description + +2. [ ] 扩展 `packages/core/src/prompt/skills/afs/list.ts`： + - 使用 `extendSchemaWithView()` 动态扩展 schema + - 增加 `view` 字段到 `AFSListInput`（可选） + - 用于列出特定 view 的文件 + +3. [ ] 扩展 `packages/core/src/prompt/skills/afs/stat.ts`： + - 使用 `extendSchemaWithView()` 动态扩展 schema + - 增加 `view` 字段到 `AFSStatInput`（可选） + - 用于查询特定 view 的状态 + +4. [ ] **不修改** `packages/core/src/prompt/skills/afs/write.ts`： + - write 不支持 view 参数 + - 只能写入 source，view 由 driver 自动生成 + +5. [ ] 保持 `packages/core/src/prompt/skills/afs/index.ts` 简洁： + - `getAFSSkills` 不需要检查 driver + - 各 skill 内部自动根据 afs.drivers 动态处理 + +**输出：** +- `@aigne/core` 支持 view 的 AFS skills + +**依赖：** +- 使用 Phase 1 创建的 `afs/core/src/view-schema.ts` 公共模块 + +--- + +### Phase 4: Loader 集成 + +**目标：** 在 agent loader 中支持 driver 配置 + +**任务：** +1. [ ] 扩展 `packages/core/src/loader/agent-yaml.ts`： + - 在 `BaseAgentSchema.afs` 中增加 `drivers` 字段 + - Schema: + ```typescript + afs?: + | boolean + | { + modules?: AFSModuleSchema[]; + drivers?: AFSDriverSchema[]; // 新增 + }; + ``` + +2. [ ] 在 `packages/core/src/loader/index.ts` 中： + - 解析 driver 配置 + - 实例化 driver 并注入到 AFS + +3. [ ] 示例配置（aigne.yaml）： + ```yaml + # 方式 1: 使用默认内置翻译 Agent（推荐） + afs: + modules: + - module: local-fs + options: + local_path: ./docs + drivers: + - driver: i18n + options: + default_source_language: zh + supported_languages: + - en + - ja + # 不配置 translation_agent，会使用内置默认翻译 Agent + # model 会从 Agent 配置中继承 + + # 方式 2: 使用自定义翻译 Agent + afs: + modules: + - module: local-fs + options: + local_path: ./docs + drivers: + - driver: i18n + options: + default_source_language: zh + supported_languages: + - en + - ja + translation_agent: ./agents/custom-translator.yaml + ``` + +**输出：** +- Loader 支持 driver 配置 + +--- + +## 十、技术细节与边界 + +### 10.1 Driver 组合问题 + +**问题：** 如何处理 `{ language: "en", format: "html" }` 这样的组合 view？ + +**V1 实现：** +- V1 版本不支持组合 view +- `I18nDriver.canHandle()` 会检查 `Object.keys(view).length === 1`，拒绝处理组合 view +- 如果传入组合 view，AFS Core 会返回错误："No driver found for view" + +**未来扩展方案：** +1. **推荐：组合 driver** + - 创建一个 `I18nFormatDriver` 明确处理 `{ language, format }` + - 内部编排：先 i18n 再 format（或并行） + - 对外仍保持"一个请求一个 driver"的语义 + +2. **不推荐：链式 driver** + - 多个 driver 依次执行（易出错，难调试） + +### 10.2 Metadata 一致性 + +**问题：** 如果物理文件被外部修改（如直接编辑 `.i18n/en/intro.md`），metadata 会失效吗？ + +**方案：** +1. 在 `read` 时校验 `derivedFrom` 是否匹配当前 `sourceRevision` +2. 如果不匹配，标记为 stale，触发重新生成 +3. 可选：监听文件系统变化（如 chokidar），自动更新 metadata + +### 10.3 并发控制（V2 功能） + +**问题：** 多个请求同时触发同一个 view 的生成，如何去重？ + +**V1 实现：** +- 暂不处理并发去重 +- 多个并发请求会触发多次处理（可能导致重复翻译） +- 通过 metadata 的 `state = 'generating'` 可以看到正在处理，但不会等待 + +**V2 优化方案：** +- 使用 in-memory 的 `processingJobs: Map>` +- key = `${path}:${JSON.stringify(view)}` +- 第一个请求创建 Promise，后续请求复用同一个 Promise + +```typescript +// V2 实现示例 +private processingJobs = new Map>(); + +async processView(path: string, view: View): Promise { + const key = `${path}:${JSON.stringify(view)}`; + + // 如果已有正在进行的任务，复用 + if (this.processingJobs.has(key)) { + return this.processingJobs.get(key)!; + } + + // 创建新任务 + const job = this._doProcessView(path, view).finally(() => { + this.processingJobs.delete(key); + }); + + this.processingJobs.set(key, job); + return job; +} +``` + +**优先级：** V2（非必需，V1 可先不实现） + +### 10.4 SourceRevision 计算 + +**方案：** +- 对于文本文件：使用内容的 SHA-256 hash +- 对于二进制文件：使用 mtime + size 组合 +- 存储为字符串：`hash:sha256:abc123...` 或 `mtime:1234567890:size:1024` + +```typescript +function computeRevision(entry: AFSEntry): string { + if (typeof entry.content === "string") { + const hash = crypto.createHash("sha256").update(entry.content).digest("hex"); + return `hash:sha256:${hash}`; + } + return `mtime:${entry.updatedAt?.getTime()}:size:${entry.metadata?.size}`; +} +``` + +--- + +## 十一、总结 + +### V1 实现范围 + +**包含功能：** +- ✅ View 抽象（仅 `language` 维度） +- ✅ i18n Driver 实现（单语言翻译，内置默认翻译 Agent） +- ✅ Metadata 管理（SQLite 存储，Source + View 两层） +- ✅ 异步 I/O（strict / fallback 两种策略，简化版） +- ✅ 物理存储映射（`.i18n/{lang}/`） +- ✅ AFS Core 集成（read/write/prefetch API） +- ✅ Skill 层集成（动态 schema 组装） +- ✅ Loader 集成（driver 配置支持） + +**V1 简化（V2 优化）：** +- ⚠️ 并发请求处理：V1 直接执行，可能重复处理；V2 增加 job 去重 +- ⚠️ generating 状态：V1 不等待，直接覆盖；V2 增加等待队列 +- ⚠️ 超时控制：V1 依赖 driver；V2 框架级超时 + +**不包含功能：** +- ❌ 组合 view（如 `{ language: "en", format: "html" }`） +- ❌ 其他 view 维度（format、policy、variant） +- ❌ 链式 driver 支持 +- ❌ 文件系统监听（自动 metadata 更新） + +### 关键设计点 + +1. **View 是投影键，不是多个文件** + - path 是唯一 identity + - language 只是 view 的一个维度（V1 唯一实现的维度） + +2. **Driver 在 Module 之后处理** + - 先通过 AFSModule 读取源文件 + - 再由 Driver 进行转换处理 + +3. **Driver 负责转换，Agent 不感知** + - Agent 只需声明"我要什么" + - Driver 负责生成和缓存 + +4. **异步 I/O（V1 简化版）** + - strict: 直接触发处理并等待 + - fallback: 后台处理，立即返回 source + - V1 不处理并发去重，V2 增加任务队列 + +5. **Metadata 分层管理** + - Source-level: 主内容版本 + - View-level: 投影状态 + +6. **物理存储对上层透明** + - `.i18n/{lang}/` 只是 driver 的实现细节 + - API 始终使用逻辑 path + view + +7. **内置默认翻译 Agent** + - i18n Driver 提供开箱即用的默认翻译 Agent + - 用户只需提供 model，无需自己实现翻译逻辑 + - 支持自定义翻译 Agent 以满足特殊需求 + +### 扩展性 + +未来可以轻松扩展： +- Format driver: `{ format: "html" }`（格式转换） +- Summary driver: `{ variant: "summary" }`（摘要生成） +- Policy driver: `{ policy: "marketing" }`（风格策略） +- 组合 driver: `{ language: "en", format: "html" }`（多维度组合） + +所有 driver 共享相同的基础设施（metadata、job queue、wait 策略）。 + +--- + +## 附录：关键文件清单 + +### AFS Core +- `afs/core/src/type.ts` - 类型定义（View, AFSDriver, ReadOptions） +- `afs/core/src/afs.ts` - AFS 主类，driver 匹配与调度，metadata 集成 +- `afs/core/src/view-schema.ts` - **新增** - 动态 View Schema 构建（buildViewSchema, extendSchemaWithView） +- `afs/core/src/metadata/type.ts` - Metadata 接口定义 +- `afs/core/src/metadata/store.ts` - SQLiteMetadataStore 实现 +- `afs/core/src/metadata/migrations/001-init.ts` - 数据库 schema +- `afs/core/src/metadata/index.ts` - Metadata 模块导出 +- `afs/core/src/view-processor.ts` - View 处理流程，job 队列，过期检测 + +### i18n Driver +- `afs/i18n-driver/src/index.ts` - I18nDriver 实现 +- `afs/i18n-driver/src/default-translation-agent.ts` - 内置默认翻译 Agent +- `afs/i18n-driver/src/storage.ts` - 物理存储映射 + +### Skills +- `packages/core/src/prompt/skills/afs/read.ts` - 使用 extendSchemaWithView 动态扩展 schema（read 支持 view） +- `packages/core/src/prompt/skills/afs/list.ts` - 使用 extendSchemaWithView 动态扩展 schema（list 可选支持 view） +- `packages/core/src/prompt/skills/afs/stat.ts` - 使用 extendSchemaWithView 动态扩展 schema（stat 可选支持 view） +- `packages/core/src/prompt/skills/afs/write.ts` - 无需修改（write 不支持 view） +- `packages/core/src/prompt/skills/afs/index.ts` - 保持简洁（getAFSSkills 不检查 driver） + +### Loader +- `packages/core/src/loader/agent-yaml.ts` - YAML schema 扩展 +- `packages/core/src/loader/index.ts` - Driver 实例化 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12fb80e21..f9006e512 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -71,6 +71,12 @@ importers: afs/core: dependencies: + '@aigne/sqlite': + specifier: workspace:^ + version: link:../../packages/sqlite + p-limit: + specifier: ^6.1.0 + version: 6.2.0 strict-event-emitter: specifier: ^0.5.1 version: 0.5.1 @@ -78,7 +84,7 @@ importers: specifier: ^1.6.1 version: 1.6.1 zod: - specifier: ^3.25.67 + specifier: ^3.24.1 version: 3.25.67 devDependencies: '@types/bun': @@ -136,12 +142,6 @@ importers: glob: specifier: ^11.0.3 version: 11.0.3 - ignore: - specifier: ^7.0.5 - version: 7.0.5 - minimatch: - specifier: ^10.1.1 - version: 10.1.1 zod: specifier: ^3.25.67 version: 3.25.67 @@ -215,7 +215,7 @@ importers: version: 13.0.1 '@arcblock/did-connect-react': specifier: ^3.2.11 - version: 3.2.11(@arcblock/ux@3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1))(@blocklet/js-sdk@1.17.4-beta-20251206-111557-41b8d449)(@blocklet/theme@3.2.11)(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@mui/material@7.3.2(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + version: 3.2.11(f02f5a02171cf12c3910ae36cc41250f) '@arcblock/ux': specifier: ^3.2.11 version: 3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1) @@ -1397,8 +1397,8 @@ importers: specifier: ^13.0.1 version: 13.0.1 openai: - specifier: ^6.14.0 - version: 6.14.0(ws@8.18.2)(zod@3.25.67) + specifier: ^6.5.0 + version: 6.6.0(ws@8.18.2)(zod@3.25.67) zod: specifier: ^3.25.67 version: 3.25.67 @@ -1618,7 +1618,7 @@ importers: version: 13.0.1 '@arcblock/did-connect-react': specifier: ^3.2.11 - version: 3.2.11(@arcblock/ux@3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1))(@blocklet/js-sdk@1.17.4-beta-20251206-111557-41b8d449)(@blocklet/theme@3.2.11)(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@mui/material@7.3.2(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + version: 3.2.11(f02f5a02171cf12c3910ae36cc41250f) '@arcblock/ux': specifier: ^3.2.11 version: 3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1) @@ -1630,7 +1630,7 @@ importers: version: 3.2.11(7d1a0dd794c522f6cb5f063dad4f7347) '@blocklet/uploader-server': specifier: ^0.3.14 - version: 0.3.14(@tus/file-store@1.0.0(@tus/server@1.0.0))(@tus/server@1.0.0)(@uppy/companion@4.15.1)(axios@1.12.2) + version: 0.3.14(@tus/file-store@1.0.0(@tus/server@1.0.0))(@tus/server@1.0.0)(@uppy/companion@4.15.1)(axios@1.12.2(debug@4.4.3)) '@mui/material': specifier: ^7.3.2 version: 7.3.2(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -2022,8 +2022,8 @@ importers: specifier: ^10.2.0 version: 10.2.0 openai: - specifier: ^6.14.0 - version: 6.14.0(ws@8.18.2)(zod@3.25.67) + specifier: ^6.6.0 + version: 6.6.0(ws@8.18.2)(zod@3.25.67) p-wait-for: specifier: ^5.0.2 version: 5.0.2 @@ -2166,9 +2166,6 @@ importers: fast-deep-equal: specifier: ^3.1.3 version: 3.1.3 - front-matter: - specifier: ^4.0.2 - version: 4.0.2 immer: specifier: ^10.1.3 version: 10.1.3 @@ -7790,9 +7787,6 @@ packages: from2@2.3.0: resolution: {integrity: sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==} - front-matter@4.0.2: - resolution: {integrity: sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==} - fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -9453,10 +9447,6 @@ packages: resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==} engines: {node: 20 || >=22} - minimatch@10.1.1: - resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==} - engines: {node: 20 || >=22} - minimatch@3.1.2: resolution: {integrity: sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==} @@ -9961,8 +9951,8 @@ packages: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} - openai@6.14.0: - resolution: {integrity: sha512-ZPD9MG5/sPpyGZ0idRoDK0P5MWEMuXe0Max/S55vuvoxqyEVkN94m9jSpE3YgNgz3WoESFvozs57dxWqAco31w==} + openai@6.6.0: + resolution: {integrity: sha512-1yWk4cBsHF5Bq9TreHYOHY7pbqdlT74COnm8vPx7WKn36StS+Hyk8DdAitnLaw67a5Cudkz5EmlFQjSrNnrA2w==} hasBin: true peerDependencies: ws: ^8.18.0 @@ -10016,6 +10006,10 @@ packages: resolution: {integrity: sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==} engines: {node: '>=10'} + p-limit@6.2.0: + resolution: {integrity: sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA==} + engines: {node: '>=18'} + p-limit@7.1.0: resolution: {integrity: sha512-7LbrpOjbzBWFHFgRtVrIAwBwW1bB7c7n914Q2+CXr1TvOfbTrVHAEH1Ya9PwDwAoKLeiRrczymYWPwaHNOQ0vA==} engines: {node: '>=20'} @@ -12889,7 +12883,7 @@ snapshots: '@ocap/wallet': 1.27.14 archiver: 7.0.1 axios: 1.12.2(debug@4.4.3) - axios-mock-adapter: 2.1.0(axios@1.12.2) + axios-mock-adapter: 2.1.0(axios@1.12.2(debug@4.4.3)) axon: 2.0.3 chalk: 4.1.2 cookie: 1.0.2 @@ -13199,7 +13193,7 @@ snapshots: - supports-color - utf-8-validate - '@arcblock/did-connect-react@3.2.11(@arcblock/ux@3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1))(@blocklet/js-sdk@1.17.4-beta-20251206-111557-41b8d449)(@blocklet/theme@3.2.11)(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@mui/material@7.3.2(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react-dom@19.1.1(react@19.1.1))(react@19.1.1)': + '@arcblock/did-connect-react@3.2.11(f02f5a02171cf12c3910ae36cc41250f)': dependencies: '@arcblock/bridge': 3.2.11 '@arcblock/did': 1.27.14 @@ -14692,10 +14686,10 @@ snapshots: - tedious - utf-8-validate - '@blocklet/did-space-react@1.2.3(6d4834af71c5d2b5f9227fe9faff3667)': + '@blocklet/did-space-react@1.2.3(17fc31b88bb3a5d6f638e9d08cbe5820)': dependencies: '@arcblock/did': 1.27.14 - '@arcblock/did-connect-react': 3.2.11(@arcblock/ux@3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1))(@blocklet/js-sdk@1.17.4-beta-20251206-111557-41b8d449)(@blocklet/theme@3.2.11)(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@mui/material@7.3.2(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + '@arcblock/did-connect-react': 3.2.11(f02f5a02171cf12c3910ae36cc41250f) '@arcblock/ux': 3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1) '@blocklet/js-sdk': 1.17.4-beta-20251206-111557-41b8d449 '@blocklet/sdk': 1.17.4-beta-20251206-111557-41b8d449 @@ -15136,13 +15130,13 @@ snapshots: '@abtnode/constant': 1.17.2 '@abtnode/util': 1.17.2 '@arcblock/bridge': 3.2.11 - '@arcblock/did-connect-react': 3.2.11(@arcblock/ux@3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1))(@blocklet/js-sdk@1.17.4-beta-20251206-111557-41b8d449)(@blocklet/theme@3.2.11)(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@mui/material@7.3.2(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@emotion/styled@11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react-dom@19.1.1(react@19.1.1))(react@19.1.1))(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + '@arcblock/did-connect-react': 3.2.11(f02f5a02171cf12c3910ae36cc41250f) '@arcblock/icons': 3.2.11(react@19.1.1) '@arcblock/react-hooks': 3.2.11 '@arcblock/ux': 3.2.11(39eb30f12e6f3867ab6d57ccfa0ed3e1) '@arcblock/ws': 1.27.12 '@blocklet/constant': 1.17.2 - '@blocklet/did-space-react': 1.2.3(6d4834af71c5d2b5f9227fe9faff3667) + '@blocklet/did-space-react': 1.2.3(17fc31b88bb3a5d6f638e9d08cbe5820) '@blocklet/js-sdk': 1.17.4-beta-20251206-111557-41b8d449 '@emotion/react': 11.14.0(@types/react@19.1.13)(react@19.1.1) '@emotion/styled': 11.14.1(@emotion/react@11.14.0(@types/react@19.1.13)(react@19.1.1))(@types/react@19.1.13)(react@19.1.1) @@ -15187,7 +15181,7 @@ snapshots: - supports-color - utf-8-validate - '@blocklet/uploader-server@0.3.14(@tus/file-store@1.0.0(@tus/server@1.0.0))(@tus/server@1.0.0)(@uppy/companion@4.15.1)(axios@1.12.2)': + '@blocklet/uploader-server@0.3.14(@tus/file-store@1.0.0(@tus/server@1.0.0))(@tus/server@1.0.0)(@uppy/companion@4.15.1)(axios@1.12.2(debug@4.4.3))': dependencies: '@abtnode/cron': 1.17.3 '@blocklet/constant': 1.17.3 @@ -15196,7 +15190,7 @@ snapshots: '@tus/file-store': 1.0.0(@tus/server@1.0.0) '@tus/server': 1.0.0 '@uppy/companion': 4.15.1 - axios: 1.12.2 + axios: 1.12.2(debug@4.4.3) body-parser: 1.20.3 exif-be-gone: 1.5.1 express-session: 1.17.3 @@ -20793,10 +20787,6 @@ snapshots: inherits: 2.0.4 readable-stream: 2.3.8 - front-matter@4.0.2: - dependencies: - js-yaml: 3.14.1 - fs-constants@1.0.0: {} fs-extra@11.3.2: @@ -20979,7 +20969,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 4.1.1 - minimatch: 10.1.1 + minimatch: 10.0.3 minipass: 7.1.2 package-json-from-dist: 1.0.1 path-scurry: 2.0.0 @@ -22873,10 +22863,6 @@ snapshots: dependencies: '@isaacs/brace-expansion': 5.0.0 - minimatch@10.1.1: - dependencies: - '@isaacs/brace-expansion': 5.0.0 - minimatch@3.1.2: dependencies: brace-expansion: 1.1.11 @@ -23382,7 +23368,7 @@ snapshots: is-inside-container: 1.0.0 wsl-utils: 0.1.0 - openai@6.14.0(ws@8.18.2)(zod@3.25.67): + openai@6.6.0(ws@8.18.2)(zod@3.25.67): optionalDependencies: ws: 8.18.2 zod: 3.25.67 @@ -23428,6 +23414,10 @@ snapshots: dependencies: yocto-queue: 0.1.0 + p-limit@6.2.0: + dependencies: + yocto-queue: 1.2.1 + p-limit@7.1.0: dependencies: yocto-queue: 1.2.1 From 3c575f1b0fccbe575f61a9cf1236e7bc7fba6ddb Mon Sep 17 00:00:00 2001 From: LBan Date: Tue, 16 Dec 2025 16:07:40 +0800 Subject: [PATCH 02/21] fix(view-processor, metadata): improve error handling and enhance metadata retrieval in SQLite store --- .../src/metadata/models/source-metadata.ts | 1 - afs/core/src/metadata/store.ts | 82 ++++- afs/core/src/view-processor.ts | 8 +- afs/core/test/view-driver.test.ts | 346 ++++++++++++++++++ 4 files changed, 416 insertions(+), 21 deletions(-) create mode 100644 afs/core/test/view-driver.test.ts diff --git a/afs/core/src/metadata/models/source-metadata.ts b/afs/core/src/metadata/models/source-metadata.ts index 29a8e267a..1584bc6a1 100644 --- a/afs/core/src/metadata/models/source-metadata.ts +++ b/afs/core/src/metadata/models/source-metadata.ts @@ -4,7 +4,6 @@ import { integer, sqliteTable, text } from "@aigne/sqlite"; * Source metadata table schema */ export const sourceMetadataTable = sqliteTable("source_metadata", { - id: integer("id").notNull().primaryKey({ autoIncrement: true }), path: text("path").notNull().primaryKey(), sourceRevision: text("source_revision").notNull(), updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), diff --git a/afs/core/src/metadata/store.ts b/afs/core/src/metadata/store.ts index 89fc68b8d..068b825be 100644 --- a/afs/core/src/metadata/store.ts +++ b/afs/core/src/metadata/store.ts @@ -28,7 +28,12 @@ export class SQLiteMetadataStore implements MetadataStore { async getSourceMetadata(path: string): Promise { const db = await this.db; const rows = await db - .select() + .select({ + path: this.sourceTable.path, + sourceRevision: this.sourceTable.sourceRevision, + updatedAt: this.sourceTable.updatedAt, + driversHint: this.sourceTable.driversHint, + }) .from(this.sourceTable) .where(eq(this.sourceTable.path, path)) .limit(1) @@ -58,7 +63,13 @@ export class SQLiteMetadataStore implements MetadataStore { driversHint: metadata.driversHint ? JSON.stringify(metadata.driversHint) : null, }) .where(eq(this.sourceTable.path, path)) - .returning() + .returning({ + path: this.sourceTable.path, + sourceRevision: this.sourceTable.sourceRevision, + updatedAt: this.sourceTable.updatedAt, + driversHint: this.sourceTable.driversHint, + createdAt: this.sourceTable.createdAt, + }) .execute(); // If no rows updated, insert new record @@ -87,7 +98,15 @@ export class SQLiteMetadataStore implements MetadataStore { const viewKey = JSON.stringify(view); const rows = await db - .select() + .select({ + path: this.viewTable.path, + view: this.viewTable.view, + state: this.viewTable.state, + derivedFrom: this.viewTable.derivedFrom, + generatedAt: this.viewTable.generatedAt, + error: this.viewTable.error, + storagePath: this.viewTable.storagePath, + }) .from(this.viewTable) .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) .limit(1) @@ -135,7 +154,18 @@ export class SQLiteMetadataStore implements MetadataStore { updatedAt: now, }) .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) - .returning() + .returning({ + id: this.viewTable.id, + path: this.viewTable.path, + view: this.viewTable.view, + state: this.viewTable.state, + derivedFrom: this.viewTable.derivedFrom, + generatedAt: this.viewTable.generatedAt, + error: this.viewTable.error, + storagePath: this.viewTable.storagePath, + createdAt: this.viewTable.createdAt, + updatedAt: this.viewTable.updatedAt, + }) .execute(); // If no rows updated, insert new record @@ -161,7 +191,15 @@ export class SQLiteMetadataStore implements MetadataStore { const db = await this.db; const rows = await db - .select() + .select({ + path: this.viewTable.path, + view: this.viewTable.view, + state: this.viewTable.state, + derivedFrom: this.viewTable.derivedFrom, + generatedAt: this.viewTable.generatedAt, + error: this.viewTable.error, + storagePath: this.viewTable.storagePath, + }) .from(this.viewTable) .where(eq(this.viewTable.path, path)) .execute(); @@ -210,7 +248,15 @@ export class SQLiteMetadataStore implements MetadataStore { const db = await this.db; const rows = await db - .select() + .select({ + path: this.viewTable.path, + view: this.viewTable.view, + state: this.viewTable.state, + derivedFrom: this.viewTable.derivedFrom, + generatedAt: this.viewTable.generatedAt, + error: this.viewTable.error, + storagePath: this.viewTable.storagePath, + }) .from(this.viewTable) .where(eq(this.viewTable.state, "stale")) .execute(); @@ -230,7 +276,15 @@ export class SQLiteMetadataStore implements MetadataStore { const db = await this.db; const rows = await db - .select() + .select({ + path: this.viewTable.path, + view: this.viewTable.view, + state: this.viewTable.state, + derivedFrom: this.viewTable.derivedFrom, + generatedAt: this.viewTable.generatedAt, + error: this.viewTable.error, + storagePath: this.viewTable.storagePath, + }) .from(this.viewTable) .where(eq(this.viewTable.state, "generating")) .execute(); @@ -267,17 +321,9 @@ export class SQLiteMetadataStore implements MetadataStore { async cleanupFailedViews(_olderThan?: Date): Promise { const db = await this.db; - // const cutoff = olderThan || new Date(Date.now() - 24 * 60 * 60 * 1000); // Default 24 hours + // Note: drizzle doesn't have a direct < operator for dates + // For now, we delete all failed views regardless of age - await db - .delete(this.viewTable) - .where( - and( - // Note: drizzle doesn't have a direct < operator for dates, we need to use sql - // For now, we'll keep this simple and delete all failed views - eq(this.viewTable.state, "failed"), - ), - ) - .execute(); + await db.delete(this.viewTable).where(eq(this.viewTable.state, "failed")).execute(); } } diff --git a/afs/core/src/view-processor.ts b/afs/core/src/view-processor.ts index 376bc43ed..e7f485dcd 100644 --- a/afs/core/src/view-processor.ts +++ b/afs/core/src/view-processor.ts @@ -96,10 +96,14 @@ export class ViewProcessor { sourceMeta = await this.metadataStore.getSourceMetadata(path); } + if (!sourceMeta) { + throw new Error(`Failed to create source metadata for ${path}`); + } + // 2. Mark as generating await this.metadataStore.setViewMetadata(path, view, { state: "generating", - derivedFrom: sourceMeta!.sourceRevision, + derivedFrom: sourceMeta.sourceRevision, }); // 3. Read source @@ -116,7 +120,7 @@ export class ViewProcessor { const result = await driver.process(module, path, view, { sourceEntry: sourceResult.data, - metadata: { derivedFrom: sourceMeta!.sourceRevision }, + metadata: { derivedFrom: sourceMeta.sourceRevision }, context, }); diff --git a/afs/core/test/view-driver.test.ts b/afs/core/test/view-driver.test.ts new file mode 100644 index 000000000..33a8e742f --- /dev/null +++ b/afs/core/test/view-driver.test.ts @@ -0,0 +1,346 @@ +import { afterEach, expect, test } from "bun:test"; +import { rmSync } from "node:fs"; +import { AFS, type AFSDriver, type AFSEntry, type AFSModule, type View } from "../src/index.js"; + +// Mock translation driver +class MockI18nDriver implements AFSDriver { + readonly name = "mock-i18n"; + readonly description = "Mock translation driver for testing"; + readonly capabilities = { + dimensions: ["language" as const], + }; + + private translations: Record> = { + en: { + 你好: "Hello", + 世界: "World", + 测试: "Test", + }, + ja: { + 你好: "こんにちは", + 世界: "世界", + 测试: "テスト", + }, + }; + + canHandle(view: View): boolean { + return !!view.language && Object.keys(view).length === 1; + } + + async process( + module: AFSModule, + path: string, + view: View, + options: { sourceEntry: AFSEntry; metadata: any; context: any }, + ): Promise<{ result: AFSEntry; message?: string }> { + const { sourceEntry } = options; + const targetLang = view.language; + if (!targetLang) { + throw new Error("Language is required for translation"); + } + + // Simple mock translation: replace Chinese words with target language + let translated = sourceEntry.content as string; + const dict = this.translations[targetLang]; + + if (dict) { + Object.entries(dict).forEach(([zh, translation]) => { + translated = translated.replace(new RegExp(zh, "g"), translation); + }); + } + + // Simulate writing to .i18n/{lang}/ directory + const storagePath = path.replace(/([^/]+)$/, `.i18n/${targetLang}/$1`); + + // Write to module storage + await module.write?.(storagePath, { content: translated }); + + return { + result: { + ...sourceEntry, + content: translated, + path, + metadata: { + ...sourceEntry.metadata, + storagePath, + view, + }, + }, + }; + } +} + +// Mock file system module +class MockFSModule implements AFSModule { + readonly name = "mock-fs"; + private files = new Map(); + + async read(path: string): Promise<{ result?: AFSEntry; message?: string }> { + const content = this.files.get(path); + if (!content) { + return { result: undefined, message: "File not found" }; + } + + return { + result: { + id: path, + path, + content, + createdAt: new Date(), + updatedAt: new Date(), + }, + }; + } + + async write( + path: string, + payload: { content: string }, + ): Promise<{ result: AFSEntry; message?: string }> { + this.files.set(path, payload.content); + + return { + result: { + id: path, + path, + content: payload.content, + createdAt: new Date(), + updatedAt: new Date(), + }, + }; + } + + async delete(path: string): Promise<{ message?: string }> { + this.files.delete(path); + return {}; + } +} + +// Cleanup test database after each test +const testDbPath = ".afs-test"; +afterEach(() => { + try { + rmSync(testDbPath, { recursive: true, force: true }); + } catch (_error) { + // Ignore cleanup errors + } +}); + +test("View driver: should translate content with view", async () => { + const mockFS = new MockFSModule(); + const mockDriver = new MockI18nDriver(); + + const afs = new AFS({ + modules: [mockFS], + drivers: [mockDriver], + metadataPath: `file:${testDbPath}/metadata.db`, + }); + + // Write source content (Chinese) + await afs.write("/modules/mock-fs/test.md", { + content: "你好，世界！这是一个测试。", + }); + + // Read English version (should trigger translation) + const enResult = await afs.read("/modules/mock-fs/test.md", { + view: { language: "en" }, + wait: "strict", + }); + + expect(enResult.result?.content).toBe("Hello，World！这是一个Test。"); + expect(enResult.result?.metadata?.view).toEqual({ language: "en" }); + + // Read Japanese version + const jaResult = await afs.read("/modules/mock-fs/test.md", { + view: { language: "ja" }, + wait: "strict", + }); + + expect(jaResult.result?.content).toBe("こんにちは，世界！这是一个テスト。"); + expect(jaResult.result?.metadata?.view).toEqual({ language: "ja" }); + + // Read source (no view) should return original + const sourceResult = await afs.read("/modules/mock-fs/test.md"); + expect(sourceResult.result?.content).toBe("你好，世界！这是一个测试。"); +}); + +test("View driver: should use cached view on second read", async () => { + const mockFS = new MockFSModule(); + const mockDriver = new MockI18nDriver(); + + // Track how many times driver.process is called + let processCallCount = 0; + const originalProcess = mockDriver.process.bind(mockDriver); + mockDriver.process = async (...args) => { + processCallCount++; + return originalProcess(...args); + }; + + const afs = new AFS({ + modules: [mockFS], + drivers: [mockDriver], + metadataPath: `file:${testDbPath}/metadata-cache.db`, + }); + + await afs.write("/modules/mock-fs/cache-test.md", { + content: "你好，世界！", + }); + + // First read: should trigger translation + await afs.read("/modules/mock-fs/cache-test.md", { + view: { language: "en" }, + wait: "strict", + }); + + expect(processCallCount).toBe(1); + + // Second read: should use cached view + const cachedResult = await afs.read("/modules/mock-fs/cache-test.md", { + view: { language: "en" }, + wait: "strict", + }); + + expect(processCallCount).toBe(1); // Still 1, not called again + expect(cachedResult.result?.content).toBe("Hello，World！"); +}); + +test("View driver: should invalidate cache when source changes", async () => { + const mockFS = new MockFSModule(); + const mockDriver = new MockI18nDriver(); + + let processCallCount = 0; + const originalProcess = mockDriver.process.bind(mockDriver); + mockDriver.process = async (...args) => { + processCallCount++; + return originalProcess(...args); + }; + + const afs = new AFS({ + modules: [mockFS], + drivers: [mockDriver], + metadataPath: `file:${testDbPath}/metadata-invalidate.db`, + }); + + // Write initial content + await afs.write("/modules/mock-fs/invalidate-test.md", { + content: "你好", + }); + + // Read English version (first translation) + const firstRead = await afs.read("/modules/mock-fs/invalidate-test.md", { + view: { language: "en" }, + wait: "strict", + }); + + expect(firstRead.result?.content).toBe("Hello"); + expect(processCallCount).toBe(1); + + // Update source content + await afs.write("/modules/mock-fs/invalidate-test.md", { + content: "你好，世界！", + }); + + // Read English version again (should re-translate) + const secondRead = await afs.read("/modules/mock-fs/invalidate-test.md", { + view: { language: "en" }, + wait: "strict", + }); + + expect(secondRead.result?.content).toBe("Hello，World！"); + expect(processCallCount).toBe(2); // Called again after source change +}); + +test("View driver: fallback mode should return source immediately", async () => { + const mockFS = new MockFSModule(); + + // Create a slow driver to test fallback behavior + class SlowDriver extends MockI18nDriver { + override async process(...args: Parameters) { + // Simulate slow translation + await new Promise((resolve) => setTimeout(resolve, 100)); + return super.process(...args); + } + } + + const slowDriver = new SlowDriver(); + + const afs = new AFS({ + modules: [mockFS], + drivers: [slowDriver], + metadataPath: `file:${testDbPath}/metadata-fallback.db`, + }); + + await afs.write("/modules/mock-fs/fallback-test.md", { + content: "你好，世界！", + }); + + // Use fallback mode: should return source immediately + const fallbackResult = await afs.read("/modules/mock-fs/fallback-test.md", { + view: { language: "en" }, + wait: "fallback", + }); + + // Should get source content immediately + expect(fallbackResult.result?.content).toBe("你好，世界！"); + expect(fallbackResult.message).toContain("being processed in background"); + + // Wait a bit for background processing + await new Promise((resolve) => setTimeout(resolve, 200)); + + // Now read again with strict mode, should get translated version + const strictResult = await afs.read("/modules/mock-fs/fallback-test.md", { + view: { language: "en" }, + wait: "strict", + }); + + expect(strictResult.result?.content).toBe("Hello，World！"); +}); + +test("View driver: prefetch should batch generate views", async () => { + const mockFS = new MockFSModule(); + const mockDriver = new MockI18nDriver(); + + let processCallCount = 0; + const originalProcess = mockDriver.process.bind(mockDriver); + mockDriver.process = async (...args) => { + processCallCount++; + return originalProcess(...args); + }; + + const afs = new AFS({ + modules: [mockFS], + drivers: [mockDriver], + metadataPath: `file:${testDbPath}/metadata-prefetch.db`, + }); + + // Create multiple files + await afs.write("/modules/mock-fs/file1.md", { content: "你好" }); + await afs.write("/modules/mock-fs/file2.md", { content: "世界" }); + await afs.write("/modules/mock-fs/file3.md", { content: "测试" }); + + // Prefetch English versions + await afs.prefetch( + ["/modules/mock-fs/file1.md", "/modules/mock-fs/file2.md", "/modules/mock-fs/file3.md"], + { view: { language: "en" }, concurrency: 2 }, + ); + + // All files should be translated + expect(processCallCount).toBe(3); + + // Read should use cached versions + const result1 = await afs.read("/modules/mock-fs/file1.md", { + view: { language: "en" }, + }); + const result2 = await afs.read("/modules/mock-fs/file2.md", { + view: { language: "en" }, + }); + const result3 = await afs.read("/modules/mock-fs/file3.md", { + view: { language: "en" }, + }); + + expect(result1.result?.content).toBe("Hello"); + expect(result2.result?.content).toBe("World"); + expect(result3.result?.content).toBe("Test"); + + // Process count should still be 3 (no additional calls) + expect(processCallCount).toBe(3); +}); From c32812cc3c30bd3e0e043b14bf50472f2a63951e Mon Sep 17 00:00:00 2001 From: LBan Date: Tue, 16 Dec 2025 19:07:00 +0800 Subject: [PATCH 03/21] feat(i18n-driver): implement initial i18n driver with built-in translation agent and storage mapping --- afs/core/src/type.ts | 1 - afs/core/src/view-processor.ts | 5 +- afs/core/test/view-driver.test.ts | 2 +- afs/i18n-driver.md | 82 ++++-- afs/i18n-driver/CHANGELOG.md | 10 + afs/i18n-driver/README.md | 55 ++++ afs/i18n-driver/package.json | 69 +++++ .../scripts/tsconfig.build.cjs.json | 8 + .../scripts/tsconfig.build.dts.json | 10 + .../scripts/tsconfig.build.esm.json | 8 + afs/i18n-driver/scripts/tsconfig.build.json | 16 ++ .../src/default-translation-agent.ts | 50 ++++ afs/i18n-driver/src/driver.ts | 123 +++++++++ afs/i18n-driver/src/index.ts | 9 + afs/i18n-driver/src/storage.ts | 34 +++ afs/i18n-driver/test/i18n-driver.test.ts | 250 ++++++++++++++++++ afs/i18n-driver/tsconfig.json | 11 + pnpm-lock.yaml | 25 ++ 18 files changed, 739 insertions(+), 29 deletions(-) create mode 100644 afs/i18n-driver/CHANGELOG.md create mode 100644 afs/i18n-driver/README.md create mode 100644 afs/i18n-driver/package.json create mode 100644 afs/i18n-driver/scripts/tsconfig.build.cjs.json create mode 100644 afs/i18n-driver/scripts/tsconfig.build.dts.json create mode 100644 afs/i18n-driver/scripts/tsconfig.build.esm.json create mode 100644 afs/i18n-driver/scripts/tsconfig.build.json create mode 100644 afs/i18n-driver/src/default-translation-agent.ts create mode 100644 afs/i18n-driver/src/driver.ts create mode 100644 afs/i18n-driver/src/index.ts create mode 100644 afs/i18n-driver/src/storage.ts create mode 100644 afs/i18n-driver/test/i18n-driver.test.ts create mode 100644 afs/i18n-driver/tsconfig.json diff --git a/afs/core/src/type.ts b/afs/core/src/type.ts index 5c2b0a7c2..f46425662 100644 --- a/afs/core/src/type.ts +++ b/afs/core/src/type.ts @@ -273,7 +273,6 @@ export interface AFSDriver { options: { sourceEntry: AFSEntry; metadata: any; - context: any; }, ): Promise<{ result: AFSEntry; message?: string }>; diff --git a/afs/core/src/view-processor.ts b/afs/core/src/view-processor.ts index e7f485dcd..bda49c4bd 100644 --- a/afs/core/src/view-processor.ts +++ b/afs/core/src/view-processor.ts @@ -74,7 +74,7 @@ export class ViewProcessor { * Process a view (generate or regenerate) * V1: Direct execution without job deduplication */ - async processView(module: AFSModule, path: string, view: View, context: any): Promise { + async processView(module: AFSModule, path: string, view: View): Promise { try { // 1. Get or create source metadata let sourceMeta = await this.metadataStore.getSourceMetadata(path); @@ -249,7 +249,6 @@ export class ViewProcessor { module: AFSModule, paths: string[], view: View, - context: any, options?: { concurrency?: number }, ): Promise { const tasksToGenerate: string[] = []; @@ -271,7 +270,7 @@ export class ViewProcessor { await Promise.all( tasksToGenerate.map((path) => limit(() => - this.processView(module, path, view, context).catch((error) => { + this.processView(module, path, view).catch((error) => { console.error(`Prefetch failed for ${path}:`, error); }), ), diff --git a/afs/core/test/view-driver.test.ts b/afs/core/test/view-driver.test.ts index 33a8e742f..4d87067ba 100644 --- a/afs/core/test/view-driver.test.ts +++ b/afs/core/test/view-driver.test.ts @@ -31,7 +31,7 @@ class MockI18nDriver implements AFSDriver { module: AFSModule, path: string, view: View, - options: { sourceEntry: AFSEntry; metadata: any; context: any }, + options: { sourceEntry: AFSEntry; metadata: any }, ): Promise<{ result: AFSEntry; message?: string }> { const { sourceEntry } = options; const targetLang = view.language; diff --git a/afs/i18n-driver.md b/afs/i18n-driver.md index 6b4b54e28..b71a3495e 100644 --- a/afs/i18n-driver.md +++ b/afs/i18n-driver.md @@ -1523,36 +1523,70 @@ export async function getAFSSkills(afs: AFS): Promise { --- -### Phase 2: i18n Driver 实现 +### Phase 2: i18n Driver 实现 ✅ 已完成 **目标：** 实现第一个 driver：i18n driver +**完成日期：** 2024-12-16 + **任务：** -1. [ ] 创建 `afs/i18n-driver/` 包： - - `src/index.ts`: 实现 `I18nDriver` 类 - - `src/default-translation-agent.ts`: 实现内置默认翻译 Agent - - `src/storage.ts`: 实现 `.i18n/{lang}/` 物理路径映射 - -2. [ ] 配置选项设计： - - `defaultSourceLanguage`: 默认源语言（可选，如 "zh"） - - `supportedLanguages`: 支持的目标语言列表（可选） - - `model`: LLM 模型实例（用于默认翻译 Agent） - - `translationAgent`: 自定义翻译 Agent（可选，不提供则使用内置默认） - - `storagePath`: 物理存储路径模板（可选，默认 `.i18n/{language}/`） - -3. [ ] 默认翻译 Agent 实现： - - 使用 AIAgent.from 创建 - - 支持多语言翻译，保持格式和技术术语 - - 输入：content, targetLanguage, sourceLanguage - - 输出：translatedContent - -4. [ ] 测试： - - 单元测试：driver 匹配、生成逻辑 - - 集成测试：与 LocalFS 集成 - - 测试默认翻译 Agent 和自定义 Agent 两种场景 +1. [x] 创建 `afs/i18n-driver/` 包： + - ✅ `src/index.ts`: 导出所有公共 API + - ✅ `src/driver.ts`: 实现 `I18nDriver` 类 + - ✅ `src/default-translation-agent.ts`: 实现内置默认翻译 Agent + - ✅ `src/storage.ts`: 实现 `/.i18n/{lang}/path` 物理路径映射 + +2. [x] 配置选项设计： + - ✅ `context`: AIGNE Context（必需，用于调用翻译 Agent） + - ✅ `defaultSourceLanguage`: 默认源语言（可选，如 "zh"） + - ✅ `supportedLanguages`: 支持的目标语言列表（可选） + - ✅ `translationAgent`: 自定义翻译 Agent（可选，不提供则使用内置默认） + - ✅ `storagePath`: 物理存储路径模板（可选，默认 `.i18n/{language}`） + - ⚠️ 删除 `model` 配置：通过 context 自动获取外层 model + +3. [x] 默认翻译 Agent 实现： + - ✅ 使用 AIAgent.from 创建 + - ✅ 支持多语言翻译，保持格式和技术术语 + - ✅ 输入：content, targetLanguage, sourceLanguage + - ✅ 输出：translatedContent + - ✅ 独立文件，方便后续优化 + +4. [x] 测试： + - ✅ 单元测试：getStoragePath 路径映射 + - ✅ 单元测试：canHandle view 匹配 + - ✅ 单元测试：supportedLanguages 过滤 + - ✅ 集成测试：I18nDriver 翻译流程 + - ✅ 集成测试：与 AFS 集成 + - ✅ **测试结果:** 7 个测试全部通过 ✅ **输出：** -- `@aigne/afs-i18n-driver` v0.0.1 +- ✅ `@aigne/afs-i18n-driver` v0.0.1 +- ✅ TypeScript 编译通过 +- ✅ Biome lint 检查通过 +- ✅ 所有单元测试通过 + +**实现亮点：** +1. **Context 驱动** - 通过 `context.newContext({ reset: true }).invoke()` 调用翻译 Agent +2. **路径映射简化** - `/.i18n/{lang}/` 前缀直接添加到原路径，保留完整目录结构 +3. **模块化设计** - 默认翻译 Agent 独立文件，支持自定义替换 +4. **完整的类型导出** - 导出 schema、类型、创建函数，便于扩展 + +**文件结构：** +``` +afs/i18n-driver/ +├── package.json +├── tsconfig.json +├── scripts/ +├── src/ +│ ├── index.ts # 导出所有公共 API +│ ├── driver.ts # I18nDriver 实现 +│ ├── default-translation-agent.ts # 内置翻译 Agent +│ └── storage.ts # 路径映射 +├── test/ +│ └── i18n-driver.test.ts +├── README.md +└── CHANGELOG.md +``` --- diff --git a/afs/i18n-driver/CHANGELOG.md b/afs/i18n-driver/CHANGELOG.md new file mode 100644 index 000000000..2b0b69ef8 --- /dev/null +++ b/afs/i18n-driver/CHANGELOG.md @@ -0,0 +1,10 @@ +# @aigne/afs-i18n-driver + +## 0.0.1 + +### Initial Release + +- I18nDriver implementation for AFS +- Built-in default translation agent +- Physical storage mapping to `.i18n/{language}/` directory +- Support for custom translation agents diff --git a/afs/i18n-driver/README.md b/afs/i18n-driver/README.md new file mode 100644 index 000000000..06b44a954 --- /dev/null +++ b/afs/i18n-driver/README.md @@ -0,0 +1,55 @@ +# @aigne/afs-i18n-driver + +AIGNE AFS driver for i18n translation. This driver enables automatic translation of files to different languages using AI. + +## Installation + +```bash +npm install @aigne/afs-i18n-driver +``` + +## Usage + +```typescript +import { AFS } from "@aigne/afs"; +import { I18nDriver } from "@aigne/afs-i18n-driver"; +import { LocalFS } from "@aigne/afs-local-fs"; +import { AIGNE } from "@aigne/core"; + +// Create context +const aigne = new AIGNE({ model: yourModel }); +const context = aigne.newContext(); + +// Create i18n driver with context +const i18nDriver = new I18nDriver({ + context, + defaultSourceLanguage: "zh", + supportedLanguages: ["en", "ja", "ko"], +}); + +// Create AFS with i18n driver +const afs = new AFS({ + modules: [new LocalFS({ localPath: "./docs" })], + drivers: [i18nDriver], +}); + +// Read translated version +const result = await afs.read("/modules/local-fs/intro.md", { + view: { language: "en" }, + wait: "strict", +}); +``` + +## Configuration + +### I18nDriverOptions + +- `context` (required): AIGNE context for invoking the translation agent +- `defaultSourceLanguage` (optional): Default source language code (e.g., "zh") +- `supportedLanguages` (optional): Array of supported target language codes +- `translationAgent` (optional): Custom translation agent (uses built-in agent if not provided) +- `storagePath` (optional): Storage path template (default: `.i18n/{language}/`) + +## License + +Elastic-2.0 diff --git a/afs/i18n-driver/package.json b/afs/i18n-driver/package.json new file mode 100644 index 000000000..8e63c4908 --- /dev/null +++ b/afs/i18n-driver/package.json @@ -0,0 +1,69 @@ +{ + "name": "@aigne/afs-i18n-driver", + "version": "0.0.1", + "description": "AIGNE AFS driver for i18n translation", + "publishConfig": { + "access": "public" + }, + "author": "Arcblock https://github.com/blocklet", + "homepage": "https://www.aigne.io/framework", + "license": "Elastic-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/AIGNE-io/aigne-framework" + }, + "bugs": { + "url": "https://github.com/AIGNE-io/aigne-framework/issues" + }, + "files": [ + "lib/cjs", + "lib/dts", + "lib/esm", + "LICENSE", + "README.md", + "CHANGELOG.md" + ], + "type": "module", + "main": "./lib/cjs/index.js", + "module": "./lib/esm/index.js", + "types": "./lib/dts/index.d.ts", + "exports": { + ".": { + "import": "./lib/esm/index.js", + "require": "./lib/cjs/index.js", + "types": "./lib/dts/index.d.ts" + }, + "./*": { + "import": "./lib/esm/*", + "require": "./lib/cjs/*", + "types": "./lib/dts/*" + } + }, + "typesVersions": { + "*": { + ".": [ + "./lib/dts/index.d.ts" + ] + } + }, + "scripts": { + "lint": "tsc --noEmit", + "build": "tsc --build scripts/tsconfig.build.json", + "clean": "rimraf lib test/coverage", + "prepublishOnly": "run-s clean build", + "test": "bun test", + "test:coverage": "bun test --coverage --coverage-reporter=lcov --coverage-reporter=text", + "postbuild": "node ../../scripts/post-build-lib.mjs" + }, + "dependencies": { + "@aigne/afs": "workspace:^", + "@aigne/core": "workspace:^", + "zod": "^3.25.67" + }, + "devDependencies": { + "@types/bun": "^1.2.22", + "npm-run-all": "^4.1.5", + "rimraf": "^6.0.1", + "typescript": "^5.9.2" + } +} diff --git a/afs/i18n-driver/scripts/tsconfig.build.cjs.json b/afs/i18n-driver/scripts/tsconfig.build.cjs.json new file mode 100644 index 000000000..b48de9c1e --- /dev/null +++ b/afs/i18n-driver/scripts/tsconfig.build.cjs.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.build.json", + "compilerOptions": { + "module": "CommonJS", + "moduleResolution": "node", + "outDir": "../lib/cjs" + } +} diff --git a/afs/i18n-driver/scripts/tsconfig.build.dts.json b/afs/i18n-driver/scripts/tsconfig.build.dts.json new file mode 100644 index 000000000..f5bb390ce --- /dev/null +++ b/afs/i18n-driver/scripts/tsconfig.build.dts.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.build.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "nodenext", + "outDir": "../lib/dts", + "declaration": true, + "emitDeclarationOnly": true + } +} diff --git a/afs/i18n-driver/scripts/tsconfig.build.esm.json b/afs/i18n-driver/scripts/tsconfig.build.esm.json new file mode 100644 index 000000000..d2d7edebe --- /dev/null +++ b/afs/i18n-driver/scripts/tsconfig.build.esm.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.build.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "nodenext", + "outDir": "../lib/esm" + } +} diff --git a/afs/i18n-driver/scripts/tsconfig.build.json b/afs/i18n-driver/scripts/tsconfig.build.json new file mode 100644 index 000000000..fabac44ae --- /dev/null +++ b/afs/i18n-driver/scripts/tsconfig.build.json @@ -0,0 +1,16 @@ +{ + "files": [], + "extends": "../tsconfig.json", + "compilerOptions": { + "composite": true, + "rootDir": "../src", + "noEmit": false, + "paths": {} + }, + "include": ["../src"], + "references": [ + { "path": "./tsconfig.build.cjs.json" }, + { "path": "./tsconfig.build.esm.json" }, + { "path": "./tsconfig.build.dts.json" } + ] +} diff --git a/afs/i18n-driver/src/default-translation-agent.ts b/afs/i18n-driver/src/default-translation-agent.ts new file mode 100644 index 000000000..61515c8d6 --- /dev/null +++ b/afs/i18n-driver/src/default-translation-agent.ts @@ -0,0 +1,50 @@ +import { type Agent, AIAgent } from "@aigne/core"; +import { z } from "zod"; + +/** + * Translation agent input schema + */ +export const translationInputSchema = z.object({ + content: z.string().describe("Content to translate"), + targetLanguage: z.string().describe("Target language code (e.g., 'en', 'ja')"), + sourceLanguage: z.string().optional().describe("Source language code"), +}); + +/** + * Translation agent output schema + */ +export const translationOutputSchema = z.object({ + translatedContent: z.string().describe("Translated content"), +}); + +export type TranslationInput = z.infer; +export type TranslationOutput = z.infer; + +/** + * Default translation agent instructions + */ +export const DEFAULT_TRANSLATION_INSTRUCTIONS = `You are a professional translator. + +Translate the provided content from the source language to the target language. +Maintain the original formatting, structure, and technical terms. +Preserve markdown syntax, code blocks, and special formatting. + +Requirements: +- Translate naturally and fluently +- Keep technical terms and proper nouns when appropriate +- Maintain the tone and style of the original content +- Do not add explanations or extra content +- Output only the translated content`; + +/** + * Create the default built-in translation agent + */ +export function createDefaultTranslationAgent(): Agent { + return AIAgent.from({ + name: "i18n_translator", + description: "Built-in translation agent for i18n driver", + instructions: DEFAULT_TRANSLATION_INSTRUCTIONS, + inputSchema: translationInputSchema, + outputSchema: translationOutputSchema, + }); +} diff --git a/afs/i18n-driver/src/driver.ts b/afs/i18n-driver/src/driver.ts new file mode 100644 index 000000000..74239f7e1 --- /dev/null +++ b/afs/i18n-driver/src/driver.ts @@ -0,0 +1,123 @@ +import type { AFSDriver, AFSEntry, AFSModule, View } from "@aigne/afs"; +import type { Agent, Context } from "@aigne/core"; +import { + createDefaultTranslationAgent, + type TranslationInput, + type TranslationOutput, +} from "./default-translation-agent.js"; +import { getStoragePath } from "./storage.js"; + +/** + * I18n Driver configuration options + */ +export interface I18nDriverOptions { + /** Context for invoking the translation agent */ + context: Context; + + /** Default source language code (e.g., "zh") */ + defaultSourceLanguage?: string; + + /** Supported target language codes */ + supportedLanguages?: string[]; + + /** Custom translation agent (uses built-in agent if not provided) */ + translationAgent?: Agent; + + /** Storage path template (default: ".i18n/{language}/") */ + storagePath?: string; +} + +/** + * I18n Driver for AFS + * + * Handles translation of files to different languages using AI. + */ +export class I18nDriver implements AFSDriver { + readonly name = "i18n"; + readonly description = "Multilingual translation driver"; + readonly capabilities = { + dimensions: ["language" as const], + }; + + private translationAgent: Agent; + + constructor(private options: I18nDriverOptions) { + // Use custom agent or create default + this.translationAgent = options.translationAgent ?? createDefaultTranslationAgent(); + } + + /** + * Check if this driver can handle the given view + * Only handles views with language dimension only + */ + canHandle(view: View): boolean { + // Must have language and only language + if (!view.language) return false; + if (Object.keys(view).length !== 1) return false; + + // Check if language is supported (if supportedLanguages is configured) + if (this.options.supportedLanguages?.length) { + return this.options.supportedLanguages.includes(view.language); + } + + return true; + } + + /** + * Process and generate the translated view + */ + async process( + module: AFSModule, + path: string, + view: View, + options: { + sourceEntry: AFSEntry; + metadata: any; + }, + ): Promise<{ result: AFSEntry; message?: string }> { + const { language } = view; + const { sourceEntry } = options; + + if (!language) { + throw new Error("Language is required for translation"); + } + + // Translate content using context from constructor options + const translatedContent = await this.translate(sourceEntry.content as string, language); + + // Get storage path + const storagePath = getStoragePath(path, language, this.options.storagePath); + + // Write translated content to storage + await module.write?.(storagePath, { content: translatedContent }); + + // Return translated entry + return { + result: { + ...sourceEntry, + content: translatedContent, + path, + metadata: { + ...sourceEntry.metadata, + storagePath, + view, + }, + }, + }; + } + + /** + * Translate content using the translation agent + */ + private async translate(content: string, targetLanguage: string): Promise { + const { translatedContent } = await this.options.context + .newContext({ reset: true }) + .invoke(this.translationAgent, { + content, + targetLanguage, + sourceLanguage: this.options.defaultSourceLanguage, + }); + + return translatedContent; + } +} diff --git a/afs/i18n-driver/src/index.ts b/afs/i18n-driver/src/index.ts new file mode 100644 index 000000000..aac9fab6b --- /dev/null +++ b/afs/i18n-driver/src/index.ts @@ -0,0 +1,9 @@ +export { + createDefaultTranslationAgent, + type TranslationInput, + type TranslationOutput, + translationInputSchema, + translationOutputSchema, +} from "./default-translation-agent.js"; +export { I18nDriver, type I18nDriverOptions } from "./driver.js"; +export { getStoragePath } from "./storage.js"; diff --git a/afs/i18n-driver/src/storage.ts b/afs/i18n-driver/src/storage.ts new file mode 100644 index 000000000..9530cb851 --- /dev/null +++ b/afs/i18n-driver/src/storage.ts @@ -0,0 +1,34 @@ +import { join } from "node:path"; + +/** + * Get storage path for a translated file + * Prepends .i18n/{language}/ to the original path + * + * @param path - Source file path (subpath from AFS module) + * @param language - Target language code + * @param template - Storage path template (default: ".i18n/{language}") + * @returns Physical storage path for the translated file + * + * @example + * getStoragePath("/docs/guide/setup.md", "en") + * // Returns: "/.i18n/en/docs/guide/setup.md" + * + * getStoragePath("/intro.md", "ja") + * // Returns: "/.i18n/ja/intro.md" + * + * getStoragePath("docs/intro.md", "en") + * // Returns: ".i18n/en/docs/intro.md" + */ +export function getStoragePath(path: string, language: string, template?: string): string { + // Handle absolute paths (starting with /) + const isAbsolute = path.startsWith("/"); + const normalizedPath = isAbsolute ? path.slice(1) : path; + + // Build the i18n directory name + const i18nDir = template ? template.replace("{language}", language) : `.i18n/${language}`; + + // Prepend i18n directory to the path + const storagePath = join(i18nDir, normalizedPath); + + return isAbsolute ? `/${storagePath}` : storagePath; +} diff --git a/afs/i18n-driver/test/i18n-driver.test.ts b/afs/i18n-driver/test/i18n-driver.test.ts new file mode 100644 index 000000000..918b8e2f8 --- /dev/null +++ b/afs/i18n-driver/test/i18n-driver.test.ts @@ -0,0 +1,250 @@ +import { afterEach, expect, test } from "bun:test"; +import assert from "node:assert"; +import { rmSync } from "node:fs"; +import { AFS, type AFSEntry, type AFSModule } from "@aigne/afs"; +import { getStoragePath, I18nDriver } from "@aigne/afs-i18n-driver"; +import { AIGNE, FunctionAgent } from "@aigne/core"; + +// Cleanup test database after each test +const testDbPath = ".afs-test"; +afterEach(() => { + try { + rmSync(testDbPath, { recursive: true, force: true }); + } catch (_error) { + // Ignore cleanup errors + } +}); + +// Mock file system module with context +class MockFSModule implements AFSModule { + readonly name = "mock-fs"; + private files = new Map(); + + constructor(public options: { context: any }) {} + + async read(path: string): Promise<{ result?: AFSEntry; message?: string }> { + const content = this.files.get(path); + if (!content) { + return { result: undefined, message: "File not found" }; + } + + return { + result: { + id: path, + path, + content, + createdAt: new Date(), + updatedAt: new Date(), + }, + }; + } + + async write( + path: string, + payload: { content: string }, + ): Promise<{ result: AFSEntry; message?: string }> { + this.files.set(path, payload.content); + + return { + result: { + id: path, + path, + content: payload.content, + createdAt: new Date(), + updatedAt: new Date(), + }, + }; + } + + async delete(path: string): Promise<{ message?: string }> { + this.files.delete(path); + return {}; + } +} + +// Mock translation agent +function createMockTranslationAgent() { + const translations: Record> = { + en: { + 你好: "Hello", + 世界: "World", + 测试: "Test", + 这是一个: "This is a", + }, + ja: { + 你好: "こんにちは", + 世界: "世界", + 测试: "テスト", + 这是一个: "これは", + }, + }; + + return FunctionAgent.from({ + name: "mock_translator", + description: "Mock translation agent for testing", + process: ({ content, targetLanguage }: { content: string; targetLanguage: string }) => { + let translated = content; + const dict = translations[targetLanguage]; + + if (dict) { + Object.entries(dict).forEach(([zh, translation]) => { + translated = translated.replaceAll(zh, translation); + }); + } + + return { translatedContent: translated }; + }, + }); +} + +test("getStoragePath should prepend .i18n/{language}/ to path", () => { + // Relative paths: .i18n/{lang}/original/path + expect(getStoragePath("docs/intro.md", "en")).toBe(".i18n/en/docs/intro.md"); + expect(getStoragePath("docs/guide/setup.md", "ja")).toBe(".i18n/ja/docs/guide/setup.md"); + + // Absolute paths: /.i18n/{lang}/original/path + expect(getStoragePath("/docs/intro.md", "en")).toBe("/.i18n/en/docs/intro.md"); + expect(getStoragePath("/docs/guide/setup.md", "ja")).toBe("/.i18n/ja/docs/guide/setup.md"); + expect(getStoragePath("/intro.md", "ko")).toBe("/.i18n/ko/intro.md"); +}); + +test("getStoragePath should support custom template", () => { + expect(getStoragePath("docs/intro.md", "en", "translations/{language}")).toBe( + "translations/en/docs/intro.md", + ); + expect(getStoragePath("/docs/intro.md", "en", "i18n/{language}")).toBe("/i18n/en/docs/intro.md"); +}); + +test("I18nDriver.canHandle should return true for language-only views", () => { + const aigne = new AIGNE(); + const driver = new I18nDriver({ context: aigne.newContext() }); + + expect(driver.canHandle({ language: "en" })).toBe(true); + expect(driver.canHandle({ language: "ja" })).toBe(true); + expect(driver.canHandle({})).toBe(false); + // @ts-expect-error - testing invalid view + expect(driver.canHandle({ language: "en", format: "html" })).toBe(false); +}); + +test("I18nDriver.canHandle should check supportedLanguages", () => { + const aigne = new AIGNE(); + const driver = new I18nDriver({ + context: aigne.newContext(), + supportedLanguages: ["en", "ja"], + }); + + expect(driver.canHandle({ language: "en" })).toBe(true); + expect(driver.canHandle({ language: "ja" })).toBe(true); + expect(driver.canHandle({ language: "ko" })).toBe(false); +}); + +test("I18nDriver should translate content using context", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockTranslator = createMockTranslationAgent(); + + const driver = new I18nDriver({ + context, + defaultSourceLanguage: "zh", + translationAgent: mockTranslator, + }); + + // Write source file + await mockFS.write("/test.md", { content: "你好，世界！这是一个测试。" }); + + // Read source + const source = await mockFS.read("/test.md"); + assert(source.result, "source.result should be defined"); + + // Process translation + const result = await driver.process( + mockFS, + "/test.md", + { language: "en" }, + { + sourceEntry: source.result, + metadata: {}, + }, + ); + + expect(result.result.content).toBe("Hello，World！This is aTest。"); + expect(result.result.metadata?.storagePath).toBe("/.i18n/en/test.md"); // root-level file + expect(result.result.metadata?.view).toEqual({ language: "en" }); +}); + +test("I18nDriver should integrate with AFS", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockTranslator = createMockTranslationAgent(); + + const driver = new I18nDriver({ + context, + defaultSourceLanguage: "zh", + translationAgent: mockTranslator, + }); + + const afs = new AFS({ + modules: [mockFS], + drivers: [driver], + metadataPath: "file:.afs-test/i18n-test.db", + }); + + // Write source content (Chinese) + await afs.write("/modules/mock-fs/doc.md", { + content: "你好，世界！", + }); + + // Read English version (should trigger translation) + const enResult = await afs.read("/modules/mock-fs/doc.md", { + view: { language: "en" }, + wait: "strict", + }); + + expect(enResult.result?.content).toBe("Hello，World！"); + expect(enResult.result?.metadata?.view).toEqual({ language: "en" }); + + // Read Japanese version + const jaResult = await afs.read("/modules/mock-fs/doc.md", { + view: { language: "ja" }, + wait: "strict", + }); + + expect(jaResult.result?.content).toBe("こんにちは，世界！"); + + // Read source (no view) should return original + const sourceResult = await afs.read("/modules/mock-fs/doc.md"); + expect(sourceResult.result?.content).toBe("你好，世界！"); +}); + +test("I18nDriver should throw error if language is missing", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockTranslator = createMockTranslationAgent(); + + const driver = new I18nDriver({ + context, + translationAgent: mockTranslator, + }); + + await mockFS.write("/test.md", { content: "test content" }); + const source = await mockFS.read("/test.md"); + assert(source.result, "source.result should be defined"); + + await expect( + driver.process( + mockFS, + "/test.md", + {}, + { + sourceEntry: source.result, + metadata: {}, + }, + ), + ).rejects.toThrow("Language is required for translation"); +}); diff --git a/afs/i18n-driver/tsconfig.json b/afs/i18n-driver/tsconfig.json new file mode 100644 index 000000000..1c7cf31c1 --- /dev/null +++ b/afs/i18n-driver/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "paths": { + "@aigne/afs-i18n-driver": ["./src"], + "@aigne/afs-i18n-driver/*": ["./src/*"] + } + }, + "exclude": ["node_modules", "lib"], + "include": ["src", "test"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f9006e512..bc7cd63e6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -128,6 +128,31 @@ importers: specifier: ^5.9.2 version: 5.9.2 + afs/i18n-driver: + dependencies: + '@aigne/afs': + specifier: workspace:^ + version: link:../core + '@aigne/core': + specifier: workspace:^ + version: link:../../packages/core + zod: + specifier: ^3.25.67 + version: 3.25.67 + devDependencies: + '@types/bun': + specifier: ^1.2.22 + version: 1.2.22(@types/react@19.1.13) + npm-run-all: + specifier: ^4.1.5 + version: 4.1.5 + rimraf: + specifier: ^6.0.1 + version: 6.0.1 + typescript: + specifier: ^5.9.2 + version: 5.9.2 + afs/local-fs: dependencies: '@aigne/afs': From 85f1b2bf24339ff67c80d49816f4e90f2bb58c41 Mon Sep 17 00:00:00 2001 From: LBan Date: Tue, 16 Dec 2025 19:34:40 +0800 Subject: [PATCH 04/21] feat(afs): enhance read functionality with AFSReadResult and view status support --- afs/core/src/type.ts | 17 ++ afs/i18n-driver.md | 79 ++++---- packages/core/src/prompt/skills/afs/read.ts | 212 +++++++++++--------- 3 files changed, 180 insertions(+), 128 deletions(-) diff --git a/afs/core/src/type.ts b/afs/core/src/type.ts index f46425662..29578c99d 100644 --- a/afs/core/src/type.ts +++ b/afs/core/src/type.ts @@ -244,6 +244,23 @@ export interface AFSContext { }; } +/** + * View status in read result + * Indicates whether the requested view was returned or fell back to source + */ +export interface ViewStatus { + fallback?: boolean; // true = returned source content, view is being generated in background +} + +/** + * Read result with optional view status + */ +export interface AFSReadResult { + result?: AFSEntry; + message?: string; + viewStatus?: ViewStatus; +} + /** * AFSDriver interface for view transformation */ diff --git a/afs/i18n-driver.md b/afs/i18n-driver.md index b71a3495e..e8d1260eb 100644 --- a/afs/i18n-driver.md +++ b/afs/i18n-driver.md @@ -1590,37 +1590,48 @@ afs/i18n-driver/ --- -### Phase 3: Skill 层集成 +### Phase 3: Skill 层集成 ✅ 已完成 **目标：** 在 `packages/core` 中集成 view 支持 +**完成日期：** 2024-12-16 + **任务：** -1. [ ] 扩展 `packages/core/src/prompt/skills/afs/read.ts`： - - 使用 `extendSchemaWithView()` 动态扩展 schema - - 增加 `view` 字段到 `AFSReadInput`（根据 drivers 自动添加） - - 增加 `wait` 字段（可选，默认使用 AFS Core 的默认值 "strict"） - - 更新 description - -2. [ ] 扩展 `packages/core/src/prompt/skills/afs/list.ts`： - - 使用 `extendSchemaWithView()` 动态扩展 schema - - 增加 `view` 字段到 `AFSListInput`（可选） - - 用于列出特定 view 的文件 - -3. [ ] 扩展 `packages/core/src/prompt/skills/afs/stat.ts`： - - 使用 `extendSchemaWithView()` 动态扩展 schema - - 增加 `view` 字段到 `AFSStatInput`（可选） - - 用于查询特定 view 的状态 - -4. [ ] **不修改** `packages/core/src/prompt/skills/afs/write.ts`： - - write 不支持 view 参数 - - 只能写入 source，view 由 driver 自动生成 - -5. [ ] 保持 `packages/core/src/prompt/skills/afs/index.ts` 简洁： - - `getAFSSkills` 不需要检查 driver - - 各 skill 内部自动根据 afs.drivers 动态处理 +1. [x] 扩展 `packages/core/src/prompt/skills/afs/read.ts`： + - ✅ 使用 `extendSchemaWithView()` 动态扩展 schema + - ✅ 增加 `view` 字段到 `AFSReadInput`（根据 drivers 自动添加） + - ✅ 增加 `wait` 字段（可选，默认使用 AFS Core 的默认值 "strict"） + - ✅ 增加 `viewStatus` 字段到 `AFSReadOutput`（指示是否 fallback） + - ✅ 更新 description（根据 driver 可用性动态生成） + +2. [x] AFS Core 扩展（支持 viewStatus 返回）： + - ✅ `type.ts`: 新增 `ViewStatus` 接口和 `AFSReadResult` 类型 + - ✅ `view-processor.ts`: `handleRead` 返回增加 `viewStatus` + - ✅ `afs.ts`: `read` 方法返回类型改为 `AFSReadResult` + +3. [x] **不实现** list/stat 的 view 支持（V1 简化）： + - ⚠️ `list.ts` 暂不支持 view（未来可扩展） + - ⚠️ `stat.ts` 不存在，暂不实现 + +4. [x] **不修改** `packages/core/src/prompt/skills/afs/write.ts`： + - ✅ write 不支持 view 参数 + - ✅ 只能写入 source，view 由 driver 自动生成 + +5. [x] 保持 `packages/core/src/prompt/skills/afs/index.ts` 简洁： + - ✅ `getAFSSkills` 不需要检查 driver + - ✅ 各 skill 内部自动根据 afs.drivers 动态处理 **输出：** -- `@aigne/core` 支持 view 的 AFS skills +- ✅ `@aigne/core` 支持 view 的 AFS skills +- ✅ TypeScript 编译通过 +- ✅ Biome lint 检查通过 +- ✅ 所有测试通过 + +**实现亮点：** +1. **动态 Schema 构建** - 根据 drivers 配置自动添加 view/wait 字段到 inputSchema +2. **viewStatus 反馈** - 输出中包含 `viewStatus.fallback` 指示是否降级到 source +3. **向后兼容** - 无 driver 时行为与之前完全一致 +4. **辅助函数** - `getDriversFromAfsConfig()` 处理各种 AFS 配置类型 **依赖：** - 使用 Phase 1 创建的 `afs/core/src/view-schema.ts` 公共模块 @@ -1845,14 +1856,14 @@ function computeRevision(entry: AFSEntry): string { ## 附录：关键文件清单 ### AFS Core -- `afs/core/src/type.ts` - 类型定义（View, AFSDriver, ReadOptions） -- `afs/core/src/afs.ts` - AFS 主类，driver 匹配与调度，metadata 集成 -- `afs/core/src/view-schema.ts` - **新增** - 动态 View Schema 构建（buildViewSchema, extendSchemaWithView） +- `afs/core/src/type.ts` - 类型定义（View, AFSDriver, ReadOptions, **ViewStatus**, **AFSReadResult**） +- `afs/core/src/afs.ts` - AFS 主类，driver 匹配与调度，metadata 集成，**read 返回 AFSReadResult** +- `afs/core/src/view-schema.ts` - 动态 View Schema 构建（buildViewSchema, extendSchemaWithView） - `afs/core/src/metadata/type.ts` - Metadata 接口定义 - `afs/core/src/metadata/store.ts` - SQLiteMetadataStore 实现 - `afs/core/src/metadata/migrations/001-init.ts` - 数据库 schema - `afs/core/src/metadata/index.ts` - Metadata 模块导出 -- `afs/core/src/view-processor.ts` - View 处理流程，job 队列，过期检测 +- `afs/core/src/view-processor.ts` - View 处理流程，**handleRead 返回 viewStatus** ### i18n Driver - `afs/i18n-driver/src/index.ts` - I18nDriver 实现 @@ -1860,11 +1871,11 @@ function computeRevision(entry: AFSEntry): string { - `afs/i18n-driver/src/storage.ts` - 物理存储映射 ### Skills -- `packages/core/src/prompt/skills/afs/read.ts` - 使用 extendSchemaWithView 动态扩展 schema（read 支持 view） -- `packages/core/src/prompt/skills/afs/list.ts` - 使用 extendSchemaWithView 动态扩展 schema（list 可选支持 view） -- `packages/core/src/prompt/skills/afs/stat.ts` - 使用 extendSchemaWithView 动态扩展 schema（stat 可选支持 view） -- `packages/core/src/prompt/skills/afs/write.ts` - 无需修改（write 不支持 view） -- `packages/core/src/prompt/skills/afs/index.ts` - 保持简洁（getAFSSkills 不检查 driver） +- `packages/core/src/prompt/skills/afs/read.ts` - ✅ **已修改** - 动态 schema（view/wait）+ viewStatus 输出 +- `packages/core/src/prompt/skills/afs/list.ts` - ⚠️ 暂不支持 view（未来可扩展） +- `packages/core/src/prompt/skills/afs/stat.ts` - ⚠️ 不存在，暂不实现 +- `packages/core/src/prompt/skills/afs/write.ts` - ✅ 无需修改（write 不支持 view） +- `packages/core/src/prompt/skills/afs/index.ts` - ✅ 保持简洁（getAFSSkills 不检查 driver） ### Loader - `packages/core/src/loader/agent-yaml.ts` - YAML schema 扩展 diff --git a/packages/core/src/prompt/skills/afs/read.ts b/packages/core/src/prompt/skills/afs/read.ts index cc7c83c1e..c17dcada2 100644 --- a/packages/core/src/prompt/skills/afs/read.ts +++ b/packages/core/src/prompt/skills/afs/read.ts @@ -1,135 +1,159 @@ -import type { AFSEntry } from "@aigne/afs"; +import { + AFS, + type AFSDriver, + type AFSEntry, + type AFSOptions, + extendSchemaWithView, + type View, + type ViewStatus, + type WaitStrategy, +} from "@aigne/afs"; import { z } from "zod"; -import type { AgentInvokeOptions, AgentOptions, Message } from "../../../agents/agent.js"; -import { AFSSkillBase } from "./base.js"; - -const DEFAULT_LINE_LIMIT = 2000; -const MAX_LINE_LENGTH = 2000; +import { + Agent, + type AgentInvokeOptions, + type AgentOptions, + type Message, +} from "../../../agents/agent.js"; export interface AFSReadInput extends Message { path: string; - offset?: number; - limit?: number; + withLineNumbers?: boolean; + view?: View; + wait?: WaitStrategy; } export interface AFSReadOutput extends Message { status: string; tool: string; path: string; + withLineNumbers?: boolean; data?: AFSEntry; message?: string; - totalLines?: number; - returnedLines?: number; - truncated?: boolean; - offset?: number; + viewStatus?: ViewStatus; } export interface AFSReadAgentOptions extends AgentOptions { afs: NonNullable["afs"]>; } -export class AFSReadAgent extends AFSSkillBase { - constructor(options: AFSReadAgentOptions) { - super({ - name: "afs_read", - description: `Read file contents from the Agentic File System (AFS) -- Returns the content of a file at the specified AFS path -- By default reads up to ${DEFAULT_LINE_LIMIT} lines, use offset/limit for large files -- Lines longer than ${MAX_LINE_LENGTH} characters will be truncated - -Usage: -- The path must be an absolute AFS path starting with "/" (e.g., "/docs/readme.md") -- Use offset to start reading from a specific line (0-based) -- Use limit to control number of lines returned (default: ${DEFAULT_LINE_LIMIT}) -- Check truncated field to know if file was partially returned`, - ...options, - inputSchema: z.object({ - path: z - .string() - .describe( - "Absolute AFS path to the file to read (e.g., '/docs/readme.md'). Must start with '/'", - ), - offset: z - .number() - .int() - .min(0) - .optional() - .describe("Line number to start reading from (0-based, default: 0)"), - limit: z - .number() - .int() - .min(1) - .max(DEFAULT_LINE_LIMIT) - .optional() - .describe(`Maximum number of lines to read (default: ${DEFAULT_LINE_LIMIT})`), - }), - outputSchema: z.object({ - status: z.string(), - tool: z.string(), - path: z.string(), - data: z.custom().optional(), - message: z.string().optional(), - totalLines: z.number().optional(), - returnedLines: z.number().optional(), - truncated: z.boolean().optional(), - offset: z.number().optional(), - }), - }); +/** + * Extract drivers from AFS config + * options.afs can be: true | AFS | AFSOptions | ((afs: AFS) => AFS) + */ +function getDriversFromAfsConfig( + afsConfig: true | AFS | AFSOptions | ((afs: AFS) => AFS), +): AFSDriver[] { + if (afsConfig === true) { + return []; + } + if (afsConfig instanceof AFS) { + return afsConfig.drivers; + } + if (typeof afsConfig === "function") { + // Function config - we can't know drivers without calling it + // Return empty array as we can't determine drivers at construction time + return []; } + // AFSOptions - return drivers if present + return afsConfig.drivers || []; +} - async process(input: AFSReadInput, _options: AgentInvokeOptions): Promise { - if (!this.afs) throw new Error("AFS is not configured for this agent."); +export class AFSReadAgent extends Agent { + constructor(options: AFSReadAgentOptions) { + // Extract drivers from AFS config + const drivers = getDriversFromAfsConfig(options.afs); + const hasDrivers = drivers.length > 0; + + // Base schema + const baseSchema = z.object({ + path: z.string().describe("Absolute file path to read"), + withLineNumbers: z + .boolean() + .optional() + .describe("Include line numbers in output (required when planning to edit the file)"), + }); - const result = await this.afs.read(input.path); + // Dynamically extend schema based on registered drivers + let inputSchema: z.ZodObject = extendSchemaWithView(baseSchema, drivers); - if (!result.data?.content || typeof result.data.content !== "string") { - return { - status: "success", - tool: "afs_read", - path: input.path, - ...result, - }; + // Add wait option only when drivers are present + if (hasDrivers) { + inputSchema = inputSchema.extend({ + wait: z + .enum(["strict", "fallback"]) + .optional() + .describe( + "Wait strategy: 'strict' waits for view generation (default), 'fallback' returns source immediately", + ), + }); } - const offset = input.offset ?? 0; - const limit = input.limit ?? DEFAULT_LINE_LIMIT; + // Build output schema + const baseOutputSchema = z.object({ + status: z.string(), + tool: z.string(), + path: z.string(), + withLineNumbers: z.boolean().optional(), + result: z.custom().optional(), + message: z.string().optional(), + }); - const allLines = result.data.content.split("\n"); - const totalLines = allLines.length; + // Add viewStatus to output schema only when drivers are present + const outputSchema = hasDrivers + ? baseOutputSchema.extend({ + viewStatus: z + .object({ + fallback: z.boolean().optional(), + }) + .optional() + .describe( + "View status indicating whether the requested view was returned or fell back to source", + ), + }) + : baseOutputSchema; + + // Determine description based on driver availability + const description = hasDrivers + ? "Read file contents with optional view projection (e.g., translated version). Use when you need to review, analyze, or understand file content." + : "Read complete file contents. Use when you need to review, analyze, or understand file content before making changes."; - // Apply offset and limit - const selectedLines = allLines.slice(offset, offset + limit); + super({ + name: "afs_read", + description, + ...options, + // Use type assertion for dynamically built schema + inputSchema: inputSchema as unknown as z.ZodType, + outputSchema: outputSchema as unknown as z.ZodType, + }); + } - // Truncate long lines - const processedLines = selectedLines.map((line) => - line.length > MAX_LINE_LENGTH ? `${line.substring(0, MAX_LINE_LENGTH)}... [truncated]` : line, - ); + async process(input: AFSReadInput, _options: AgentInvokeOptions): Promise { + if (!this.afs) throw new Error("AFS is not configured for this agent."); - const returnedLines = processedLines.length; - const truncated = offset > 0 || offset + limit < totalLines; + const result = await this.afs.read(input.path, { + view: input.view, + wait: input.wait, + }); - const processedContent = processedLines.join("\n"); + let content = result.data?.content; - let message: string | undefined; - if (truncated) { - const startLine = offset + 1; - const endLine = offset + returnedLines; - message = `Showing lines ${startLine}-${endLine} of ${totalLines}. Use offset/limit to read more.`; + if (input.withLineNumbers && typeof content === "string") { + content = content + .split("\n") + .map((line, idx) => `${idx + 1}| ${line}`) + .join("\n"); } return { status: "success", tool: "afs_read", path: input.path, - totalLines, - returnedLines, - truncated, - offset, - message, + withLineNumbers: input.withLineNumbers, ...result, - data: { + data: result.data && { ...result.data, - content: processedContent, + content, }, }; } From 9b8c5a331fa9f65f3a3385d80cd4660c61c793f6 Mon Sep 17 00:00:00 2001 From: LBan Date: Tue, 16 Dec 2025 21:08:51 +0800 Subject: [PATCH 05/21] feat(i18n-driver): add context support for translation and enhance driver integration --- afs/i18n-driver/src/driver.ts | 28 +- afs/i18n-driver/test/i18n-driver.test.ts | 45 ++- packages/cli/package.json | 1 + packages/cli/src/utils/load-aigne.ts | 7 + packages/core/src/loader/agent-yaml.ts | 318 ++++++++++++++++---- packages/core/src/loader/index.ts | 190 +++++++----- packages/core/src/prompt/skills/afs/read.ts | 1 + pnpm-lock.yaml | 3 + 8 files changed, 437 insertions(+), 156 deletions(-) diff --git a/afs/i18n-driver/src/driver.ts b/afs/i18n-driver/src/driver.ts index 74239f7e1..e96b97496 100644 --- a/afs/i18n-driver/src/driver.ts +++ b/afs/i18n-driver/src/driver.ts @@ -11,9 +11,6 @@ import { getStoragePath } from "./storage.js"; * I18n Driver configuration options */ export interface I18nDriverOptions { - /** Context for invoking the translation agent */ - context: Context; - /** Default source language code (e.g., "zh") */ defaultSourceLanguage?: string; @@ -41,7 +38,7 @@ export class I18nDriver implements AFSDriver { private translationAgent: Agent; - constructor(private options: I18nDriverOptions) { + constructor(private options: I18nDriverOptions = {}) { // Use custom agent or create default this.translationAgent = options.translationAgent ?? createDefaultTranslationAgent(); } @@ -73,17 +70,26 @@ export class I18nDriver implements AFSDriver { options: { sourceEntry: AFSEntry; metadata: any; + context?: Context; }, ): Promise<{ result: AFSEntry; message?: string }> { const { language } = view; - const { sourceEntry } = options; + const { sourceEntry, context } = options; if (!language) { throw new Error("Language is required for translation"); } - // Translate content using context from constructor options - const translatedContent = await this.translate(sourceEntry.content as string, language); + if (!context) { + throw new Error("Context is required for translation. Pass context via read options."); + } + + // Translate content using context from process options + const translatedContent = await this.translate( + sourceEntry.content as string, + language, + context, + ); // Get storage path const storagePath = getStoragePath(path, language, this.options.storagePath); @@ -109,8 +115,12 @@ export class I18nDriver implements AFSDriver { /** * Translate content using the translation agent */ - private async translate(content: string, targetLanguage: string): Promise { - const { translatedContent } = await this.options.context + private async translate( + content: string, + targetLanguage: string, + context: Context, + ): Promise { + const { translatedContent } = await context .newContext({ reset: true }) .invoke(this.translationAgent, { content, diff --git a/afs/i18n-driver/test/i18n-driver.test.ts b/afs/i18n-driver/test/i18n-driver.test.ts index 918b8e2f8..391969c31 100644 --- a/afs/i18n-driver/test/i18n-driver.test.ts +++ b/afs/i18n-driver/test/i18n-driver.test.ts @@ -116,8 +116,7 @@ test("getStoragePath should support custom template", () => { }); test("I18nDriver.canHandle should return true for language-only views", () => { - const aigne = new AIGNE(); - const driver = new I18nDriver({ context: aigne.newContext() }); + const driver = new I18nDriver(); expect(driver.canHandle({ language: "en" })).toBe(true); expect(driver.canHandle({ language: "ja" })).toBe(true); @@ -127,9 +126,7 @@ test("I18nDriver.canHandle should return true for language-only views", () => { }); test("I18nDriver.canHandle should check supportedLanguages", () => { - const aigne = new AIGNE(); const driver = new I18nDriver({ - context: aigne.newContext(), supportedLanguages: ["en", "ja"], }); @@ -146,7 +143,6 @@ test("I18nDriver should translate content using context", async () => { const mockTranslator = createMockTranslationAgent(); const driver = new I18nDriver({ - context, defaultSourceLanguage: "zh", translationAgent: mockTranslator, }); @@ -158,7 +154,7 @@ test("I18nDriver should translate content using context", async () => { const source = await mockFS.read("/test.md"); assert(source.result, "source.result should be defined"); - // Process translation + // Process translation (context is passed via options) const result = await driver.process( mockFS, "/test.md", @@ -166,6 +162,7 @@ test("I18nDriver should translate content using context", async () => { { sourceEntry: source.result, metadata: {}, + context, }, ); @@ -182,7 +179,6 @@ test("I18nDriver should integrate with AFS", async () => { const mockTranslator = createMockTranslationAgent(); const driver = new I18nDriver({ - context, defaultSourceLanguage: "zh", translationAgent: mockTranslator, }); @@ -198,10 +194,11 @@ test("I18nDriver should integrate with AFS", async () => { content: "你好，世界！", }); - // Read English version (should trigger translation) + // Read English version (should trigger translation, context is passed via options) const enResult = await afs.read("/modules/mock-fs/doc.md", { view: { language: "en" }, wait: "strict", + context, }); expect(enResult.result?.content).toBe("Hello，World！"); @@ -211,6 +208,7 @@ test("I18nDriver should integrate with AFS", async () => { const jaResult = await afs.read("/modules/mock-fs/doc.md", { view: { language: "ja" }, wait: "strict", + context, }); expect(jaResult.result?.content).toBe("こんにちは，世界！"); @@ -228,7 +226,6 @@ test("I18nDriver should throw error if language is missing", async () => { const mockTranslator = createMockTranslationAgent(); const driver = new I18nDriver({ - context, translationAgent: mockTranslator, }); @@ -244,7 +241,37 @@ test("I18nDriver should throw error if language is missing", async () => { { sourceEntry: source.result, metadata: {}, + context, }, ), ).rejects.toThrow("Language is required for translation"); }); + +test("I18nDriver should throw error if context is missing", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockTranslator = createMockTranslationAgent(); + + const driver = new I18nDriver({ + translationAgent: mockTranslator, + }); + + await mockFS.write("/test.md", { content: "test content" }); + const source = await mockFS.read("/test.md"); + assert(source.result, "source.result should be defined"); + + await expect( + driver.process( + mockFS, + "/test.md", + { language: "en" }, + { + sourceEntry: source.result, + metadata: {}, + // no context provided + }, + ), + ).rejects.toThrow("Context is required for translation"); +}); diff --git a/packages/cli/package.json b/packages/cli/package.json index 2ff7cd799..ba7a82830 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -59,6 +59,7 @@ "dependencies": { "@aigne/afs": "workspace:^", "@aigne/afs-history": "workspace:^", + "@aigne/afs-i18n-driver": "workspace:^", "@aigne/afs-local-fs": "workspace:^", "@aigne/agent-library": "workspace:^", "@aigne/agentic-memory": "workspace:^", diff --git a/packages/cli/src/utils/load-aigne.ts b/packages/cli/src/utils/load-aigne.ts index 7aa038f0a..a9b5781a9 100644 --- a/packages/cli/src/utils/load-aigne.ts +++ b/packages/cli/src/utils/load-aigne.ts @@ -98,6 +98,13 @@ export async function loadAIGNE({ load: (options) => import("@aigne/afs-local-fs").then((m) => m.LocalFS.load(options)), }, ], + availableDrivers: [ + { + driver: "i18n", + create: (options) => + import("@aigne/afs-i18n-driver").then((m) => new m.I18nDriver(options)), + }, + ], }, metadata: { ...metadata, cliVersion: AIGNE_CLI_VERSION }, }); diff --git a/packages/core/src/loader/agent-yaml.ts b/packages/core/src/loader/agent-yaml.ts index e83ea131d..6b625330f 100644 --- a/packages/core/src/loader/agent-yaml.ts +++ b/packages/core/src/loader/agent-yaml.ts @@ -3,8 +3,12 @@ import { jsonSchemaToZod } from "@aigne/json-schema-to-zod"; import { nodejs } from "@aigne/platform-helpers/nodejs/index.js"; import { parse } from "yaml"; import { type ZodType, z } from "zod"; -import type { AgentHooks, TaskRenderMode } from "../agents/agent.js"; +import type { AgentClass, AgentHooks, FunctionAgentFn, TaskRenderMode } from "../agents/agent.js"; +import { AIAgentToolChoice } from "../agents/ai-agent.js"; +import { type Role, roleSchema } from "../agents/chat-model.js"; +import { ProcessMode, type ReflectionMode } from "../agents/team-agent.js"; import { tryOrThrow } from "../utils/type-utils.js"; +import { codeToFunctionAgentFn } from "./function-agent.js"; import type { LoadOptions } from "./index.js"; import { camelizeSchema, @@ -13,6 +17,7 @@ import { imageModelSchema, inputOutputSchema, optionalize, + preprocessSchema, } from "./schema.js"; export interface HooksSchema { @@ -38,30 +43,14 @@ export type AFSModuleSchema = options?: Record; }; -export interface AFSContextPresetSchema { - view?: string; - select?: { - agent: NestAgentSchema; - }; - per?: { - agent: NestAgentSchema; - }; - dedupe?: { - agent: NestAgentSchema; - }; -} - -export interface AFSContextSchema { - search?: { - presets?: Record; - }; - list?: { - presets?: Record; - }; -} +export type AFSDriverSchema = + | string + | { + driver: string; + options?: Record; + }; -export interface AgentSchema { - type: string; +export interface BaseAgentSchema { name?: string; description?: string; model?: z.infer; @@ -82,14 +71,77 @@ export interface AgentSchema { }; afs?: | boolean - | (Omit & { + | (Omit & { modules?: AFSModuleSchema[]; - context?: AFSContextSchema; + drivers?: AFSDriverSchema[]; }); shareAFS?: boolean; - [key: string]: unknown; } +export type Instructions = { role: Exclude; content: string; path: string }[]; + +export interface AIAgentSchema extends BaseAgentSchema { + type: "ai"; + instructions?: Instructions; + autoReorderSystemMessages?: boolean; + autoMergeSystemMessages?: boolean; + inputKey?: string; + inputFileKey?: string; + outputKey?: string; + outputFileKey?: string; + toolChoice?: AIAgentToolChoice; + toolCallsConcurrency?: number; + keepTextInToolUses?: boolean; +} + +export interface ImageAgentSchema extends BaseAgentSchema { + type: "image"; + instructions: Instructions; + inputFileKey?: string; +} + +export interface MCPAgentSchema extends BaseAgentSchema { + type: "mcp"; + url?: string; + command?: string; + args?: string[]; +} + +export interface TeamAgentSchema extends BaseAgentSchema { + type: "team"; + mode?: ProcessMode; + iterateOn?: string; + concurrency?: number; + iterateWithPreviousOutput?: boolean; + includeAllStepsOutput?: boolean; + reflection?: Omit & { reviewer: NestAgentSchema }; +} + +export interface TransformAgentSchema extends BaseAgentSchema { + type: "transform"; + jsonata: string; +} + +export interface FunctionAgentSchema extends BaseAgentSchema { + type: "function"; + process: FunctionAgentFn; +} + +export interface ThirdAgentSchema extends BaseAgentSchema { + agentClass?: AgentClass; + type: ""; // type is a non-empty string, here set to empty string to avoid type conflict + [key: string]: any; +} + +export type AgentSchema = + | AIAgentSchema + | ImageAgentSchema + | MCPAgentSchema + | TeamAgentSchema + | TransformAgentSchema + | FunctionAgentSchema + | ThirdAgentSchema; + export async function parseAgentFile( path: string, data: any, @@ -132,7 +184,57 @@ export async function loadAgentFromYamlFile(path: string, options: LoadOptions) return agent; } -export const getAgentSchema = ({ filepath }: { filepath: string; options?: LoadOptions }) => { +const instructionItemSchema = z.union([ + z.object({ + role: roleSchema.default("system"), + url: z.string(), + }), + z.object({ + role: roleSchema.default("system"), + content: z.string(), + }), +]); + +const parseInstructionItem = + ({ filepath }: { filepath: string }) => + async ({ role, ...v }: z.infer): Promise => { + if (role === "tool") + throw new Error(`'tool' role is not allowed in instruction item in agent file ${filepath}`); + + if ("content" in v && typeof v.content === "string") { + return { role, content: v.content, path: filepath }; + } + if ("url" in v && typeof v.url === "string") { + const url = nodejs.path.isAbsolute(v.url) + ? v.url + : nodejs.path.join(nodejs.path.dirname(filepath), v.url); + return nodejs.fs.readFile(url, "utf8").then((content) => ({ role, content, path: url })); + } + throw new Error( + `Invalid instruction item in agent file ${filepath}. Expected 'content' or 'url' property`, + ); + }; + +export const getInstructionsSchema = ({ filepath }: { filepath: string }) => + z + .union([z.string(), instructionItemSchema, z.array(instructionItemSchema)]) + .transform(async (v): Promise => { + if (typeof v === "string") return [{ role: "system", content: v, path: filepath }]; + + if (Array.isArray(v)) { + return Promise.all(v.map((item) => parseInstructionItem({ filepath })(item))); + } + + return [await parseInstructionItem({ filepath })(v)]; + }) as unknown as ZodType; + +export const getAgentSchema = ({ + filepath, + options, +}: { + filepath: string; + options?: LoadOptions; +}) => { const agentSchema: ZodType = z.lazy(() => { const nestAgentSchema: ZodType = z.lazy(() => z.union([ @@ -161,34 +263,7 @@ export const getAgentSchema = ({ filepath }: { filepath: string; options?: LoadO }), ); - const afsContextPresetsSchema = z.object({ - presets: optionalize( - z.record( - z.string(), - z.object({ - view: optionalize(z.string()), - select: optionalize( - z.object({ - agent: nestAgentSchema, - }), - ), - per: optionalize( - z.object({ - agent: nestAgentSchema, - }), - ), - dedupe: optionalize( - z.object({ - agent: nestAgentSchema, - }), - ), - }), - ), - ), - }); - const baseAgentSchema = z.object({ - type: z.string(), name: optionalize(z.string()), alias: optionalize(z.array(z.string())), description: optionalize(z.string()), @@ -198,11 +273,11 @@ export const getAgentSchema = ({ filepath }: { filepath: string; options?: LoadO taskRenderMode: optionalize(z.union([z.literal("hide"), z.literal("collapse")])), inputSchema: optionalize(inputOutputSchema({ path: filepath })).transform((v) => v ? jsonSchemaToZod(v) : undefined, - ) as unknown as ZodType, + ) as unknown as ZodType, defaultInput: optionalize(defaultInputSchema), outputSchema: optionalize(inputOutputSchema({ path: filepath })).transform((v) => v ? jsonSchemaToZod(v) : undefined, - ) as unknown as ZodType, + ) as unknown as ZodType, includeInputInOutput: optionalize(z.boolean()), hooks: optionalize(z.union([hooksSchema, z.array(hooksSchema)])), skills: optionalize(z.array(nestAgentSchema)), @@ -235,11 +310,18 @@ export const getAgentSchema = ({ filepath }: { filepath: string; options?: LoadO ]), ), ), - context: optionalize( - z.object({ - search: optionalize(afsContextPresetsSchema), - list: optionalize(afsContextPresetsSchema), - }), + drivers: optionalize( + z.array( + z.union([ + z.string(), + camelizeSchema( + z.object({ + driver: z.string(), + options: optionalize(z.record(z.any())), + }), + ), + ]), + ), ), }), ), @@ -248,7 +330,117 @@ export const getAgentSchema = ({ filepath }: { filepath: string; options?: LoadO shareAFS: optionalize(z.boolean()), }); - return camelizeSchema(baseAgentSchema.passthrough()); + const instructionsSchema = getInstructionsSchema({ filepath: filepath }); + + return camelizeSchema( + preprocessSchema( + async (json: unknown) => { + if ( + typeof json === "object" && + json && + "type" in json && + typeof json.type === "string" && + !["ai", "image", "mcp", "team", "transform", "function"].includes(json.type) + ) { + if (!options?.require) + throw new Error( + `Module loader is not provided to load agent type module ${json.type} from ${filepath}`, + ); + const Mod = await options.require(json.type, { parent: filepath }); + if (typeof Mod?.default?.prototype?.constructor !== "function") { + throw new Error( + `The agent type module ${json.type} does not export a default Agent class`, + ); + } + + Object.assign(json, { agentClass: Mod.default }); + } + + return json; + }, + z.union([ + z + .object({ + type: z.string() as z.ZodType<"">, + agentClass: z.custom( + (v) => typeof v?.prototype?.constructor === "function", + ), + }) + .extend(baseAgentSchema.shape) + .passthrough(), + z.discriminatedUnion("type", [ + z + .object({ + type: z.literal("ai"), + instructions: optionalize(instructionsSchema), + autoReorderSystemMessages: optionalize(z.boolean()), + autoMergeSystemMessages: optionalize(z.boolean()), + inputKey: optionalize(z.string()), + outputKey: optionalize(z.string()), + inputFileKey: optionalize(z.string()), + outputFileKey: optionalize(z.string()), + toolChoice: optionalize(z.nativeEnum(AIAgentToolChoice)), + toolCallsConcurrency: optionalize(z.number().int().min(0)), + keepTextInToolUses: optionalize(z.boolean()), + catchToolsError: optionalize(z.boolean()), + structuredStreamMode: optionalize(z.boolean()), + }) + .extend(baseAgentSchema.shape), + z + .object({ + type: z.literal("image"), + instructions: instructionsSchema, + inputFileKey: optionalize(z.string()), + }) + .extend(baseAgentSchema.shape), + z + .object({ + type: z.literal("mcp"), + url: optionalize(z.string()), + command: optionalize(z.string()), + args: optionalize(z.array(z.string())), + }) + .extend(baseAgentSchema.shape), + z + .object({ + type: z.literal("team"), + mode: optionalize(z.nativeEnum(ProcessMode)), + iterateOn: optionalize(z.string()), + concurrency: optionalize(z.number().int().min(1)), + iterateWithPreviousOutput: optionalize(z.boolean()), + includeAllStepsOutput: optionalize(z.boolean()), + reflection: camelizeSchema( + optionalize( + z.object({ + reviewer: nestAgentSchema, + isApproved: z.string(), + maxIterations: optionalize(z.number().int().min(1)), + returnLastOnMaxIterations: optionalize(z.boolean()), + customErrorMessage: optionalize(z.string()), + }), + ), + ), + }) + .extend(baseAgentSchema.shape), + z + .object({ + type: z.literal("transform"), + jsonata: z.string(), + }) + .extend(baseAgentSchema.shape), + z + .object({ + type: z.literal("function"), + process: z.preprocess( + (v) => (typeof v === "string" ? codeToFunctionAgentFn(v) : v), + z.custom(), + ) as ZodType, + }) + .extend(baseAgentSchema.shape), + ]), + ]), + ), + ); }); return agentSchema; diff --git a/packages/core/src/loader/index.ts b/packages/core/src/loader/index.ts index c799d07f8..8b811efeb 100644 --- a/packages/core/src/loader/index.ts +++ b/packages/core/src/loader/index.ts @@ -1,13 +1,20 @@ -import { AFS, type AFSContext, type AFSContextPreset, type AFSModule } from "@aigne/afs"; +import { AFS, type AFSDriver, type AFSModule } from "@aigne/afs"; import { nodejs } from "@aigne/platform-helpers/nodejs/index.js"; import { parse } from "yaml"; import { type ZodType, z } from "zod"; -import { Agent, type AgentHooks, type AgentOptions } from "../agents/agent.js"; +import { Agent, type AgentHooks, type AgentOptions, FunctionAgent } from "../agents/agent.js"; +import { AIAgent } from "../agents/ai-agent.js"; import type { ChatModel } from "../agents/chat-model.js"; +import { ImageAgent } from "../agents/image-agent.js"; import type { ImageModel } from "../agents/image-model.js"; +import { MCPAgent } from "../agents/mcp-agent.js"; +import { TeamAgent } from "../agents/team-agent.js"; +import { TransformAgent } from "../agents/transform-agent.js"; import type { AIGNEOptions } from "../aigne/aigne.js"; import type { AIGNECLIAgent } from "../aigne/type.js"; import type { MemoryAgent, MemoryAgentOptions } from "../memory/memory.js"; +import { PromptBuilder } from "../prompt/prompt-builder.js"; +import { ChatMessagesTemplate, parseChatMessages } from "../prompt/template.js"; import { isAgent } from "../utils/agent-utils.js"; import { flat, @@ -19,12 +26,11 @@ import { } from "../utils/type-utils.js"; import { loadAgentFromJsFile } from "./agent-js.js"; import { - type AFSContextPresetSchema, type HooksSchema, + type Instructions, loadAgentFromYamlFile, type NestAgentSchema, } from "./agent-yaml.js"; -import { builtinAgents } from "./agents.js"; import { camelizeSchema, chatModelSchema, imageModelSchema, optionalize } from "./schema.js"; const AIGNE_FILE_NAME = ["aigne.yaml", "aigne.yml"]; @@ -44,22 +50,18 @@ export interface LoadOptions { availableModules?: { module: string; alias?: string[]; - load: (options: { filepath: string; parsed?: object }) => PromiseOrValue; + create: (options?: Record) => PromiseOrValue; + }[]; + availableDrivers?: { + driver: string; + alias?: string[]; + create: (options?: Record) => PromiseOrValue; }[]; }; aigne?: z.infer; require?: (modulePath: string, options: { parent?: string }) => Promise; } -export interface AgentLoadOptions extends LoadOptions { - loadNestAgent: ( - path: string, - agent: NestAgentSchema, - options: LoadOptions, - agentOptions?: AgentOptions & Record, - ) => Promise; -} - export async function load(path: string, options: LoadOptions = {}): Promise { const { aigne, rootDir } = await loadAIGNEFile(path); @@ -223,45 +225,8 @@ export async function parseAgent( } else if (agent.afs === true) { afs = new AFS(); } else if (agent.afs) { - afs = new AFS({}); - - const loadAFSContextPresets = async ( - presets: Record, - ): Promise> => { - return Object.fromEntries( - await Promise.all( - Object.entries(presets).map>(async ([key, value]) => { - const [select, per, dedupe] = await Promise.all( - [value.select, value.per, value.dedupe].map(async (item) => { - if (!item?.agent) return undefined; - const agent = await loadNestAgent(path, item.agent, options, { afs }); - return { - invoke: (input: any, options: any) => - options.context.invoke(agent, input, { - ...options, - streaming: false, - }), - }; - }), - ); - - return [key, { ...value, select, per, dedupe }]; - }), - ), - ); - }; - - const context: AFSContext = { - search: { - presets: await loadAFSContextPresets(agent.afs.context?.search?.presets || {}), - }, - list: { - presets: await loadAFSContextPresets(agent.afs.context?.list?.presets || {}), - }, - }; - - afs.options.context = context; - + // Create modules + const modules: AFSModule[] = []; for (const m of agent.afs.modules || []) { const moduleName = typeof m === "string" ? m : m.module; @@ -270,13 +235,26 @@ export async function parseAgent( ); if (!mod) throw new Error(`AFS module not found: ${typeof m === "string" ? m : m.module}`); - const module = await mod.load({ - filepath: path, - parsed: typeof m === "string" ? {} : m.options, - }); + const module = await mod.create(typeof m === "string" ? {} : m.options); + modules.push(module); + } + + // Create drivers + const drivers: AFSDriver[] = []; + for (const d of agent.afs.drivers || []) { + const driverName = typeof d === "string" ? d : d.driver; + + const drv = options?.afs?.availableDrivers?.find( + (drv) => drv.driver === driverName || drv.alias?.includes(driverName), + ); + if (!drv) throw new Error(`AFS driver not found: ${typeof d === "string" ? d : d.driver}`); - afs.mount(module); + const driver = await drv.create(typeof d === "string" ? {} : d.options); + drivers.push(driver); } + + // Create AFS with modules and drivers + afs = new AFS({ modules, drivers }); } const skills = @@ -313,30 +291,79 @@ export async function parseAgent( afs: afs || agentOptions?.afs, }; - let agentClass = builtinAgents[agent.type]; + let instructions: PromptBuilder | undefined; + if ("instructions" in agent && agent.instructions && ["ai", "image"].includes(agent.type)) { + instructions = instructionsToPromptBuilder(agent.instructions); + } - if (!agentClass) { - if (!options?.require) - throw new Error( - `Module loader is not provided to load agent type module ${agent.type} from ${path}`, - ); - const Mod = await options.require(agent.type, { parent: path }); - if (typeof Mod?.default?.prototype?.constructor !== "function") { - throw new Error(`The agent type module ${agent.type} does not export a default Agent class`); + switch (agent.type) { + case "ai": { + return AIAgent.from({ + ...baseOptions, + instructions, + }); } + case "image": { + if (!instructions) + throw new Error(`Missing required instructions for image agent at path: ${path}`); - agentClass = Mod.default; + return ImageAgent.from({ + ...baseOptions, + instructions, + }); + } + case "mcp": { + if (agent.url) { + return MCPAgent.from({ + ...baseOptions, + url: agent.url, + }); + } + if (agent.command) { + return MCPAgent.from({ + ...baseOptions, + command: agent.command, + args: agent.args, + }); + } + throw new Error(`Missing url or command in mcp agent: ${path}`); + } + case "team": { + return TeamAgent.from({ + ...baseOptions, + mode: agent.mode, + iterateOn: agent.iterateOn, + reflection: agent.reflection && { + ...agent.reflection, + reviewer: await loadNestAgent(path, agent.reflection.reviewer, options), + }, + }); + } + case "transform": { + return TransformAgent.from({ + ...baseOptions, + jsonata: agent.jsonata, + }); + } + case "function": { + return FunctionAgent.from({ + ...baseOptions, + process: agent.process, + }); + } } - if (!agentClass) { - throw new Error(`Unsupported agent type: ${agent.type} from ${path}`); + if ("agentClass" in agent && agent.agentClass) { + return await agent.agentClass.load({ + filepath: path, + parsed: baseOptions, + options, + }); } - return await agentClass.load({ - filepath: path, - parsed: baseOptions, - options: { ...options, loadNestAgent }, - }); + throw new Error( + `Unsupported agent type: ${"type" in agent ? agent.type : "unknown"} at path: ${path}`, + ); } async function loadMemory( @@ -437,3 +464,16 @@ export async function findAIGNEFile(path: string): Promise { `aigne.yaml not found in ${path}. Please ensure you are in the correct directory or provide a valid path.`, ); } + +export function instructionsToPromptBuilder(instructions: Instructions) { + return new PromptBuilder({ + instructions: ChatMessagesTemplate.from( + parseChatMessages( + instructions.map((i) => ({ + ...i, + options: { workingDir: nodejs.path.dirname(i.path) }, + })), + ), + ), + }); +} diff --git a/packages/core/src/prompt/skills/afs/read.ts b/packages/core/src/prompt/skills/afs/read.ts index c17dcada2..9da27778b 100644 --- a/packages/core/src/prompt/skills/afs/read.ts +++ b/packages/core/src/prompt/skills/afs/read.ts @@ -134,6 +134,7 @@ export class AFSReadAgent extends Agent { const result = await this.afs.read(input.path, { view: input.view, wait: input.wait, + context: _options.context, }); let content = result.data?.content; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bc7cd63e6..784d9405a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1932,6 +1932,9 @@ importers: '@aigne/afs-history': specifier: workspace:^ version: link:../../afs/history + '@aigne/afs-i18n-driver': + specifier: workspace:^ + version: link:../../afs/i18n-driver '@aigne/afs-local-fs': specifier: workspace:^ version: link:../../afs/local-fs From 4657e50d7352bf612c77aef50e9d3ca0fc9c09c0 Mon Sep 17 00:00:00 2001 From: LBan Date: Wed, 17 Dec 2025 09:53:55 +0800 Subject: [PATCH 06/21] feat(afs): add test for loading AIAgent with AFS drivers and modules from YAML --- .../core/test-agents/test-afs-driver.yaml | 13 +++++ packages/core/test/loader/agent-yaml.test.ts | 49 +++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 packages/core/test-agents/test-afs-driver.yaml diff --git a/packages/core/test-agents/test-afs-driver.yaml b/packages/core/test-agents/test-afs-driver.yaml new file mode 100644 index 000000000..d7ba8273a --- /dev/null +++ b/packages/core/test-agents/test-afs-driver.yaml @@ -0,0 +1,13 @@ +type: ai +afs: + modules: + - module: local-fs + options: + local_path: ./test-docs + drivers: + - driver: i18n + options: + default_source_language: zh + supported_languages: + - en + - ja diff --git a/packages/core/test/loader/agent-yaml.test.ts b/packages/core/test/loader/agent-yaml.test.ts index fdc6f3973..5be62841d 100644 --- a/packages/core/test/loader/agent-yaml.test.ts +++ b/packages/core/test/loader/agent-yaml.test.ts @@ -341,6 +341,55 @@ test("loadAgentFromYaml should load AIAgent with AFS correctly", async () => { expect(agent.afs).toBeInstanceOf(AFS); }); +test("loadAgentFromYaml should load AIAgent with AFS drivers correctly", async () => { + // Track options passed to create functions + let moduleOptions: any; + let driverOptions: any; + + // Mock module for testing + const createMockModule = (options: any) => { + moduleOptions = options; + return { + name: "local-fs", + description: "Mock local-fs module", + }; + }; + + // Mock driver for testing + const createMockDriver = (options: any) => { + driverOptions = options; + return { + name: "i18n", + description: "Mock i18n driver", + capabilities: { dimensions: ["language" as const] }, + canHandle: (view: { language?: string }) => !!view.language, + process: async () => ({ result: { id: "test", path: "/test", content: "translated" } }), + }; + }; + + const agent = await loadAgent( + join(import.meta.dirname, "../../test-agents/test-afs-driver.yaml"), + { + afs: { + availableModules: [{ module: "local-fs", create: createMockModule }], + availableDrivers: [{ driver: "i18n", create: createMockDriver }], + }, + }, + ); + + assert(agent instanceof AIAgent, "agent should be an instance of AIAgent"); + expect(agent.afs).toBeInstanceOf(AFS); + expect(agent.afs?.drivers).toHaveLength(1); + expect(agent.afs?.drivers[0]?.name).toBe("i18n"); + + // Verify options are passed as-is from YAML (not camelized) + expect(moduleOptions).toEqual({ local_path: "./test-docs" }); + expect(driverOptions).toEqual({ + default_source_language: "zh", + supported_languages: ["en", "ja"], + }); +}); + test("loadAgentFromYaml should inline function correctly", async () => { const agent = await loadAgent( join(import.meta.dirname, "../../test-agents/test-inline-function.yaml"), From 942203ce02e65bf85939f33c5926479be1a33bd3 Mon Sep 17 00:00:00 2001 From: LBan Date: Wed, 17 Dec 2025 14:54:10 +0800 Subject: [PATCH 07/21] fix: polish code --- afs/core/src/afs.ts | 2 +- afs/i18n-driver/src/default-translation-agent.ts | 8 ++++++++ afs/i18n-driver/src/driver.ts | 12 +++++------- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/afs/core/src/afs.ts b/afs/core/src/afs.ts index f755f7e53..ce752e7f5 100644 --- a/afs/core/src/afs.ts +++ b/afs/core/src/afs.ts @@ -65,7 +65,7 @@ export class AFS extends Emitter implements AFSRoot { // Initialize metadata store and view processor if drivers are present if (this._drivers.length > 0) { - const metadataPath = options?.metadataPath || "file:.afs/metadata.db"; + const metadataPath = options?.metadataPath || "file:./.afs/metadata.db"; this.metadataStore = new SQLiteMetadataStore({ url: metadataPath }); this.viewProcessor = new ViewProcessor(this.metadataStore, this._drivers); diff --git a/afs/i18n-driver/src/default-translation-agent.ts b/afs/i18n-driver/src/default-translation-agent.ts index 61515c8d6..9b4852316 100644 --- a/afs/i18n-driver/src/default-translation-agent.ts +++ b/afs/i18n-driver/src/default-translation-agent.ts @@ -29,6 +29,14 @@ Translate the provided content from the source language to the target language. Maintain the original formatting, structure, and technical terms. Preserve markdown syntax, code blocks, and special formatting. +Original content: +Source language: {{ sourceLanguage }} + +{{ content }} + + +Target language: {{ targetLanguage }} + Requirements: - Translate naturally and fluently - Keep technical terms and proper nouns when appropriate diff --git a/afs/i18n-driver/src/driver.ts b/afs/i18n-driver/src/driver.ts index e96b97496..068ab121f 100644 --- a/afs/i18n-driver/src/driver.ts +++ b/afs/i18n-driver/src/driver.ts @@ -120,13 +120,11 @@ export class I18nDriver implements AFSDriver { targetLanguage: string, context: Context, ): Promise { - const { translatedContent } = await context - .newContext({ reset: true }) - .invoke(this.translationAgent, { - content, - targetLanguage, - sourceLanguage: this.options.defaultSourceLanguage, - }); + const { translatedContent } = await context.invoke(this.translationAgent, { + content, + targetLanguage, + sourceLanguage: this.options.defaultSourceLanguage, + }); return translatedContent; } From 760c539c6d9f0140ecdc13c90a07cdc0f1998a75 Mon Sep 17 00:00:00 2001 From: LBan Date: Wed, 17 Dec 2025 15:27:11 +0800 Subject: [PATCH 08/21] feat(afs): introduce storage configuration for AFS options and update related tests --- afs/core/src/afs.ts | 7 +- afs/core/test/view-driver.test.ts | 10 +- afs/i18n-driver.md | 135 +++++++++++------- afs/i18n-driver/test/i18n-driver.test.ts | 2 +- packages/core/src/loader/agent-yaml.ts | 7 + packages/core/src/loader/index.ts | 8 +- .../core/test-agents/test-afs-storage.yaml | 15 ++ packages/core/test/loader/agent-yaml.test.ts | 34 +++++ 8 files changed, 154 insertions(+), 64 deletions(-) create mode 100644 packages/core/test-agents/test-afs-storage.yaml diff --git a/afs/core/src/afs.ts b/afs/core/src/afs.ts index ce752e7f5..bba416aa8 100644 --- a/afs/core/src/afs.ts +++ b/afs/core/src/afs.ts @@ -41,7 +41,9 @@ export interface AFSOptions { modules?: AFSModule[]; context?: AFSContext; drivers?: AFSDriver[]; - metadataPath?: string; // SQLite database path for metadata, default: ".afs/metadata.db" + storage?: { + url: string; // Storage path for AFS data, default: ".afs" + }; } export class AFS extends Emitter implements AFSRoot { @@ -65,7 +67,8 @@ export class AFS extends Emitter implements AFSRoot { // Initialize metadata store and view processor if drivers are present if (this._drivers.length > 0) { - const metadataPath = options?.metadataPath || "file:./.afs/metadata.db"; + const storageUrl = options?.storage?.url || ".afs"; + const metadataPath = `file:${storageUrl}/metadata.db`; this.metadataStore = new SQLiteMetadataStore({ url: metadataPath }); this.viewProcessor = new ViewProcessor(this.metadataStore, this._drivers); diff --git a/afs/core/test/view-driver.test.ts b/afs/core/test/view-driver.test.ts index 4d87067ba..fe226f1c4 100644 --- a/afs/core/test/view-driver.test.ts +++ b/afs/core/test/view-driver.test.ts @@ -132,7 +132,7 @@ test("View driver: should translate content with view", async () => { const afs = new AFS({ modules: [mockFS], drivers: [mockDriver], - metadataPath: `file:${testDbPath}/metadata.db`, + storage: { url: testDbPath }, }); // Write source content (Chinese) @@ -178,7 +178,7 @@ test("View driver: should use cached view on second read", async () => { const afs = new AFS({ modules: [mockFS], drivers: [mockDriver], - metadataPath: `file:${testDbPath}/metadata-cache.db`, + storage: { url: `${testDbPath}/cache` }, }); await afs.write("/modules/mock-fs/cache-test.md", { @@ -217,7 +217,7 @@ test("View driver: should invalidate cache when source changes", async () => { const afs = new AFS({ modules: [mockFS], drivers: [mockDriver], - metadataPath: `file:${testDbPath}/metadata-invalidate.db`, + storage: { url: `${testDbPath}/invalidate` }, }); // Write initial content @@ -266,7 +266,7 @@ test("View driver: fallback mode should return source immediately", async () => const afs = new AFS({ modules: [mockFS], drivers: [slowDriver], - metadataPath: `file:${testDbPath}/metadata-fallback.db`, + storage: { url: `${testDbPath}/fallback` }, }); await afs.write("/modules/mock-fs/fallback-test.md", { @@ -309,7 +309,7 @@ test("View driver: prefetch should batch generate views", async () => { const afs = new AFS({ modules: [mockFS], drivers: [mockDriver], - metadataPath: `file:${testDbPath}/metadata-prefetch.db`, + storage: { url: `${testDbPath}/prefetch` }, }); // Create multiple files diff --git a/afs/i18n-driver.md b/afs/i18n-driver.md index e8d1260eb..319c8dc13 100644 --- a/afs/i18n-driver.md +++ b/afs/i18n-driver.md @@ -1460,66 +1460,93 @@ export async function getAFSSkills(afs: AFS): Promise { ## 九、实施计划 -### Phase 1: 基础架构（AFS Core） +### Phase 1: 基础架构（AFS Core） ✅ 已完成 **目标：** 在 `afs/core` 中建立 view 和 driver 机制 +**完成日期：** 2024-12-16 + **任务：** -1. [ ] 扩展 `afs/core/src/type.ts`： - - 定义 `View` 类型（V1 只包含 `language`） - - 定义 `AFSDriver` 接口 - - 扩展 `ReadOptions` 支持 view（`WriteOptions` 不需要修改） - -2. [ ] 扩展 `afs/core/src/afs.ts`： - - 增加 `drivers` 字段和注册机制 - - 实现 `read/stat/list` 支持 view 参数（`write` 不支持 view） - - 实现 driver 匹配逻辑 - - 实现 `prefetch` API - -3. [ ] 创建 `afs/core/src/metadata/`： - - `metadata/type.ts`: 定义 `SourceMetadata`, `ViewMetadata`, `MetadataStore` 接口 - - `metadata/store.ts`: 实现 `SQLiteMetadataStore` 类 - - `metadata/migrations/001-init.ts`: 创建 source_metadata 和 view_metadata 表 - - `metadata/index.ts`: 导出所有接口和实现 - -4. [ ] Metadata Store 核心功能： - - `getSourceMetadata()`: 查询 source metadata - - `setSourceMetadata()`: 更新/创建 source metadata - - `getViewMetadata()`: 查询 view metadata - - `setViewMetadata()`: 更新/创建 view metadata（支持部分更新） - - `markViewsAsStale()`: 批量标记 view 为 stale - - `listViewMetadata()`: 列出某个 path 的所有 view - - `cleanupOrphanedViewMetadata()`: 清理孤立的 view metadata - -5. [ ] 创建 `afs/core/src/view-processor.ts`（V1 简化版）： - - 实现 `processView()`: 异步处理 view 生成（无 job 去重） - - 实现 `isViewStale()`: 检查 view 是否过期 - - 实现 `computeRevision()`: 计算 sourceRevision - - 实现 `readViewResult()`: 从 storagePath 读取 view 结果 - - 实现 strict / fallback 策略 - - **V1 暂不实现：** job 去重、等待队列（留待 V2） - -6. [ ] 创建 `afs/core/src/view-schema.ts`： - - 实现 `buildViewSchema()`: 根据 drivers 动态构建 view schema - - 实现 `extendSchemaWithView()`: 为基础 schema 扩展 view 字段 - - 支持所有 view dimensions（language, format, policy, variant） - - **注意：** 这是 AFS Core 的基础功能，供 Skill 层使用 - -7. [ ] 扩展 `afs/core/src/afs.ts` 集成 Metadata： - - 在 constructor 中初始化 MetadataStore（数据库路径：`.afs/metadata.db`） - - 配置项：`metadataPath`（可选，默认 `.afs/metadata.db`） - - `read()` 中使用 metadata 判断 view 状态 - - `write()` 中更新 source metadata 并标记 view 为 stale - - `delete()` 中清理相关 metadata - -8. [ ] AFS 配置扩展： - - 增加 `AFSOptions.metadataPath` 配置项 - - 默认值：`.afs/metadata.db`（相对于 workspace 根目录） - - 支持自定义路径（绝对或相对路径） +1. [x] 扩展 `afs/core/src/type.ts`： + - ✅ 定义 `View` 类型（V1 实现 `language`，预留 format/policy/variant） + - ✅ 定义 `AFSDriver` 接口（canHandle, process, capabilities, onMount） + - ✅ 扩展 `ReadOptions` 支持 view 和 wait 策略 + - ✅ 定义 `WaitStrategy` 类型（strict/fallback） + +2. [x] 扩展 `afs/core/src/afs.ts`： + - ✅ 增加 `drivers` 字段和注册机制 + - ✅ 实现 `read` 支持 view 参数（`write` 不支持 view） + - ✅ 实现 driver 匹配逻辑 + - ✅ 实现 `prefetch` API（支持并发控制） + - ✅ Driver onMount 生命周期 + - ✅ Write/delete 自动更新 metadata + +3. [x] 创建 `afs/core/src/metadata/`： + - ✅ `metadata/type.ts`: 定义 `SourceMetadata`, `ViewMetadata`, `MetadataStore` 接口 + - ✅ `metadata/store.ts`: 使用 **drizzle-orm** 实现 `SQLiteMetadataStore` 类 + - ✅ `metadata/models/`: drizzle schema 定义（source-metadata, view-metadata） + - ✅ `metadata/migrations/001-init.ts`: 创建 source_metadata 和 view_metadata 表 + - ✅ `metadata/index.ts`: 导出所有接口和实现 + +4. [x] Metadata Store 核心功能： + - ✅ `getSourceMetadata()`: 查询 source metadata + - ✅ `setSourceMetadata()`: 更新/创建 source metadata（使用 drizzle update/insert） + - ✅ `getViewMetadata()`: 查询 view metadata + - ✅ `setViewMetadata()`: 更新/创建 view metadata（支持部分更新） + - ✅ `markViewsAsStale()`: 批量标记 view 为 stale + - ✅ `listViewMetadata()`: 列出某个 path 的所有 view + - ✅ `cleanupOrphanedViewMetadata()`: 清理孤立的 view metadata + - ✅ 使用 drizzle query builder 替代手写 SQL + +5. [x] 创建 `afs/core/src/view-processor.ts`（V1 简化版）： + - ✅ 实现 `processView()`: 异步处理 view 生成（无 job 去重） + - ✅ 实现 `isViewStale()`: 检查 view 是否过期 + - ✅ 实现 `computeRevision()`: 计算 sourceRevision（hash:sha256 / mtime:size） + - ✅ 实现 `readViewResult()`: 从 storagePath 读取 view 结果 + - ✅ 实现 strict / fallback 策略 + - ✅ 实现 `prefetch()`: 批量生成（使用 **p-limit** 控制并发） + - ⚠️ **V1 暂不实现：** job 去重、等待队列（留待 V2） + +6. [x] 创建 `afs/core/src/view-schema.ts`： + - ✅ 实现 `buildViewSchema()`: 根据 drivers 动态构建 view schema + - ✅ 实现 `extendSchemaWithView()`: 为基础 schema 扩展 view 字段 + - ✅ 支持所有 view dimensions（language, format, policy, variant） + - ✅ **注意：** 这是 AFS Core 的基础功能，供 Skill 层使用 + +7. [x] 扩展 `afs/core/src/afs.ts` 集成 Metadata： + - ✅ 在 constructor 中初始化 MetadataStore（数据库路径：`.afs/metadata.db`） + - ✅ 配置项：`storage.url`（可选，默认 `.afs`） + - ✅ `read()` 中使用 metadata 判断 view 状态 + - ✅ `write()` 中更新 source metadata 并标记 view 为 stale + - ✅ `delete()` 中清理相关 metadata + - ✅ Driver onMount 生命周期 + +8. [x] AFS 配置扩展： + - ✅ 增加 `AFSOptions.storage.url` 配置项 + - ✅ 默认值：`.afs`（相对于 workspace 根目录） + - ✅ metadata.db 存放在 `{storage.url}/metadata.db` + +9. [x] 单元测试（`afs/core/test/view-driver.test.ts`）： + - ✅ MockI18nDriver 翻译驱动 + - ✅ 多语言 view 读取测试 + - ✅ View 缓存机制测试 + - ✅ Source 变更后 view 失效测试 + - ✅ Fallback 模式测试 + - ✅ Prefetch 批量生成测试 + - ✅ **测试结果:** 5 个测试全部通过 ✅ **输出：** -- `@aigne/afs` v2.0.0（breaking change） -- 向后兼容：不传 view 时行为与当前一致 +- ✅ `@aigne/afs` Phase 1 完成 +- ✅ 向后兼容：不传 view 时行为与当前一致 +- ✅ TypeScript 编译通过 +- ✅ Biome lint 检查通过 +- ✅ 所有单元测试通过 + +**实现亮点：** +1. **Drizzle ORM 集成** - 使用 schema 定义和 query builder API，类型安全 +2. **并发控制优化** - 使用 `p-limit` 管理 prefetch 并发（可配置并发数） +3. **严格的类型检查** - 移除所有非空断言，通过所有 lint 检查 +4. **完整的测试覆盖** - 5 个核心场景，20 个断言 --- diff --git a/afs/i18n-driver/test/i18n-driver.test.ts b/afs/i18n-driver/test/i18n-driver.test.ts index 391969c31..34b54ce36 100644 --- a/afs/i18n-driver/test/i18n-driver.test.ts +++ b/afs/i18n-driver/test/i18n-driver.test.ts @@ -186,7 +186,7 @@ test("I18nDriver should integrate with AFS", async () => { const afs = new AFS({ modules: [mockFS], drivers: [driver], - metadataPath: "file:.afs-test/i18n-test.db", + storage: { url: ".afs-test/i18n-test" }, }); // Write source content (Chinese) diff --git a/packages/core/src/loader/agent-yaml.ts b/packages/core/src/loader/agent-yaml.ts index 6b625330f..463058526 100644 --- a/packages/core/src/loader/agent-yaml.ts +++ b/packages/core/src/loader/agent-yaml.ts @@ -323,6 +323,13 @@ export const getAgentSchema = ({ ]), ), ), + storage: optionalize( + camelizeSchema( + z.object({ + url: z.string(), + }), + ), + ), }), ), ]), diff --git a/packages/core/src/loader/index.ts b/packages/core/src/loader/index.ts index 8b811efeb..48ef02c3d 100644 --- a/packages/core/src/loader/index.ts +++ b/packages/core/src/loader/index.ts @@ -253,8 +253,12 @@ export async function parseAgent( drivers.push(driver); } - // Create AFS with modules and drivers - afs = new AFS({ modules, drivers }); + // Create AFS with modules, drivers, and storage options + afs = new AFS({ + modules, + drivers, + storage: agent.afs.storage, + }); } const skills = diff --git a/packages/core/test-agents/test-afs-storage.yaml b/packages/core/test-agents/test-afs-storage.yaml new file mode 100644 index 000000000..3f170f61d --- /dev/null +++ b/packages/core/test-agents/test-afs-storage.yaml @@ -0,0 +1,15 @@ +type: ai +afs: + storage: + url: .custom-afs + modules: + - module: local-fs + options: + local_path: ./test-docs + drivers: + - driver: i18n + options: + default_source_language: zh + supported_languages: + - en + - ja diff --git a/packages/core/test/loader/agent-yaml.test.ts b/packages/core/test/loader/agent-yaml.test.ts index 5be62841d..0a1e62106 100644 --- a/packages/core/test/loader/agent-yaml.test.ts +++ b/packages/core/test/loader/agent-yaml.test.ts @@ -390,6 +390,40 @@ test("loadAgentFromYaml should load AIAgent with AFS drivers correctly", async ( }); }); +test("loadAgentFromYaml should load AIAgent with AFS storage config correctly", async () => { + // Mock module for testing + const createMockModule = () => ({ + name: "local-fs", + description: "Mock local-fs module", + }); + + // Mock driver for testing + const createMockDriver = () => ({ + name: "i18n", + description: "Mock i18n driver", + capabilities: { dimensions: ["language" as const] }, + canHandle: (view: { language?: string }) => !!view.language, + process: async () => ({ result: { id: "test", path: "/test", content: "translated" } }), + }); + + const agent = await loadAgent( + join(import.meta.dirname, "../../test-agents/test-afs-storage.yaml"), + { + afs: { + availableModules: [{ module: "local-fs", create: createMockModule }], + availableDrivers: [{ driver: "i18n", create: createMockDriver }], + }, + }, + ); + + assert(agent instanceof AIAgent, "agent should be an instance of AIAgent"); + expect(agent.afs).toBeInstanceOf(AFS); + + // Verify that AFS was created with storage config + // The storage config should be passed through to AFS constructor + expect(agent.afs?.drivers).toHaveLength(1); +}); + test("loadAgentFromYaml should inline function correctly", async () => { const agent = await loadAgent( join(import.meta.dirname, "../../test-agents/test-inline-function.yaml"), From 0c63610b408b7ee923e2add14fdf24b17bc2a59a Mon Sep 17 00:00:00 2001 From: LBan Date: Wed, 17 Dec 2025 15:53:43 +0800 Subject: [PATCH 09/21] refactor(afs, metadata): update metadata handling to include module in source and view operations --- afs/core/src/metadata/migrations/001-init.ts | 11 +- .../src/metadata/models/source-metadata.ts | 24 ++-- afs/core/src/metadata/models/view-metadata.ts | 3 +- afs/core/src/metadata/store.ts | 108 ++++++++++++------ afs/core/src/metadata/type.ts | 31 +++-- 5 files changed, 116 insertions(+), 61 deletions(-) diff --git a/afs/core/src/metadata/migrations/001-init.ts b/afs/core/src/metadata/migrations/001-init.ts index 7b91fb434..362d288ca 100644 --- a/afs/core/src/metadata/migrations/001-init.ts +++ b/afs/core/src/metadata/migrations/001-init.ts @@ -10,16 +10,20 @@ export const init = { // Source metadata table sql` CREATE TABLE IF NOT EXISTS source_metadata ( - path TEXT PRIMARY KEY, + id INTEGER PRIMARY KEY AUTOINCREMENT, + module TEXT NOT NULL, + path TEXT NOT NULL, source_revision TEXT NOT NULL, updated_at INTEGER NOT NULL, drivers_hint TEXT, created_at INTEGER NOT NULL )`, + sql`CREATE UNIQUE INDEX IF NOT EXISTS unique_module_path ON source_metadata(module, path)`, // View metadata table sql` CREATE TABLE IF NOT EXISTS view_metadata ( id INTEGER PRIMARY KEY AUTOINCREMENT, + module TEXT NOT NULL, path TEXT NOT NULL, view TEXT NOT NULL, state TEXT NOT NULL, @@ -28,13 +32,12 @@ CREATE TABLE IF NOT EXISTS view_metadata ( error TEXT, storage_path TEXT, created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - UNIQUE(path, view) + updated_at INTEGER NOT NULL )`, // Indexes sql`CREATE INDEX IF NOT EXISTS idx_view_path ON view_metadata(path)`, sql`CREATE INDEX IF NOT EXISTS idx_view_state ON view_metadata(state)`, sql`CREATE INDEX IF NOT EXISTS idx_view_derived_from ON view_metadata(derived_from)`, - sql`CREATE UNIQUE INDEX IF NOT EXISTS unique_path_view ON view_metadata(path, view)`, + sql`CREATE UNIQUE INDEX IF NOT EXISTS unique_module_path_view ON view_metadata(module, path, view)`, ], }; diff --git a/afs/core/src/metadata/models/source-metadata.ts b/afs/core/src/metadata/models/source-metadata.ts index 1584bc6a1..db5f8ac29 100644 --- a/afs/core/src/metadata/models/source-metadata.ts +++ b/afs/core/src/metadata/models/source-metadata.ts @@ -1,15 +1,23 @@ -import { integer, sqliteTable, text } from "@aigne/sqlite"; +import { index, integer, sqliteTable, text } from "@aigne/sqlite"; /** * Source metadata table schema */ -export const sourceMetadataTable = sqliteTable("source_metadata", { - path: text("path").notNull().primaryKey(), - sourceRevision: text("source_revision").notNull(), - updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), - driversHint: text("drivers_hint"), // JSON array stored as text - createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(), -}); +export const sourceMetadataTable = sqliteTable( + "source_metadata", + { + id: integer("id").notNull().primaryKey({ autoIncrement: true }), + module: text("module").notNull(), + path: text("path").notNull(), + sourceRevision: text("source_revision").notNull(), + updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), + driversHint: text("drivers_hint"), // JSON array stored as text + createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(), + }, + (table) => ({ + uniqueModulePath: index("unique_module_path").on(table.module, table.path), + }), +); export type SourceMetadataRow = typeof sourceMetadataTable.$inferSelect; export type SourceMetadataInsert = typeof sourceMetadataTable.$inferInsert; diff --git a/afs/core/src/metadata/models/view-metadata.ts b/afs/core/src/metadata/models/view-metadata.ts index d67df84ab..0f7a2db24 100644 --- a/afs/core/src/metadata/models/view-metadata.ts +++ b/afs/core/src/metadata/models/view-metadata.ts @@ -7,6 +7,7 @@ export const viewMetadataTable = sqliteTable( "view_metadata", { id: integer("id").notNull().primaryKey({ autoIncrement: true }), + module: text("module").notNull(), path: text("path").notNull(), view: text("view").notNull(), // JSON stringified View object state: text("state").notNull(), // 'ready' | 'stale' | 'generating' | 'failed' @@ -21,7 +22,7 @@ export const viewMetadataTable = sqliteTable( pathIdx: index("idx_view_path").on(table.path), stateIdx: index("idx_view_state").on(table.state), derivedFromIdx: index("idx_view_derived_from").on(table.derivedFrom), - uniquePathView: index("unique_path_view").on(table.path, table.view), + uniqueModulePathView: index("unique_module_path_view").on(table.module, table.path, table.view), }), ); diff --git a/afs/core/src/metadata/store.ts b/afs/core/src/metadata/store.ts index 068b825be..e623d4040 100644 --- a/afs/core/src/metadata/store.ts +++ b/afs/core/src/metadata/store.ts @@ -25,17 +25,18 @@ export class SQLiteMetadataStore implements MetadataStore { } // Source metadata operations - async getSourceMetadata(path: string): Promise { + async getSourceMetadata(module: string, path: string): Promise { const db = await this.db; const rows = await db .select({ + module: this.sourceTable.module, path: this.sourceTable.path, sourceRevision: this.sourceTable.sourceRevision, updatedAt: this.sourceTable.updatedAt, driversHint: this.sourceTable.driversHint, }) .from(this.sourceTable) - .where(eq(this.sourceTable.path, path)) + .where(and(eq(this.sourceTable.module, module), eq(this.sourceTable.path, path))) .limit(1) .execute(); @@ -43,6 +44,7 @@ export class SQLiteMetadataStore implements MetadataStore { if (!row) return null; return { + module: row.module, path: row.path, sourceRevision: row.sourceRevision, updatedAt: row.updatedAt, @@ -50,7 +52,11 @@ export class SQLiteMetadataStore implements MetadataStore { }; } - async setSourceMetadata(path: string, metadata: Omit): Promise { + async setSourceMetadata( + module: string, + path: string, + metadata: Omit, + ): Promise { const db = await this.db; const now = new Date(); @@ -62,13 +68,9 @@ export class SQLiteMetadataStore implements MetadataStore { updatedAt: metadata.updatedAt, driversHint: metadata.driversHint ? JSON.stringify(metadata.driversHint) : null, }) - .where(eq(this.sourceTable.path, path)) + .where(and(eq(this.sourceTable.module, module), eq(this.sourceTable.path, path))) .returning({ - path: this.sourceTable.path, - sourceRevision: this.sourceTable.sourceRevision, - updatedAt: this.sourceTable.updatedAt, - driversHint: this.sourceTable.driversHint, - createdAt: this.sourceTable.createdAt, + id: this.sourceTable.id, }) .execute(); @@ -77,6 +79,7 @@ export class SQLiteMetadataStore implements MetadataStore { await db .insert(this.sourceTable) .values({ + module, path, sourceRevision: metadata.sourceRevision, updatedAt: metadata.updatedAt, @@ -87,18 +90,22 @@ export class SQLiteMetadataStore implements MetadataStore { } } - async deleteSourceMetadata(path: string): Promise { + async deleteSourceMetadata(module: string, path: string): Promise { const db = await this.db; - await db.delete(this.sourceTable).where(eq(this.sourceTable.path, path)).execute(); + await db + .delete(this.sourceTable) + .where(and(eq(this.sourceTable.module, module), eq(this.sourceTable.path, path))) + .execute(); } // View metadata operations - async getViewMetadata(path: string, view: View): Promise { + async getViewMetadata(module: string, path: string, view: View): Promise { const db = await this.db; const viewKey = JSON.stringify(view); const rows = await db .select({ + module: this.viewTable.module, path: this.viewTable.path, view: this.viewTable.view, state: this.viewTable.state, @@ -108,7 +115,13 @@ export class SQLiteMetadataStore implements MetadataStore { storagePath: this.viewTable.storagePath, }) .from(this.viewTable) - .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) + .where( + and( + eq(this.viewTable.module, module), + eq(this.viewTable.path, path), + eq(this.viewTable.view, viewKey), + ), + ) .limit(1) .execute(); @@ -116,6 +129,7 @@ export class SQLiteMetadataStore implements MetadataStore { if (!row) return null; return { + module: row.module, path: row.path, view: JSON.parse(row.view), state: row.state as ViewState, @@ -126,13 +140,18 @@ export class SQLiteMetadataStore implements MetadataStore { }; } - async setViewMetadata(path: string, view: View, metadata: Partial): Promise { + async setViewMetadata( + module: string, + path: string, + view: View, + metadata: Partial, + ): Promise { const db = await this.db; const viewKey = JSON.stringify(view); const now = new Date(); // Get existing record - const existing = await this.getViewMetadata(path, view); + const existing = await this.getViewMetadata(module, path, view); const merged = { state: metadata.state || existing?.state || "stale", @@ -153,18 +172,15 @@ export class SQLiteMetadataStore implements MetadataStore { storagePath: merged.storagePath, updatedAt: now, }) - .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) + .where( + and( + eq(this.viewTable.module, module), + eq(this.viewTable.path, path), + eq(this.viewTable.view, viewKey), + ), + ) .returning({ id: this.viewTable.id, - path: this.viewTable.path, - view: this.viewTable.view, - state: this.viewTable.state, - derivedFrom: this.viewTable.derivedFrom, - generatedAt: this.viewTable.generatedAt, - error: this.viewTable.error, - storagePath: this.viewTable.storagePath, - createdAt: this.viewTable.createdAt, - updatedAt: this.viewTable.updatedAt, }) .execute(); @@ -173,6 +189,7 @@ export class SQLiteMetadataStore implements MetadataStore { await db .insert(this.viewTable) .values({ + module, path, view: viewKey, state: merged.state, @@ -187,11 +204,12 @@ export class SQLiteMetadataStore implements MetadataStore { } } - async listViewMetadata(path: string): Promise { + async listViewMetadata(module: string, path: string): Promise { const db = await this.db; const rows = await db .select({ + module: this.viewTable.module, path: this.viewTable.path, view: this.viewTable.view, state: this.viewTable.state, @@ -201,10 +219,11 @@ export class SQLiteMetadataStore implements MetadataStore { storagePath: this.viewTable.storagePath, }) .from(this.viewTable) - .where(eq(this.viewTable.path, path)) + .where(and(eq(this.viewTable.module, module), eq(this.viewTable.path, path))) .execute(); return rows.map((row) => ({ + module: row.module, path: row.path, view: JSON.parse(row.view), state: row.state as ViewState, @@ -215,22 +234,31 @@ export class SQLiteMetadataStore implements MetadataStore { })); } - async deleteViewMetadata(path: string, view?: View): Promise { + async deleteViewMetadata(module: string, path: string, view?: View): Promise { const db = await this.db; if (view) { const viewKey = JSON.stringify(view); await db .delete(this.viewTable) - .where(and(eq(this.viewTable.path, path), eq(this.viewTable.view, viewKey))) + .where( + and( + eq(this.viewTable.module, module), + eq(this.viewTable.path, path), + eq(this.viewTable.view, viewKey), + ), + ) .execute(); } else { - await db.delete(this.viewTable).where(eq(this.viewTable.path, path)).execute(); + await db + .delete(this.viewTable) + .where(and(eq(this.viewTable.module, module), eq(this.viewTable.path, path))) + .execute(); } } // Batch operations - async markViewsAsStale(path: string): Promise { + async markViewsAsStale(module: string, path: string): Promise { const db = await this.db; const now = new Date(); @@ -240,7 +268,7 @@ export class SQLiteMetadataStore implements MetadataStore { state: "stale", updatedAt: now, }) - .where(eq(this.viewTable.path, path)) + .where(and(eq(this.viewTable.module, module), eq(this.viewTable.path, path))) .execute(); } @@ -249,6 +277,7 @@ export class SQLiteMetadataStore implements MetadataStore { const rows = await db .select({ + module: this.viewTable.module, path: this.viewTable.path, view: this.viewTable.view, state: this.viewTable.state, @@ -262,6 +291,7 @@ export class SQLiteMetadataStore implements MetadataStore { .execute(); return rows.map((row) => ({ + module: row.module, path: row.path, view: JSON.parse(row.view), state: row.state as ViewState, @@ -277,6 +307,7 @@ export class SQLiteMetadataStore implements MetadataStore { const rows = await db .select({ + module: this.viewTable.module, path: this.viewTable.path, view: this.viewTable.view, state: this.viewTable.state, @@ -290,6 +321,7 @@ export class SQLiteMetadataStore implements MetadataStore { .execute(); return rows.map((row) => ({ + module: row.module, path: row.path, view: JSON.parse(row.view), state: row.state as ViewState, @@ -304,17 +336,17 @@ export class SQLiteMetadataStore implements MetadataStore { async cleanupOrphanedViewMetadata(): Promise { const db = await this.db; - // Get all distinct paths from view_metadata + // Get all distinct (module, path) pairs from view_metadata const rows = await db - .selectDistinct({ path: this.viewTable.path }) + .selectDistinct({ module: this.viewTable.module, path: this.viewTable.path }) .from(this.viewTable) .execute(); - for (const { path } of rows) { - const sourceMeta = await this.getSourceMetadata(path); + for (const { module, path } of rows) { + const sourceMeta = await this.getSourceMetadata(module, path); if (!sourceMeta) { - // Source doesn't exist, delete all view metadata for this path - await this.deleteViewMetadata(path); + // Source doesn't exist, delete all view metadata for this module + path + await this.deleteViewMetadata(module, path); } } } diff --git a/afs/core/src/metadata/type.ts b/afs/core/src/metadata/type.ts index cf116a4fd..5eab12524 100644 --- a/afs/core/src/metadata/type.ts +++ b/afs/core/src/metadata/type.ts @@ -10,10 +10,11 @@ import type { View } from "../type.js"; export type ViewState = "ready" | "stale" | "generating" | "failed"; /** - * Source-level metadata (per path) + * Source-level metadata (per module + path) * Tracks the main content version */ export interface SourceMetadata { + module: string; path: string; sourceRevision: string; // Content hash or mtime:size identifier updatedAt: Date; @@ -21,10 +22,11 @@ export interface SourceMetadata { } /** - * View-level metadata (per path + view combination) + * View-level metadata (per module + path + view combination) * Tracks the state of each view projection */ export interface ViewMetadata { + module: string; path: string; view: View; state: ViewState; @@ -40,18 +42,27 @@ export interface ViewMetadata { */ export interface MetadataStore { // Source metadata operations - getSourceMetadata(path: string): Promise; - setSourceMetadata(path: string, metadata: Omit): Promise; - deleteSourceMetadata(path: string): Promise; + getSourceMetadata(module: string, path: string): Promise; + setSourceMetadata( + module: string, + path: string, + metadata: Omit, + ): Promise; + deleteSourceMetadata(module: string, path: string): Promise; // View metadata operations - getViewMetadata(path: string, view: View): Promise; - setViewMetadata(path: string, view: View, metadata: Partial): Promise; - listViewMetadata(path: string): Promise; - deleteViewMetadata(path: string, view?: View): Promise; + getViewMetadata(module: string, path: string, view: View): Promise; + setViewMetadata( + module: string, + path: string, + view: View, + metadata: Partial, + ): Promise; + listViewMetadata(module: string, path: string): Promise; + deleteViewMetadata(module: string, path: string, view?: View): Promise; // Batch operations - markViewsAsStale(path: string): Promise; + markViewsAsStale(module: string, path: string): Promise; listStaleViews(): Promise; listGeneratingViews(): Promise; From bb1e7c70c3c326335da2bc288630fbf4f332c38f Mon Sep 17 00:00:00 2001 From: LBan Date: Thu, 25 Dec 2025 19:55:59 +0800 Subject: [PATCH 10/21] fix: polish code --- afs/core/src/afs.ts | 8 ++--- afs/core/src/type.ts | 20 ++---------- afs/core/src/view-processor.ts | 59 +++++++++++++++++++--------------- pnpm-lock.yaml | 42 ++++++++++++++++-------- 4 files changed, 68 insertions(+), 61 deletions(-) diff --git a/afs/core/src/afs.ts b/afs/core/src/afs.ts index bba416aa8..e0bd398d4 100644 --- a/afs/core/src/afs.ts +++ b/afs/core/src/afs.ts @@ -28,8 +28,8 @@ import { type AFSWriteEntryPayload, type AFSWriteOptions, type AFSWriteResult, - type View, afsEntrySchema, + type View, } from "./type.js"; import { ViewProcessor } from "./view-processor.js"; @@ -174,7 +174,7 @@ export class AFS extends Emitter implements AFSRoot { for (const { module, modulePath, subpath } of modules) { // If view is requested and we have a view processor, use it if (options?.view && this.viewProcessor) { - const res = await this.viewProcessor.handleRead(module, subpath, options); + const res = await this.viewProcessor.handleRead(module, subpath, options, options.context); if (res?.data) { return { @@ -216,7 +216,7 @@ export class AFS extends Emitter implements AFSRoot { // Update metadata if view processor is available if (this.viewProcessor) { - await this.viewProcessor.handleWrite(module.subpath, res.data); + await this.viewProcessor.handleWrite(module.module.name, module.subpath, res.data); } return { @@ -236,7 +236,7 @@ export class AFS extends Emitter implements AFSRoot { // Clean up metadata if view processor is available if (this.viewProcessor) { - await this.viewProcessor.handleDelete(module.subpath); + await this.viewProcessor.handleDelete(module.module.name, module.subpath); } return result; diff --git a/afs/core/src/type.ts b/afs/core/src/type.ts index 29578c99d..f96204fc3 100644 --- a/afs/core/src/type.ts +++ b/afs/core/src/type.ts @@ -244,23 +244,6 @@ export interface AFSContext { }; } -/** - * View status in read result - * Indicates whether the requested view was returned or fell back to source - */ -export interface ViewStatus { - fallback?: boolean; // true = returned source content, view is being generated in background -} - -/** - * Read result with optional view status - */ -export interface AFSReadResult { - result?: AFSEntry; - message?: string; - viewStatus?: ViewStatus; -} - /** * AFSDriver interface for view transformation */ @@ -290,8 +273,9 @@ export interface AFSDriver { options: { sourceEntry: AFSEntry; metadata: any; + context: any; }, - ): Promise<{ result: AFSEntry; message?: string }>; + ): Promise<{ data: AFSEntry; message?: string }>; /** * Optional: Called when driver is mounted to AFS diff --git a/afs/core/src/view-processor.ts b/afs/core/src/view-processor.ts index bda49c4bd..bed498a7a 100644 --- a/afs/core/src/view-processor.ts +++ b/afs/core/src/view-processor.ts @@ -1,7 +1,14 @@ import { createHash } from "node:crypto"; import pLimit from "p-limit"; import type { MetadataStore, ViewMetadata } from "./metadata/index.js"; -import type { AFSDriver, AFSEntry, AFSModule, AFSReadOptions, AFSReadResult, View } from "./type.js"; +import type { + AFSDriver, + AFSEntry, + AFSModule, + AFSReadOptions, + AFSReadResult, + View, +} from "./type.js"; /** * View processor for handling view generation and caching @@ -51,8 +58,8 @@ export class ViewProcessor { /** * Check if a view is stale (outdated) */ - async isViewStale(path: string, view: View): Promise { - const viewMeta = await this.metadataStore.getViewMetadata(path, view); + async isViewStale(module: AFSModule, path: string, view: View): Promise { + const viewMeta = await this.metadataStore.getViewMetadata(module.name, path, view); if (!viewMeta) return true; // Missing view is stale if (viewMeta.state === "stale" || viewMeta.state === "failed") { @@ -64,7 +71,7 @@ export class ViewProcessor { } // Check if derivedFrom matches current sourceRevision - const sourceMeta = await this.metadataStore.getSourceMetadata(path); + const sourceMeta = await this.metadataStore.getSourceMetadata(module.name, path); if (!sourceMeta) return true; return viewMeta.derivedFrom !== sourceMeta.sourceRevision; @@ -74,10 +81,10 @@ export class ViewProcessor { * Process a view (generate or regenerate) * V1: Direct execution without job deduplication */ - async processView(module: AFSModule, path: string, view: View): Promise { + async processView(module: AFSModule, path: string, view: View, context: any): Promise { try { // 1. Get or create source metadata - let sourceMeta = await this.metadataStore.getSourceMetadata(path); + let sourceMeta = await this.metadataStore.getSourceMetadata(module.name, path); if (!sourceMeta) { // Read source to create metadata @@ -87,13 +94,13 @@ export class ViewProcessor { } const sourceRevision = this.computeRevision(sourceResult.data); - await this.metadataStore.setSourceMetadata(path, { + await this.metadataStore.setSourceMetadata(module.name, path, { sourceRevision, updatedAt: new Date(), driversHint: this.drivers.map((d) => d.name), }); - sourceMeta = await this.metadataStore.getSourceMetadata(path); + sourceMeta = await this.metadataStore.getSourceMetadata(module.name, path); } if (!sourceMeta) { @@ -101,7 +108,7 @@ export class ViewProcessor { } // 2. Mark as generating - await this.metadataStore.setViewMetadata(path, view, { + await this.metadataStore.setViewMetadata(module.name, path, view, { state: "generating", derivedFrom: sourceMeta.sourceRevision, }); @@ -125,17 +132,17 @@ export class ViewProcessor { }); // 5. Update to ready - await this.metadataStore.setViewMetadata(path, view, { + await this.metadataStore.setViewMetadata(module.name, path, view, { state: "ready", generatedAt: new Date(), - storagePath: result.result.metadata?.storagePath, + storagePath: result.data.metadata?.storagePath, error: undefined, }); - return result.result; + return result.data; } catch (error: any) { // 6. Mark as failed - await this.metadataStore.setViewMetadata(path, view, { + await this.metadataStore.setViewMetadata(module.name, path, view, { state: "failed", error: error.message, }); @@ -181,8 +188,8 @@ export class ViewProcessor { } // 1. Query view metadata - const viewMeta = await this.metadataStore.getViewMetadata(path, options.view); - const isStale = await this.isViewStale(path, options.view); + const viewMeta = await this.metadataStore.getViewMetadata(module.name, path, options.view); + const isStale = await this.isViewStale(module, path, options.view); // 2. If view is ready and not stale, return it if (viewMeta?.state === "ready" && !isStale) { @@ -215,14 +222,14 @@ export class ViewProcessor { /** * Update source metadata after write */ - async handleWrite(path: string, entry: AFSEntry): Promise { + async handleWrite(module: string, path: string, entry: AFSEntry): Promise { const newRevision = this.computeRevision(entry); // Get old metadata - const oldMeta = await this.metadataStore.getSourceMetadata(path); + const oldMeta = await this.metadataStore.getSourceMetadata(module, path); // Update source metadata - await this.metadataStore.setSourceMetadata(path, { + await this.metadataStore.setSourceMetadata(module, path, { sourceRevision: newRevision, updatedAt: new Date(), driversHint: this.drivers.map((d) => d.name), @@ -230,16 +237,16 @@ export class ViewProcessor { // If revision changed, mark all views as stale if (!oldMeta || oldMeta.sourceRevision !== newRevision) { - await this.metadataStore.markViewsAsStale(path); + await this.metadataStore.markViewsAsStale(module, path); } } /** * Clean up metadata after delete */ - async handleDelete(path: string): Promise { - await this.metadataStore.deleteViewMetadata(path); - await this.metadataStore.deleteSourceMetadata(path); + async handleDelete(module: string, path: string): Promise { + await this.metadataStore.deleteViewMetadata(module, path); + await this.metadataStore.deleteSourceMetadata(module, path); } /** @@ -249,14 +256,14 @@ export class ViewProcessor { module: AFSModule, paths: string[], view: View, - options?: { concurrency?: number }, + options?: { concurrency?: number; context?: any }, ): Promise { const tasksToGenerate: string[] = []; // Check which paths need generation for (const path of paths) { - const isStale = await this.isViewStale(path, view); - const viewMeta = await this.metadataStore.getViewMetadata(path, view); + const isStale = await this.isViewStale(module, path, view); + const viewMeta = await this.metadataStore.getViewMetadata(module.name, path, view); if (isStale || !viewMeta || viewMeta.state !== "ready") { tasksToGenerate.push(path); @@ -270,7 +277,7 @@ export class ViewProcessor { await Promise.all( tasksToGenerate.map((path) => limit(() => - this.processView(module, path, view).catch((error) => { + this.processView(module, path, view, options?.context).catch((error) => { console.error(`Prefetch failed for ${path}:`, error); }), ), diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 784d9405a..e5b58e5ce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,7 +84,7 @@ importers: specifier: ^1.6.1 version: 1.6.1 zod: - specifier: ^3.24.1 + specifier: ^3.25.67 version: 3.25.67 devDependencies: '@types/bun': @@ -167,6 +167,12 @@ importers: glob: specifier: ^11.0.3 version: 11.0.3 + ignore: + specifier: ^7.0.5 + version: 7.0.5 + minimatch: + specifier: ^10.1.1 + version: 10.1.1 zod: specifier: ^3.25.67 version: 3.25.67 @@ -1422,8 +1428,8 @@ importers: specifier: ^13.0.1 version: 13.0.1 openai: - specifier: ^6.5.0 - version: 6.6.0(ws@8.18.2)(zod@3.25.67) + specifier: ^6.14.0 + version: 6.15.0(ws@8.18.2)(zod@3.25.67) zod: specifier: ^3.25.67 version: 3.25.67 @@ -2050,8 +2056,8 @@ importers: specifier: ^10.2.0 version: 10.2.0 openai: - specifier: ^6.6.0 - version: 6.6.0(ws@8.18.2)(zod@3.25.67) + specifier: ^6.14.0 + version: 6.15.0(ws@8.18.2)(zod@3.25.67) p-wait-for: specifier: ^5.0.2 version: 5.0.2 @@ -2194,6 +2200,9 @@ importers: fast-deep-equal: specifier: ^3.1.3 version: 3.1.3 + front-matter: + specifier: ^4.0.2 + version: 4.0.2 immer: specifier: ^10.1.3 version: 10.1.3 @@ -7815,6 +7824,9 @@ packages: from2@2.3.0: resolution: {integrity: sha512-OMcX/4IC/uqEPVgGeyfN22LJk6AZrMkRZHxcHBMBvHScDGgwTm2GT2Wkgtocyd3JfZffjj2kYUDXXII0Fk9W0g==} + front-matter@4.0.2: + resolution: {integrity: sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==} + fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} @@ -9471,8 +9483,8 @@ packages: minimalistic-crypto-utils@1.0.1: resolution: {integrity: sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==} - minimatch@10.0.3: - resolution: {integrity: sha512-IPZ167aShDZZUMdRk66cyQAW3qr0WzbHkPdMYa8bzZhlHhO3jALbKdxcaak7W9FfT2rZNpQuUu4Od7ILEpXSaw==} + minimatch@10.1.1: + resolution: {integrity: sha512-enIvLvRAFZYXJzkCYG5RKmPfrFArdLv+R+lbQ53BmIMLIry74bjKzX6iHAm8WYamJkhSSEabrWN5D97XnKObjQ==} engines: {node: 20 || >=22} minimatch@3.1.2: @@ -9979,8 +9991,8 @@ packages: resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} engines: {node: '>=18'} - openai@6.6.0: - resolution: {integrity: sha512-1yWk4cBsHF5Bq9TreHYOHY7pbqdlT74COnm8vPx7WKn36StS+Hyk8DdAitnLaw67a5Cudkz5EmlFQjSrNnrA2w==} + openai@6.15.0: + resolution: {integrity: sha512-F1Lvs5BoVvmZtzkUEVyh8mDQPPFolq4F+xdsx/DO8Hee8YF3IGAlZqUIsF+DVGhqf4aU0a3bTghsxB6OIsRy1g==} hasBin: true peerDependencies: ws: ^8.18.0 @@ -18060,7 +18072,7 @@ snapshots: '@ts-morph/common@0.28.0': dependencies: - minimatch: 10.0.3 + minimatch: 10.1.1 path-browserify: 1.0.1 tinyglobby: 0.2.15 @@ -20815,6 +20827,10 @@ snapshots: inherits: 2.0.4 readable-stream: 2.3.8 + front-matter@4.0.2: + dependencies: + js-yaml: 3.14.1 + fs-constants@1.0.0: {} fs-extra@11.3.2: @@ -20997,7 +21013,7 @@ snapshots: dependencies: foreground-child: 3.3.1 jackspeak: 4.1.1 - minimatch: 10.0.3 + minimatch: 10.1.1 minipass: 7.1.2 package-json-from-dist: 1.0.1 path-scurry: 2.0.0 @@ -22887,7 +22903,7 @@ snapshots: minimalistic-crypto-utils@1.0.1: {} - minimatch@10.0.3: + minimatch@10.1.1: dependencies: '@isaacs/brace-expansion': 5.0.0 @@ -23396,7 +23412,7 @@ snapshots: is-inside-container: 1.0.0 wsl-utils: 0.1.0 - openai@6.6.0(ws@8.18.2)(zod@3.25.67): + openai@6.15.0(ws@8.18.2)(zod@3.25.67): optionalDependencies: ws: 8.18.2 zod: 3.25.67 From 15894fe9af7db3d07349db759c25eba15ceb9ee8 Mon Sep 17 00:00:00 2001 From: LBan Date: Thu, 25 Dec 2025 20:40:03 +0800 Subject: [PATCH 11/21] fix: polish code --- packages/core/src/loader/agent-yaml.ts | 57 +++++++++++++++++++++++++- packages/core/src/loader/index.ts | 50 +++++++++++++++++++++- 2 files changed, 105 insertions(+), 2 deletions(-) diff --git a/packages/core/src/loader/agent-yaml.ts b/packages/core/src/loader/agent-yaml.ts index 463058526..5aee867eb 100644 --- a/packages/core/src/loader/agent-yaml.ts +++ b/packages/core/src/loader/agent-yaml.ts @@ -50,6 +50,28 @@ export type AFSDriverSchema = options?: Record; }; +export interface AFSContextPresetSchema { + view?: string; + select?: { + agent: NestAgentSchema; + }; + per?: { + agent: NestAgentSchema; + }; + dedupe?: { + agent: NestAgentSchema; + }; +} + +export interface AFSContextSchema { + search?: { + presets?: Record; + }; + list?: { + presets?: Record; + }; +} + export interface BaseAgentSchema { name?: string; description?: string; @@ -71,9 +93,10 @@ export interface BaseAgentSchema { }; afs?: | boolean - | (Omit & { + | (Omit & { modules?: AFSModuleSchema[]; drivers?: AFSDriverSchema[]; + context?: AFSContextSchema; }); shareAFS?: boolean; } @@ -263,6 +286,32 @@ export const getAgentSchema = ({ }), ); + const afsContextPresetsSchema = z.object({ + presets: optionalize( + z.record( + z.string(), + z.object({ + view: optionalize(z.string()), + select: optionalize( + z.object({ + agent: nestAgentSchema, + }), + ), + per: optionalize( + z.object({ + agent: nestAgentSchema, + }), + ), + dedupe: optionalize( + z.object({ + agent: nestAgentSchema, + }), + ), + }), + ), + ), + }); + const baseAgentSchema = z.object({ name: optionalize(z.string()), alias: optionalize(z.array(z.string())), @@ -330,6 +379,12 @@ export const getAgentSchema = ({ }), ), ), + context: optionalize( + z.object({ + search: optionalize(afsContextPresetsSchema), + list: optionalize(afsContextPresetsSchema), + }), + ), }), ), ]), diff --git a/packages/core/src/loader/index.ts b/packages/core/src/loader/index.ts index 48ef02c3d..e949287d3 100644 --- a/packages/core/src/loader/index.ts +++ b/packages/core/src/loader/index.ts @@ -1,4 +1,4 @@ -import { AFS, type AFSDriver, type AFSModule } from "@aigne/afs"; +import { AFS, type AFSContext, type AFSContextPreset, type AFSDriver, type AFSModule } from "@aigne/afs"; import { nodejs } from "@aigne/platform-helpers/nodejs/index.js"; import { parse } from "yaml"; import { type ZodType, z } from "zod"; @@ -26,6 +26,7 @@ import { } from "../utils/type-utils.js"; import { loadAgentFromJsFile } from "./agent-js.js"; import { + type AFSContextPresetSchema, type HooksSchema, type Instructions, loadAgentFromYamlFile, @@ -62,6 +63,15 @@ export interface LoadOptions { require?: (modulePath: string, options: { parent?: string }) => Promise; } +export interface AgentLoadOptions extends LoadOptions { + loadNestAgent: ( + path: string, + agent: NestAgentSchema, + options: LoadOptions, + agentOptions?: AgentOptions & Record, + ) => Promise; +} + export async function load(path: string, options: LoadOptions = {}): Promise { const { aigne, rootDir } = await loadAIGNEFile(path); @@ -259,6 +269,44 @@ export async function parseAgent( drivers, storage: agent.afs.storage, }); + + // Load and configure AFSContext presets + const loadAFSContextPresets = async ( + presets: Record, + ): Promise> => { + return Object.fromEntries( + await Promise.all( + Object.entries(presets).map>(async ([key, value]) => { + const [select, per, dedupe] = await Promise.all( + [value.select, value.per, value.dedupe].map(async (item) => { + if (!item?.agent) return undefined; + const agent = await loadNestAgent(path, item.agent, options, { afs }); + return { + invoke: (input: any, options: any) => + options.context.invoke(agent, input, { + ...options, + streaming: false, + }), + }; + }), + ); + + return [key, { ...value, select, per, dedupe }]; + }), + ), + ); + }; + + const context: AFSContext = { + search: { + presets: await loadAFSContextPresets(agent.afs.context?.search?.presets || {}), + }, + list: { + presets: await loadAFSContextPresets(agent.afs.context?.list?.presets || {}), + }, + }; + + afs.options.context = context; } const skills = From 3e237d03f7e33ed996eda47cca766f6c9072dc34 Mon Sep 17 00:00:00 2001 From: LBan Date: Thu, 25 Dec 2025 20:49:16 +0800 Subject: [PATCH 12/21] fix: polish code --- packages/core/src/loader/agent-yaml.ts | 244 +------------------------ packages/core/src/loader/index.ts | 109 ++--------- 2 files changed, 29 insertions(+), 324 deletions(-) diff --git a/packages/core/src/loader/agent-yaml.ts b/packages/core/src/loader/agent-yaml.ts index 5aee867eb..674f28bf4 100644 --- a/packages/core/src/loader/agent-yaml.ts +++ b/packages/core/src/loader/agent-yaml.ts @@ -3,12 +3,8 @@ import { jsonSchemaToZod } from "@aigne/json-schema-to-zod"; import { nodejs } from "@aigne/platform-helpers/nodejs/index.js"; import { parse } from "yaml"; import { type ZodType, z } from "zod"; -import type { AgentClass, AgentHooks, FunctionAgentFn, TaskRenderMode } from "../agents/agent.js"; -import { AIAgentToolChoice } from "../agents/ai-agent.js"; -import { type Role, roleSchema } from "../agents/chat-model.js"; -import { ProcessMode, type ReflectionMode } from "../agents/team-agent.js"; +import type { AgentHooks, TaskRenderMode } from "../agents/agent.js"; import { tryOrThrow } from "../utils/type-utils.js"; -import { codeToFunctionAgentFn } from "./function-agent.js"; import type { LoadOptions } from "./index.js"; import { camelizeSchema, @@ -17,7 +13,6 @@ import { imageModelSchema, inputOutputSchema, optionalize, - preprocessSchema, } from "./schema.js"; export interface HooksSchema { @@ -72,7 +67,8 @@ export interface AFSContextSchema { }; } -export interface BaseAgentSchema { +export interface AgentSchema { + type: string; name?: string; description?: string; model?: z.infer; @@ -99,72 +95,9 @@ export interface BaseAgentSchema { context?: AFSContextSchema; }); shareAFS?: boolean; + [key: string]: unknown; } -export type Instructions = { role: Exclude; content: string; path: string }[]; - -export interface AIAgentSchema extends BaseAgentSchema { - type: "ai"; - instructions?: Instructions; - autoReorderSystemMessages?: boolean; - autoMergeSystemMessages?: boolean; - inputKey?: string; - inputFileKey?: string; - outputKey?: string; - outputFileKey?: string; - toolChoice?: AIAgentToolChoice; - toolCallsConcurrency?: number; - keepTextInToolUses?: boolean; -} - -export interface ImageAgentSchema extends BaseAgentSchema { - type: "image"; - instructions: Instructions; - inputFileKey?: string; -} - -export interface MCPAgentSchema extends BaseAgentSchema { - type: "mcp"; - url?: string; - command?: string; - args?: string[]; -} - -export interface TeamAgentSchema extends BaseAgentSchema { - type: "team"; - mode?: ProcessMode; - iterateOn?: string; - concurrency?: number; - iterateWithPreviousOutput?: boolean; - includeAllStepsOutput?: boolean; - reflection?: Omit & { reviewer: NestAgentSchema }; -} - -export interface TransformAgentSchema extends BaseAgentSchema { - type: "transform"; - jsonata: string; -} - -export interface FunctionAgentSchema extends BaseAgentSchema { - type: "function"; - process: FunctionAgentFn; -} - -export interface ThirdAgentSchema extends BaseAgentSchema { - agentClass?: AgentClass; - type: ""; // type is a non-empty string, here set to empty string to avoid type conflict - [key: string]: any; -} - -export type AgentSchema = - | AIAgentSchema - | ImageAgentSchema - | MCPAgentSchema - | TeamAgentSchema - | TransformAgentSchema - | FunctionAgentSchema - | ThirdAgentSchema; - export async function parseAgentFile( path: string, data: any, @@ -207,57 +140,7 @@ export async function loadAgentFromYamlFile(path: string, options: LoadOptions) return agent; } -const instructionItemSchema = z.union([ - z.object({ - role: roleSchema.default("system"), - url: z.string(), - }), - z.object({ - role: roleSchema.default("system"), - content: z.string(), - }), -]); - -const parseInstructionItem = - ({ filepath }: { filepath: string }) => - async ({ role, ...v }: z.infer): Promise => { - if (role === "tool") - throw new Error(`'tool' role is not allowed in instruction item in agent file ${filepath}`); - - if ("content" in v && typeof v.content === "string") { - return { role, content: v.content, path: filepath }; - } - if ("url" in v && typeof v.url === "string") { - const url = nodejs.path.isAbsolute(v.url) - ? v.url - : nodejs.path.join(nodejs.path.dirname(filepath), v.url); - return nodejs.fs.readFile(url, "utf8").then((content) => ({ role, content, path: url })); - } - throw new Error( - `Invalid instruction item in agent file ${filepath}. Expected 'content' or 'url' property`, - ); - }; - -export const getInstructionsSchema = ({ filepath }: { filepath: string }) => - z - .union([z.string(), instructionItemSchema, z.array(instructionItemSchema)]) - .transform(async (v): Promise => { - if (typeof v === "string") return [{ role: "system", content: v, path: filepath }]; - - if (Array.isArray(v)) { - return Promise.all(v.map((item) => parseInstructionItem({ filepath })(item))); - } - - return [await parseInstructionItem({ filepath })(v)]; - }) as unknown as ZodType; - -export const getAgentSchema = ({ - filepath, - options, -}: { - filepath: string; - options?: LoadOptions; -}) => { +export const getAgentSchema = ({ filepath }: { filepath: string; options?: LoadOptions }) => { const agentSchema: ZodType = z.lazy(() => { const nestAgentSchema: ZodType = z.lazy(() => z.union([ @@ -313,6 +196,7 @@ export const getAgentSchema = ({ }); const baseAgentSchema = z.object({ + type: z.string(), name: optionalize(z.string()), alias: optionalize(z.array(z.string())), description: optionalize(z.string()), @@ -322,11 +206,11 @@ export const getAgentSchema = ({ taskRenderMode: optionalize(z.union([z.literal("hide"), z.literal("collapse")])), inputSchema: optionalize(inputOutputSchema({ path: filepath })).transform((v) => v ? jsonSchemaToZod(v) : undefined, - ) as unknown as ZodType, + ) as unknown as ZodType, defaultInput: optionalize(defaultInputSchema), outputSchema: optionalize(inputOutputSchema({ path: filepath })).transform((v) => v ? jsonSchemaToZod(v) : undefined, - ) as unknown as ZodType, + ) as unknown as ZodType, includeInputInOutput: optionalize(z.boolean()), hooks: optionalize(z.union([hooksSchema, z.array(hooksSchema)])), skills: optionalize(z.array(nestAgentSchema)), @@ -392,117 +276,7 @@ export const getAgentSchema = ({ shareAFS: optionalize(z.boolean()), }); - const instructionsSchema = getInstructionsSchema({ filepath: filepath }); - - return camelizeSchema( - preprocessSchema( - async (json: unknown) => { - if ( - typeof json === "object" && - json && - "type" in json && - typeof json.type === "string" && - !["ai", "image", "mcp", "team", "transform", "function"].includes(json.type) - ) { - if (!options?.require) - throw new Error( - `Module loader is not provided to load agent type module ${json.type} from ${filepath}`, - ); - const Mod = await options.require(json.type, { parent: filepath }); - if (typeof Mod?.default?.prototype?.constructor !== "function") { - throw new Error( - `The agent type module ${json.type} does not export a default Agent class`, - ); - } - - Object.assign(json, { agentClass: Mod.default }); - } - - return json; - }, - z.union([ - z - .object({ - type: z.string() as z.ZodType<"">, - agentClass: z.custom( - (v) => typeof v?.prototype?.constructor === "function", - ), - }) - .extend(baseAgentSchema.shape) - .passthrough(), - z.discriminatedUnion("type", [ - z - .object({ - type: z.literal("ai"), - instructions: optionalize(instructionsSchema), - autoReorderSystemMessages: optionalize(z.boolean()), - autoMergeSystemMessages: optionalize(z.boolean()), - inputKey: optionalize(z.string()), - outputKey: optionalize(z.string()), - inputFileKey: optionalize(z.string()), - outputFileKey: optionalize(z.string()), - toolChoice: optionalize(z.nativeEnum(AIAgentToolChoice)), - toolCallsConcurrency: optionalize(z.number().int().min(0)), - keepTextInToolUses: optionalize(z.boolean()), - catchToolsError: optionalize(z.boolean()), - structuredStreamMode: optionalize(z.boolean()), - }) - .extend(baseAgentSchema.shape), - z - .object({ - type: z.literal("image"), - instructions: instructionsSchema, - inputFileKey: optionalize(z.string()), - }) - .extend(baseAgentSchema.shape), - z - .object({ - type: z.literal("mcp"), - url: optionalize(z.string()), - command: optionalize(z.string()), - args: optionalize(z.array(z.string())), - }) - .extend(baseAgentSchema.shape), - z - .object({ - type: z.literal("team"), - mode: optionalize(z.nativeEnum(ProcessMode)), - iterateOn: optionalize(z.string()), - concurrency: optionalize(z.number().int().min(1)), - iterateWithPreviousOutput: optionalize(z.boolean()), - includeAllStepsOutput: optionalize(z.boolean()), - reflection: camelizeSchema( - optionalize( - z.object({ - reviewer: nestAgentSchema, - isApproved: z.string(), - maxIterations: optionalize(z.number().int().min(1)), - returnLastOnMaxIterations: optionalize(z.boolean()), - customErrorMessage: optionalize(z.string()), - }), - ), - ), - }) - .extend(baseAgentSchema.shape), - z - .object({ - type: z.literal("transform"), - jsonata: z.string(), - }) - .extend(baseAgentSchema.shape), - z - .object({ - type: z.literal("function"), - process: z.preprocess( - (v) => (typeof v === "string" ? codeToFunctionAgentFn(v) : v), - z.custom(), - ) as ZodType, - }) - .extend(baseAgentSchema.shape), - ]), - ]), - ), - ); + return camelizeSchema(baseAgentSchema.passthrough()); }); return agentSchema; diff --git a/packages/core/src/loader/index.ts b/packages/core/src/loader/index.ts index e949287d3..86226bdcc 100644 --- a/packages/core/src/loader/index.ts +++ b/packages/core/src/loader/index.ts @@ -2,19 +2,12 @@ import { AFS, type AFSContext, type AFSContextPreset, type AFSDriver, type AFSMo import { nodejs } from "@aigne/platform-helpers/nodejs/index.js"; import { parse } from "yaml"; import { type ZodType, z } from "zod"; -import { Agent, type AgentHooks, type AgentOptions, FunctionAgent } from "../agents/agent.js"; -import { AIAgent } from "../agents/ai-agent.js"; +import { Agent, type AgentHooks, type AgentOptions } from "../agents/agent.js"; import type { ChatModel } from "../agents/chat-model.js"; -import { ImageAgent } from "../agents/image-agent.js"; import type { ImageModel } from "../agents/image-model.js"; -import { MCPAgent } from "../agents/mcp-agent.js"; -import { TeamAgent } from "../agents/team-agent.js"; -import { TransformAgent } from "../agents/transform-agent.js"; import type { AIGNEOptions } from "../aigne/aigne.js"; import type { AIGNECLIAgent } from "../aigne/type.js"; import type { MemoryAgent, MemoryAgentOptions } from "../memory/memory.js"; -import { PromptBuilder } from "../prompt/prompt-builder.js"; -import { ChatMessagesTemplate, parseChatMessages } from "../prompt/template.js"; import { isAgent } from "../utils/agent-utils.js"; import { flat, @@ -28,10 +21,10 @@ import { loadAgentFromJsFile } from "./agent-js.js"; import { type AFSContextPresetSchema, type HooksSchema, - type Instructions, loadAgentFromYamlFile, type NestAgentSchema, } from "./agent-yaml.js"; +import { builtinAgents } from "./agents.js"; import { camelizeSchema, chatModelSchema, imageModelSchema, optionalize } from "./schema.js"; const AIGNE_FILE_NAME = ["aigne.yaml", "aigne.yml"]; @@ -343,79 +336,30 @@ export async function parseAgent( afs: afs || agentOptions?.afs, }; - let instructions: PromptBuilder | undefined; - if ("instructions" in agent && agent.instructions && ["ai", "image"].includes(agent.type)) { - instructions = instructionsToPromptBuilder(agent.instructions); - } + let agentClass = builtinAgents[agent.type]; - switch (agent.type) { - case "ai": { - return AIAgent.from({ - ...baseOptions, - instructions, - }); - } - case "image": { - if (!instructions) - throw new Error(`Missing required instructions for image agent at path: ${path}`); - - return ImageAgent.from({ - ...baseOptions, - instructions, - }); - } - case "mcp": { - if (agent.url) { - return MCPAgent.from({ - ...baseOptions, - url: agent.url, - }); - } - if (agent.command) { - return MCPAgent.from({ - ...baseOptions, - command: agent.command, - args: agent.args, - }); - } - throw new Error(`Missing url or command in mcp agent: ${path}`); - } - case "team": { - return TeamAgent.from({ - ...baseOptions, - mode: agent.mode, - iterateOn: agent.iterateOn, - reflection: agent.reflection && { - ...agent.reflection, - reviewer: await loadNestAgent(path, agent.reflection.reviewer, options), - }, - }); - } - case "transform": { - return TransformAgent.from({ - ...baseOptions, - jsonata: agent.jsonata, - }); - } - case "function": { - return FunctionAgent.from({ - ...baseOptions, - process: agent.process, - }); + if (!agentClass) { + if (!options?.require) + throw new Error( + `Module loader is not provided to load agent type module ${agent.type} from ${path}`, + ); + const Mod = await options.require(agent.type, { parent: path }); + if (typeof Mod?.default?.prototype?.constructor !== "function") { + throw new Error(`The agent type module ${agent.type} does not export a default Agent class`); } + + agentClass = Mod.default; } - if ("agentClass" in agent && agent.agentClass) { - return await agent.agentClass.load({ - filepath: path, - parsed: baseOptions, - options, - }); + if (!agentClass) { + throw new Error(`Unsupported agent type: ${agent.type} from ${path}`); } - throw new Error( - `Unsupported agent type: ${"type" in agent ? agent.type : "unknown"} at path: ${path}`, - ); + return await agentClass.load({ + filepath: path, + parsed: baseOptions, + options: { ...options, loadNestAgent }, + }); } async function loadMemory( @@ -516,16 +460,3 @@ export async function findAIGNEFile(path: string): Promise { `aigne.yaml not found in ${path}. Please ensure you are in the correct directory or provide a valid path.`, ); } - -export function instructionsToPromptBuilder(instructions: Instructions) { - return new PromptBuilder({ - instructions: ChatMessagesTemplate.from( - parseChatMessages( - instructions.map((i) => ({ - ...i, - options: { workingDir: nodejs.path.dirname(i.path) }, - })), - ), - ), - }); -} From 089387b15a016ba8749e1de608423b18af2368b7 Mon Sep 17 00:00:00 2001 From: LBan Date: Thu, 25 Dec 2025 21:08:44 +0800 Subject: [PATCH 13/21] fix: polish code --- afs/i18n-driver/src/driver.ts | 26 ++++++++++++++++++++++++-- packages/cli/src/utils/load-aigne.ts | 4 ++-- packages/core/src/loader/index.ts | 22 +++++++++++++++++----- 3 files changed, 43 insertions(+), 9 deletions(-) diff --git a/afs/i18n-driver/src/driver.ts b/afs/i18n-driver/src/driver.ts index 068ab121f..66d530b7c 100644 --- a/afs/i18n-driver/src/driver.ts +++ b/afs/i18n-driver/src/driver.ts @@ -1,5 +1,7 @@ import type { AFSDriver, AFSEntry, AFSModule, View } from "@aigne/afs"; import type { Agent, Context } from "@aigne/core"; +import { optionalize } from "@aigne/core/loader/schema.js"; +import { z } from "zod"; import { createDefaultTranslationAgent, type TranslationInput, @@ -23,6 +25,16 @@ export interface I18nDriverOptions { /** Storage path template (default: ".i18n/{language}/") */ storagePath?: string; } +const i18nDriverOptionsSchema = z.object({ + defaultSourceLanguage: optionalize(z.string()), + supportedLanguages: optionalize(z.array(z.string())), + // translationAgent: optionalize( + // z.custom>( + // (value) => value instanceof Agent, + // ), + // ), + storagePath: optionalize(z.string()), +}); /** * I18n Driver for AFS @@ -38,6 +50,16 @@ export class I18nDriver implements AFSDriver { private translationAgent: Agent; + static schema() { + return i18nDriverOptionsSchema; + } + + static async load({ parsed }: { parsed?: object }) { + const valid = await I18nDriver.schema().passthrough().parseAsync(parsed); + + return new I18nDriver(valid); + } + constructor(private options: I18nDriverOptions = {}) { // Use custom agent or create default this.translationAgent = options.translationAgent ?? createDefaultTranslationAgent(); @@ -72,7 +94,7 @@ export class I18nDriver implements AFSDriver { metadata: any; context?: Context; }, - ): Promise<{ result: AFSEntry; message?: string }> { + ): Promise<{ data: AFSEntry; message?: string }> { const { language } = view; const { sourceEntry, context } = options; @@ -99,7 +121,7 @@ export class I18nDriver implements AFSDriver { // Return translated entry return { - result: { + data: { ...sourceEntry, content: translatedContent, path, diff --git a/packages/cli/src/utils/load-aigne.ts b/packages/cli/src/utils/load-aigne.ts index a9b5781a9..05d8a295c 100644 --- a/packages/cli/src/utils/load-aigne.ts +++ b/packages/cli/src/utils/load-aigne.ts @@ -101,8 +101,8 @@ export async function loadAIGNE({ availableDrivers: [ { driver: "i18n", - create: (options) => - import("@aigne/afs-i18n-driver").then((m) => new m.I18nDriver(options)), + load: (options) => + import("@aigne/afs-i18n-driver").then((m) => m.I18nDriver.load(options)), }, ], }, diff --git a/packages/core/src/loader/index.ts b/packages/core/src/loader/index.ts index 86226bdcc..1997381a2 100644 --- a/packages/core/src/loader/index.ts +++ b/packages/core/src/loader/index.ts @@ -1,4 +1,10 @@ -import { AFS, type AFSContext, type AFSContextPreset, type AFSDriver, type AFSModule } from "@aigne/afs"; +import { + AFS, + type AFSContext, + type AFSContextPreset, + type AFSDriver, + type AFSModule, +} from "@aigne/afs"; import { nodejs } from "@aigne/platform-helpers/nodejs/index.js"; import { parse } from "yaml"; import { type ZodType, z } from "zod"; @@ -44,12 +50,12 @@ export interface LoadOptions { availableModules?: { module: string; alias?: string[]; - create: (options?: Record) => PromiseOrValue; + load: (options: { filepath: string; parsed?: object }) => PromiseOrValue; }[]; availableDrivers?: { driver: string; alias?: string[]; - create: (options?: Record) => PromiseOrValue; + load: (options: { parsed?: object }) => PromiseOrValue; }[]; }; aigne?: z.infer; @@ -238,7 +244,11 @@ export async function parseAgent( ); if (!mod) throw new Error(`AFS module not found: ${typeof m === "string" ? m : m.module}`); - const module = await mod.create(typeof m === "string" ? {} : m.options); + const module = await mod.load({ + filepath: path, + parsed: typeof m === "string" ? {} : m.options, + }); + modules.push(module); } @@ -252,7 +262,9 @@ export async function parseAgent( ); if (!drv) throw new Error(`AFS driver not found: ${typeof d === "string" ? d : d.driver}`); - const driver = await drv.create(typeof d === "string" ? {} : d.options); + const driver = await drv.load({ + parsed: typeof d === "string" ? {} : d.options, + }); drivers.push(driver); } From 0918b7353b5cba277af94e6127ab623603e90e7e Mon Sep 17 00:00:00 2001 From: LBan Date: Thu, 25 Dec 2025 21:49:56 +0800 Subject: [PATCH 14/21] fix: polish code --- packages/core/src/prompt/skills/afs/read.ts | 226 ++++++++++---------- 1 file changed, 117 insertions(+), 109 deletions(-) diff --git a/packages/core/src/prompt/skills/afs/read.ts b/packages/core/src/prompt/skills/afs/read.ts index 9da27778b..35a856520 100644 --- a/packages/core/src/prompt/skills/afs/read.ts +++ b/packages/core/src/prompt/skills/afs/read.ts @@ -1,24 +1,15 @@ -import { - AFS, - type AFSDriver, - type AFSEntry, - type AFSOptions, - extendSchemaWithView, - type View, - type ViewStatus, - type WaitStrategy, -} from "@aigne/afs"; +import type { AFSEntry, View, ViewStatus, WaitStrategy } from "@aigne/afs"; import { z } from "zod"; -import { - Agent, - type AgentInvokeOptions, - type AgentOptions, - type Message, -} from "../../../agents/agent.js"; +import type { AgentInvokeOptions, AgentOptions, Message } from "../../../agents/agent.js"; +import { AFSSkillBase } from "./base.js"; + +const DEFAULT_LINE_LIMIT = 2000; +const MAX_LINE_LENGTH = 2000; export interface AFSReadInput extends Message { path: string; - withLineNumbers?: boolean; + offset?: number; + limit?: number; view?: View; wait?: WaitStrategy; } @@ -27,9 +18,12 @@ export interface AFSReadOutput extends Message { status: string; tool: string; path: string; - withLineNumbers?: boolean; data?: AFSEntry; message?: string; + totalLines?: number; + returnedLines?: number; + truncated?: boolean; + offset?: number; viewStatus?: ViewStatus; } @@ -37,94 +31,77 @@ export interface AFSReadAgentOptions extends AgentOptions["afs"]>; } -/** - * Extract drivers from AFS config - * options.afs can be: true | AFS | AFSOptions | ((afs: AFS) => AFS) - */ -function getDriversFromAfsConfig( - afsConfig: true | AFS | AFSOptions | ((afs: AFS) => AFS), -): AFSDriver[] { - if (afsConfig === true) { - return []; - } - if (afsConfig instanceof AFS) { - return afsConfig.drivers; - } - if (typeof afsConfig === "function") { - // Function config - we can't know drivers without calling it - // Return empty array as we can't determine drivers at construction time - return []; - } - // AFSOptions - return drivers if present - return afsConfig.drivers || []; -} - -export class AFSReadAgent extends Agent { +export class AFSReadAgent extends AFSSkillBase { constructor(options: AFSReadAgentOptions) { - // Extract drivers from AFS config - const drivers = getDriversFromAfsConfig(options.afs); - const hasDrivers = drivers.length > 0; - - // Base schema - const baseSchema = z.object({ - path: z.string().describe("Absolute file path to read"), - withLineNumbers: z - .boolean() - .optional() - .describe("Include line numbers in output (required when planning to edit the file)"), - }); - - // Dynamically extend schema based on registered drivers - let inputSchema: z.ZodObject = extendSchemaWithView(baseSchema, drivers); - - // Add wait option only when drivers are present - if (hasDrivers) { - inputSchema = inputSchema.extend({ + super({ + name: "afs_read", + description: `Read file contents from the Agentic File System (AFS) +- Returns the content of a file at the specified AFS path +- By default reads up to ${DEFAULT_LINE_LIMIT} lines, use offset/limit for large files +- Lines longer than ${MAX_LINE_LENGTH} characters will be truncated +- Supports optional view projection (e.g., translated version) when drivers are available + +Usage: +- The path must be an absolute AFS path starting with "/" (e.g., "/docs/readme.md") +- Use offset to start reading from a specific line (0-based) +- Use limit to control number of lines returned (default: ${DEFAULT_LINE_LIMIT}) +- Check truncated field to know if file was partially returned +- Use view parameter to request a specific projection (e.g., {language: "zh"} for Chinese translation) +- Use wait parameter to control view generation behavior ("strict" waits for view, "fallback" returns source immediately)`, + ...options, + inputSchema: z.object({ + path: z + .string() + .describe( + "Absolute AFS path to the file to read (e.g., '/docs/readme.md'). Must start with '/'", + ), + offset: z + .number() + .int() + .min(0) + .optional() + .describe("Line number to start reading from (0-based, default: 0)"), + limit: z + .number() + .int() + .min(1) + .max(DEFAULT_LINE_LIMIT) + .optional() + .describe(`Maximum number of lines to read (default: ${DEFAULT_LINE_LIMIT})`), + view: z + .object({ + language: z.string().optional(), + }) + .optional() + .describe( + "Optional view projection (e.g., {language: 'zh'} for Chinese translation). Requires configured drivers.", + ), wait: z .enum(["strict", "fallback"]) .optional() .describe( - "Wait strategy: 'strict' waits for view generation (default), 'fallback' returns source immediately", + "Wait strategy for view generation: 'strict' waits for completion (default), 'fallback' returns source immediately", ), - }); - } - - // Build output schema - const baseOutputSchema = z.object({ - status: z.string(), - tool: z.string(), - path: z.string(), - withLineNumbers: z.boolean().optional(), - result: z.custom().optional(), - message: z.string().optional(), - }); - - // Add viewStatus to output schema only when drivers are present - const outputSchema = hasDrivers - ? baseOutputSchema.extend({ - viewStatus: z - .object({ - fallback: z.boolean().optional(), - }) - .optional() - .describe( - "View status indicating whether the requested view was returned or fell back to source", - ), - }) - : baseOutputSchema; - - // Determine description based on driver availability - const description = hasDrivers - ? "Read file contents with optional view projection (e.g., translated version). Use when you need to review, analyze, or understand file content." - : "Read complete file contents. Use when you need to review, analyze, or understand file content before making changes."; - - super({ - name: "afs_read", - description, - ...options, - // Use type assertion for dynamically built schema - inputSchema: inputSchema as unknown as z.ZodType, - outputSchema: outputSchema as unknown as z.ZodType, + }), + outputSchema: z.object({ + status: z.string(), + tool: z.string(), + path: z.string(), + data: z.custom().optional(), + message: z.string().optional(), + totalLines: z.number().optional(), + returnedLines: z.number().optional(), + truncated: z.boolean().optional(), + offset: z.number().optional(), + viewStatus: z + .object({ + fallback: z.boolean().optional(), + }) + .optional() + .describe( + "View status indicating whether the requested view was returned or fell back to source", + ), + }), }); } @@ -137,24 +114,55 @@ export class AFSReadAgent extends Agent { context: _options.context, }); - let content = result.data?.content; + if (!result.data?.content || typeof result.data.content !== "string") { + return { + status: "success", + tool: "afs_read", + path: input.path, + ...result, + }; + } + + const offset = input.offset ?? 0; + const limit = input.limit ?? DEFAULT_LINE_LIMIT; + + const allLines = result.data.content.split("\n"); + const totalLines = allLines.length; + + // Apply offset and limit + const selectedLines = allLines.slice(offset, offset + limit); + + // Truncate long lines + const processedLines = selectedLines.map((line) => + line.length > MAX_LINE_LENGTH ? `${line.substring(0, MAX_LINE_LENGTH)}... [truncated]` : line, + ); + + const returnedLines = processedLines.length; + const truncated = offset > 0 || offset + limit < totalLines; + + const processedContent = processedLines.join("\n"); - if (input.withLineNumbers && typeof content === "string") { - content = content - .split("\n") - .map((line, idx) => `${idx + 1}| ${line}`) - .join("\n"); + let message: string | undefined; + if (truncated) { + const startLine = offset + 1; + const endLine = offset + returnedLines; + message = `Showing lines ${startLine}-${endLine} of ${totalLines}. Use offset/limit to read more.`; } return { status: "success", tool: "afs_read", path: input.path, - withLineNumbers: input.withLineNumbers, + totalLines, + returnedLines, + truncated, + offset, + message, + viewStatus: result.viewStatus, ...result, - data: result.data && { + data: { ...result.data, - content, + content: processedContent, }, }; } From cec1e607822f4ee9fb68efdae7ac22c577b2c58d Mon Sep 17 00:00:00 2001 From: LBan Date: Thu, 25 Dec 2025 22:05:42 +0800 Subject: [PATCH 15/21] fix: polish code --- afs/core/test/view-driver.test.ts | 42 ++++++++++---------- afs/i18n-driver/test/i18n-driver.test.ts | 36 ++++++++--------- packages/core/test/loader/agent-yaml.test.ts | 23 ++++++----- 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/afs/core/test/view-driver.test.ts b/afs/core/test/view-driver.test.ts index fe226f1c4..ec362751e 100644 --- a/afs/core/test/view-driver.test.ts +++ b/afs/core/test/view-driver.test.ts @@ -31,8 +31,8 @@ class MockI18nDriver implements AFSDriver { module: AFSModule, path: string, view: View, - options: { sourceEntry: AFSEntry; metadata: any }, - ): Promise<{ result: AFSEntry; message?: string }> { + options: { sourceEntry: AFSEntry; metadata: any; context: any }, + ): Promise<{ data: AFSEntry; message?: string }> { const { sourceEntry } = options; const targetLang = view.language; if (!targetLang) { @@ -56,7 +56,7 @@ class MockI18nDriver implements AFSDriver { await module.write?.(storagePath, { content: translated }); return { - result: { + data: { ...sourceEntry, content: translated, path, @@ -75,14 +75,14 @@ class MockFSModule implements AFSModule { readonly name = "mock-fs"; private files = new Map(); - async read(path: string): Promise<{ result?: AFSEntry; message?: string }> { + async read(path: string): Promise<{ data?: AFSEntry; message?: string }> { const content = this.files.get(path); if (!content) { - return { result: undefined, message: "File not found" }; + return { data: undefined, message: "File not found" }; } return { - result: { + data: { id: path, path, content, @@ -95,11 +95,11 @@ class MockFSModule implements AFSModule { async write( path: string, payload: { content: string }, - ): Promise<{ result: AFSEntry; message?: string }> { + ): Promise<{ data: AFSEntry; message?: string }> { this.files.set(path, payload.content); return { - result: { + data: { id: path, path, content: payload.content, @@ -146,8 +146,8 @@ test("View driver: should translate content with view", async () => { wait: "strict", }); - expect(enResult.result?.content).toBe("Hello，World！这是一个Test。"); - expect(enResult.result?.metadata?.view).toEqual({ language: "en" }); + expect(enResult.data?.content).toBe("Hello，World！这是一个Test。"); + expect(enResult.data?.metadata?.view).toEqual({ language: "en" }); // Read Japanese version const jaResult = await afs.read("/modules/mock-fs/test.md", { @@ -155,12 +155,12 @@ test("View driver: should translate content with view", async () => { wait: "strict", }); - expect(jaResult.result?.content).toBe("こんにちは，世界！这是一个テスト。"); - expect(jaResult.result?.metadata?.view).toEqual({ language: "ja" }); + expect(jaResult.data?.content).toBe("こんにちは，世界！这是一个テスト。"); + expect(jaResult.data?.metadata?.view).toEqual({ language: "ja" }); // Read source (no view) should return original const sourceResult = await afs.read("/modules/mock-fs/test.md"); - expect(sourceResult.result?.content).toBe("你好，世界！这是一个测试。"); + expect(sourceResult.data?.content).toBe("你好，世界！这是一个测试。"); }); test("View driver: should use cached view on second read", async () => { @@ -200,7 +200,7 @@ test("View driver: should use cached view on second read", async () => { }); expect(processCallCount).toBe(1); // Still 1, not called again - expect(cachedResult.result?.content).toBe("Hello，World！"); + expect(cachedResult.data?.content).toBe("Hello，World！"); }); test("View driver: should invalidate cache when source changes", async () => { @@ -231,7 +231,7 @@ test("View driver: should invalidate cache when source changes", async () => { wait: "strict", }); - expect(firstRead.result?.content).toBe("Hello"); + expect(firstRead.data?.content).toBe("Hello"); expect(processCallCount).toBe(1); // Update source content @@ -245,7 +245,7 @@ test("View driver: should invalidate cache when source changes", async () => { wait: "strict", }); - expect(secondRead.result?.content).toBe("Hello，World！"); + expect(secondRead.data?.content).toBe("Hello，World！"); expect(processCallCount).toBe(2); // Called again after source change }); @@ -280,7 +280,7 @@ test("View driver: fallback mode should return source immediately", async () => }); // Should get source content immediately - expect(fallbackResult.result?.content).toBe("你好，世界！"); + expect(fallbackResult.data?.content).toBe("你好，世界！"); expect(fallbackResult.message).toContain("being processed in background"); // Wait a bit for background processing @@ -292,7 +292,7 @@ test("View driver: fallback mode should return source immediately", async () => wait: "strict", }); - expect(strictResult.result?.content).toBe("Hello，World！"); + expect(strictResult.data?.content).toBe("Hello，World！"); }); test("View driver: prefetch should batch generate views", async () => { @@ -337,9 +337,9 @@ test("View driver: prefetch should batch generate views", async () => { view: { language: "en" }, }); - expect(result1.result?.content).toBe("Hello"); - expect(result2.result?.content).toBe("World"); - expect(result3.result?.content).toBe("Test"); + expect(result1.data?.content).toBe("Hello"); + expect(result2.data?.content).toBe("World"); + expect(result3.data?.content).toBe("Test"); // Process count should still be 3 (no additional calls) expect(processCallCount).toBe(3); diff --git a/afs/i18n-driver/test/i18n-driver.test.ts b/afs/i18n-driver/test/i18n-driver.test.ts index 34b54ce36..6ac719b4c 100644 --- a/afs/i18n-driver/test/i18n-driver.test.ts +++ b/afs/i18n-driver/test/i18n-driver.test.ts @@ -22,14 +22,14 @@ class MockFSModule implements AFSModule { constructor(public options: { context: any }) {} - async read(path: string): Promise<{ result?: AFSEntry; message?: string }> { + async read(path: string): Promise<{ data?: AFSEntry; message?: string }> { const content = this.files.get(path); if (!content) { - return { result: undefined, message: "File not found" }; + return { data: undefined, message: "File not found" }; } return { - result: { + data: { id: path, path, content, @@ -42,11 +42,11 @@ class MockFSModule implements AFSModule { async write( path: string, payload: { content: string }, - ): Promise<{ result: AFSEntry; message?: string }> { + ): Promise<{ data: AFSEntry; message?: string }> { this.files.set(path, payload.content); return { - result: { + data: { id: path, path, content: payload.content, @@ -152,7 +152,7 @@ test("I18nDriver should translate content using context", async () => { // Read source const source = await mockFS.read("/test.md"); - assert(source.result, "source.result should be defined"); + assert(source.data, "source.data should be defined"); // Process translation (context is passed via options) const result = await driver.process( @@ -160,15 +160,15 @@ test("I18nDriver should translate content using context", async () => { "/test.md", { language: "en" }, { - sourceEntry: source.result, + sourceEntry: source.data, metadata: {}, context, }, ); - expect(result.result.content).toBe("Hello，World！This is aTest。"); - expect(result.result.metadata?.storagePath).toBe("/.i18n/en/test.md"); // root-level file - expect(result.result.metadata?.view).toEqual({ language: "en" }); + expect(result.data.content).toBe("Hello，World！This is aTest。"); + expect(result.data.metadata?.storagePath).toBe("/.i18n/en/test.md"); // root-level file + expect(result.data.metadata?.view).toEqual({ language: "en" }); }); test("I18nDriver should integrate with AFS", async () => { @@ -201,8 +201,8 @@ test("I18nDriver should integrate with AFS", async () => { context, }); - expect(enResult.result?.content).toBe("Hello，World！"); - expect(enResult.result?.metadata?.view).toEqual({ language: "en" }); + expect(enResult.data?.content).toBe("Hello，World！"); + expect(enResult.data?.metadata?.view).toEqual({ language: "en" }); // Read Japanese version const jaResult = await afs.read("/modules/mock-fs/doc.md", { @@ -211,11 +211,11 @@ test("I18nDriver should integrate with AFS", async () => { context, }); - expect(jaResult.result?.content).toBe("こんにちは，世界！"); + expect(jaResult.data?.content).toBe("こんにちは，世界！"); // Read source (no view) should return original const sourceResult = await afs.read("/modules/mock-fs/doc.md"); - expect(sourceResult.result?.content).toBe("你好，世界！"); + expect(sourceResult.data?.content).toBe("你好，世界！"); }); test("I18nDriver should throw error if language is missing", async () => { @@ -231,7 +231,7 @@ test("I18nDriver should throw error if language is missing", async () => { await mockFS.write("/test.md", { content: "test content" }); const source = await mockFS.read("/test.md"); - assert(source.result, "source.result should be defined"); + assert(source.data, "source.data should be defined"); await expect( driver.process( @@ -239,7 +239,7 @@ test("I18nDriver should throw error if language is missing", async () => { "/test.md", {}, { - sourceEntry: source.result, + sourceEntry: source.data, metadata: {}, context, }, @@ -260,7 +260,7 @@ test("I18nDriver should throw error if context is missing", async () => { await mockFS.write("/test.md", { content: "test content" }); const source = await mockFS.read("/test.md"); - assert(source.result, "source.result should be defined"); + assert(source.data, "source.data should be defined"); await expect( driver.process( @@ -268,7 +268,7 @@ test("I18nDriver should throw error if context is missing", async () => { "/test.md", { language: "en" }, { - sourceEntry: source.result, + sourceEntry: source.data, metadata: {}, // no context provided }, diff --git a/packages/core/test/loader/agent-yaml.test.ts b/packages/core/test/loader/agent-yaml.test.ts index 0a1e62106..ae629c9b1 100644 --- a/packages/core/test/loader/agent-yaml.test.ts +++ b/packages/core/test/loader/agent-yaml.test.ts @@ -363,7 +363,7 @@ test("loadAgentFromYaml should load AIAgent with AFS drivers correctly", async ( description: "Mock i18n driver", capabilities: { dimensions: ["language" as const] }, canHandle: (view: { language?: string }) => !!view.language, - process: async () => ({ result: { id: "test", path: "/test", content: "translated" } }), + process: async () => ({ data: { id: "test", path: "/test", content: "translated" } }), }; }; @@ -371,8 +371,8 @@ test("loadAgentFromYaml should load AIAgent with AFS drivers correctly", async ( join(import.meta.dirname, "../../test-agents/test-afs-driver.yaml"), { afs: { - availableModules: [{ module: "local-fs", create: createMockModule }], - availableDrivers: [{ driver: "i18n", create: createMockDriver }], + availableModules: [{ module: "local-fs", load: createMockModule }], + availableDrivers: [{ driver: "i18n", load: createMockDriver }], }, }, ); @@ -383,10 +383,15 @@ test("loadAgentFromYaml should load AIAgent with AFS drivers correctly", async ( expect(agent.afs?.drivers[0]?.name).toBe("i18n"); // Verify options are passed as-is from YAML (not camelized) - expect(moduleOptions).toEqual({ local_path: "./test-docs" }); + expect(moduleOptions).toEqual({ + filepath: expect.stringContaining("test-afs-driver.yaml"), + parsed: { local_path: "./test-docs" }, + }); expect(driverOptions).toEqual({ - default_source_language: "zh", - supported_languages: ["en", "ja"], + parsed: { + default_source_language: "zh", + supported_languages: ["en", "ja"], + }, }); }); @@ -403,15 +408,15 @@ test("loadAgentFromYaml should load AIAgent with AFS storage config correctly", description: "Mock i18n driver", capabilities: { dimensions: ["language" as const] }, canHandle: (view: { language?: string }) => !!view.language, - process: async () => ({ result: { id: "test", path: "/test", content: "translated" } }), + process: async () => ({ data: { id: "test", path: "/test", content: "translated" } }), }); const agent = await loadAgent( join(import.meta.dirname, "../../test-agents/test-afs-storage.yaml"), { afs: { - availableModules: [{ module: "local-fs", create: createMockModule }], - availableDrivers: [{ driver: "i18n", create: createMockDriver }], + availableModules: [{ module: "local-fs", load: createMockModule }], + availableDrivers: [{ driver: "i18n", load: createMockDriver }], }, }, ); From cc6446264df1d4095ea3067996bf677651224fa1 Mon Sep 17 00:00:00 2001 From: LBan Date: Fri, 26 Dec 2025 15:13:40 +0800 Subject: [PATCH 16/21] feat(afs): implement SlotScanner for parsing image slots and enhance metadata handling --- afs/core/src/afs.ts | 4 +- afs/core/src/metadata/migrations/001-init.ts | 36 +- afs/core/src/metadata/models/deps-metadata.ts | 23 ++ afs/core/src/metadata/models/index.ts | 2 + .../src/metadata/models/slots-metadata.ts | 25 ++ .../src/metadata/models/source-metadata.ts | 2 + afs/core/src/metadata/store.ts | 271 ++++++++++++++- afs/core/src/metadata/type.ts | 45 +++ afs/core/src/slot-scanner.ts | 122 +++++++ afs/core/src/type.ts | 21 +- afs/core/src/utils.ts | 12 + afs/core/src/view-key.ts | 42 +++ afs/core/src/view-processor.ts | 65 +++- afs/core/test/slot-scanner.test.ts | 312 ++++++++++++++++++ afs/core/test/view-key.test.ts | 195 +++++++++++ afs/i18n-driver/test/i18n-driver.test.ts | 1 - 16 files changed, 1146 insertions(+), 32 deletions(-) create mode 100644 afs/core/src/metadata/models/deps-metadata.ts create mode 100644 afs/core/src/metadata/models/slots-metadata.ts create mode 100644 afs/core/src/slot-scanner.ts create mode 100644 afs/core/src/utils.ts create mode 100644 afs/core/src/view-key.ts create mode 100644 afs/core/test/slot-scanner.test.ts create mode 100644 afs/core/test/view-key.test.ts diff --git a/afs/core/src/afs.ts b/afs/core/src/afs.ts index e0bd398d4..587e673e2 100644 --- a/afs/core/src/afs.ts +++ b/afs/core/src/afs.ts @@ -216,7 +216,7 @@ export class AFS extends Emitter implements AFSRoot { // Update metadata if view processor is available if (this.viewProcessor) { - await this.viewProcessor.handleWrite(module.module.name, module.subpath, res.data); + await this.viewProcessor.handleWrite(module.module, module.subpath, res.data); } return { @@ -236,7 +236,7 @@ export class AFS extends Emitter implements AFSRoot { // Clean up metadata if view processor is available if (this.viewProcessor) { - await this.viewProcessor.handleDelete(module.module.name, module.subpath); + await this.viewProcessor.handleDelete(module.module, module.subpath); } return result; diff --git a/afs/core/src/metadata/migrations/001-init.ts b/afs/core/src/metadata/migrations/001-init.ts index 362d288ca..5b2470435 100644 --- a/afs/core/src/metadata/migrations/001-init.ts +++ b/afs/core/src/metadata/migrations/001-init.ts @@ -2,7 +2,7 @@ import type { SQL } from "@aigne/sqlite"; import { sql } from "@aigne/sqlite"; /** - * Initial migration: Create source_metadata and view_metadata tables + * Initial migration: Create source_metadata, view_metadata, afs_slots, and afs_deps_meta tables */ export const init = { hash: "001-init", @@ -16,9 +16,12 @@ CREATE TABLE IF NOT EXISTS source_metadata ( source_revision TEXT NOT NULL, updated_at INTEGER NOT NULL, drivers_hint TEXT, + kind TEXT, + attrs_json TEXT, created_at INTEGER NOT NULL )`, sql`CREATE UNIQUE INDEX IF NOT EXISTS unique_module_path ON source_metadata(module, path)`, + // View metadata table sql` CREATE TABLE IF NOT EXISTS view_metadata ( @@ -34,10 +37,39 @@ CREATE TABLE IF NOT EXISTS view_metadata ( created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL )`, - // Indexes sql`CREATE INDEX IF NOT EXISTS idx_view_path ON view_metadata(path)`, sql`CREATE INDEX IF NOT EXISTS idx_view_state ON view_metadata(state)`, sql`CREATE INDEX IF NOT EXISTS idx_view_derived_from ON view_metadata(derived_from)`, sql`CREATE UNIQUE INDEX IF NOT EXISTS unique_module_path_view ON view_metadata(module, path, view)`, + + // Image slots table + sql` +CREATE TABLE IF NOT EXISTS afs_slots ( + owner_path TEXT NOT NULL, + slot_id TEXT NOT NULL, + owner_revision TEXT NOT NULL, + slot_type TEXT NOT NULL, + desc TEXT NOT NULL, + intent_key TEXT NOT NULL, + asset_path TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (owner_path, slot_id) +)`, + sql`CREATE INDEX IF NOT EXISTS idx_slots_asset_path ON afs_slots(asset_path)`, + sql`CREATE INDEX IF NOT EXISTS idx_slots_intent_key ON afs_slots(intent_key)`, + + // Dependencies metadata table + sql` +CREATE TABLE IF NOT EXISTS afs_deps_meta ( + out_path TEXT NOT NULL, + out_view_key TEXT NOT NULL, + in_path TEXT NOT NULL, + in_revision TEXT NOT NULL, + role TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (out_path, out_view_key, in_path) +)`, + sql`CREATE INDEX IF NOT EXISTS idx_deps_in_path ON afs_deps_meta(in_path)`, + sql`CREATE INDEX IF NOT EXISTS idx_deps_out_path ON afs_deps_meta(out_path, out_view_key)`, ], }; diff --git a/afs/core/src/metadata/models/deps-metadata.ts b/afs/core/src/metadata/models/deps-metadata.ts new file mode 100644 index 000000000..25b6e3f75 --- /dev/null +++ b/afs/core/src/metadata/models/deps-metadata.ts @@ -0,0 +1,23 @@ +import { integer, primaryKey, sqliteTable, text } from "@aigne/sqlite"; + +/** + * Dependencies metadata table schema + * Tracks dependencies between view outputs and their inputs + */ +export const depsMetadataTable = sqliteTable( + "afs_deps_meta", + { + outPath: text("out_path").notNull(), // output artifact path + outViewKey: text("out_view_key").notNull(), // output viewKey (normalized) + inPath: text("in_path").notNull(), // input dependency path + inRevision: text("in_revision").notNull(), // input sourceRevision at dependency time + role: text("role").notNull(), // "owner-context" | "source" | "lexicon" | "policy" + updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), + }, + (table) => ({ + pk: primaryKey({ columns: [table.outPath, table.outViewKey, table.inPath] }), + }), +); + +export type DepsMetadataRow = typeof depsMetadataTable.$inferSelect; +export type DepsMetadataInsert = typeof depsMetadataTable.$inferInsert; diff --git a/afs/core/src/metadata/models/index.ts b/afs/core/src/metadata/models/index.ts index 8544b73c2..0112143d4 100644 --- a/afs/core/src/metadata/models/index.ts +++ b/afs/core/src/metadata/models/index.ts @@ -1,2 +1,4 @@ +export * from "./deps-metadata.js"; +export * from "./slots-metadata.js"; export * from "./source-metadata.js"; export * from "./view-metadata.js"; diff --git a/afs/core/src/metadata/models/slots-metadata.ts b/afs/core/src/metadata/models/slots-metadata.ts new file mode 100644 index 000000000..d7daf82fb --- /dev/null +++ b/afs/core/src/metadata/models/slots-metadata.ts @@ -0,0 +1,25 @@ +import { integer, primaryKey, sqliteTable, text } from "@aigne/sqlite"; + +/** + * Slots metadata table schema + * Tracks image slots declared in documents + */ +export const slotsMetadataTable = sqliteTable( + "afs_slots", + { + ownerPath: text("owner_path").notNull(), + slotId: text("slot_id").notNull(), + ownerRevision: text("owner_revision").notNull(), + slotType: text("slot_type").notNull(), // v1: "image" + desc: text("desc").notNull(), // prompt seed for image generation + intentKey: text("intent_key").notNull(), // hash(normalize(desc)) or explicit key + assetPath: text("asset_path").notNull(), // .afs/images/by-intent/ + updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), + }, + (table) => ({ + pk: primaryKey({ columns: [table.ownerPath, table.slotId] }), + }), +); + +export type SlotsMetadataRow = typeof slotsMetadataTable.$inferSelect; +export type SlotsMetadataInsert = typeof slotsMetadataTable.$inferInsert; diff --git a/afs/core/src/metadata/models/source-metadata.ts b/afs/core/src/metadata/models/source-metadata.ts index db5f8ac29..301302fd0 100644 --- a/afs/core/src/metadata/models/source-metadata.ts +++ b/afs/core/src/metadata/models/source-metadata.ts @@ -12,6 +12,8 @@ export const sourceMetadataTable = sqliteTable( sourceRevision: text("source_revision").notNull(), updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), driversHint: text("drivers_hint"), // JSON array stored as text + kind: text("kind"), // "doc" | "image" | "unknown" (hint for driver selection) + attrsJson: text("attrs_json"), // JSON object for extended attributes (mime, size, width/height, etc.) createdAt: integer("created_at", { mode: "timestamp_ms" }).notNull(), }, (table) => ({ diff --git a/afs/core/src/metadata/store.ts b/afs/core/src/metadata/store.ts index e623d4040..6f49b5be4 100644 --- a/afs/core/src/metadata/store.ts +++ b/afs/core/src/metadata/store.ts @@ -1,8 +1,21 @@ import { and, eq, initDatabase } from "@aigne/sqlite"; import type { View } from "../type.js"; +import { normalizeViewKey } from "../view-key.js"; import { migrate } from "./migrate.js"; -import { sourceMetadataTable, viewMetadataTable } from "./models/index.js"; -import type { MetadataStore, SourceMetadata, ViewMetadata, ViewState } from "./type.js"; +import { + depsMetadataTable, + slotsMetadataTable, + sourceMetadataTable, + viewMetadataTable, +} from "./models/index.js"; +import type { + DependencyMetadata, + MetadataStore, + SlotMetadata, + SourceMetadata, + ViewMetadata, + ViewState, +} from "./type.js"; export interface SQLiteMetadataStoreOptions { url?: string; @@ -11,10 +24,27 @@ export interface SQLiteMetadataStoreOptions { /** * SQLite-based metadata store implementation */ +/** + * Parse normalized viewKey back to View object + * Format: "language=en;format=png" -> {language:"en", format:"png"} + */ +function parseViewKey(viewKey: string): View { + const view: View = {}; + viewKey.split(";").forEach((pair) => { + const [key, value] = pair.split("="); + if (key && value) { + view[key as keyof View] = value; + } + }); + return view; +} + export class SQLiteMetadataStore implements MetadataStore { private db: ReturnType; private sourceTable = sourceMetadataTable; private viewTable = viewMetadataTable; + private slotsTable = slotsMetadataTable; + private depsTable = depsMetadataTable; constructor(options?: SQLiteMetadataStoreOptions) { this.db = initDatabase({ url: options?.url }).then(async (db) => { @@ -34,6 +64,8 @@ export class SQLiteMetadataStore implements MetadataStore { sourceRevision: this.sourceTable.sourceRevision, updatedAt: this.sourceTable.updatedAt, driversHint: this.sourceTable.driversHint, + kind: this.sourceTable.kind, + attrsJson: this.sourceTable.attrsJson, }) .from(this.sourceTable) .where(and(eq(this.sourceTable.module, module), eq(this.sourceTable.path, path))) @@ -49,6 +81,8 @@ export class SQLiteMetadataStore implements MetadataStore { sourceRevision: row.sourceRevision, updatedAt: row.updatedAt, driversHint: row.driversHint ? JSON.parse(row.driversHint) : undefined, + kind: row.kind as "doc" | "image" | "unknown" | undefined, + attrs: row.attrsJson ? JSON.parse(row.attrsJson) : undefined, }; } @@ -67,6 +101,8 @@ export class SQLiteMetadataStore implements MetadataStore { sourceRevision: metadata.sourceRevision, updatedAt: metadata.updatedAt, driversHint: metadata.driversHint ? JSON.stringify(metadata.driversHint) : null, + kind: metadata.kind, + attrsJson: metadata.attrs ? JSON.stringify(metadata.attrs) : null, }) .where(and(eq(this.sourceTable.module, module), eq(this.sourceTable.path, path))) .returning({ @@ -84,6 +120,8 @@ export class SQLiteMetadataStore implements MetadataStore { sourceRevision: metadata.sourceRevision, updatedAt: metadata.updatedAt, driversHint: metadata.driversHint ? JSON.stringify(metadata.driversHint) : null, + kind: metadata.kind, + attrsJson: metadata.attrs ? JSON.stringify(metadata.attrs) : null, createdAt: now, }) .execute(); @@ -101,7 +139,7 @@ export class SQLiteMetadataStore implements MetadataStore { // View metadata operations async getViewMetadata(module: string, path: string, view: View): Promise { const db = await this.db; - const viewKey = JSON.stringify(view); + const viewKey = normalizeViewKey(view); const rows = await db .select({ @@ -131,7 +169,7 @@ export class SQLiteMetadataStore implements MetadataStore { return { module: row.module, path: row.path, - view: JSON.parse(row.view), + view: parseViewKey(row.view), state: row.state as ViewState, derivedFrom: row.derivedFrom, generatedAt: row.generatedAt || undefined, @@ -147,7 +185,7 @@ export class SQLiteMetadataStore implements MetadataStore { metadata: Partial, ): Promise { const db = await this.db; - const viewKey = JSON.stringify(view); + const viewKey = normalizeViewKey(view); const now = new Date(); // Get existing record @@ -225,7 +263,7 @@ export class SQLiteMetadataStore implements MetadataStore { return rows.map((row) => ({ module: row.module, path: row.path, - view: JSON.parse(row.view), + view: parseViewKey(row.view), state: row.state as ViewState, derivedFrom: row.derivedFrom, generatedAt: row.generatedAt || undefined, @@ -238,7 +276,7 @@ export class SQLiteMetadataStore implements MetadataStore { const db = await this.db; if (view) { - const viewKey = JSON.stringify(view); + const viewKey = normalizeViewKey(view); await db .delete(this.viewTable) .where( @@ -293,7 +331,7 @@ export class SQLiteMetadataStore implements MetadataStore { return rows.map((row) => ({ module: row.module, path: row.path, - view: JSON.parse(row.view), + view: parseViewKey(row.view), state: row.state as ViewState, derivedFrom: row.derivedFrom, generatedAt: row.generatedAt || undefined, @@ -323,7 +361,7 @@ export class SQLiteMetadataStore implements MetadataStore { return rows.map((row) => ({ module: row.module, path: row.path, - view: JSON.parse(row.view), + view: parseViewKey(row.view), state: row.state as ViewState, derivedFrom: row.derivedFrom, generatedAt: row.generatedAt || undefined, @@ -358,4 +396,219 @@ export class SQLiteMetadataStore implements MetadataStore { await db.delete(this.viewTable).where(eq(this.viewTable.state, "failed")).execute(); } + + // Slot metadata operations + async getSlot(_module: string, ownerPath: string, slotId: string): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.slotsTable) + .where(and(eq(this.slotsTable.ownerPath, ownerPath), eq(this.slotsTable.slotId, slotId))) + .limit(1) + .execute(); + + const row = rows[0]; + if (!row) return null; + + return { + ownerPath: row.ownerPath, + slotId: row.slotId, + ownerRevision: row.ownerRevision, + slotType: row.slotType as "image", + desc: row.desc, + intentKey: row.intentKey, + assetPath: row.assetPath, + updatedAt: new Date(row.updatedAt), + }; + } + + async listSlots(_module: string, ownerPath: string): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.slotsTable) + .where(eq(this.slotsTable.ownerPath, ownerPath)) + .execute(); + + return rows.map((row) => ({ + ownerPath: row.ownerPath, + slotId: row.slotId, + ownerRevision: row.ownerRevision, + slotType: row.slotType as "image", + desc: row.desc, + intentKey: row.intentKey, + assetPath: row.assetPath, + updatedAt: new Date(row.updatedAt), + })); + } + + async getSlotByAssetPath(_module: string, assetPath: string): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.slotsTable) + .where(eq(this.slotsTable.assetPath, assetPath)) + .limit(1) + .execute(); + + const row = rows[0]; + if (!row) return null; + + return { + ownerPath: row.ownerPath, + slotId: row.slotId, + ownerRevision: row.ownerRevision, + slotType: row.slotType as "image", + desc: row.desc, + intentKey: row.intentKey, + assetPath: row.assetPath, + updatedAt: new Date(row.updatedAt), + }; + } + + async upsertSlot(_module: string, slot: Omit): Promise { + const db = await this.db; + const now = new Date(); + + // Try to update first + const updated = await db + .update(this.slotsTable) + .set({ + ownerRevision: slot.ownerRevision, + slotType: slot.slotType, + desc: slot.desc, + intentKey: slot.intentKey, + assetPath: slot.assetPath, + updatedAt: now, + }) + .where( + and(eq(this.slotsTable.ownerPath, slot.ownerPath), eq(this.slotsTable.slotId, slot.slotId)), + ) + .returning({ ownerPath: this.slotsTable.ownerPath }) + .execute(); + + // If no rows updated, insert new record + if (!updated.length) { + await db + .insert(this.slotsTable) + .values({ + ownerPath: slot.ownerPath, + slotId: slot.slotId, + ownerRevision: slot.ownerRevision, + slotType: slot.slotType, + desc: slot.desc, + intentKey: slot.intentKey, + assetPath: slot.assetPath, + updatedAt: now, + }) + .execute(); + } + } + + async deleteSlots(_module: string, ownerPath: string): Promise { + const db = await this.db; + + await db.delete(this.slotsTable).where(eq(this.slotsTable.ownerPath, ownerPath)).execute(); + } + + // Dependency metadata operations + async setDependency(_module: string, dep: Omit): Promise { + const db = await this.db; + const now = new Date(); + + // Try to update first + const updated = await db + .update(this.depsTable) + .set({ + inRevision: dep.inRevision, + role: dep.role, + updatedAt: now, + }) + .where( + and( + eq(this.depsTable.outPath, dep.outPath), + eq(this.depsTable.outViewKey, dep.outViewKey), + eq(this.depsTable.inPath, dep.inPath), + ), + ) + .returning({ outPath: this.depsTable.outPath }) + .execute(); + + // If no rows updated, insert new record + if (!updated.length) { + await db + .insert(this.depsTable) + .values({ + outPath: dep.outPath, + outViewKey: dep.outViewKey, + inPath: dep.inPath, + inRevision: dep.inRevision, + role: dep.role, + updatedAt: now, + }) + .execute(); + } + } + + async listDependenciesByInput(_module: string, inPath: string): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.depsTable) + .where(eq(this.depsTable.inPath, inPath)) + .execute(); + + return rows.map((row) => ({ + outPath: row.outPath, + outViewKey: row.outViewKey, + inPath: row.inPath, + inRevision: row.inRevision, + role: row.role as DependencyMetadata["role"], + updatedAt: new Date(row.updatedAt), + })); + } + + async listDependenciesByOutput( + _module: string, + outPath: string, + outViewKey: string, + ): Promise { + const db = await this.db; + + const rows = await db + .select() + .from(this.depsTable) + .where(and(eq(this.depsTable.outPath, outPath), eq(this.depsTable.outViewKey, outViewKey))) + .execute(); + + return rows.map((row) => ({ + outPath: row.outPath, + outViewKey: row.outViewKey, + inPath: row.inPath, + inRevision: row.inRevision, + role: row.role as DependencyMetadata["role"], + updatedAt: new Date(row.updatedAt), + })); + } + + async deleteDependenciesByOutput( + _module: string, + outPath: string, + outViewKey?: string, + ): Promise { + const db = await this.db; + + if (outViewKey) { + await db + .delete(this.depsTable) + .where(and(eq(this.depsTable.outPath, outPath), eq(this.depsTable.outViewKey, outViewKey))) + .execute(); + } else { + await db.delete(this.depsTable).where(eq(this.depsTable.outPath, outPath)).execute(); + } + } } diff --git a/afs/core/src/metadata/type.ts b/afs/core/src/metadata/type.ts index 5eab12524..a6ee5e0be 100644 --- a/afs/core/src/metadata/type.ts +++ b/afs/core/src/metadata/type.ts @@ -19,6 +19,8 @@ export interface SourceMetadata { sourceRevision: string; // Content hash or mtime:size identifier updatedAt: Date; driversHint?: string[]; // Optional: driver names that may process this file + kind?: "doc" | "image" | "unknown"; // Resource type hint for driver selection + attrs?: Record; // Extended attributes (mime, size, width/height, etc.) } /** @@ -36,6 +38,32 @@ export interface ViewMetadata { storagePath?: string; // Physical storage path (e.g., '.i18n/en/...') } +/** + * Slot metadata (image slot declared in documents) + */ +export interface SlotMetadata { + ownerPath: string; + slotId: string; + ownerRevision: string; + slotType: "image"; // v1: only image + desc: string; // prompt seed + intentKey: string; // hash(normalize(desc)) or explicit key + assetPath: string; // .afs/images/by-intent/ + updatedAt: Date; +} + +/** + * Dependency metadata (view output depends on inputs) + */ +export interface DependencyMetadata { + outPath: string; + outViewKey: string; // normalized viewKey + inPath: string; + inRevision: string; // sourceRevision at dependency time + role: "owner-context" | "source" | "lexicon" | "policy"; + updatedAt: Date; +} + /** * Metadata Store interface * Manages source and view metadata using SQLite @@ -69,4 +97,21 @@ export interface MetadataStore { // Cleanup operations cleanupOrphanedViewMetadata(): Promise; cleanupFailedViews(olderThan?: Date): Promise; + + // Slot metadata operations + getSlot(module: string, ownerPath: string, slotId: string): Promise; + listSlots(module: string, ownerPath: string): Promise; + getSlotByAssetPath(module: string, assetPath: string): Promise; + upsertSlot(module: string, slot: Omit): Promise; + deleteSlots(module: string, ownerPath: string): Promise; + + // Dependency metadata operations + setDependency(module: string, dep: Omit): Promise; + listDependenciesByInput(module: string, inPath: string): Promise; + listDependenciesByOutput( + module: string, + outPath: string, + outViewKey: string, + ): Promise; + deleteDependenciesByOutput(module: string, outPath: string, outViewKey?: string): Promise; } diff --git a/afs/core/src/slot-scanner.ts b/afs/core/src/slot-scanner.ts new file mode 100644 index 000000000..28ef5672b --- /dev/null +++ b/afs/core/src/slot-scanner.ts @@ -0,0 +1,122 @@ +import type { MetadataStore } from "./metadata/index.js"; +import type { AFSModule, ImageSlot } from "./type.js"; +import { sha256Hash } from "./utils.js"; + +/** + * Normalize description for intentKey computation + * - Trim whitespace + * - Convert to lowercase + * - Collapse multiple spaces to single space + */ +function normalizeDesc(desc: string): string { + return desc.trim().toLowerCase().replace(/\s+/g, " "); +} + +/** + * Compute intentKey from description or use explicit key + */ +async function computeIntentKey(desc: string, key?: string): Promise { + if (key) { + // Explicit key takes precedence + return key; + } + + // Hash normalized description + const normalized = normalizeDesc(desc); + return sha256Hash(normalized); +} + +/** + * Slot scanner for detecting image slots in documents + */ +export class SlotScanner { + // Slot pattern: or + private static SLOT_PATTERN = + //g; + + constructor(private metadataStore: MetadataStore) {} + + /** + * Scan document content for image slots + * @param module AFSModule instance + * @param ownerPath Document path + * @param content Document content (markdown text) + * @param ownerRevision Source revision of the owner document + * @returns Array of parsed image slots + */ + async scan( + module: AFSModule, + ownerPath: string, + content: string, + ownerRevision: string, + ): Promise { + const slots: ImageSlot[] = []; + const seenIds = new Set(); + + // Reset regex state + SlotScanner.SLOT_PATTERN.lastIndex = 0; + + let match: RegExpExecArray | null = SlotScanner.SLOT_PATTERN.exec(content); + while (match !== null) { + const [_fullMatch, id, key, desc] = match; + + // Validate: captured groups must be present + if (!id || !desc) { + continue; // Skip malformed slots + } + + // Validate: id must be unique within owner + if (seenIds.has(id)) { + throw new Error(`Duplicate slot id "${id}" in ${ownerPath}`); + } + seenIds.add(id); + + // Compute intentKey + const intentKey = await computeIntentKey(desc, key); + const assetPath = `.afs/images/by-intent/${intentKey}`; + + slots.push({ id, desc, key, intentKey, assetPath }); + + // Upsert slot to database + await this.metadataStore.upsertSlot(module.name, { + ownerPath, + slotId: id, + ownerRevision, + slotType: "image", + desc, + intentKey, + assetPath, + }); + + // Ensure image node exists in source_metadata + await this.ensureImageNode(module, assetPath, intentKey); + + // Get next match + match = SlotScanner.SLOT_PATTERN.exec(content); + } + + return slots; + } + + /** + * Ensure image node exists in source_metadata + * Sets kind="image" and sourceRevision="intent:" + */ + private async ensureImageNode( + module: AFSModule, + assetPath: string, + intentKey: string, + ): Promise { + const existing = await this.metadataStore.getSourceMetadata(module.name, assetPath); + + if (!existing) { + // Create new source metadata for the image node + await this.metadataStore.setSourceMetadata(module.name, assetPath, { + sourceRevision: `intent:${intentKey}`, + updatedAt: new Date(), + kind: "image", + driversHint: ["image-generate"], + }); + } + } +} diff --git a/afs/core/src/type.ts b/afs/core/src/type.ts index f96204fc3..9c2164c81 100644 --- a/afs/core/src/type.ts +++ b/afs/core/src/type.ts @@ -43,14 +43,14 @@ export interface AFSSearchResult { /** * View represents different projections of the same file - * V1: Only language dimension is implemented - * Future: format, policy, variant dimensions + * V1: language and format dimensions are implemented + * Future: policy, variant dimensions */ export interface View { language?: string; // Target language for translation (e.g., "en", "zh", "ja") - // format?: string; // Future: format conversion (e.g., "html", "pdf") - // policy?: string; // Future: content style policy (e.g., "technical", "marketing") - // variant?: string; // Future: content variant (e.g., "summary", "toc", "index") + format?: string; // Format conversion (e.g., "png", "webp", "html", "pdf") + variant?: string; // Content variant (e.g., "summary", "toc", "index") + policy?: string; // Content style policy (e.g., "technical", "marketing") } /** @@ -68,6 +68,17 @@ export interface ViewStatus { fallback?: boolean; // true = returned source content, view is being generated in background } +/** + * Image slot parsed from document + */ +export interface ImageSlot { + id: string; // slot identifier (unique within owner) + desc: string; // original description (prompt seed) + key?: string; // optional explicit key for cross-document reuse + intentKey: string; // computed hash or explicit key + assetPath: string; // .afs/images/by-intent/ +} + export interface AFSReadOptions { view?: View; wait?: WaitStrategy; diff --git a/afs/core/src/utils.ts b/afs/core/src/utils.ts new file mode 100644 index 000000000..45cf7a7cd --- /dev/null +++ b/afs/core/src/utils.ts @@ -0,0 +1,12 @@ +/** + * Cross-platform SHA-256 hash function + * Uses Web Crypto API which is available in both Node.js and browsers + */ +export async function sha256Hash(data: string): Promise { + const encoder = new TextEncoder(); + const dataBuffer = encoder.encode(data); + const hashBuffer = await crypto.subtle.digest("SHA-256", dataBuffer); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + const hashHex = hashArray.map((b) => b.toString(16).padStart(2, "0")).join(""); + return hashHex.substring(0, 16); +} diff --git a/afs/core/src/view-key.ts b/afs/core/src/view-key.ts new file mode 100644 index 000000000..775838bd6 --- /dev/null +++ b/afs/core/src/view-key.ts @@ -0,0 +1,42 @@ +import type { View } from "./type.js"; + +/** + * Normalize ViewKey to a consistent string format + * + * Rules: + * 1. Only allow whitelisted keys: language | format | variant | policy + * 2. Normalize values: trim() + toLowerCase() + * 3. Fixed key order: language → format → variant → policy + * 4. Format: k=v;k=v (keys without values are omitted) + * + * This ensures stable primary key in view_metadata table: + * - {format:"PNG", language:"EN"} → "language=en;format=png" + * - {language:"en", format:"png"} → "language=en;format=png" + * (same viewKey regardless of input key order) + */ +export function normalizeViewKey(view: View): string { + const pairs: string[] = []; + + // Fixed order: language → format → variant → policy + const trimmedLanguage = view.language?.trim().toLowerCase(); + if (trimmedLanguage) { + pairs.push(`language=${trimmedLanguage}`); + } + + const trimmedFormat = view.format?.trim().toLowerCase(); + if (trimmedFormat) { + pairs.push(`format=${trimmedFormat}`); + } + + const trimmedVariant = view.variant?.trim().toLowerCase(); + if (trimmedVariant) { + pairs.push(`variant=${trimmedVariant}`); + } + + const trimmedPolicy = view.policy?.trim().toLowerCase(); + if (trimmedPolicy) { + pairs.push(`policy=${trimmedPolicy}`); + } + + return pairs.join(";"); +} diff --git a/afs/core/src/view-processor.ts b/afs/core/src/view-processor.ts index bed498a7a..68d30fde6 100644 --- a/afs/core/src/view-processor.ts +++ b/afs/core/src/view-processor.ts @@ -1,6 +1,6 @@ -import { createHash } from "node:crypto"; import pLimit from "p-limit"; import type { MetadataStore, ViewMetadata } from "./metadata/index.js"; +import { SlotScanner } from "./slot-scanner.js"; import type { AFSDriver, AFSEntry, @@ -9,16 +9,21 @@ import type { AFSReadResult, View, } from "./type.js"; +import { sha256Hash } from "./utils.js"; /** * View processor for handling view generation and caching * V1 implementation: Simplified without job deduplication */ export class ViewProcessor { + private slotScanner: SlotScanner; + constructor( private metadataStore: MetadataStore, private drivers: AFSDriver[], - ) {} + ) { + this.slotScanner = new SlotScanner(metadataStore); + } /** * Find a driver that can handle the given view @@ -44,9 +49,9 @@ export class ViewProcessor { * - For text content: SHA-256 hash * - For binary/other: mtime + size */ - computeRevision(entry: AFSEntry): string { + async computeRevision(entry: AFSEntry): Promise { if (typeof entry.content === "string") { - const hash = createHash("sha256").update(entry.content).digest("hex").substring(0, 16); + const hash = await sha256Hash(entry.content); return `hash:sha256:${hash}`; } @@ -93,7 +98,7 @@ export class ViewProcessor { throw new Error(`Source file not found: ${path}`); } - const sourceRevision = this.computeRevision(sourceResult.data); + const sourceRevision = await this.computeRevision(sourceResult.data); await this.metadataStore.setSourceMetadata(module.name, path, { sourceRevision, updatedAt: new Date(), @@ -222,14 +227,14 @@ export class ViewProcessor { /** * Update source metadata after write */ - async handleWrite(module: string, path: string, entry: AFSEntry): Promise { - const newRevision = this.computeRevision(entry); + async handleWrite(module: AFSModule, path: string, entry: AFSEntry): Promise { + const newRevision = await this.computeRevision(entry); // Get old metadata - const oldMeta = await this.metadataStore.getSourceMetadata(module, path); + const oldMeta = await this.metadataStore.getSourceMetadata(module.name, path); // Update source metadata - await this.metadataStore.setSourceMetadata(module, path, { + await this.metadataStore.setSourceMetadata(module.name, path, { sourceRevision: newRevision, updatedAt: new Date(), driversHint: this.drivers.map((d) => d.name), @@ -237,16 +242,50 @@ export class ViewProcessor { // If revision changed, mark all views as stale if (!oldMeta || oldMeta.sourceRevision !== newRevision) { - await this.metadataStore.markViewsAsStale(module, path); + await this.metadataStore.markViewsAsStale(module.name, path); + + // Mark dependent views as stale (for images that depend on this document) + await this.markDependentViewsStale(module.name, path); + } + + // Scan for image slots if content is text + if (typeof entry.content === "string") { + await this.slotScanner.scan(module, path, entry.content, newRevision); + } + } + + /** + * Mark views that depend on the given input path as stale + * Used for dependency propagation (e.g., images depend on owner document context) + */ + private async markDependentViewsStale(module: string, inPath: string): Promise { + // Query all dependencies where this path is an input + const deps = await this.metadataStore.listDependenciesByInput(module, inPath); + + // Mark each dependent view as stale + for (const dep of deps) { + // Parse viewKey back to View object + const view: View = {}; + dep.outViewKey.split(";").forEach((pair) => { + const [key, value] = pair.split("="); + if (key && value) { + view[key as keyof View] = value; + } + }); + + await this.metadataStore.setViewMetadata(module, dep.outPath, view, { + state: "stale", + }); } } /** * Clean up metadata after delete */ - async handleDelete(module: string, path: string): Promise { - await this.metadataStore.deleteViewMetadata(module, path); - await this.metadataStore.deleteSourceMetadata(module, path); + async handleDelete(module: AFSModule, path: string): Promise { + await this.metadataStore.deleteViewMetadata(module.name, path); + await this.metadataStore.deleteSourceMetadata(module.name, path); + await this.metadataStore.deleteSlots(module.name, path); } /** diff --git a/afs/core/test/slot-scanner.test.ts b/afs/core/test/slot-scanner.test.ts new file mode 100644 index 000000000..302a62cb3 --- /dev/null +++ b/afs/core/test/slot-scanner.test.ts @@ -0,0 +1,312 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { rmSync } from "node:fs"; +import { SQLiteMetadataStore } from "../src/metadata/index.js"; +import { SlotScanner } from "../src/slot-scanner.js"; +import type { AFSModule } from "../src/type.js"; + +// Mock module for testing +class MockModule implements AFSModule { + readonly name = "test-module"; + private files = new Map(); + + async read(path: string) { + const content = this.files.get(path); + return content ? { data: { id: path, path, content } } : { data: undefined }; + } + + async write(path: string, payload: { content: string }) { + this.files.set(path, payload.content); + return { data: { id: path, path, content: payload.content } }; + } +} + +describe("SlotScanner", () => { + const testDbPath = ".test-slot-scanner.db"; + let metadataStore: SQLiteMetadataStore; + let scanner: SlotScanner; + let module: MockModule; + + beforeEach(() => { + metadataStore = new SQLiteMetadataStore({ url: `file:${testDbPath}` }); + scanner = new SlotScanner(metadataStore); + module = new MockModule(); + }); + + afterEach(() => { + rmSync(testDbPath, { force: true }); + }); + + describe("Basic slot parsing", () => { + test("should parse single image slot", async () => { + const content = ` +# Document + +Some text here. + + + +More text. +`; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + + expect(slots.length).toBe(1); + const slot = slots[0]; + expect(slot).toBeDefined(); + expect(slot?.id).toBe("hero"); + expect(slot?.desc).toBe("A beautiful sunset over mountains"); + expect(slot?.key).toBeUndefined(); + expect(slot?.intentKey).toBeDefined(); + expect(slot?.assetPath).toBe(`.afs/images/by-intent/${slot?.intentKey}`); + }); + + test("should parse multiple image slots", async () => { + const content = ` + + + +`; + + const slots = await scanner.scan(module, "/multi.md", content, "rev1"); + + expect(slots.length).toBe(3); + expect(slots[0]?.id).toBe("img1"); + expect(slots[1]?.id).toBe("img2"); + expect(slots[2]?.id).toBe("img3"); + }); + + test("should parse slot with explicit key", async () => { + const content = ``; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + + expect(slots.length).toBe(1); + expect(slots[0]?.key).toBe("company-logo"); + expect(slots[0]?.intentKey).toBe("company-logo"); // Explicit key takes precedence + }); + + test("should handle slots with various id formats", async () => { + const content = ` + + + + + +`; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + + expect(slots.length).toBe(5); + expect(slots.map((s) => s.id)).toEqual([ + "simple", + "with-dash", + "with_underscore", + "with.dot", + "mix-all_these.123", + ]); + }); + }); + + describe("IntentKey computation", () => { + test("should generate same intentKey for same description", async () => { + const content1 = ``; + const content2 = ``; + + const slots1 = await scanner.scan(module, "/doc1.md", content1, "rev1"); + const slots2 = await scanner.scan(module, "/doc2.md", content2, "rev1"); + + expect(slots1[0]?.intentKey).toBe(slots2[0]?.intentKey); + }); + + test("should generate same intentKey for descriptions differing only in whitespace/case", async () => { + const content1 = ``; + const content2 = ``; + + const slots1 = await scanner.scan(module, "/doc1.md", content1, "rev1"); + const slots2 = await scanner.scan(module, "/doc2.md", content2, "rev1"); + + // Normalization: trim, lowercase, collapse spaces + expect(slots1[0]?.intentKey).toBe(slots2[0]?.intentKey); + }); + + test("should generate different intentKey for different descriptions", async () => { + const content = ` + + +`; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + + expect(slots[0]?.intentKey).not.toBe(slots[1]?.intentKey); + }); + + test("should use explicit key instead of hash", async () => { + const content1 = ``; + const content2 = ``; + + const slots1 = await scanner.scan(module, "/doc1.md", content1, "rev1"); + const slots2 = await scanner.scan(module, "/doc2.md", content2, "rev1"); + + // Same explicit key = same intentKey, even though descriptions differ + expect(slots1[0]?.intentKey).toBe("shared-logo"); + expect(slots2[0]?.intentKey).toBe("shared-logo"); + expect(slots1[0]?.intentKey).toBe(slots2[0]?.intentKey); + }); + }); + + describe("Error handling", () => { + test("should reject duplicate slot ids within same document", async () => { + const content = ` + + +`; + + await expect(scanner.scan(module, "/test.md", content, "rev1")).rejects.toThrow( + 'Duplicate slot id "dup"', + ); + }); + + test("should allow same id in different documents", async () => { + const content = ``; + + const slots1 = await scanner.scan(module, "/doc1.md", content, "rev1"); + const slots2 = await scanner.scan(module, "/doc2.md", content, "rev1"); + + expect(slots1.length).toBe(1); + expect(slots2.length).toBe(1); + expect(slots1[0]?.id).toBe("hero"); + expect(slots2[0]?.id).toBe("hero"); + }); + + test("should skip malformed slots", async () => { + const content = ` + + + + +`; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + + // Only the valid one should be parsed + expect(slots.length).toBe(1); + expect(slots[0]?.id).toBe("valid"); + }); + }); + + describe("Database integration", () => { + test("should upsert slot metadata to database", async () => { + const content = ``; + + await scanner.scan(module, "/test.md", content, "rev1"); + + const slot = await metadataStore.getSlot(module.name, "/test.md", "test-img"); + + expect(slot).toBeDefined(); + expect(slot?.slotId).toBe("test-img"); + expect(slot?.ownerPath).toBe("/test.md"); + expect(slot?.desc).toBe("Test image"); + expect(slot?.slotType).toBe("image"); + }); + + test("should create image node in source_metadata", async () => { + const content = ``; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + const slot = slots[0]; + expect(slot).toBeDefined(); + const imagePath = slot?.assetPath; + + const sourceMeta = await metadataStore.getSourceMetadata(module.name, imagePath ?? ""); + + expect(sourceMeta).toBeDefined(); + expect(sourceMeta?.kind).toBe("image"); + expect(sourceMeta?.sourceRevision).toBe(`intent:${slot?.intentKey}`); + expect(sourceMeta?.driversHint).toContain("image-generate"); + }); + + test("should update slot when content changes", async () => { + const content1 = ``; + const content2 = ``; + + await scanner.scan(module, "/test.md", content1, "rev1"); + await scanner.scan(module, "/test.md", content2, "rev2"); + + const slot = await metadataStore.getSlot(module.name, "/test.md", "img"); + + expect(slot?.desc).toBe("New description"); + expect(slot?.ownerRevision).toBe("rev2"); + }); + + test("should list all slots for a document", async () => { + const content = ` + + + +`; + + await scanner.scan(module, "/test.md", content, "rev1"); + + const slots = await metadataStore.listSlots(module.name, "/test.md"); + + expect(slots.length).toBe(3); + expect(slots.map((s) => s.slotId).sort()).toEqual(["img1", "img2", "img3"]); + }); + + test("should query slot by asset path", async () => { + const content = ``; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + const assetPath = slots[0]?.assetPath ?? ""; + + const slot = await metadataStore.getSlotByAssetPath(module.name, assetPath); + + expect(slot).toBeDefined(); + expect(slot?.slotId).toBe("test"); + expect(slot?.assetPath).toBe(assetPath); + }); + }); + + describe("Intent-based deduplication", () => { + test("should reuse same asset path for identical descriptions", async () => { + const content1 = ``; + const content2 = ``; // Case/whitespace differ + + const slots1 = await scanner.scan(module, "/doc1.md", content1, "rev1"); + const slots2 = await scanner.scan(module, "/doc2.md", content2, "rev1"); + + // Same normalized description = same intentKey = same assetPath + expect(slots1[0]?.assetPath).toBe(slots2[0]?.assetPath); + + // Both slots should point to the same image node + const allSlots = [ + ...(await metadataStore.listSlots(module.name, "/doc1.md")), + ...(await metadataStore.listSlots(module.name, "/doc2.md")), + ]; + + expect(allSlots.length).toBe(2); + expect(allSlots[0]?.assetPath).toBe(allSlots[1]?.assetPath); + }); + + test("should create multiple image nodes for different intents", async () => { + const content = ` + + + +`; + + const slots = await scanner.scan(module, "/test.md", content, "rev1"); + + const assetPaths = slots.map((s) => s.assetPath); + + // All different descriptions = different asset paths + expect(new Set(assetPaths).size).toBe(3); + + // All should have corresponding image nodes + for (const assetPath of assetPaths) { + const meta = await metadataStore.getSourceMetadata(module.name, assetPath); + expect(meta?.kind).toBe("image"); + } + }); + }); +}); diff --git a/afs/core/test/view-key.test.ts b/afs/core/test/view-key.test.ts new file mode 100644 index 000000000..63773f7df --- /dev/null +++ b/afs/core/test/view-key.test.ts @@ -0,0 +1,195 @@ +import { describe, expect, test } from "bun:test"; +import type { View } from "../src/type.js"; +import { normalizeViewKey } from "../src/view-key.js"; + +describe("normalizeViewKey", () => { + describe("Basic normalization", () => { + test("should normalize single dimension", () => { + const view: View = { language: "en" }; + expect(normalizeViewKey(view)).toBe("language=en"); + }); + + test("should normalize multiple dimensions in fixed order", () => { + const view: View = { + language: "zh", + format: "png", + }; + expect(normalizeViewKey(view)).toBe("language=zh;format=png"); + }); + + test("should use fixed order regardless of input order", () => { + // Different input orders should produce the same output + const view1: View = { format: "png", language: "en" }; + const view2: View = { language: "en", format: "png" }; + + const key1 = normalizeViewKey(view1); + const key2 = normalizeViewKey(view2); + + expect(key1).toBe(key2); + expect(key1).toBe("language=en;format=png"); + }); + + test("should handle all four dimensions", () => { + const view: View = { + policy: "technical", + variant: "summary", + format: "html", + language: "ja", + }; + + // Fixed order: language → format → variant → policy + expect(normalizeViewKey(view)).toBe( + "language=ja;format=html;variant=summary;policy=technical", + ); + }); + }); + + describe("Case and whitespace normalization", () => { + test("should convert to lowercase", () => { + const view: View = { language: "EN", format: "PNG" }; + expect(normalizeViewKey(view)).toBe("language=en;format=png"); + }); + + test("should trim whitespace", () => { + const view: View = { language: " en ", format: " png " }; + expect(normalizeViewKey(view)).toBe("language=en;format=png"); + }); + + test("should handle mixed case and whitespace", () => { + const view: View = { + language: " ZH ", + format: "WebP ", + variant: " Summary", + }; + expect(normalizeViewKey(view)).toBe("language=zh;format=webp;variant=summary"); + }); + }); + + describe("Empty and partial views", () => { + test("should handle empty view", () => { + const view: View = {}; + expect(normalizeViewKey(view)).toBe(""); + }); + + test("should omit undefined dimensions", () => { + const view: View = { language: "en", format: undefined }; + expect(normalizeViewKey(view)).toBe("language=en"); + }); + + test("should handle only middle dimension", () => { + const view: View = { format: "png" }; + expect(normalizeViewKey(view)).toBe("format=png"); + }); + + test("should handle only last dimension", () => { + const view: View = { policy: "marketing" }; + expect(normalizeViewKey(view)).toBe("policy=marketing"); + }); + + test("should handle sparse dimensions", () => { + const view: View = { language: "en", policy: "technical" }; + // format and variant omitted + expect(normalizeViewKey(view)).toBe("language=en;policy=technical"); + }); + }); + + describe("Consistency guarantees", () => { + test("same view object should produce same key", () => { + const view: View = { language: "zh", format: "png" }; + const key1 = normalizeViewKey(view); + const key2 = normalizeViewKey(view); + expect(key1).toBe(key2); + }); + + test("equivalent views with different casing should produce same key", () => { + const view1: View = { language: "EN", format: "PNG" }; + const view2: View = { language: "en", format: "png" }; + expect(normalizeViewKey(view1)).toBe(normalizeViewKey(view2)); + }); + + test("equivalent views with different whitespace should produce same key", () => { + const view1: View = { language: "en", format: "png" }; + const view2: View = { language: " en ", format: "png " }; + expect(normalizeViewKey(view1)).toBe(normalizeViewKey(view2)); + }); + + test("should be stable across different key insertion orders", () => { + // Simulate different object creation patterns + const view1: View = {}; + view1.language = "zh"; + view1.format = "webp"; + view1.variant = "thumbnail"; + + const view2: View = {}; + view2.variant = "thumbnail"; + view2.language = "zh"; + view2.format = "webp"; + + expect(normalizeViewKey(view1)).toBe(normalizeViewKey(view2)); + }); + }); + + describe("Real-world use cases", () => { + test("i18n translation view", () => { + const view: View = { language: "ja" }; + expect(normalizeViewKey(view)).toBe("language=ja"); + }); + + test("image format conversion", () => { + const view: View = { format: "webp" }; + expect(normalizeViewKey(view)).toBe("format=webp"); + }); + + test("translated image", () => { + const view: View = { language: "zh", format: "png" }; + expect(normalizeViewKey(view)).toBe("language=zh;format=png"); + }); + + test("document summary in different language", () => { + const view: View = { language: "en", variant: "summary" }; + expect(normalizeViewKey(view)).toBe("language=en;variant=summary"); + }); + + test("marketing content in specific format", () => { + const view: View = { + language: "zh", + format: "html", + policy: "marketing", + }; + expect(normalizeViewKey(view)).toBe("language=zh;format=html;policy=marketing"); + }); + + test("full view specification", () => { + const view: View = { + language: "ja", + format: "pdf", + variant: "toc", + policy: "technical", + }; + expect(normalizeViewKey(view)).toBe("language=ja;format=pdf;variant=toc;policy=technical"); + }); + }); + + describe("Edge cases", () => { + test("should handle empty strings", () => { + const view: View = { language: "", format: "png" }; + // Empty string is falsy after trim, should be omitted + expect(normalizeViewKey(view)).toBe("format=png"); + }); + + test("should handle only whitespace", () => { + const view: View = { language: " ", format: "png" }; + // Only whitespace becomes empty after trim, should be omitted + expect(normalizeViewKey(view)).toBe("format=png"); + }); + + test("should handle special characters in values", () => { + const view: View = { + language: "zh-CN", + format: "image/png", + variant: "v1.0", + }; + expect(normalizeViewKey(view)).toBe("language=zh-cn;format=image/png;variant=v1.0"); + }); + }); +}); diff --git a/afs/i18n-driver/test/i18n-driver.test.ts b/afs/i18n-driver/test/i18n-driver.test.ts index 6ac719b4c..cc2fc86aa 100644 --- a/afs/i18n-driver/test/i18n-driver.test.ts +++ b/afs/i18n-driver/test/i18n-driver.test.ts @@ -121,7 +121,6 @@ test("I18nDriver.canHandle should return true for language-only views", () => { expect(driver.canHandle({ language: "en" })).toBe(true); expect(driver.canHandle({ language: "ja" })).toBe(true); expect(driver.canHandle({})).toBe(false); - // @ts-expect-error - testing invalid view expect(driver.canHandle({ language: "en", format: "html" })).toBe(false); }); From 55629682f4afb48cd05dd22f7e4c583d7227fb8e Mon Sep 17 00:00:00 2001 From: LBan Date: Fri, 26 Dec 2025 18:11:14 +0800 Subject: [PATCH 17/21] feat(image-driver): add AFS image generation driver with support for AI-generated images and metadata handling --- afs/core/src/afs.ts | 103 +++++ afs/core/src/index.ts | 1 + afs/core/src/metadata/migrations/001-init.ts | 1 + .../src/metadata/models/slots-metadata.ts | 1 + afs/core/src/metadata/store.ts | 5 + afs/core/src/metadata/type.ts | 1 + afs/core/src/slot-scanner.ts | 24 +- afs/core/src/type.ts | 1 + afs/core/src/view-processor.ts | 31 +- afs/image-driver/package.json | 70 ++++ .../scripts/tsconfig.build.cjs.json | 8 + .../scripts/tsconfig.build.dts.json | 10 + .../scripts/tsconfig.build.esm.json | 8 + afs/image-driver/scripts/tsconfig.build.json | 16 + .../src/default-generation-agent.ts | 92 ++++ afs/image-driver/src/driver.ts | 211 ++++++++++ afs/image-driver/src/index.ts | 10 + afs/image-driver/src/storage.ts | 27 ++ afs/image-driver/test/image-driver.test.ts | 394 ++++++++++++++++++ afs/image-driver/tsconfig.json | 11 + pnpm-lock.yaml | 28 ++ 21 files changed, 1047 insertions(+), 6 deletions(-) create mode 100644 afs/image-driver/package.json create mode 100644 afs/image-driver/scripts/tsconfig.build.cjs.json create mode 100644 afs/image-driver/scripts/tsconfig.build.dts.json create mode 100644 afs/image-driver/scripts/tsconfig.build.esm.json create mode 100644 afs/image-driver/scripts/tsconfig.build.json create mode 100644 afs/image-driver/src/default-generation-agent.ts create mode 100644 afs/image-driver/src/driver.ts create mode 100644 afs/image-driver/src/index.ts create mode 100644 afs/image-driver/src/storage.ts create mode 100644 afs/image-driver/test/image-driver.test.ts create mode 100644 afs/image-driver/tsconfig.json diff --git a/afs/core/src/afs.ts b/afs/core/src/afs.ts index 587e673e2..fea706fd4 100644 --- a/afs/core/src/afs.ts +++ b/afs/core/src/afs.ts @@ -31,6 +31,7 @@ import { afsEntrySchema, type View, } from "./type.js"; +import { normalizeViewKey } from "./view-key.js"; import { ViewProcessor } from "./view-processor.js"; const DEFAULT_MAX_DEPTH = 1; @@ -483,6 +484,108 @@ export class AFS extends Emitter implements AFSRoot { return metadataSuffix; } + /** + * Get slot metadata by owner path and slot ID + * @param ownerPath - Document path that declares the slot + * @param slotId - Slot ID + * @returns Slot metadata or null if not found + */ + async getSlot(ownerPath: string, slotId: string) { + if (!this.metadataStore) { + throw new Error("MetadataStore not initialized. Drivers must be configured to use slots."); + } + + const module = this.findModules(ownerPath, { exactMatch: true })[0]; + if (!module) { + throw new Error(`No module found for path: ${ownerPath}`); + } + + return await this.metadataStore.getSlot(module.module.name, module.subpath, slotId); + } + + /** + * Read image by slot (convenience method) + * @param ownerPath - Document path that declares the slot + * @param slotId - Slot ID + * @param options - Read options with view specification + * @returns AFSReadResult with the image + */ + async getImageBySlot( + ownerPath: string, + slotId: string, + options?: AFSReadOptions, + ): Promise { + const slot = await this.getSlot(ownerPath, slotId); + if (!slot) { + throw new Error(`Slot "${slotId}" not found in document: ${ownerPath}`); + } + + const module = this.findModules(ownerPath, { exactMatch: true })[0]; + if (!module) { + throw new Error(`No module found for path: ${ownerPath}`); + } + + // Construct full path for the image + const imagePath = joinURL(MODULES_ROOT_DIR, module.module.name, slot.assetPath); + + return await this.read(imagePath, options); + } + + /** + * Render slots in document content by replacing slot markers with image references + * @param ownerPath - Document path + * @param content - Document content with slot markers + * @param options - Rendering options + * @returns Content with slots replaced by image references + */ + async renderSlots( + ownerPath: string, + content: string, + options?: { + view?: View; + format?: (slot: any, imagePath: string) => string; + }, + ): Promise { + if (!this.metadataStore) { + throw new Error("MetadataStore not initialized. Drivers must be configured to use slots."); + } + + const module = this.findModules(ownerPath, { exactMatch: true })[0]; + if (!module) { + throw new Error(`No module found for path: ${ownerPath}`); + } + + // Get all slots for this document + const slots = await this.metadataStore.listSlots(module.module.name, module.subpath); + + let rendered = content; + + // Replace each slot marker with image reference + for (const slot of slots) { + // Construct image path based on view + let imagePath = slot.assetPath; + if (options?.view) { + const viewKey = normalizeViewKey(options.view); + const format = options.view.format || "png"; + imagePath = `${slot.assetPath}/${viewKey}/${slot.slug}.${format}`; + } + + // Use custom format function or default markdown image syntax + const replacement = options?.format + ? options.format(slot, imagePath) + : `![${slot.desc}](${imagePath})`; + + // Replace slot marker with image reference + const slotPattern = new RegExp( + ``, + "g", + ); + rendered = rendered.replace(slotPattern, replacement); + } + + return rendered; + } + /** * Prefetch views for batch generation * @param pathOrGlob - Single path or array of paths (glob support TBD) diff --git a/afs/core/src/index.ts b/afs/core/src/index.ts index fa84e5695..3bbd1bf6c 100644 --- a/afs/core/src/index.ts +++ b/afs/core/src/index.ts @@ -1,5 +1,6 @@ export * from "./afs.js"; export * from "./metadata/index.js"; export * from "./type.js"; +export * from "./view-key.js"; export * from "./view-processor.js"; export * from "./view-schema.js"; diff --git a/afs/core/src/metadata/migrations/001-init.ts b/afs/core/src/metadata/migrations/001-init.ts index 5b2470435..1124f1468 100644 --- a/afs/core/src/metadata/migrations/001-init.ts +++ b/afs/core/src/metadata/migrations/001-init.ts @@ -52,6 +52,7 @@ CREATE TABLE IF NOT EXISTS afs_slots ( desc TEXT NOT NULL, intent_key TEXT NOT NULL, asset_path TEXT NOT NULL, + slug TEXT NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (owner_path, slot_id) )`, diff --git a/afs/core/src/metadata/models/slots-metadata.ts b/afs/core/src/metadata/models/slots-metadata.ts index d7daf82fb..2c42970af 100644 --- a/afs/core/src/metadata/models/slots-metadata.ts +++ b/afs/core/src/metadata/models/slots-metadata.ts @@ -14,6 +14,7 @@ export const slotsMetadataTable = sqliteTable( desc: text("desc").notNull(), // prompt seed for image generation intentKey: text("intent_key").notNull(), // hash(normalize(desc)) or explicit key assetPath: text("asset_path").notNull(), // .afs/images/by-intent/ + slug: text("slug").notNull(), // human-readable name for file storage updatedAt: integer("updated_at", { mode: "timestamp_ms" }).notNull(), }, (table) => ({ diff --git a/afs/core/src/metadata/store.ts b/afs/core/src/metadata/store.ts index 6f49b5be4..374dece90 100644 --- a/afs/core/src/metadata/store.ts +++ b/afs/core/src/metadata/store.ts @@ -419,6 +419,7 @@ export class SQLiteMetadataStore implements MetadataStore { desc: row.desc, intentKey: row.intentKey, assetPath: row.assetPath, + slug: row.slug, updatedAt: new Date(row.updatedAt), }; } @@ -440,6 +441,7 @@ export class SQLiteMetadataStore implements MetadataStore { desc: row.desc, intentKey: row.intentKey, assetPath: row.assetPath, + slug: row.slug, updatedAt: new Date(row.updatedAt), })); } @@ -465,6 +467,7 @@ export class SQLiteMetadataStore implements MetadataStore { desc: row.desc, intentKey: row.intentKey, assetPath: row.assetPath, + slug: row.slug, updatedAt: new Date(row.updatedAt), }; } @@ -482,6 +485,7 @@ export class SQLiteMetadataStore implements MetadataStore { desc: slot.desc, intentKey: slot.intentKey, assetPath: slot.assetPath, + slug: slot.slug, updatedAt: now, }) .where( @@ -502,6 +506,7 @@ export class SQLiteMetadataStore implements MetadataStore { desc: slot.desc, intentKey: slot.intentKey, assetPath: slot.assetPath, + slug: slot.slug, updatedAt: now, }) .execute(); diff --git a/afs/core/src/metadata/type.ts b/afs/core/src/metadata/type.ts index a6ee5e0be..82b239d01 100644 --- a/afs/core/src/metadata/type.ts +++ b/afs/core/src/metadata/type.ts @@ -49,6 +49,7 @@ export interface SlotMetadata { desc: string; // prompt seed intentKey: string; // hash(normalize(desc)) or explicit key assetPath: string; // .afs/images/by-intent/ + slug: string; // human-readable name for file storage updatedAt: Date; } diff --git a/afs/core/src/slot-scanner.ts b/afs/core/src/slot-scanner.ts index 28ef5672b..9ec6e9069 100644 --- a/afs/core/src/slot-scanner.ts +++ b/afs/core/src/slot-scanner.ts @@ -12,6 +12,24 @@ function normalizeDesc(desc: string): string { return desc.trim().toLowerCase().replace(/\s+/g, " "); } +/** + * Generate human-readable slug from description + * - Convert to lowercase + * - Remove non-alphanumeric characters (except spaces and hyphens) + * - Replace spaces with hyphens + * - Limit to 50 characters + */ +function generateSlug(desc: string): string { + return desc + .toLowerCase() + .trim() + .replace(/[^\w\s-]/g, "") + .replace(/\s+/g, "-") + .replace(/-+/g, "-") + .substring(0, 50) + .replace(/^-|-$/g, ""); // Remove leading/trailing hyphens +} + /** * Compute intentKey from description or use explicit key */ @@ -71,9 +89,10 @@ export class SlotScanner { } seenIds.add(id); - // Compute intentKey + // Compute intentKey and slug const intentKey = await computeIntentKey(desc, key); - const assetPath = `.afs/images/by-intent/${intentKey}`; + const slug = generateSlug(desc); + const assetPath = `/.afs/images/by-intent/${intentKey}`; slots.push({ id, desc, key, intentKey, assetPath }); @@ -86,6 +105,7 @@ export class SlotScanner { desc, intentKey, assetPath, + slug, }); // Ensure image node exists in source_metadata diff --git a/afs/core/src/type.ts b/afs/core/src/type.ts index 9c2164c81..c317f7905 100644 --- a/afs/core/src/type.ts +++ b/afs/core/src/type.ts @@ -285,6 +285,7 @@ export interface AFSDriver { sourceEntry: AFSEntry; metadata: any; context: any; + metadataStore?: any; // MetadataStore instance (for drivers that need access to metadata) }, ): Promise<{ data: AFSEntry; message?: string }>; diff --git a/afs/core/src/view-processor.ts b/afs/core/src/view-processor.ts index 68d30fde6..b68436cef 100644 --- a/afs/core/src/view-processor.ts +++ b/afs/core/src/view-processor.ts @@ -118,10 +118,28 @@ export class ViewProcessor { derivedFrom: sourceMeta.sourceRevision, }); - // 3. Read source - const sourceResult = await module.read?.(path); - if (!sourceResult?.data) { - throw new Error(`Source file not found: ${path}`); + // 3. Read source (skip for virtual nodes like images) + let sourceResult: { data?: AFSEntry } | undefined; + + // For image nodes (kind="image"), source content is not needed + // The driver will fetch context from owner document via slot metadata + if (sourceMeta.kind === "image") { + // Create a placeholder entry for image nodes + sourceResult = { + data: { + id: path, + path, + content: "", // No actual content for virtual image nodes + createdAt: new Date(), + updatedAt: new Date(), + }, + }; + } else { + // For regular documents, read the actual source + sourceResult = await module.read?.(path); + if (!sourceResult?.data) { + throw new Error(`Source file not found: ${path}`); + } } // 4. Find and call driver @@ -130,10 +148,15 @@ export class ViewProcessor { throw new Error(`No driver found for view: ${JSON.stringify(view)}`); } + if (!sourceResult?.data) { + throw new Error(`Failed to get source entry for ${path}`); + } + const result = await driver.process(module, path, view, { sourceEntry: sourceResult.data, metadata: { derivedFrom: sourceMeta.sourceRevision }, context, + metadataStore: this.metadataStore, }); // 5. Update to ready diff --git a/afs/image-driver/package.json b/afs/image-driver/package.json new file mode 100644 index 000000000..0f8420bd4 --- /dev/null +++ b/afs/image-driver/package.json @@ -0,0 +1,70 @@ +{ + "name": "@aigne/afs-image-driver", + "version": "0.0.1", + "description": "AIGNE AFS driver for AI image generation", + "publishConfig": { + "access": "public" + }, + "author": "Arcblock https://github.com/blocklet", + "homepage": "https://www.aigne.io/framework", + "license": "Elastic-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/AIGNE-io/aigne-framework" + }, + "bugs": { + "url": "https://github.com/AIGNE-io/aigne-framework/issues" + }, + "files": [ + "lib/cjs", + "lib/dts", + "lib/esm", + "LICENSE", + "README.md", + "CHANGELOG.md" + ], + "type": "module", + "main": "./lib/cjs/index.js", + "module": "./lib/esm/index.js", + "types": "./lib/dts/index.d.ts", + "exports": { + ".": { + "import": "./lib/esm/index.js", + "require": "./lib/cjs/index.js", + "types": "./lib/dts/index.d.ts" + }, + "./*": { + "import": "./lib/esm/*", + "require": "./lib/cjs/*", + "types": "./lib/dts/*" + } + }, + "typesVersions": { + "*": { + ".": [ + "./lib/dts/index.d.ts" + ] + } + }, + "scripts": { + "lint": "tsc --noEmit", + "build": "tsc --build scripts/tsconfig.build.json", + "clean": "rimraf lib test/coverage", + "prepublishOnly": "run-s clean build", + "test": "bun test", + "test:coverage": "bun test --coverage --coverage-reporter=lcov --coverage-reporter=text", + "postbuild": "node ../../scripts/post-build-lib.mjs" + }, + "dependencies": { + "@aigne/afs": "workspace:^", + "@aigne/core": "workspace:^", + "@aigne/gemini": "workspace:^", + "zod": "^3.25.67" + }, + "devDependencies": { + "@types/bun": "^1.2.22", + "npm-run-all": "^4.1.5", + "rimraf": "^6.0.1", + "typescript": "^5.9.2" + } +} diff --git a/afs/image-driver/scripts/tsconfig.build.cjs.json b/afs/image-driver/scripts/tsconfig.build.cjs.json new file mode 100644 index 000000000..b48de9c1e --- /dev/null +++ b/afs/image-driver/scripts/tsconfig.build.cjs.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.build.json", + "compilerOptions": { + "module": "CommonJS", + "moduleResolution": "node", + "outDir": "../lib/cjs" + } +} diff --git a/afs/image-driver/scripts/tsconfig.build.dts.json b/afs/image-driver/scripts/tsconfig.build.dts.json new file mode 100644 index 000000000..f5bb390ce --- /dev/null +++ b/afs/image-driver/scripts/tsconfig.build.dts.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.build.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "nodenext", + "outDir": "../lib/dts", + "declaration": true, + "emitDeclarationOnly": true + } +} diff --git a/afs/image-driver/scripts/tsconfig.build.esm.json b/afs/image-driver/scripts/tsconfig.build.esm.json new file mode 100644 index 000000000..d2d7edebe --- /dev/null +++ b/afs/image-driver/scripts/tsconfig.build.esm.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.build.json", + "compilerOptions": { + "module": "NodeNext", + "moduleResolution": "nodenext", + "outDir": "../lib/esm" + } +} diff --git a/afs/image-driver/scripts/tsconfig.build.json b/afs/image-driver/scripts/tsconfig.build.json new file mode 100644 index 000000000..fabac44ae --- /dev/null +++ b/afs/image-driver/scripts/tsconfig.build.json @@ -0,0 +1,16 @@ +{ + "files": [], + "extends": "../tsconfig.json", + "compilerOptions": { + "composite": true, + "rootDir": "../src", + "noEmit": false, + "paths": {} + }, + "include": ["../src"], + "references": [ + { "path": "./tsconfig.build.cjs.json" }, + { "path": "./tsconfig.build.esm.json" }, + { "path": "./tsconfig.build.dts.json" } + ] +} diff --git a/afs/image-driver/src/default-generation-agent.ts b/afs/image-driver/src/default-generation-agent.ts new file mode 100644 index 000000000..014b33775 --- /dev/null +++ b/afs/image-driver/src/default-generation-agent.ts @@ -0,0 +1,92 @@ +import { GeminiImageModel } from "@aigne/gemini"; +import { z } from "zod"; + +/** + * Image generation agent input schema + */ +export const imageGenerationInputSchema = z.object({ + desc: z.string().describe("Image description (prompt seed)"), + context: z.string().optional().describe("Owner document context to enrich the generation"), +}); + +/** + * Image generation agent output schema + */ +export const imageGenerationOutputSchema = z.object({ + imageData: z.string().describe("Generated image data (base64 encoded)"), + mimeType: z.string().optional().describe("Image MIME type"), +}); + +export type ImageGenerationInput = z.infer; +export type ImageGenerationOutput = z.infer; + +/** + * Build prompt for image generation + * Combines description with owner document context + */ +function buildPrompt(desc: string, context?: string): string { + if (!context) { + return desc; + } + + // Combine context and description for richer generation + return `Based on the following document context, generate an image that matches the description. + +Document Context: +${context} + +Image Description: ${desc} + +Please generate an image that: +- Accurately represents the description +- Is consistent with the document context +- Has professional quality and clear visual elements`; +} + +/** + * Create the default built-in image generation agent + */ +export function createDefaultImageGenerationAgent(options?: { + model?: string; + apiKey?: string; +}): GeminiImageModel { + return new GeminiImageModel({ + apiKey: options?.apiKey, + model: options?.model || "gemini-2.5-flash", + description: "Built-in image generation agent for image-driver", + }); +} + +/** + * Generate image using the agent + * Handles prompt construction and output format + */ +export async function generateImage( + agent: GeminiImageModel, + input: ImageGenerationInput, + format: string = "png", +): Promise { + const prompt = buildPrompt(input.desc, input.context); + + // Determine output MIME type based on format + const mimeType = format === "webp" ? "image/webp" : "image/png"; + + const result = await agent.invoke({ + prompt, + outputFileType: "file", + modelOptions: { + outputMimeType: mimeType, + }, + }); + + // Extract first image from results + const firstImage = result.images[0]; + if (!firstImage || firstImage.type !== "file") { + throw new Error("Failed to generate image: no image data returned"); + } + + return { + imageData: firstImage.data, + mimeType: firstImage.mimeType || mimeType, + }; +} diff --git a/afs/image-driver/src/driver.ts b/afs/image-driver/src/driver.ts new file mode 100644 index 000000000..39d66933b --- /dev/null +++ b/afs/image-driver/src/driver.ts @@ -0,0 +1,211 @@ +import type { AFSDriver, AFSEntry, AFSModule, View } from "@aigne/afs"; +import { normalizeViewKey } from "@aigne/afs"; +import type { Context } from "@aigne/core"; +import { optionalize } from "@aigne/core/loader/schema.js"; +import type { GeminiImageModel } from "@aigne/gemini"; +import { z } from "zod"; +import { + createDefaultImageGenerationAgent, + generateImage, + type ImageGenerationInput, +} from "./default-generation-agent.js"; +import { getStoragePath } from "./storage.js"; + +/** + * Image Generate Driver configuration options + */ +export interface ImageGenerateDriverOptions { + /** Custom image generation agent (uses built-in agent if not provided) */ + imageGenerationAgent?: GeminiImageModel; + + /** Model to use for generation (default: "gemini-2.5-flash") */ + model?: string; + + /** API key for Gemini */ + apiKey?: string; + + /** Maximum retries on failure (default: 3) */ + maxRetries?: number; +} + +const imageGenerateDriverOptionsSchema = z.object({ + model: optionalize(z.string()), + apiKey: optionalize(z.string()), + maxRetries: optionalize(z.number()), +}); + +/** + * Image Generate Driver for AFS + * + * Handles AI-powered image generation based on slots in documents. + */ +export class ImageGenerateDriver implements AFSDriver { + readonly name = "image-generate"; + readonly description = "AI image generation driver"; + readonly capabilities = { + dimensions: ["format" as const], + }; + + private imageGenerationAgent: GeminiImageModel; + private maxRetries: number; + + static schema() { + return imageGenerateDriverOptionsSchema; + } + + static async load({ parsed }: { parsed?: object }) { + const valid = await ImageGenerateDriver.schema().passthrough().parseAsync(parsed); + return new ImageGenerateDriver(valid); + } + + constructor(private options: ImageGenerateDriverOptions = {}) { + // Use custom agent or create default + this.imageGenerationAgent = + options.imageGenerationAgent ?? + createDefaultImageGenerationAgent({ + model: options.model, + apiKey: options.apiKey, + }); + this.maxRetries = options.maxRetries ?? 3; + } + + /** + * Check if this driver can handle the given view + * Only handles views with format dimension only (for Phase 2) + */ + canHandle(view: View): boolean { + // Must have format + if (!view.format) return false; + + // Phase 2: Only support format dimension (no language, variant, policy) + const dimensions = Object.keys(view).filter((key) => key !== "format"); + if (dimensions.length > 0) return false; + + // Only support png format in Phase 2 + if (view.format !== "png") return false; + + return true; + } + + /** + * Process and generate the image view + */ + async process( + module: AFSModule, + path: string, + view: View, + options: { + sourceEntry: AFSEntry; + metadata: any; + context?: Context; + metadataStore?: any; + }, + ): Promise<{ data: AFSEntry; message?: string }> { + const { format } = view; + const { sourceEntry, context, metadataStore } = options; + + if (!format) { + throw new Error("Format is required for image generation"); + } + + if (!context) { + throw new Error("Context is required for image generation. Pass context via read options."); + } + + if (!metadataStore) { + throw new Error("MetadataStore not found in options"); + } + + // Query slot information from afs_slots + const slot = await metadataStore.getSlotByAssetPath(module.name, path); + if (!slot) { + throw new Error(`No slot found for asset path: ${path}`); + } + + // Read owner document content for context + const ownerResult = await module.read?.(slot.ownerPath); + if (!ownerResult?.data) { + throw new Error(`Failed to read owner document: ${slot.ownerPath}`); + } + + const ownerContent = ownerResult.data.content as string; + + // Generate image with retry logic + const result = await this.generateWithRetry( + { + desc: slot.desc, + context: ownerContent, + }, + format, + ); + + // Compute storage path using slug for readable filename + const viewKey = normalizeViewKey(view); + const storagePath = getStoragePath(path, viewKey, slot.slug, format); + + // Write image to storage + const imageBuffer = Buffer.from(result.imageData, "base64"); + await module.write?.(storagePath, { content: imageBuffer }); + + // Record dependency relationship (image depends on owner document) + const ownerMeta = await metadataStore.getSourceMetadata(module.name, slot.ownerPath); + if (ownerMeta) { + await metadataStore.setDependency(module.name, { + outPath: path, + outViewKey: viewKey, + inPath: slot.ownerPath, + inRevision: ownerMeta.sourceRevision, + role: "owner-context", + }); + } + + // Return generated entry + return { + data: { + ...sourceEntry, + content: imageBuffer, + path, + metadata: { + ...sourceEntry.metadata, + storagePath, + view, + mimeType: result.mimeType, + }, + }, + message: `Image generated successfully for slot "${slot.slotId}"`, + }; + } + + /** + * Generate image with retry logic + */ + private async generateWithRetry( + input: ImageGenerationInput, + format: string, + ): Promise<{ imageData: string; mimeType?: string }> { + let lastError: Error | undefined; + + for (let attempt = 1; attempt <= this.maxRetries; attempt++) { + try { + return await generateImage(this.imageGenerationAgent, input, format); + } catch (error: any) { + lastError = error; + console.warn( + `[ImageGenerateDriver] Attempt ${attempt}/${this.maxRetries} failed:`, + error.message, + ); + + // If still have retries left, wait with exponential backoff + if (attempt < this.maxRetries) { + const delay = Math.pow(2, attempt - 1) * 1000; // 1s, 2s, 4s + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + } + + // All retries failed + throw new Error( + `Failed to generate image after ${this.maxRetries} attempts: ${lastError?.message}`, + ); + } +} diff --git a/afs/image-driver/src/index.ts b/afs/image-driver/src/index.ts new file mode 100644 index 000000000..8f5a1f8c5 --- /dev/null +++ b/afs/image-driver/src/index.ts @@ -0,0 +1,10 @@ +export { + createDefaultImageGenerationAgent, + generateImage, + type ImageGenerationInput, + type ImageGenerationOutput, + imageGenerationInputSchema, + imageGenerationOutputSchema, +} from "./default-generation-agent.js"; +export { ImageGenerateDriver, type ImageGenerateDriverOptions } from "./driver.js"; +export { getStoragePath } from "./storage.js"; diff --git a/afs/image-driver/src/storage.ts b/afs/image-driver/src/storage.ts new file mode 100644 index 000000000..c83b2a9db --- /dev/null +++ b/afs/image-driver/src/storage.ts @@ -0,0 +1,27 @@ +import { join } from "node:path"; + +/** + * Get storage path for a generated image + * Format: //. + * + * @param assetPath - Logical asset path (e.g., ".afs/images/by-intent/abc123") + * @param viewKey - Normalized view key (e.g., "format=png;variant=original") + * @param slug - Human-readable filename (e.g., "company-logo") + * @param format - Image format (e.g., "png", "webp") + * @returns Physical storage path for the generated image + * + * @example + * getStoragePath(".afs/images/by-intent/abc123", "format=png", "company-logo", "png") + * // Returns: ".afs/images/by-intent/abc123/format=png/company-logo.png" + * + * getStoragePath(".afs/images/by-intent/xyz789", "format=webp;variant=thumbnail", "diagram", "webp") + * // Returns: ".afs/images/by-intent/xyz789/format=webp;variant=thumbnail/diagram.webp" + */ +export function getStoragePath( + assetPath: string, + viewKey: string, + slug: string, + format: string, +): string { + return join(assetPath, viewKey, `${slug}.${format}`); +} diff --git a/afs/image-driver/test/image-driver.test.ts b/afs/image-driver/test/image-driver.test.ts new file mode 100644 index 000000000..b22648b37 --- /dev/null +++ b/afs/image-driver/test/image-driver.test.ts @@ -0,0 +1,394 @@ +import { afterEach, expect, spyOn, test } from "bun:test"; +import assert from "node:assert"; +import { rmSync } from "node:fs"; +import { AFS, type AFSEntry, type AFSModule } from "@aigne/afs"; +import { GeminiImageModel } from "@aigne/gemini"; +import { AIGNE } from "@aigne/core"; +import { getStoragePath, ImageGenerateDriver } from "@aigne/afs-image-driver"; + +// Cleanup test database after each test +const testDbPath = ".afs-test"; +afterEach(() => { + try { + rmSync(testDbPath, { recursive: true, force: true }); + } catch (_error) { + // Ignore cleanup errors + } +}); + +// Mock file system module +class MockFSModule implements AFSModule { + readonly name = "mock-fs"; + private files = new Map(); + + constructor(public options: { context: any }) {} + + async read(path: string): Promise<{ data?: AFSEntry; message?: string }> { + const content = this.files.get(path); + if (!content) { + return { data: undefined, message: "File not found" }; + } + + return { + data: { + id: path, + path, + content, + createdAt: new Date(), + updatedAt: new Date(), + }, + }; + } + + async write( + path: string, + payload: { content: string | Buffer }, + ): Promise<{ data: AFSEntry; message?: string }> { + this.files.set(path, payload.content); + + return { + data: { + id: path, + path, + content: payload.content, + createdAt: new Date(), + updatedAt: new Date(), + }, + }; + } + + async delete(path: string): Promise<{ message?: string }> { + this.files.delete(path); + return {}; + } +} + +// Mock image generation agent +function createMockImageGenerationAgent() { + const agent = new GeminiImageModel({ + apiKey: "mock-api-key", + model: "gemini-2.5-flash", + }); + + // Mock the invoke method to return a fake base64 image + spyOn(agent, "invoke").mockResolvedValue({ + images: [ + { + type: "file" as const, + data: "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==", // 1x1 transparent PNG + mimeType: "image/png", + }, + ], + usage: { + inputTokens: 10, + outputTokens: 20, + }, + model: "gemini-2.5-flash", + }); + + return agent; +} + +test("getStoragePath should generate correct image storage path with slug", () => { + expect(getStoragePath(".afs/images/by-intent/abc123", "format=png", "company-logo", "png")).toBe( + ".afs/images/by-intent/abc123/format=png/company-logo.png", + ); + + expect( + getStoragePath(".afs/images/by-intent/xyz789", "format=webp;variant=thumbnail", "diagram", "webp"), + ).toBe(".afs/images/by-intent/xyz789/format=webp;variant=thumbnail/diagram.webp"); +}); + +test("ImageGenerateDriver.canHandle should return true for format-only views", () => { + const driver = new ImageGenerateDriver(); + + // Should handle png format only + expect(driver.canHandle({ format: "png" })).toBe(true); + + // Should not handle other formats in Phase 2 + expect(driver.canHandle({ format: "webp" })).toBe(false); + expect(driver.canHandle({ format: "jpg" })).toBe(false); + + // Should not handle empty view + expect(driver.canHandle({})).toBe(false); + + // Should not handle views with other dimensions (Phase 2 restriction) + expect(driver.canHandle({ format: "png", language: "en" })).toBe(false); + expect(driver.canHandle({ format: "png", variant: "thumbnail" })).toBe(false); +}); + +test("ImageGenerateDriver should generate image with mock agent", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockAgent = createMockImageGenerationAgent(); + + const driver = new ImageGenerateDriver({ + imageGenerationAgent: mockAgent, + }); + + const afs = new AFS({ + modules: [mockFS], + drivers: [driver], + storage: { url: ".afs-test/image-driver-test" }, + }); + + // Write document with image slot + await afs.write("/modules/mock-fs/intro.md", { + content: `# Introduction + +This is a test document. + + + +More content here.`, + }); + + // Read the generated image (should trigger generation) + // Note: path is relative to module, not with /modules/mock-fs prefix + const imageResult = await afs.read("/modules/mock-fs/.afs/images/by-intent/2f101b8d45f157d1", { + view: { format: "png" }, + wait: "strict", + context, + }); + + // Verify image was generated + expect(imageResult.data?.content).toBeInstanceOf(Buffer); + expect(imageResult.data?.metadata?.view).toEqual({ format: "png" }); + expect(imageResult.data?.metadata?.mimeType).toBe("image/png"); + + // Verify the mock agent was called with correct prompt + expect(mockAgent.invoke).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: expect.stringContaining("system architecture diagram"), + outputFileType: "file", + }), + ); +}); + +test("ImageGenerateDriver should use owner document context", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockAgent = createMockImageGenerationAgent(); + + const driver = new ImageGenerateDriver({ + imageGenerationAgent: mockAgent, + }); + + const afs = new AFS({ + modules: [mockFS], + drivers: [driver], + storage: { url: ".afs-test/image-context-test" }, + }); + + const ownerContent = `# Product Documentation + +Our product is a cloud-based AI platform. + + + +Key features include ML, NLP, and computer vision.`; + + // Write document with image slot + await afs.write("/modules/mock-fs/product.md", { + content: ownerContent, + }); + + // Read the generated image + await afs.read("/modules/mock-fs/.afs/images/by-intent/d2b2787a39359f32", { + view: { format: "png" }, + wait: "strict", + context, + }); + + // Verify the mock agent was called with both description and context + expect(mockAgent.invoke).toHaveBeenCalledWith( + expect.objectContaining({ + prompt: expect.stringContaining("company logo"), + }), + ); + + // The prompt should contain owner document context + const callArgs = (mockAgent.invoke as any).mock.calls[0][0]; + expect(callArgs.prompt).toContain("cloud-based AI platform"); +}); + +test("ImageGenerateDriver should record dependency on owner document", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockAgent = createMockImageGenerationAgent(); + + const driver = new ImageGenerateDriver({ + imageGenerationAgent: mockAgent, + }); + + const afs = new AFS({ + modules: [mockFS], + drivers: [driver], + storage: { url: ".afs-test/image-deps-test" }, + }); + + // Write document with slot + await afs.write("/modules/mock-fs/guide.md", { + content: `# Guide\n\n`, + }); + + // Generate image + await afs.read("/modules/mock-fs/.afs/images/by-intent/75f99bc454c630b4", { + view: { format: "png" }, + wait: "strict", + context, + }); + + // Verify dependency was recorded + const metadataStore = (afs as any).metadataStore; + const deps = await metadataStore.listDependenciesByOutput( + "mock-fs", + "/.afs/images/by-intent/75f99bc454c630b4", + "format=png", + ); + + expect(deps.length).toBe(1); + expect(deps[0].inPath).toBe("/guide.md"); // Path is relative to module + expect(deps[0].role).toBe("owner-context"); +}); + +test("ImageGenerateDriver should throw error if slot not found", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockAgent = createMockImageGenerationAgent(); + + const driver = new ImageGenerateDriver({ + imageGenerationAgent: mockAgent, + }); + + const afs = new AFS({ + modules: [mockFS], + drivers: [driver], + storage: { url: ".afs-test/image-no-slot-test" }, + }); + + // Try to read image without creating a slot first + // This will fail at ViewProcessor level (no source_metadata for this path) + await expect( + afs.read("/modules/mock-fs/.afs/images/by-intent/nonexistent", { + view: { format: "png" }, + wait: "strict", + context, + }), + ).rejects.toThrow("Source file not found"); +}); + +test("ImageGenerateDriver should throw error if context is missing", async () => { + const mockFS = new MockFSModule({ context: null }); + const mockAgent = createMockImageGenerationAgent(); + + const driver = new ImageGenerateDriver({ + imageGenerationAgent: mockAgent, + }); + + await mockFS.write("/test.md", { content: "content" }); + const source = await mockFS.read("/test.md"); + assert(source.data, "source.data should be defined"); + + await expect( + driver.process( + mockFS, + ".afs/images/by-intent/test", + { format: "png" }, + { + sourceEntry: source.data, + metadata: {}, + // no context provided + }, + ), + ).rejects.toThrow("Context is required for image generation"); +}); + +test("SlotScanner should generate slug from description", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockAgent = createMockImageGenerationAgent(); + + const driver = new ImageGenerateDriver({ + imageGenerationAgent: mockAgent, + }); + + const afs = new AFS({ + modules: [mockFS], + drivers: [driver], + storage: { url: ".afs-test/slug-test" }, + }); + + // Write document with descriptive slot + await afs.write("/modules/mock-fs/doc.md", { + content: `# Document\n\n`, + }); + + // Get slot metadata + const slot = await afs.getSlot("/modules/mock-fs/doc.md", "hero"); + expect(slot).toBeDefined(); + expect(slot?.slug).toBe("system-architecture-diagram"); + expect(slot?.desc).toBe("system architecture diagram"); +}); + +test("AFS helper methods should work correctly", async () => { + const aigne = new AIGNE(); + const context = aigne.newContext(); + + const mockFS = new MockFSModule({ context }); + const mockAgent = createMockImageGenerationAgent(); + + const driver = new ImageGenerateDriver({ + imageGenerationAgent: mockAgent, + }); + + const afs = new AFS({ + modules: [mockFS], + drivers: [driver], + storage: { url: ".afs-test/helper-test" }, + }); + + const docContent = `# Product Guide + +This is our product. + + + +End of document.`; + + // Write document + await afs.write("/modules/mock-fs/product.md", { content: docContent }); + + // Test getSlot + const slot = await afs.getSlot("/modules/mock-fs/product.md", "logo"); + expect(slot).toBeDefined(); + expect(slot?.slotId).toBe("logo"); + expect(slot?.desc).toBe("company logo"); + expect(slot?.slug).toBe("company-logo"); + + // Test getImageBySlot + const imageResult = await afs.getImageBySlot("/modules/mock-fs/product.md", "logo", { + view: { format: "png" }, + wait: "strict", + context, + }); + expect(imageResult.data?.content).toBeInstanceOf(Buffer); + + // Test renderSlots + const rendered = await afs.renderSlots("/modules/mock-fs/product.md", docContent, { + view: { format: "png" }, + }); + expect(rendered).toContain("![company logo](/.afs/images/by-intent/"); + expect(rendered).toContain("/format=png/company-logo.png)"); + expect(rendered).not.toContain(" +``` + +**可选的 key 参数**（跨文档复用）： + +```html + +``` + +**约束**： +- 必须单行 +- 双引号 +- `id`: `[a-z0-9._-]+`，同一 ownerPath 内唯一 +- `key`: 可选，`[a-z0-9._-]+`，用于跨文档复用同一图片意图 +- `desc`: 图片生成的 prompt seed + +### 2. Intent Key 计算 + +**规则**： + +```typescript +function computeIntentKey(desc: string, key?: string): string { + if (key) { + // 显式 key 优先 + return key; + } + + // 规范化 desc：去除多余空格、转小写 + const normalized = desc.trim().toLowerCase().replace(/\s+/g, " "); + + // SHA-256 hash (取前 16 字符) + return sha256(normalized).substring(0, 16); +} +``` + +**示例**： +- `desc="System Architecture Diagram"` → `intentKey="a1b2c3d4e5f6g7h8"` +- `key="company-logo"` → `intentKey="company-logo"` + +**注意**：intentKey 仅用于去重和路径生成，不影响实际的图片生成内容（仍使用原始 desc） + +### 3. 图片节点路径 + +> ⚠️ **重要变更**：为避免与用户文件夹冲突，图片资源统一存储在 `.afs` 目录下 + +**Asset Identity**（逻辑路径）： + +``` +.afs/images/by-intent/ +``` + +**物化存储**（物理路径，实现细节）： + +``` +.afs/images/by-intent///image. +``` + +**示例**： +- 逻辑路径：`.afs/images/by-intent/a1b2c3d4e5f6g7h8` +- 物理路径：`.afs/images/by-intent/a1b2c3d4e5f6g7h8/format=png;variant=original/image.png` + +**ViewKey 示例**： +- `{format:"png", variant:"original"}` → `format=png;variant=original` +- `{format:"webp", variant:"thumbnail", language:"en"}` → `format=webp;language=en;variant=thumbnail` + +### 4. 元数据扩展 + +#### 修改表：`source_metadata`（新增 kind 字段） + +> 参考设计文档 [i18n+image.md#4.1](./i18n+image.md#L83-L96) + +**新增字段**： + +| 字段 | 类型 | 说明 | 默认值 | +|------|------|------|--------| +| kind | TEXT | 资源类型提示：`"doc"` \| `"image"` \| `"unknown"` | `"unknown"` | +| attrs_json | TEXT | 扩展属性（JSON），如 mime、size、width/height 等 | `null` | + +**kind 推断规则**（deterministic）： +- `write(path, string)` → `kind="doc"` +- `write(path, Buffer)` + mime sniff → `kind="image"` +- SlotScanner 创建图片节点 → `kind="image"` +- 其他 → `kind="unknown"` + +**kind 用途**（hint，非真相）： +- Driver candidate 剪枝（提高匹配效率） +- 生命周期管理（prefetch/GC/统计） +- **不允许**作为唯一判断条件 + +**sourceRevision 策略**： +- `kind="doc"`: content hash（或 mtime+size） +- `kind="image"` (by-intent): `intent:`（因为 identity 即意图） +- `kind="image"` (上传): binary hash + +#### 新增表：`afs_slots` + +| 字段 | 类型 | 说明 | +|------|------|------| +| owner_path (PK1) | TEXT | 引用者文档路径 | +| slot_id (PK2) | TEXT | slot 标识符（owner 内唯一） | +| owner_revision | TEXT | 扫描时 owner 的 sourceRevision | +| slot_type | TEXT | v1 固定 `"image"` | +| desc | TEXT | 图片描述（prompt seed） | +| intent_key | TEXT | hash(normalize(desc)) 或 key | +| asset_path | TEXT | `.afs/images/by-intent/` | +| updated_at | INTEGER | 更新时间戳 | + +#### 新增表：`afs_deps_meta` + +| 字段 | 类型 | 说明 | +|------|------|------| +| out_path (PK1) | TEXT | 产物路径 | +| out_view_key (PK2) | TEXT | 产物 viewKey | +| in_path (PK3) | TEXT | 输入依赖路径 | +| in_revision | TEXT | 输入当时的 sourceRevision | +| role | TEXT | `"owner-context"` \| `"source"` | +| updated_at | INTEGER | 更新时间戳 | + +**v1 最小依赖追踪**： +- 图片 view 产物依赖 ownerPath（role=owner-context） +- i18n view 产物依赖 source doc（role=source） + +--- + +## 🚀 分阶段实现计划 + +### Phase 1: 基础设施搭建（M1 + M2）✅ **已完成** + +**目标**：扩展元数据存储 + 实现 Slot Scanner + +**完成时间**：2025-12-26 + +#### 任务清单 + +- [x] **1.1 扩展元数据 Schema** ✅ + - [x] 修改 `afs/core/src/metadata/models/source-metadata.ts`： + - 添加 `kind: text("kind")` 字段 + - 添加 `attrsJson: text("attrs_json")` 字段 + - [x] 创建 `afs/core/src/metadata/models/slots-metadata.ts` + - [x] 创建 `afs/core/src/metadata/models/deps-metadata.ts` + - [x] 更新 `afs/core/src/metadata/models/index.ts` 导出新表 + - [x] **合并迁移**：直接修改 `001-init.ts`（避免多个迁移文件） + - [x] 更新 `MetadataStore` 接口和 `SourceMetadata` 类型，添加 kind/attrs 字段 + - [x] 更新 `MetadataStore` 接口，添加 slots 和 deps 相关方法 + - [x] 实现 `normalizeViewKey(view: View): string` 函数（k=v;k=v 格式） + - [x] 更新现有 `getViewMetadata` / `setViewMetadata` 使用 `normalizeViewKey` + +- [x] **1.2 实现 MetadataStore 新方法** ✅ + - [x] `getSlot(ownerPath, slotId)`: 查询单个 slot + - [x] `listSlots(ownerPath)`: 列出文档的所有 slots + - [x] `getSlotByAssetPath(assetPath)`: 反向查询 slot（供 driver 使用） + - [x] `upsertSlot(...)`: 插入或更新 slot + - [x] `deleteSlots(ownerPath)`: 删除文档的所有 slots + - [x] `setDependency(...)`: 记录依赖关系 + - [x] `listDependenciesByInput(inPath)`: 查询依赖某个输入的所有产物 + - [x] `listDependenciesByOutput(outPath, outViewKey)`: 查询产物的所有依赖 + +- [x] **1.3 实现 Slot Scanner** ✅ + - [x] 创建 `afs/core/src/slot-scanner.ts` + - [x] 实现正则解析：`SLOT_PATTERN = //g` + - [x] 实现 `computeIntentKey(desc, key?)` 函数（支持显式 key 和描述哈希） + - [x] 实现 `scan(module, ownerPath, content, ownerRevision)` 方法 + - [x] 实现 `ensureImageNode()`: 创建图片节点的 source_metadata，设置 `kind="image"`, `sourceRevision="intent:"` + - [x] 添加 slot 格式验证（id/key 必须符合 `[a-z0-9._-]+`） + +- [x] **1.4 集成到 ViewProcessor** ✅ + - [x] 在 `ViewProcessor` 构造函数中初始化 `SlotScanner` + - [x] 修改 `handleWrite()` 方法，在文本内容写入后触发 `slotScanner.scan()` + - [x] 实现 `markDependentViewsStale(module, inPath)` 方法 + - [x] **重构签名**：`handleWrite/handleDelete` 接收 `AFSModule` 对象而非字符串 + - [x] 集成依赖追踪：文档更新时自动标记依赖图片为 stale + +- [x] **1.5 单元测试** ✅ + - [x] 创建 `test/slot-scanner.test.ts`（18 个测试，覆盖率 100%） + - [x] 创建 `test/view-key.test.ts`（25 个测试） + - [x] 测试 slot 解析（正常 case、边界 case、错误格式、id 冲突） + - [x] 测试 intentKey 计算（规范化、去重、显式 key） + - [x] 测试 metadata 操作（upsert、query、delete、列表） + - [x] 测试依赖追踪和去重复用 + +- [x] **1.6 代码重构与质量保证** ✅ + - [x] 创建 `utils.ts`，提取 `sha256Hash` 公共函数 + - [x] 移动 `ImageSlot` 类型到 `type.ts` + - [x] 合并迁移脚本到 `001-init.ts`（删除 002 迁移） + - [x] 修复 TypeScript 严格空值检查问题（数组索引使用可选链） + - [x] 通过 Biome lint 检查（31 个文件，0 错误） + +**验收标准（全部达成）**： +- ✅ 写入包含 slot 的文档后，`afs_slots` 表中有对应记录 +- ✅ 图片节点的 `source_metadata` 被自动创建，`kind="image"` +- ✅ Slot 格式验证正常工作（重复 id 抛错，malformed slot 跳过） +- ✅ `normalizeViewKey()` 函数通过测试，相同 view 不同键顺序产生相同 viewKey +- ✅ 所有 68 个测试通过（43 个新增 + 25 个现有） +- ✅ 构建成功，代码质量检查通过 + +**交付成果**： +- **新增文件**（7 个）： + - `src/utils.ts` - 公共工具函数 + - `src/view-key.ts` - ViewKey 规范化 + - `src/slot-scanner.ts` - Slot 扫描器 + - `src/metadata/models/slots-metadata.ts` - Slots 表定义 + - `src/metadata/models/deps-metadata.ts` - 依赖表定义 + - `test/slot-scanner.test.ts` - Slot 扫描器测试（18 个测试） + - `test/view-key.test.ts` - ViewKey 规范化测试（25 个测试） +- **更新文件**（6 个）： + - `src/metadata/migrations/001-init.ts` - 合并所有表定义 + - `src/metadata/type.ts` - 新增 3 个接口，9 个方法签名 + - `src/metadata/store.ts` - 实现 9 个新方法 + - `src/type.ts` - 新增 ImageSlot 接口，启用 format/variant/policy 字段 + - `src/view-processor.ts` - 集成 SlotScanner，实现依赖追踪 + - `src/afs.ts` - 更新 write/delete 调用签名 + +**实际耗时**：1 天（含重构和测试） + +**下一步**：Phase 2 - 图片生成 Driver 实现 + +--- + +### Phase 2: 图片生成 Driver（M3）✅ **已完成** + +**目标**：实现 ImageGenerateDriver，支持基础图片生成（不带 language） + +**完成时间**：2025-12-26 + +#### 任务清单 + +- [x] **2.1 创建 image-driver 包** ✅ + - [x] 创建目录 `afs/image-driver/` + - [x] 参考 `afs/i18n-driver/package.json` 创建 `package.json`： + - 包名：`@aigne/afs-image-driver` + - 依赖：`@aigne/afs`, `@aigne/core`, `@aigne/gemini`, `zod` + - 构建脚本：参考 i18n-driver 的配置 + - [x] 参考 `afs/i18n-driver/tsconfig.json` 创建 `tsconfig.json` + - [x] 创建 `scripts/` 目录，复制构建配置文件： + - `tsconfig.build.json` + - `tsconfig.build.cjs.json` + - `tsconfig.build.esm.json` + - `tsconfig.build.dts.json` + - [x] 创建 `src/driver.ts` + - [x] 创建 `src/storage.ts`（存储路径计算） + - [x] 创建 `src/index.ts` + +- [x] **2.2 实现 ImageGenerateDriver** ✅ + - [x] 定义 `ImageGenerateDriverOptions` 接口 + - [x] 实现 `canHandle(view)` 方法：Phase 2 只处理 `{format:"png"}` 组合 + - [x] 实现 `process()` 方法主流程： + 1. 从 `afs_slots` 查询 slot 信息 + 2. 读取 owner 文档内容（提供生成上下文） + 3. 调用 AI Agent 生成图片 + 4. 计算并写入物理存储路径 + 5. 记录依赖关系到 `afs_deps_meta` + 6. 返回 AFSEntry + - [x] 实现存储路径计算：`//.`（优化版：使用 slug 作为文件名） + +- [x] **2.3 实现默认图片生成 Agent** ✅ + - [x] 创建 `src/default-generation-agent.ts` + - [x] 定义 `ImageGenerationInput` / `ImageGenerationOutput` 接口 + - [x] 使用 `GeminiImageModel` 实现 Agent（参考 `models/gemini/src/gemini-image-model.ts`）： + - 默认模型：`gemini-2.5-flash`（Gemini 文生图模型） + - [x] Prompt 工程：结合 slot.desc + owner context（约 300 字上下文） + - [x] 支持 format 参数（png），通过 `outputFileType` 配置 + - [x] 实现重试逻辑：失败时自动重试最多 3 次，指数退避间隔（1s, 2s, 4s） + - [x] 参考测试：`models/gemini/test/gemini-image-model.test.ts` + +- [x] **2.4 Driver 注册与测试** ✅ + - [x] 创建集成测试 `test/image-driver.test.ts`（9 个测试） + - [x] 测试完整流程：write slot → read image → verify generation + - [x] 测试依赖追踪：update owner → image becomes stale + - [x] Mock AI Agent 进行测试（避免实际 API 调用） + +- [x] **2.5 Slug 优化方案** ✅（附加优化） + - [x] 添加 `slug` 字段到 `afs_slots` schema + - [x] 实现 `generateSlug()` 函数从描述生成人类可读的文件名 + - [x] 修改 `getStoragePath()` 使用 slug：`//.` + - [x] 添加 AFS 辅助方法： + - `getSlot(ownerPath, slotId)` - 获取 slot 元数据 + - `getImageBySlot(ownerPath, slotId, options)` - 通过 slot 读取图片 + - `renderSlots(ownerPath, content, options)` - 替换文档中的 slot 标记 + - [x] 更新 MetadataStore 方法支持 slug 字段 + - [x] 测试 slug 生成和辅助方法 + +**验收标准（全部达成）**： +- ✅ 读取图片 asset 时，driver 被正确匹配 +- ✅ 图片生成完成后，物理文件存在 +- ✅ `view_metadata.state = "ready"`，storagePath 正确 +- ✅ `afs_deps_meta` 记录了依赖关系 +- ✅ 重试机制正常工作：失败时自动重试最多 3 次 +- ✅ Slug 优化：文件名可读（如 `company-logo.png` 而非 `image.png`） +- ✅ 辅助方法：可以通过 slot ID 直接访问图片和渲染文档 + +**交付成果**： +- **新增包**：`@aigne/afs-image-driver` +- **新增文件**（8 个）： + - `afs/image-driver/package.json` + - `afs/image-driver/tsconfig.json` + - `afs/image-driver/src/driver.ts` - ImageGenerateDriver 实现（212 行） + - `afs/image-driver/src/default-generation-agent.ts` - 默认生成 Agent（45 行） + - `afs/image-driver/src/storage.ts` - 存储路径计算（27 行） + - `afs/image-driver/src/index.ts` - 导出接口 + - `afs/image-driver/test/image-driver.test.ts` - 集成测试（9 个测试，394 行） + - `afs/image-driver/scripts/*` - 构建配置文件（4 个） +- **更新文件**（5 个）： + - `afs/core/src/metadata/models/slots-metadata.ts` - 添加 slug 字段 + - `afs/core/src/metadata/type.ts` - SlotMetadata 接口添加 slug + - `afs/core/src/metadata/store.ts` - 所有 slot 方法支持 slug + - `afs/core/src/slot-scanner.ts` - 生成并存储 slug + - `afs/core/src/afs.ts` - 添加 3 个辅助方法（getSlot, getImageBySlot, renderSlots） + +**测试结果**： +- 所有 9 个测试通过 ✅ +- 覆盖功能： + - 存储路径计算（带 slug） + - Driver canHandle 匹配逻辑 + - 完整图片生成流程 + - Owner 文档上下文使用 + - 依赖关系记录 + - 错误处理（slot 不存在、context 缺失） + - Slug 生成（如 `"system architecture diagram"` → `"system-architecture-diagram"`） + - AFS 辅助方法（getSlot, getImageBySlot, renderSlots） + +**实际耗时**：1 天（包含 slug 优化） + +**关键技术决策**： +- ✅ 使用 gemini-2.5-flash 模型（而非 imagen-4.0） +- ✅ 默认使用完整 owner 文档作为上下文（约 300 字） +- ✅ Phase 2 只支持 png 格式（variant、language 留待后续） +- ✅ 简单重试策略：失败时重试所有错误类型 +- ✅ Slug 优化：解决文件名可读性和 slot 替换便利性问题 +- ✅ 路径稳定性：保持 intentKey 作为路径标识，slug 仅用于文件名 + +**下一步**：Phase 3 - Fallback 与增强功能 + +**预估工作量**：3-4 天 → **实际：1 天** ✅ + +--- + +### Phase 3: Fallback 与增强功能（M4） + +**目标**：实现 fallback 策略 + 依赖传播优化 + +#### 任务清单 + +- [ ] **3.1 Fallback 提示消息** + - [ ] 修改 `ViewProcessor.handleRead()`，对图片路径在 fallback 模式返回提示字符串 + - [ ] 提示消息：`"图片还未准备好，正在生成中..."`（或类似文案） + - [ ] 添加 `isImagePath(path)` 辅助函数（检测 `.afs/images/` 路径） + - [ ] 测试 fallback 模式：立即返回提示 + 后台生成 + +- [ ] **3.2 依赖传播优化** + - [ ] 实现精细化 stale 检查：比较 `in_revision` 与当前 ownerRevision + - [ ] 仅在 revision 真正变化时标记 stale + - [ ] 添加批量依赖传播测试 + +- [ ] **3.3 ViewKey 规范化改进**（可选） + - [ ] 实现 `normalizeViewKey(view)` 函数（k=v;k=v 格式） + - [ ] 统一键顺序：`language → format → variant → policy` + - [ ] 值规范化：trim、lowercase + - [ ] 迁移现有 JSON.stringify 使用 + +- [ ] **3.4 错误处理与重试机制** + - [ ] 参考 i18n driver 的重试逻辑（`afs/i18n-driver/src/driver.ts`） + - [ ] 实现自动重试：失败时最多重试 3 次 + - [ ] 重试间隔：指数退避（1s, 2s, 4s） + - [ ] 3 次重试后仍失败，标记 `state="failed"` + - [ ] 记录详细错误信息到 `view_metadata.error` + - [ ] 支持手动清除 failed 状态后重新生成 + +**验收标准**： +- ✅ wait="fallback" 时立即返回提示消息"图片还未准备好，正在生成中..." +- ✅ owner 文档更新后，图片自动标记 stale +- ✅ 生成失败有清晰的错误提示，记录到 `view_metadata.error` +- ✅ 重试逻辑完善：指数退避间隔，3 次后标记 failed + +**预估工作量**：2-3 天 + +--- + +### Phase 4: 多语言图片支持（M5，可选） + +**目标**：支持图片的多语言 view（如 `{format:"png", language:"en"}`） + +#### 任务清单 + +- [ ] **4.1 扩展 Driver 能力** + - [ ] 修改 `ImageGenerateDriver.canHandle()`，支持 `language` 维度 + - [ ] 更新 `capabilities.dimensions` 为 `["format", "variant", "language"]` + +- [ ] **4.2 多语言生成逻辑** + - [ ] 修改 AI Agent，传递 `language` 参数 + - [ ] Prompt 工程：引导生成特定语言的文字标注 + - [ ] 测试不同语言的图片生成 + +- [ ] **4.3 Fallback 语言链** + - [ ] 请求 `language="en"` 但缺失时，fallback 到无语言版本 + - [ ] 实现语言 fallback 链逻辑 + - [ ] 测试多语言 fallback 场景 + +**验收标准**： +- ✅ 可以生成带语言标注的图片（如中文/英文版本） +- ✅ 语言缺失时能正确 fallback +- ✅ 不同语言的图片独立存储 + +**预估工作量**：2-3 天 + +--- + +### Phase 5: 优化与生产就绪（M6） + +**目标**：性能优化、GC 机制、监控 + +#### 任务清单 + +- [ ] **5.1 批量预生成（Prefetch）** + - [ ] 支持批量图片生成：`afs.prefetch(paths, {view: {format:"png"}})` + - [ ] 并发控制（已有 p-limit） + - [ ] 进度回调 + +- [ ] **5.2 垃圾回收（GC）** + - [ ] 实现 `cleanupUnusedAssets()`：删除无 slot 引用的图片节点 + - [ ] 引用计数：检测 `afs_slots` 中是否还有引用 + - [ ] 物理文件清理策略 + +- [ ] **5.3 监控与诊断** + - [ ] 添加统计接口：`getStats()` - 返回 slot/view/deps 数量 + - [ ] 性能监控：记录生成耗时 + - [ ] 健康检查：检测 stale/failed 视图数量 + +- [ ] **5.4 文档与示例** + - [ ] 更新 `afs/README.md` + - [ ] 编写 image-driver 使用文档 + - [ ] 创建完整示例项目（包含 slot → 生成 → 发布流程） + +**验收标准**： +- ✅ 可以批量预生成所有图片 +- ✅ GC 能清理未引用的资源 +- ✅ 有完整的使用文档 + +**预估工作量**：3-4 天 + +--- + +## 📐 关键技术细节 + +### 1. Slot 解析正则表达式 + +```typescript +const SLOT_PATTERN = //g; + +// 匹配示例： +// ✅ +// ✅ +// ❌ (id 包含空格) +``` + +### 2. Intent Key 规范化 + +```typescript +function normalizeDesc(desc: string): string { + return desc + .trim() // 去除首尾空格 + .toLowerCase() // 转小写 + .replace(/\s+/g, " "); // 多个空格合并为一个 +} + +// 示例： +// " System Architecture " → "system architecture" +// "COMPANY LOGO" → "company logo" +``` + +### 3. ViewKey 序列化规则 + +> ⚠️ **重要**：Phase 1 实现规范化格式，确保键顺序一致性 + +```typescript +/** + * 规范化 ViewKey 序列化 + * 规则： + * 1. 仅允许白名单键：language | format | variant | policy + * 2. 值规范化：trim() + toLowerCase() + * 3. 键排序固定：language → format → variant → policy + * 4. 格式：k=v;k=v（无值的键不出现） + */ +function normalizeViewKey(view: View): string { + const pairs: string[] = []; + + // 固定顺序：language → format → variant → policy + if (view.language) pairs.push(`language=${view.language.trim().toLowerCase()}`); + if (view.format) pairs.push(`format=${view.format.trim().toLowerCase()}`); + if (view.variant) pairs.push(`variant=${view.variant.trim().toLowerCase()}`); + if (view.policy) pairs.push(`policy=${view.policy.trim().toLowerCase()}`); + + return pairs.join(';'); +} + +// 示例： +// {format:"PNG", variant:"ORIGINAL"} → "format=png;variant=original" +// {language:"en", format:"webp"} → "language=en;format=webp" +// {language:"zh", format:"png", variant:"original"} → "language=zh;format=png;variant=original" +``` + +**重要性**： +- 决定 `view_metadata` 表主键的稳定性 +- 避免 `{format:"png", language:"en"}` 和 `{language:"en", format:"png"}` 被识别为不同 view +- 所有 view 相关操作必须使用此函数序列化 + +### 4. 重试机制实现 + +> 参考 i18n driver 的重试逻辑 + +```typescript +async function processViewWithRetry( + module: AFSModule, + path: string, + view: View, + context: any, + maxRetries = 3 +): Promise { + let lastError: Error | undefined; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + // 尝试生成 + const result = await driver.process(module, path, view, { + sourceEntry, + metadata: { derivedFrom: sourceMeta.sourceRevision }, + context, + }); + + return result.data; + } catch (error: any) { + lastError = error; + console.warn(`Attempt ${attempt}/${maxRetries} failed:`, error.message); + + // 如果还有重试机会，等待后重试 + if (attempt < maxRetries) { + const delay = Math.pow(2, attempt - 1) * 1000; // 1s, 2s, 4s + await new Promise(resolve => setTimeout(resolve, delay)); + } + } + } + + // 所有重试都失败 + throw new Error( + `Failed after ${maxRetries} attempts: ${lastError?.message}` + ); +} +``` + +### 5. Fallback 提示消息 + +```typescript +// 在 ViewProcessor.handleRead() 中 +if (wait === "fallback") { + // 触发后台生成 + this.processView(module, path, options.view, context).catch(...); + + // 检测是否为图片路径 + if (isImagePath(path)) { + return { + data: { + path, + content: "图片还未准备好，正在生成中...", + metadata: { placeholder: true } + }, + message: "Image is being generated in background", + viewStatus: { fallback: true } + }; + } + + // 非图片，返回源内容（如 doc） + const sourceResult = await module.read?.(path); + return { ... }; +} + +function isImagePath(path: string): boolean { + return path.startsWith('.afs/images/'); +} +``` + +### 6. 依赖传播示例 + +```typescript +// 场景：用户修改包含 slot 的文档 +await afs.write("/docs/intro.md", updatedContent); + +// 触发流程： +// 1. ViewProcessor.handleWrite() +// - 检测 sourceRevision 变化 +// - 调用 markDependentViewsStale("/docs/intro.md") +// +// 2. markDependentViewsStale() +// - 查询: SELECT * FROM afs_deps_meta WHERE in_path = "/docs/intro.md" +// - 结果: [{ outPath: "assets/images/by-intent/a1b2", outViewKey: "format=png;..." }] +// - 标记: UPDATE view_metadata SET state = "stale" WHERE ... +// +// 3. 下次读取图片时 +// - isViewStale() 返回 true +// - 重新调用 driver.process() 生成新图片 +``` + +### 7. 物理存储路径规则 + +``` +项目根目录/ +└── modules/ + └── doc-smith/ + ├── docs/ + │ └── intro.md # 源文档 + ├── .i18n/ + │ └── en/ + │ └── docs/ + │ └── intro.md # i18n view 产物 + └── .afs/ + ├── metadata.db # SQLite 元数据 + └── images/ + └── by-intent/ + └── a1b2c3d4/ # intentKey (图片节点) + ├── format=png;variant=original/ + │ └── image.png # 基础图片 + ├── format=webp;variant=thumbnail/ + │ └── image.webp # 缩略图 + └── format=png;language=en;variant=original/ + └── image.png # 英文版图片 +``` + +**说明**： +- 应用层 identity：`.afs/images/by-intent/a1b2c3d4` +- 物化存储：`.afs/images/by-intent/a1b2c3d4//image.` +- 应用层永远不应该直接拼接物化路径，必须通过 `afs.read(path, {view})` 访问 +- `.afs/` 目录统一管理 AFS 框架的内部资源，避免与用户文件冲突 + +--- + +## 🧪 测试策略 + +### 单元测试 + +- **SlotScanner** + - 正则解析准确性 + - intentKey 计算一致性 + - 格式验证（边界 case） + +- **ImageGenerateDriver** + - canHandle() 匹配逻辑 + - 存储路径计算 + - Mock Agent 测试 + +### 集成测试 + +- **完整流程** + 1. Write 文档（包含 slot） + 2. Verify slots 表记录 + 3. Read 图片 asset + 4. Verify 生成完成 + 5. Update 文档 + 6. Verify 图片 stale + +- **依赖追踪** + 1. 生成图片 + 2. Verify deps 记录 + 3. 修改 owner + 4. Verify 图片标记 stale + +- **Fallback 行为** + 1. Read 未生成的图片（wait="fallback"） + 2. Verify 立即返回 placeholder + 3. Wait 后台生成完成 + 4. Read 再次返回真实图片 + +### 性能测试 + +- 批量 slot 扫描（1000+ slots） +- 并发图片生成（concurrency=10） +- 依赖传播性能（大量依赖关系） + +--- + +## ⚠️ 注意事项与风险 + +### 1. ViewKey 兼容性 + +**问题**：当前代码使用 `JSON.stringify(view)`，改为 `k=v;k=v` 格式会导致旧数据不兼容。 + +**解决方案**： +- Phase 1-3 先保持 JSON.stringify +- Phase 4 实现新格式时添加迁移逻辑 +- 或者提供配置开关，允许选择序列化方式 + +### 2. Intent Key 冲突 + +**问题**：SHA-256 理论上可能冲突（极低概率）。 + +**解决方案**： +- 插入 `afs_slots` 时检测 intentKey 冲突 +- 如果 desc 不同但 intentKey 相同，抛出错误 +- 建议用户使用显式 `key` 参数避免冲突 + +### 3. 图片生成成本 + +**问题**：AI 图片生成成本高、耗时长。 + +**解决方案**： +- 默认使用 fallback 模式，后台异步生成 +- 提供 prefetch 接口，发布前批量生成 +- 考虑缓存策略（intentKey 相同则永久缓存） + +### 4. 物理文件清理 + +**问题**：slot 被删除后，图片文件可能成为孤儿。 + +**解决方案**： +- Phase 5 实现 GC 机制 +- 定期扫描无引用的 asset +- 提供手动清理接口 + +### 5. 并发生成控制 + +**问题**：大量图片同时生成可能超出 API 配额。 + +**解决方案**： +- prefetch 使用 p-limit 控制并发 +- 添加队列机制（可选） +- 支持限流配置 + +--- + +## 📚 参考资料 + +- [i18n+image.md](./i18n+image.md) - 完整工程设计文档 +- [afs/core/src/view-processor.ts](../afs/core/src/view-processor.ts) - ViewProcessor 实现 +- [afs/i18n-driver/src/driver.ts](../afs/i18n-driver/src/driver.ts) - I18nDriver 参考实现 +- [afs/core/src/metadata/store.ts](../afs/core/src/metadata/store.ts) - SQLite MetadataStore + +--- + +## 🎯 下一步行动 + +**建议优先级**： + +1. **Review 本文档** - 确认设计方向和任务拆分 +2. **Phase 1 实施** - 先搭建基础设施（metadata + slot scanner） +3. **中间验证** - 确保 slot 扫描流程正常工作后再继续 +4. **Phase 2 实施** - 实现图片生成 driver +5. **迭代优化** - 根据实际使用情况调整 Phase 3-5 + +**已确定决策**： + +- ✅ **图片路径**：使用 `.afs/images/by-intent/`（避免与用户文件冲突） +- ✅ **source_metadata.kind 字段**：需要在 Phase 1 添加 +- ✅ **AI 服务**：使用 Gemini 模型（`imagen-4.0-generate-001` 或 `gemini-2.5-flash`） +- ✅ **包结构**：参考 `i18n-driver` 包的结构和配置 +- ✅ **ViewKey 序列化**：实现 `k=v;k=v` 格式，确保键顺序一致（Phase 1 实现） +- ✅ **Fallback 策略**：返回提示字符串"图片还未准备好"（短期不使用真正的 fallback 模式） +- ✅ **重试机制**：失败时自动重试 3 次，参考 i18n driver 的翻译重试逻辑 + +请 review 本计划，如有调整建议请提出！ diff --git a/afs/i18n+image.md b/afs/i18n+image.md new file mode 100644 index 000000000..afdc42682 --- /dev/null +++ b/afs/i18n+image.md @@ -0,0 +1,403 @@ +# AFS View + Driver Materialization（i18n + image）工程设计 v1 + +## 1. 目标与范围 + +### 1.1 目标 + +* 把 **多语言翻译** 与 **图片生成/派生** 都统一为 AFS 的 **view 物化能力**： + * identity = `path` + * projection = `view` + * materialization = driver 产物（落盘/缓存/异步 job） + +* 应用层（DocSmith 等）只负责： + * 写入 source-of-truth（doc 内容） + * 声明图片需求（slot） + +* 框架层（AFS）负责： + * 解析 slot、去重复用、创建图片节点 + * view 状态管理、strict/fallback/prefetch + * 多语言 i18n driver + * image driver（generate/derive），图片可选多语言 view + +### 1.2 v1 范围 + +* View 统一模型（language/format/variant/policy） +* sqlite 元数据：source/view/slots/deps（deps 可先最小实现） +* i18n driver：doc 的 language view +* image pipeline：slot scanner + image generate + image derive +* kind：保留，作为 hint（非真相），提供推断/纠错机制 + +--- + +## 2. 对外 API（v1） + +```ts +type View = { + language?: string; // "en" | "zh" | ... + format?: string; // doc: "md"|"html"|"pdf" ; image: "png"|"webp" + // variant?: string; // doc: "summary"|"toc" ; image: "original"|"thumbnail" + // policy?: string; // "technical"|"marketing" ... +}; + +// 下面这些接口参数名只是设计，以代码中实现为这准 +type ReadOptions = { view?: View; wait?: "strict" | "fallback" }; + +afs.read(path: string, opts?: ReadOptions): Promise; +afs.write(path: string, content: any, opts?: { view?: View }): Promise; +afs.stat(path: string, opts?: { view?: View }): Promise; +afs.list(pathOrGlob: string, opts?: { view?: View }): Promise; +afs.prefetch(pathOrGlob: string|string[], opts: { view: View }): Promise; +``` + +### 2.1 约束（强制） + +* `write(path, {view})` 只允许 driver/框架写 view 产物；应用写 source 不带 view。 +* `read(path, {view})` 是唯一获取 view 的入口；应用不拼 `.i18n/` 或图片缓存路径。 + +--- + +## 3. ViewKey 规范化（必须做） + +### 3.1 ViewKey 序列化规则 + +* 仅允许白名单 keys：`language|format|variant|policy` +* 值 normalize：`trim()`，`lowercase()`（语言代码等） +* key 排序固定：`language -> format -> variant -> policy` +* 序列化格式：`k=v;k=v`（无值的 key 不出现） + +示例： + +* `{language:"en"}` → `language=en` +* `{format:"webp",variant:"thumbnail"}` → `format=webp;variant=thumbnail` +* `{language:"zh",format:"png",variant:"original"}` → `language=zh;format=png;variant=original` + +> 这决定了 `afs_view_meta` 的主键稳定性，必须统一。 + +--- + +## 4. 元数据存储（sqlite） + +> 你们已经在 i18n 用了 sqlite 两表（source/view），这里扩展为 **4 表（v1 推荐）**： +> `source_meta` / `view_meta` / `slots` / `deps`（deps 可先最小实现） + +### 4.1 `afs_source_meta`（按 path） 代码中这个表已经存在了，需要合并考虑，不要覆盖现有实现 + +| 字段 | 类型 | 说明 | | | | +| --------------- | ------- | -------------------------------- | ------- | -------------- | --------- | +| path (PK) | TEXT | identity | | | | +| kind | TEXT | `doc | image | unknown`（hint） | | +| updated_at | INTEGER | unix ms | | | | +| attrs_json | TEXT | 可选扩展（mime、size、width/height…） | | | | + +**source_revision 建议：** + +* doc：content hash（或 mtime+size） +* image（by-intent）：`intentKey`（因为 identity=意图） +* image（上传）：binary hash + +### 4.2 `afs_view_meta`（按 path + viewKey） 代码中这个表已经存在了，需要合并考虑，不要覆盖现有实现 + +| 字段 | 类型 | 说明 | | | | +| -------------- | ------- | ------------------ | ----- | ---------- | ------- | +| path (PK1) | TEXT | identity | | | | +| view_key (PK2) | TEXT | viewKey | | | | +| state | TEXT | `ready | stale | generating | failed` | +| derived_from | TEXT | 对应 source_revision | | | | +| generated_at | INTEGER | unix ms | | | | +| driver_id | TEXT | 物化 driver（诊断用） | | | | +| error | TEXT | 可选错误 | | | | + +### 4.3 `afs_slots`（slot → asset 绑定） + +| 字段 | 类型 | 说明 | +| ---------------- | ------- | ------------------------------------- | +| owner_path (PK1) | TEXT | 引用者节点 | +| slot_id (PK2) | TEXT | 稳定锚点 | +| owner_revision | TEXT | 扫描时 owner 的 source_revision | +| slot_type | TEXT | v1 固定 `image` | +| desc | TEXT | prompt seed | +| intent_key | TEXT | hash(normalize(desc)) | +| asset_path | TEXT | `assets/images/by-intent/` | +| updated_at | INTEGER | unix ms | + +> v1 的复用：desc 相同 → intentKey 相同 → assetPath 相同。 + +### 4.4 `afs_deps_meta`（产物 view 依赖） + +| 字段 | 类型 | 说明 | | | | +| ------------------ | ------- | --------------------- | ------- | ------ | -- | +| out_path (PK1) | TEXT | 产物 path | | | | +| out_view_key (PK2) | TEXT | 产物 viewKey | | | | +| in_path (PK3) | TEXT | 输入依赖 path | | | | +| in_revision | TEXT | 输入当时的 source_revision | | | | +| role | TEXT | `owner-context | lexicon | policy | …` | +| updated_at | INTEGER | unix ms | | | | + +**v1 最小 deps：** + +* image view 产物依赖 ownerPath（role=owner-context） +* i18n view 产物依赖 source doc（role=source） + +--- + +## 5. kind：来源、推断、纠错、使用 + +### 5.1 kind 定位 + +* kind 是 **hint**，用于： + + * driver candidate 剪枝 + * 生命周期管理（prefetch/GC/统计） +* kind 不是真相，不允许成为唯一判断条件。 + +### 5.2 kind 写入来源优先级 + +1. **AFS/Scanner 创建**的节点（高可信）：例如 `assets/images/by-intent/*` → `kind=image` +2. **Driver 写入** view 产物时可补充 attrs（mime/size），但不改 source kind +3. 应用写入 doc：AFS 根据 `content` 类型检测 + +### 5.3 推断规则（deterministic） + +* `write(path, string)` → kind=doc +* `write(path, Buffer)` + mime sniff → kind=image +* path pattern `assets/images/by-intent/` → image +* 其它 → unknown + +### 5.4 读时校验（兜底触发条件） + +仅在以下情况触发 sniff： + +* kind=unknown +* resolver 发现多个 driver 候选且 kind 无法剪枝 +* driver 执行前发现输入不符合预期（例如 image driver 打开后不是图片） + +触发后： + +* 更新 `afs_source_meta.kind `（自愈） +* 重新 resolve 一次 + +--- + +## 6. Slot 协议与 Scanner（deterministic pass） + +### 6.1 slot 格式（单一规范） + +``` + +``` + +约束： + +* 必须单行 +* 双引号 +* id：`[a-z0-9._-]+`，同 ownerPath 内唯一 +* desc：允许任意文本；若包含 `"` 需要 `\"`（或 v1 直接规定 desc 不允许双引号） +* key is optional. Use a short, stable token ([a-z0-9._-]+) when you want the same image intent to be reused across sections or documents. + +### 6.2 Scanner 触发时机（v1） + +* 在 `afs.write(ownerPath)` 完成后，AFS core enqueue `scan(ownerPath)` +* 可选：`afs.prefetch(ownerPath, {view:{variant:"images"}})` 主动触发 scan+生成 + +### 6.3 Scanner 算法（伪代码） + +```ts +function scanOwnerForImageSlots(ownerPath): + content = afs.read(ownerPath) // source read + ownerMeta = source_meta(ownerPath) // must exist + + slots = parseSlots(content) // regex + strict validation + for slot in slots: + intentKey = hash(normalize(slot.desc)) + assetPath = `assets/images/by-intent/${intentKey}` + + upsert afs_slots(ownerPath, slot.id, ownerMeta.source_revision, "image", + slot.desc, intentKey, assetPath) + + ensure source_meta(assetPath): + kind=image, kind_source=scanner, conf=100 + source_revision=intentKey + + // mark baseline views missing/stale (optional) + ensure view_meta(assetPath, viewKeyFor({variant:"original",format:"png"})): + if not exists -> state=missing (store as stale or add explicit missing) +``` + +> **状态字段里你们现在用 missing/stale/ready**。sqlite 里如果只存 4 态，建议： + +* 不存在 = missing +* 存在且 state=stale 表示需要重建 + （这样不用额外枚举 missing） + +--- + +## 7. Driver 体系与 resolve（避免 pipeline） + +### 7.1 Driver 注册（能力声明） + +每个 driver 声明： + +* `id` +* `supportedKinds`（可选剪枝） +* `match(view)`：是否能处理某些维度子集 +* `materialize(path, view, ctx)`：产物生成 + +建议 v1 至少有： + +* `i18n.driver`（doc + language） +* `image.generate.driver`（image + variant=original + format=png/webp + 可选 language） + +### 7.2 Resolve 流程（伪代码） + +```ts +async function read(path, {view, wait}): + viewKey = normalizeViewKey(view) + + // 1) fast-path: view ready + vm = view_meta.get(path, viewKey) + if (vm?.state === "ready"): + return readMaterialized(path, viewKey) + + // 2) determine source + kind hint + sm = source_meta.get(path) || inferSourceMetaIfNeeded(path) + kind = sm.kind + + // 3) find capable driver (UNIQUE) + candidates = drivers.filter(d => d.match(view) && d.supportsKind(kind)) + driver = pickUnique(candidates) // must be unique; otherwise fail or require composite + + // 4) materialization control + if (wait === "fallback"): + kickJobDedup(path, viewKey, driver) + return fallbackResult(path, view) // doc: source default lang; image: best-effort + + // strict + return await waitForJob(path, viewKey, () => kickJobDedup(...)) +``` + +### 7.3 “唯一 driver”策略（必须） + +* `pickUnique()` 不能返回多个 +* 多维组合（如 `language+format`）必须通过**组合 driver**显式注册，例如： + + * `doc.i18n+format.driver`（内部先 i18n 再 format），对外仍是一个 driver + +> 这和你们 i18n 讨论里“避免 middleware 叠加”完全一致。 + +--- + +## 8. Materialization：状态机、wait 策略 + +### 8.1 状态机（view_meta.state） + +* `ready`：可直接读产物 +* `stale`：可读旧产物（可选）但应重建；v1 建议 stale 时 strict 等重建，fallback 返回旧/降级 +* `generating`：已有 job 在跑 +* `failed`：生成失败（可重试） + +> “missing” 用 `view_meta` 不存在表示。 + +### 8.2 strict / fallback + +* `strict`：缺失/过期 → 等待 materialize 完成（promise） +* `fallback`： + + * doc（i18n）：直接返回主语言内容，同时后台翻译 + * image： + + * 如果请求 `language=xx` 缺失 → 先返回不带 language 的图（若 ready），否则返回占位 + * 同时后台生成目标语言图 + +--- + +## 10. deps：过期传播（把“上下文变化”纳入一致性） + +这是图片方案落地后最容易变成技术债的点，所以 v1 就建议做“最小 deps”。 + +### 10.1 image view 生成完成时写 deps + +在 `image.generate.driver.materialize()` 完成后： + +* 写 `deps(out=assetPath+viewKey, in=ownerPath+ownerRevision, role=owner-context)` + +### 10.2 ownerPath 更新后传播 stale（两种实现） + +A) 简单实现（v1 推荐） +`write(ownerPath)` 后： + +* scanner 重新扫描 slot（更新 ownerRevision） +* 同时查询 deps：`SELECT out_path,out_view_key WHERE in_path=ownerPath` + 将这些 out 的 view_meta 标记 stale + +B) 更精细（v2） +比较 `in_revision` 与当前 ownerRevision，只有不一致才 stale + +--- + +## 11. 物化产物落盘（Workspace 约定） +`modules/doc-smith` + +`modules` 是 afs 读取的前缀 +`doc-smith` 是 module 名称 + +### 11.1 Doc i18n + +* source：`modules/doc-smith/docs/`（主语言） +* 物化：`modules/doc-smith/.i18n/docs//...`（落盘策略，不是抽象） +* meta：`.afs/metadata.db` + +### 11.2 Image + +* asset identity：`modules/doc-smith/assets/images/by-intent/` +* 物化 view 存储（示例策略）： + + * `modules/doc-smith/assets/images/by-intent///image.png` + （实现层细节，抽象层不暴露） + +--- + +## 12. 关键实现清单（任务拆分） + +### M1：View/Meta/Resolve 基建 - 已实现了部分版本，需要结合现有代码实现 + +* [ ] ViewKey 规范化实现 +* [ ] sqlite：source_meta / view_meta / slots / deps 表 +* [ ] kind 推断（write-time）+ 兜底校验入口 +* [ ] read/write/prefetch 的框架骨架（含 wait） + +### M2：i18n driver（doc + language） - 已实现了部分版本，需要结合现有代码实现 + +* [ ] i18n.materialize：翻译生成、写入 view 产物、更新 view_meta +* [ ] strict/fallback 行为落地 +* [ ] deps：doc language view 依赖 source doc + +### M3：image pipeline（scan + generate + derive） + +* [ ] slot parser（regex + 语法校验） +* [ ] scanner：写 slots + ensure image node +* [ ] image.generate.driver：original png（先不做 language） +* [ ] deps：image view 依赖 ownerPath（最小实现） + +### M4：图片多语言（可选） + +* [ ] image.generate 支持 view.language +* [ ] fallback：无语言图兜底 +* [ ] 渲染器默认策略：优先 language view + fallback + +--- + +## 13. 最小可运行示例（行为预期） + +### 13.1 应用写 doc（包含 slot） + +* `afs.write("docs/intro.md", "......")` +* AFS 自动 scan： + + * upsert slots：`owner=docs/intro.md, slot=img-001 → assetPath=assets/images/by-intent/` + * ensure `source_meta(assetPath).kind=image` + +### 13.2 发布器渲染请求图片 + +* `assetPath = lookupSlots("docs/intro.md","img-001")` +* `afs.read(assetPath, { view:{format:"webp",variant:"thumbnail",language:"en"}, wait:"fallback" })` +* 若缺：立即返回无语言图或占位，同时后台生成语言版缩略图