diff --git a/apps/api/package.json b/apps/api/package.json index e8525b4..7bd1a5a 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -31,7 +31,8 @@ "@xpenser/contracts": "0.1.0", "@xpenser/timezone": "0.1.0", "knex": "3.1.0", - "pg": "8.13.3" + "pg": "8.13.3", + "sharp": "0.34.5" }, "devDependencies": { "@types/node": "22.15.3", diff --git a/apps/api/src/api/endpoints.ts b/apps/api/src/api/endpoints.ts index c1a64f1..16dad81 100644 --- a/apps/api/src/api/endpoints.ts +++ b/apps/api/src/api/endpoints.ts @@ -1,5 +1,5 @@ import { api, PrincipalSchema } from '@xpenser/contracts'; -import { ConfigToken, DbToken, LoggerToken } from '../di/tokens.js'; +import { ConfigToken, DbToken, KnexToken, LoggerToken } from '../di/tokens.js'; export const RegisterEndpoint = api.auth.register .inject({ db: DbToken, config: ConfigToken }) @@ -265,7 +265,7 @@ export const EnrichVendorEndpoint = api.vendors.enrich export const ListTransactionsEndpoint = api.transactions.list .authorize(PrincipalSchema) - .inject({ db: DbToken }) + .inject({ db: DbToken, knex: KnexToken }) .summary('List transactions') .description('Lists transactions owned by the authenticated user.') .tags('transactions') @@ -297,6 +297,54 @@ export const DeleteTransactionEndpoint = api.transactions.delete .tags('transactions') .operationId('deleteTransaction'); +export const GetTransactionScanImageEndpoint = api.transactions.scanImage + .authorize(PrincipalSchema) + .inject({ knex: KnexToken }) + .summary('Get scanned transaction image') + .description( + 'Returns the original scanner image attached to a confirmed transaction.' + ) + .tags('transactions') + .operationId('getTransactionScanImage'); + +export const CreateTransactionScanEndpoint = api.transactionScans.create + .authorize(PrincipalSchema) + .inject({ db: DbToken, config: ConfigToken }) + .summary('Scan transaction image') + .description( + 'Extracts draft transactions from an uploaded receipt, invoice, bank app screenshot, or statement image.' + ) + .tags('transaction-scans') + .operationId('createTransactionScan'); + +export const StartTransactionScanJobEndpoint = api.transactionScans.start + .authorize(PrincipalSchema) + .inject({ db: DbToken, config: ConfigToken }) + .summary('Start transaction image scan') + .description( + 'Starts an asynchronous multimodal scan job and returns a short-lived progress token.' + ) + .tags('transaction-scans') + .operationId('startTransactionScanJob'); + +export const TransactionScanProgressEndpoint = api.transactionScans.progress + .summary('Transaction image scan progress') + .description( + 'Streams progress and the final scan result for a short-lived scan job token.' + ) + .tags('transaction-scans') + .operationId('transactionScanProgress'); + +export const DecideTransactionScanItemEndpoint = api.transactionScans.decide + .authorize(PrincipalSchema) + .inject({ db: DbToken }) + .summary('Record transaction scan decision') + .description( + 'Records whether a scanned draft was confirmed or discarded, including user corrections for future scans.' + ) + .tags('transaction-scans') + .operationId('decideTransactionScanItem'); + export const DashboardSummaryEndpoint = api.dashboard.summary .authorize(PrincipalSchema) .inject({ db: DbToken }) @@ -385,7 +433,14 @@ export const endpoints = { list: ListTransactionsEndpoint, create: CreateTransactionEndpoint, update: UpdateTransactionEndpoint, - delete: DeleteTransactionEndpoint + delete: DeleteTransactionEndpoint, + scanImage: GetTransactionScanImageEndpoint + }, + transactionScans: { + create: CreateTransactionScanEndpoint, + start: StartTransactionScanJobEndpoint, + progress: TransactionScanProgressEndpoint, + decide: DecideTransactionScanItemEndpoint }, dashboard: { summary: DashboardSummaryEndpoint, diff --git a/apps/api/src/api/handlers/index.ts b/apps/api/src/api/handlers/index.ts index 00d807f..346d4df 100644 --- a/apps/api/src/api/handlers/index.ts +++ b/apps/api/src/api/handlers/index.ts @@ -29,12 +29,19 @@ import { telegramStatusHandler, telegramTokenHandler } from './telegram.js'; +import { + createTransactionScanHandler, + decideTransactionScanItemHandler, + startTransactionScanJobHandler, + transactionScanProgressHandler +} from './transaction-scans.js'; import { categoryTrendHandler, createTransactionHandler, dashboardSummaryHandler, dashboardWindowHandler, deleteTransactionHandler, + getTransactionScanImageHandler, listTransactionsHandler, statsOverviewHandler, statsWindowHandler, @@ -98,7 +105,14 @@ export const handlers = { list: listTransactionsHandler, create: createTransactionHandler, update: updateTransactionHandler, - delete: deleteTransactionHandler + delete: deleteTransactionHandler, + scanImage: getTransactionScanImageHandler + }, + transactionScans: { + create: createTransactionScanHandler, + start: startTransactionScanJobHandler, + progress: transactionScanProgressHandler, + decide: decideTransactionScanItemHandler }, dashboard: { summary: dashboardSummaryHandler, diff --git a/apps/api/src/api/handlers/transaction-scans.ts b/apps/api/src/api/handlers/transaction-scans.ts new file mode 100644 index 0000000..7e89a8f --- /dev/null +++ b/apps/api/src/api/handlers/transaction-scans.ts @@ -0,0 +1,79 @@ +import { + ActionResult, + type Handler, + type SubscriptionHandler +} from '@cleverbrush/server'; +import { + startTransactionScanJob, + subscribeTransactionScanJob +} from '../../application/transaction-scan-jobs.js'; +import { + recordTransactionScanDecision, + scanTransactionsFromImage, + TransactionScanInputError, + TransactionScanNotFoundError +} from '../../application/transaction-scans.js'; +import type { + CreateTransactionScanEndpoint, + DecideTransactionScanItemEndpoint, + StartTransactionScanJobEndpoint, + TransactionScanProgressEndpoint +} from '../endpoints.js'; + +export const createTransactionScanHandler: Handler< + typeof CreateTransactionScanEndpoint +> = async ({ body, principal }, { db, config }) => { + try { + const scan = await scanTransactionsFromImage( + db, + config, + principal.userId, + body + ); + return ActionResult.created( + scan, + `/api/transaction-scans/${scan.scanId}` + ); + } catch (err) { + if (err instanceof TransactionScanInputError) { + return ActionResult.badRequest({ message: err.message }); + } + throw err; + } +}; + +export const startTransactionScanJobHandler: Handler< + typeof StartTransactionScanJobEndpoint +> = async ({ body, principal }, { db, config }) => { + const job = startTransactionScanJob(db, config, principal.userId, body); + return ActionResult.accepted(job); +}; + +export const transactionScanProgressHandler: SubscriptionHandler< + typeof TransactionScanProgressEndpoint +> = async function* ({ query, signal }) { + yield* subscribeTransactionScanJob(query, signal); +}; + +export const decideTransactionScanItemHandler: Handler< + typeof DecideTransactionScanItemEndpoint +> = async ({ body, params, principal }, { db }) => { + try { + await recordTransactionScanDecision( + db, + principal.userId, + params.scanId, + params.itemId, + body + ); + return ActionResult.noContent(); + } catch (err) { + if (err instanceof TransactionScanInputError) { + return ActionResult.badRequest({ message: err.message }); + } + if (err instanceof TransactionScanNotFoundError) { + return ActionResult.notFound({ message: err.message }); + } + throw err; + } +}; diff --git a/apps/api/src/api/handlers/transactions.ts b/apps/api/src/api/handlers/transactions.ts index dcd809b..95f3a19 100644 --- a/apps/api/src/api/handlers/transactions.ts +++ b/apps/api/src/api/handlers/transactions.ts @@ -5,6 +5,7 @@ import { dashboardSummary, dashboardWindow, deleteTransaction, + getTransactionScanImage, listTransactions, statsOverview, statsWindow, @@ -19,6 +20,7 @@ import type { DashboardSummaryEndpoint, DashboardWindowEndpoint, DeleteTransactionEndpoint, + GetTransactionScanImageEndpoint, ListTransactionsEndpoint, StatsOverviewEndpoint, StatsWindowEndpoint, @@ -27,8 +29,8 @@ import type { export const listTransactionsHandler: Handler< typeof ListTransactionsEndpoint -> = async ({ query, principal }, { db }) => { - return listTransactions(db, principal.userId, query); +> = async ({ query, principal }, { db, knex }) => { + return listTransactions(db, principal.userId, query, knex); }; export const createTransactionHandler: Handler< @@ -93,6 +95,19 @@ export const deleteTransactionHandler: Handler< } }; +export const getTransactionScanImageHandler: Handler< + typeof GetTransactionScanImageEndpoint +> = async ({ params, principal }, { knex }) => { + try { + return await getTransactionScanImage(knex, principal.userId, params.id); + } catch (err) { + if (err instanceof TransactionNotFoundError) { + return ActionResult.notFound({ message: err.message }); + } + throw err; + } +}; + export const dashboardSummaryHandler: Handler< typeof DashboardSummaryEndpoint > = async ({ query, principal }, { db }) => { diff --git a/apps/api/src/application/openai.test.ts b/apps/api/src/application/openai.test.ts index 58f0b20..7c5569b 100644 --- a/apps/api/src/application/openai.test.ts +++ b/apps/api/src/application/openai.test.ts @@ -1,11 +1,16 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import type { Config } from '../config.js'; -import { generateStructuredJson, OpenAIConfigError } from './openai.js'; +import { + generateStructuredJson, + generateStructuredJsonFromContent, + OpenAIConfigError +} from './openai.js'; const config = { openai: { apiKey: 'sk-test', - reportModel: 'gpt-5-mini' + reportModel: 'gpt-5-mini', + transactionScanModel: 'gpt-5.5' } } as Config; @@ -47,6 +52,69 @@ describe('generateStructuredJson', () => { }) }) ); + expect( + JSON.parse(String(fetchSpy.mock.calls[0]?.[1]?.body)) + ).toMatchObject({ + store: false, + text: { + format: { + name: 'test_schema', + strict: true, + type: 'json_schema' + } + } + }); + }); + + it('requests structured JSON with image input content', async () => { + const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + output_text: JSON.stringify({ count: 2 }) + }) + } as Response); + + await expect( + generateStructuredJsonFromContent<{ count: number }>(config, { + content: [ + { type: 'input_text', text: 'Extract transactions.' }, + { + type: 'input_image', + image_url: 'data:image/png;base64,abc', + detail: 'original' + } + ], + model: config.openai.reportModel, + schema: { + type: 'object', + properties: { count: { type: 'number' } }, + required: ['count'], + additionalProperties: false + }, + schemaName: 'image_schema', + system: 'Write JSON.' + }) + ).resolves.toEqual({ count: 2 }); + + expect( + JSON.parse(String(fetchSpy.mock.calls[0]?.[1]?.body)) + ).toMatchObject({ + input: [ + expect.any(Object), + { + role: 'user', + content: [ + { type: 'input_text', text: 'Extract transactions.' }, + { + type: 'input_image', + image_url: 'data:image/png;base64,abc', + detail: 'original' + } + ] + } + ] + }); }); it('throws when OpenAI is not configured', async () => { diff --git a/apps/api/src/application/openai.ts b/apps/api/src/application/openai.ts index f0daac7..9024116 100644 --- a/apps/api/src/application/openai.ts +++ b/apps/api/src/application/openai.ts @@ -12,6 +12,21 @@ type StructuredJsonOptions = { readonly system: string; }; +type InputContentPart = + | { + readonly text: string; + readonly type: 'input_text'; + } + | { + readonly detail?: 'auto' | 'high' | 'low' | 'original'; + readonly image_url: string; + readonly type: 'input_image'; + }; + +type StructuredJsonContentOptions = Omit & { + readonly content: readonly InputContentPart[]; +}; + function responseOutputText(json: unknown): string { if ( typeof json === 'object' && @@ -55,6 +70,24 @@ function responseOutputText(json: unknown): string { export async function generateStructuredJson( config: Config, options: StructuredJsonOptions +): Promise { + return generateStructuredJsonFromContent(config, { + content: [ + { + type: 'input_text', + text: JSON.stringify(options.input) + } + ], + model: options.model, + schema: options.schema, + schemaName: options.schemaName, + system: options.system + }); +} + +export async function generateStructuredJsonFromContent( + config: Config, + options: StructuredJsonContentOptions ): Promise { if (!config.openai.apiKey) { throw new OpenAIConfigError('OPENAI_API_KEY is not set.'); @@ -74,15 +107,11 @@ export async function generateStructuredJson( }, { role: 'user', - content: [ - { - type: 'input_text', - text: JSON.stringify(options.input) - } - ] + content: options.content } ], model: options.model, + store: false, text: { format: { name: options.schemaName, diff --git a/apps/api/src/application/transaction-scan-jobs.ts b/apps/api/src/application/transaction-scan-jobs.ts new file mode 100644 index 0000000..09d7176 --- /dev/null +++ b/apps/api/src/application/transaction-scan-jobs.ts @@ -0,0 +1,284 @@ +import { randomBytes, randomUUID } from 'node:crypto'; +import type { + TransactionScanBody, + TransactionScanJobResponse, + TransactionScanProgressEvent, + TransactionScanProgressQuery, + TransactionScanResponse +} from '@xpenser/contracts'; +import type { Config } from '../config.js'; +import type { AppDb } from '../db/schemas.js'; +import { + scanTransactionsFromImage, + TransactionScanInputError +} from './transaction-scans.js'; + +const jobTtlMs = 30 * 60 * 1_000; + +type MutableScanJob = { + readonly events: TransactionScanProgressEvent[]; + readonly id: string; + readonly listeners: Set<() => void>; + readonly token: string; + readonly userId: number; + deleteTimer: ReturnType | null; + done: boolean; + expiresAt: number; +}; + +type ProgressStage = Exclude< + TransactionScanProgressEvent['stage'], + 'complete' | 'failed' | 'queued' +>; + +const jobs = new Map(); + +function scheduleDelete(job: MutableScanJob): void { + if (job.deleteTimer) { + return; + } + + job.deleteTimer = setTimeout(() => { + if (jobs.get(job.id) === job) { + jobs.delete(job.id); + } + }, jobTtlMs); + job.deleteTimer.unref?.(); +} + +function cleanupExpiredJobs(): void { + const now = Date.now(); + for (const [jobId, job] of jobs) { + if (job.expiresAt <= now) { + jobs.delete(jobId); + if (job.deleteTimer) { + clearTimeout(job.deleteTimer); + } + } + } +} + +function notify(job: MutableScanJob): void { + for (const listener of job.listeners) { + listener(); + } + job.listeners.clear(); +} + +function event({ + error = null, + job, + message, + progress, + scan = null, + stage +}: { + readonly error?: string | null; + readonly job: MutableScanJob; + readonly message: string; + readonly progress: number; + readonly scan?: TransactionScanResponse | null; + readonly stage: TransactionScanProgressEvent['stage']; +}): TransactionScanProgressEvent { + return { + jobId: job.id, + stage, + message, + progress, + scan, + error + }; +} + +function emit(job: MutableScanJob, nextEvent: TransactionScanProgressEvent) { + if (job.done) { + return; + } + + job.events.push(nextEvent); + if (nextEvent.stage === 'complete' || nextEvent.stage === 'failed') { + job.done = true; + job.expiresAt = Date.now() + jobTtlMs; + scheduleDelete(job); + } + notify(job); +} + +function progressMessage(stage: ProgressStage): string { + switch (stage) { + case 'preparing': + return 'Loading categories, vendors, and prior scan corrections.'; + case 'analyzing': + return 'Reading image details with AI.'; + case 'saving': + return 'Saving scan suggestions for review.'; + } + return 'Scanning image.'; +} + +function progressValue(stage: ProgressStage): number { + switch (stage) { + case 'preparing': + return 15; + case 'analyzing': + return 45; + case 'saving': + return 85; + } + return 50; +} + +function failureMessage(err: unknown): string { + return err instanceof TransactionScanInputError + ? err.message + : 'Could not scan the image. Try again.'; +} + +async function runJob( + job: MutableScanJob, + db: AppDb, + config: Config, + body: TransactionScanBody +): Promise { + try { + const scan = await scanTransactionsFromImage( + db, + config, + job.userId, + body, + { + onProgress: stage => + emit( + job, + event({ + job, + message: progressMessage(stage), + progress: progressValue(stage), + stage + }) + ) + } + ); + const count = scan.drafts.length; + emit( + job, + event({ + job, + message: + count === 1 + ? 'Found 1 transaction for review.' + : `Found ${count} transactions for review.`, + progress: 100, + scan, + stage: 'complete' + }) + ); + } catch (err) { + emit( + job, + event({ + error: failureMessage(err), + job, + message: 'Scan failed.', + progress: 100, + stage: 'failed' + }) + ); + } +} + +function createJob(userId: number): MutableScanJob { + return { + events: [], + id: randomUUID(), + listeners: new Set(), + token: randomBytes(32).toString('base64url'), + userId, + deleteTimer: null, + done: false, + expiresAt: Date.now() + jobTtlMs + }; +} + +export function startTransactionScanJob( + db: AppDb, + config: Config, + userId: number, + body: TransactionScanBody +): TransactionScanJobResponse { + cleanupExpiredJobs(); + const job = createJob(userId); + jobs.set(job.id, job); + emit( + job, + event({ + job, + message: 'Scan queued.', + progress: 0, + stage: 'queued' + }) + ); + void runJob(job, db, config, body); + return { jobId: job.id, token: job.token }; +} + +function authorizedJob(query: TransactionScanProgressQuery) { + cleanupExpiredJobs(); + const job = jobs.get(query.jobId); + return job && job.token === query.token ? job : undefined; +} + +function waitForEvent(job: MutableScanJob, signal: AbortSignal): Promise { + if (signal.aborted || job.done) { + return Promise.resolve(); + } + + return new Promise(resolve => { + let resolved = false; + const listener = () => { + if (resolved) { + return; + } + resolved = true; + job.listeners.delete(listener); + signal.removeEventListener('abort', abortListener); + resolve(); + }; + const abortListener = () => listener(); + job.listeners.add(listener); + signal.addEventListener('abort', abortListener, { once: true }); + }); +} + +export async function* subscribeTransactionScanJob( + query: TransactionScanProgressQuery, + signal: AbortSignal +): AsyncGenerator { + const job = authorizedJob(query); + if (!job) { + yield { + jobId: query.jobId, + stage: 'failed', + message: 'Scan job was not found.', + progress: 100, + scan: null, + error: 'Scan job was not found.' + }; + return; + } + + let index = 0; + while (!signal.aborted) { + while (index < job.events.length) { + const nextEvent = job.events[index]; + index += 1; + if (nextEvent) { + yield nextEvent; + } + } + if (job.done) { + return; + } + await waitForEvent(job, signal); + } +} diff --git a/apps/api/src/application/transaction-scans.test.ts b/apps/api/src/application/transaction-scans.test.ts new file mode 100644 index 0000000..6c88a92 --- /dev/null +++ b/apps/api/src/application/transaction-scans.test.ts @@ -0,0 +1,705 @@ +import type { + Category, + Transaction, + TransactionScanDecisionBody, + Vendor +} from '@xpenser/contracts'; +import sharp from 'sharp'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { Config } from '../config.js'; +import type { + AppDb, + TransactionDb, + TransactionScanDb, + TransactionScanImageDb, + TransactionScanItemDb, + UserDb +} from '../db/schemas.js'; +import { + prepareScanImagesForVision, + recordTransactionScanDecision, + scanTransactionsFromImage +} from './transaction-scans.js'; + +const mocks = vi.hoisted(() => ({ + generateStructuredJsonFromContent: vi.fn(), + listCategories: vi.fn(), + listTransactions: vi.fn(), + listVendors: vi.fn() +})); + +vi.mock('./categories.js', () => ({ + listCategories: mocks.listCategories +})); + +vi.mock('./openai.js', () => ({ + generateStructuredJsonFromContent: mocks.generateStructuredJsonFromContent +})); + +vi.mock('./transactions.js', () => ({ + listTransactions: mocks.listTransactions +})); + +vi.mock('./vendors.js', () => ({ + listVendors: mocks.listVendors +})); + +const timestamp = new Date('2026-06-01T12:00:00.000Z'); + +type TestQuery = Promise & { + first: () => Promise; + update: (values: Partial) => Promise; + where: ( + selector: (row: T) => TValue, + value: TValue + ) => TestQuery; +}; + +function testQuery(rows: T[]): TestQuery { + const query = Promise.resolve(rows) as TestQuery; + query.where = (selector: (row: T) => TValue, value: TValue) => + testQuery(rows.filter(row => selector(row) === value)); + query.first = async () => rows[0]; + query.update = async (values: Partial) => { + for (const row of rows) { + Object.assign(row, values); + } + return rows; + }; + return query; +} + +function user(overrides: Partial = {}): UserDb { + return { + id: 1, + email: 'jane@example.com', + emailVerified: true, + role: 'user', + authProvider: 'local', + defaultCurrency: 'USD', + countryCode: 'US', + timezone: 'UTC', + weeklyEmailReportEnabled: true, + monthlyEmailReportEnabled: true, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function category(overrides: Partial = {}): Category { + return { + id: 7, + name: 'Groceries', + type: 'expense', + kind: 'normal', + parentId: null, + displayName: 'Groceries', + inUse: true, + hasChildren: false, + archivedAt: null, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function vendor(overrides: Partial = {}): Vendor { + return { + id: 5, + name: 'Walmart', + displayName: 'Walmart', + domain: 'walmart.com', + transactionCount: 3, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function transaction(overrides: Partial = {}): Transaction { + return { + id: 99, + categoryId: 7, + vendorId: 5, + vendorName: 'Walmart', + categoryName: 'Groceries', + categoryDisplayName: 'Groceries', + categoryParentId: null, + categoryKind: 'normal', + type: 'expense', + amount: 12.34, + currency: 'USD', + defaultCurrencyAmount: 12.34, + defaultCurrency: 'USD', + exchangeRate: 1, + exchangeRateDate: '2026-06-01', + occurredAt: timestamp, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function transactionRow(overrides: Partial = {}): TransactionDb { + return { + id: 99, + userId: 1, + categoryId: 7, + vendorId: 5, + type: 'expense', + amount: 12.34, + currency: 'USD', + defaultCurrencyAmount: 12.34, + defaultCurrency: 'USD', + exchangeRate: 1, + exchangeRateDate: '2026-06-01', + occurredAt: timestamp, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function scanItem( + overrides: Partial = {} +): TransactionScanItemDb { + return { + id: 20, + scanId: 10, + userId: 1, + draftJson: '{}', + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function scan(overrides: Partial = {}): TransactionScanDb { + return { + id: 10, + userId: 1, + documentKind: 'receipt', + imageHash: + '6105d6cc76af400325e94d588ce511be5bfdbb73b437dc51eca43917d7a43e3d', + model: 'gpt-5.5', + warningsJson: '[]', + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function scanImage( + overrides: Partial = {} +): TransactionScanImageDb { + return { + id: 30, + scanId: 10, + userId: 1, + imageHash: + '6105d6cc76af400325e94d588ce511be5bfdbb73b437dc51eca43917d7a43e3d', + mimeType: 'image/png', + fileName: 'receipt.png', + sizeBytes: 5, + imageBase64: Buffer.from('image').toString('base64'), + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +function testDb({ + scanImages = [], + scanItems = [], + scans = [scan()], + transactions = [], + users = [user()] +}: { + readonly scanImages?: TransactionScanImageDb[]; + readonly scanItems?: TransactionScanItemDb[]; + readonly scans?: TransactionScanDb[]; + readonly transactions?: TransactionDb[]; + readonly users?: UserDb[]; +} = {}): AppDb { + return { + users: { + find: vi.fn(async (id: number) => + users.find(candidate => candidate.id === id) + ) + }, + transactionScans: { + where: vi.fn( + ( + selector: (row: TransactionScanDb) => TValue, + value: TValue + ) => testQuery(scans).where(selector, value) + ), + insert: vi.fn(async value => { + const created = scan({ + id: 10, + ...value + }); + scans.push(created); + return created; + }) + }, + transactionScanItems: { + where: vi.fn( + ( + selector: (row: TransactionScanItemDb) => TValue, + value: TValue + ) => testQuery(scanItems).where(selector, value) + ), + insert: vi.fn(async value => { + const created = scanItem({ + id: scanItems.length + 20, + ...value + }); + scanItems.push(created); + return created; + }) + }, + transactionScanImages: { + where: vi.fn( + ( + selector: (row: TransactionScanImageDb) => TValue, + value: TValue + ) => testQuery(scanImages).where(selector, value) + ), + insert: vi.fn(async value => { + const created = scanImage({ + id: scanImages.length + 30, + ...value + }); + scanImages.push(created); + return created; + }) + }, + transactions: { + where: vi.fn( + ( + selector: (row: TransactionDb) => TValue, + value: TValue + ) => testQuery(transactions).where(selector, value) + ) + } + } as unknown as AppDb; +} + +const config = { + openai: { + apiKey: 'sk-test', + reportModel: 'gpt-5-mini', + transactionScanModel: 'gpt-5.5' + } +} as Config; + +describe('transaction image scans', () => { + afterEach(() => { + vi.restoreAllMocks(); + mocks.generateStructuredJsonFromContent.mockReset(); + mocks.listCategories.mockReset(); + mocks.listTransactions.mockReset(); + mocks.listVendors.mockReset(); + }); + + it('creates sanitized draft transactions from image scan output', async () => { + mocks.listCategories.mockResolvedValue([category()]); + mocks.listVendors.mockResolvedValue([vendor()]); + mocks.listTransactions.mockResolvedValue({ items: [transaction()] }); + mocks.generateStructuredJsonFromContent.mockResolvedValue({ + documentKind: 'receipt', + warnings: ['Check tax lines.'], + visibleTotal: 12.34, + visibleTotalCurrency: 'USD', + transactions: [ + { + amount: 12.34, + categoryId: 7, + confidence: { + amount: 'high', + category: 'medium', + currency: 'high', + date: 'medium', + overall: 'medium', + vendor: 'high' + }, + currency: 'usd', + evidence: 'Walmart 12.34', + lineItemSubtotal: 12.34, + lineItems: [], + note: 'Milk - 4.34\nBread - 8.00', + occurredDate: '2026-06-01', + occurredTime: null, + suggestedCategoryKind: null, + suggestedCategoryName: null, + suggestedCategoryParentId: null, + suggestedCategoryReason: null, + suggestedCategoryType: null, + suggestedVendorName: null, + transactionType: 'expense', + vendorId: 5 + } + ] + }); + + const scanItems: TransactionScanItemDb[] = []; + const db = testDb({ scanItems }); + const result = await scanTransactionsFromImage(db, config, 1, { + imageBase64: Buffer.from('image').toString('base64'), + mimeType: 'image/png', + fileName: 'receipt.png' + }); + + expect(result).toMatchObject({ + scanId: 10, + documentKind: 'receipt', + warnings: ['Check tax lines.'], + drafts: [ + { + id: 20, + amount: 12.34, + categoryId: 7, + currency: 'USD', + vendorId: 5, + possibleDuplicateTransactionIds: [99] + } + ] + }); + expect(scanItems).toHaveLength(1); + expect(JSON.parse(scanItems[0]?.draftJson ?? '{}')).toMatchObject({ + amount: 12.34, + categoryId: 7, + currency: 'USD', + note: 'Milk - 4.34\nBread - 8.00', + vendorId: 5 + }); + expect( + mocks.generateStructuredJsonFromContent.mock.calls[0]?.[1].content + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + image_url: expect.stringContaining( + 'data:image/png;base64,' + ), + type: 'input_image' + }) + ]) + ); + expect( + JSON.parse( + String( + mocks.generateStructuredJsonFromContent.mock.calls[0]?.[1] + .content[0]?.text + ) + ).outputRules.note + ).toContain('multiline notes'); + }); + + it('suggests category creation instead of trusting low-confidence category IDs', async () => { + mocks.listCategories.mockResolvedValue([category()]); + mocks.listVendors.mockResolvedValue([vendor()]); + mocks.listTransactions.mockResolvedValue({ items: [] }); + mocks.generateStructuredJsonFromContent.mockResolvedValue({ + documentKind: 'receipt', + warnings: [], + visibleTotal: 18.5, + visibleTotalCurrency: 'USD', + transactions: [ + { + amount: 18.5, + categoryId: 7, + confidence: { + amount: 'high', + category: 'low', + currency: 'high', + date: 'medium', + overall: 'medium', + vendor: 'low' + }, + currency: 'USD', + evidence: 'Hardware supplies 18.50', + lineItemSubtotal: 18.5, + lineItems: [ + { + amount: 18.5, + description: 'Hardware supplies', + quantity: 1 + } + ], + note: null, + occurredDate: '2026-06-01', + occurredTime: null, + suggestedCategoryKind: 'normal', + suggestedCategoryName: 'Home supplies', + suggestedCategoryParentId: null, + suggestedCategoryReason: + 'No existing category matches hardware supplies.', + suggestedCategoryType: 'expense', + suggestedVendorName: 'Hardware Shop', + transactionType: 'expense', + vendorId: null + } + ] + }); + + const result = await scanTransactionsFromImage(testDb(), config, 1, { + imageBase64: Buffer.from('image').toString('base64'), + mimeType: 'image/png' + }); + + expect(result.drafts[0]).toMatchObject({ + categoryId: null, + suggestedCategory: { + name: 'Home supplies', + type: 'expense', + parentId: null, + kind: 'normal', + reason: 'No existing category matches hardware supplies.' + }, + confidence: { + category: 'low' + } + }); + }); + + it('keeps split invoice drafts linked to one scan with itemized notes', async () => { + mocks.listCategories.mockResolvedValue([ + category({ id: 7, name: 'Groceries', displayName: 'Groceries' }), + category({ id: 8, name: 'Medicine', displayName: 'Medicine' }) + ]); + mocks.listVendors.mockResolvedValue([vendor()]); + mocks.listTransactions.mockResolvedValue({ items: [] }); + mocks.generateStructuredJsonFromContent.mockResolvedValue({ + documentKind: 'invoice', + warnings: [], + visibleTotal: 33.0, + visibleTotalCurrency: 'USD', + transactions: [ + { + amount: 21.0, + categoryId: 7, + confidence: { + amount: 'high', + category: 'high', + currency: 'high', + date: 'high', + overall: 'high', + vendor: 'high' + }, + currency: 'USD', + evidence: 'Apples 10.00 Milk 11.00', + lineItemSubtotal: 21, + lineItems: [ + { + amount: 10, + description: 'Apples', + quantity: 1 + }, + { amount: 11, description: 'Milk', quantity: 1 } + ], + note: 'Apples - 10.00\nMilk - 11.00', + occurredDate: '2026-06-02', + occurredTime: null, + suggestedCategoryKind: null, + suggestedCategoryName: null, + suggestedCategoryParentId: null, + suggestedCategoryReason: null, + suggestedCategoryType: null, + suggestedVendorName: null, + transactionType: 'expense', + vendorId: 5 + }, + { + amount: 12.0, + categoryId: 8, + confidence: { + amount: 'high', + category: 'high', + currency: 'high', + date: 'high', + overall: 'high', + vendor: 'high' + }, + currency: 'USD', + evidence: 'Bandages 12.00', + lineItemSubtotal: 12, + lineItems: [ + { amount: 12, description: 'Bandages', quantity: 1 } + ], + note: 'Bandages - 12.00', + occurredDate: '2026-06-02', + occurredTime: null, + suggestedCategoryKind: null, + suggestedCategoryName: null, + suggestedCategoryParentId: null, + suggestedCategoryReason: null, + suggestedCategoryType: null, + suggestedVendorName: null, + transactionType: 'expense', + vendorId: 5 + } + ] + }); + + const scanItems: TransactionScanItemDb[] = []; + const result = await scanTransactionsFromImage( + testDb({ scanItems }), + config, + 1, + { + imageBase64: Buffer.from('image').toString('base64'), + mimeType: 'image/png', + fileName: 'invoice.png' + } + ); + + expect(result).toMatchObject({ + scanId: 10, + documentKind: 'invoice', + drafts: [ + { + amount: 21, + categoryId: 7, + note: 'Apples - 10.00\nMilk - 11.00' + }, + { + amount: 12, + categoryId: 8, + note: 'Bandages - 12.00' + } + ] + }); + expect(scanItems).toHaveLength(2); + expect(scanItems.map(item => item.scanId)).toEqual([10, 10]); + }); + + it('splits very tall images into ordered vision tiles', async () => { + const buffer = await sharp({ + create: { + background: '#ffffff', + channels: 3, + height: 7661, + width: 1440 + } + }) + .png() + .toBuffer(); + + const prepared = await prepareScanImagesForVision(buffer, 'image/png'); + + expect(prepared.images).toHaveLength(4); + expect(prepared.images[0]).toMatchObject({ + description: 'tile 1 of 4', + mimeType: 'image/jpeg', + width: 1440 + }); + expect(prepared.promptContext.preprocessing).toContain( + 'ordered overlapping tiles' + ); + expect(prepared.warnings[0]).toContain('split into 4 ordered tiles'); + }); + + it('records confirmed scan corrections', async () => { + const item = scanItem(); + const db = testDb({ + scanItems: [item], + transactions: [transactionRow({ id: 42 })] + }); + const body: TransactionScanDecisionBody = { + decision: 'confirmed', + transactionId: 42, + correctedTransaction: { + amount: 19.99, + categoryId: 7, + currency: 'USD', + occurredAt: timestamp, + vendorId: 5, + note: 'Corrected' + } + }; + + await recordTransactionScanDecision(db, 1, 10, 20, body); + + expect(item).toMatchObject({ + decision: 'confirmed', + transactionId: 42 + }); + expect(JSON.parse(item.correctedJson ?? '{}')).toMatchObject({ + amount: 19.99, + categoryId: 7, + note: 'Corrected' + }); + expect(item.decidedAt).toBeInstanceOf(Date); + }); + + it('stores the original image for confirmed scan transactions', async () => { + const item = scanItem(); + const scanImages: TransactionScanImageDb[] = []; + const db = testDb({ + scanImages, + scanItems: [item], + transactions: [transactionRow({ id: 42 })] + }); + + await recordTransactionScanDecision(db, 1, 10, 20, { + decision: 'confirmed', + transactionId: 42, + correctedTransaction: { + amount: 19.99, + categoryId: 7, + currency: 'USD', + occurredAt: timestamp, + vendorId: null, + note: null + }, + attachment: { + imageBase64: Buffer.from('image').toString('base64'), + mimeType: 'image/png', + fileName: 'receipt.png' + } + }); + + expect(scanImages).toHaveLength(1); + expect(scanImages[0]).toMatchObject({ + scanId: 10, + userId: 1, + mimeType: 'image/png', + fileName: 'receipt.png', + sizeBytes: 5, + imageBase64: Buffer.from('image').toString('base64') + }); + }); + + it('rejects scan attachments that do not match the original scan hash', async () => { + const item = scanItem(); + const scanImages: TransactionScanImageDb[] = []; + const db = testDb({ + scanImages, + scanItems: [item], + transactions: [transactionRow({ id: 42 })] + }); + + await expect( + recordTransactionScanDecision(db, 1, 10, 20, { + decision: 'confirmed', + transactionId: 42, + correctedTransaction: { + amount: 19.99, + categoryId: 7, + currency: 'USD', + occurredAt: timestamp, + vendorId: null, + note: null + }, + attachment: { + imageBase64: Buffer.from('other').toString('base64'), + mimeType: 'image/png', + fileName: 'receipt.png' + } + }) + ).rejects.toThrow('Confirmed scan image did not match'); + expect(scanImages).toHaveLength(0); + }); +}); diff --git a/apps/api/src/application/transaction-scans.ts b/apps/api/src/application/transaction-scans.ts new file mode 100644 index 0000000..1c95f8e --- /dev/null +++ b/apps/api/src/application/transaction-scans.ts @@ -0,0 +1,1195 @@ +import { createHash } from 'node:crypto'; +import type { + Category, + Transaction, + TransactionScanBody, + TransactionScanDecisionBody, + TransactionScanDraft, + TransactionScanResponse, + Vendor +} from '@xpenser/contracts'; +import { FieldLimits, TransactionScanLimits } from '@xpenser/contracts'; +import { + dateToLocalDateParam, + localDateTimeInputToDate +} from '@xpenser/timezone'; +import sharp from 'sharp'; +import type { Config } from '../config.js'; +import type { + AppDb, + TransactionScanDb, + TransactionScanItemDb, + UserDb +} from '../db/schemas.js'; +import { listCategories } from './categories.js'; +import { generateStructuredJsonFromContent } from './openai.js'; +import { listTransactions } from './transactions.js'; +import { listVendors } from './vendors.js'; + +export class TransactionScanInputError extends Error {} +export class TransactionScanNotFoundError extends Error {} + +const maxImageBytes = TransactionScanLimits.maxImageBytes; +const maxDrafts = 25; +const maxContextVendors = 100; +const maxRecentTransactions = 100; +const maxCorrectionExamples = 10; +const visionLongSideLimit = 6000; +const extremeAspectRatio = 4; +const tileLongSide = 2400; +const tileOverlap = 160; +const maxVisionTiles = 8; + +const confidenceValues = new Set(['high', 'medium', 'low']); +const documentKinds = new Set([ + 'bank_app', + 'bank_statement', + 'invoice', + 'receipt', + 'other' +]); + +type Confidence = 'high' | 'low' | 'medium'; +type TransactionType = 'expense' | 'income'; + +type RawFieldConfidence = { + readonly amount?: unknown; + readonly category?: unknown; + readonly currency?: unknown; + readonly date?: unknown; + readonly overall?: unknown; + readonly vendor?: unknown; +}; + +type RawScannedTransaction = { + readonly amount?: unknown; + readonly categoryId?: unknown; + readonly confidence?: RawFieldConfidence; + readonly currency?: unknown; + readonly evidence?: unknown; + readonly lineItemSubtotal?: unknown; + readonly lineItems?: unknown; + readonly note?: unknown; + readonly occurredDate?: unknown; + readonly occurredTime?: unknown; + readonly suggestedCategoryKind?: unknown; + readonly suggestedCategoryName?: unknown; + readonly suggestedCategoryParentId?: unknown; + readonly suggestedCategoryReason?: unknown; + readonly suggestedCategoryType?: unknown; + readonly suggestedVendorName?: unknown; + readonly transactionType?: unknown; + readonly vendorId?: unknown; +}; + +type RawScanResult = { + readonly documentKind?: unknown; + readonly transactions?: unknown; + readonly visibleTotal?: unknown; + readonly visibleTotalCurrency?: unknown; + readonly warnings?: unknown; +}; + +type ScanImageInput = { + readonly buffer: Buffer; + readonly description: string; + readonly height?: number; + readonly mimeType: TransactionScanBody['mimeType']; + readonly width?: number; +}; + +type PreparedScanImages = { + readonly images: readonly ScanImageInput[]; + readonly promptContext: { + readonly originalHeight?: number; + readonly originalWidth?: number; + readonly preprocessing: string; + readonly tiles: readonly { + readonly description: string; + readonly height?: number; + readonly width?: number; + }[]; + }; + readonly warnings: readonly string[]; +}; + +type PromptCategory = Pick< + Category, + 'displayName' | 'id' | 'kind' | 'name' | 'parentId' | 'parentName' | 'type' +> & { + readonly effectiveType: TransactionType; +}; + +type PromptVendor = Pick< + Vendor, + | 'displayName' + | 'domain' + | 'id' + | 'name' + | 'resolvedName' + | 'suggestedCategoryDisplayName' + | 'suggestedCategoryId' + | 'transactionCount' +>; + +type CorrectionExample = { + readonly decision: string; + readonly draft: unknown; + readonly corrected: unknown; +}; + +type ScanProgressStage = 'analyzing' | 'preparing' | 'saving'; + +type ScanProgressOptions = { + readonly onProgress?: (stage: ScanProgressStage) => void; +}; + +type TransactionScanItemQuery = Promise & { + readonly first: () => Promise; + readonly update: ( + values: Partial + ) => Promise; + readonly where: ( + selector: (row: TransactionScanItemDb) => TValue, + value: TValue + ) => TransactionScanItemQuery; +}; + +type TransactionScanItemTable = { + readonly insert: ( + value: Pick + ) => Promise; + readonly where: ( + selector: (row: TransactionScanItemDb) => TValue, + value: TValue + ) => TransactionScanItemQuery; +}; + +function scanItemTable(db: AppDb): TransactionScanItemTable { + return db.transactionScanItems as unknown as TransactionScanItemTable; +} + +const scanResultSchema = { + additionalProperties: false, + properties: { + documentKind: { + enum: ['bank_app', 'bank_statement', 'invoice', 'receipt', 'other'], + type: 'string' + }, + warnings: { + items: { type: 'string' }, + type: 'array' + }, + visibleTotal: { type: ['number', 'null'] }, + visibleTotalCurrency: { type: ['string', 'null'] }, + transactions: { + items: { + additionalProperties: false, + properties: { + amount: { type: ['number', 'null'] }, + categoryId: { type: ['number', 'null'] }, + confidence: { + additionalProperties: false, + properties: { + amount: { + enum: ['high', 'medium', 'low'], + type: 'string' + }, + category: { + enum: ['high', 'medium', 'low'], + type: 'string' + }, + currency: { + enum: ['high', 'medium', 'low'], + type: 'string' + }, + date: { + enum: ['high', 'medium', 'low'], + type: 'string' + }, + overall: { + enum: ['high', 'medium', 'low'], + type: 'string' + }, + vendor: { + enum: ['high', 'medium', 'low'], + type: 'string' + } + }, + required: [ + 'amount', + 'category', + 'currency', + 'date', + 'overall', + 'vendor' + ], + type: 'object' + }, + currency: { type: ['string', 'null'] }, + evidence: { type: 'string' }, + lineItemSubtotal: { type: ['number', 'null'] }, + lineItems: { + items: { + additionalProperties: false, + properties: { + amount: { type: ['number', 'null'] }, + description: { type: 'string' }, + quantity: { type: ['number', 'null'] } + }, + required: ['amount', 'description', 'quantity'], + type: 'object' + }, + type: 'array' + }, + note: { type: ['string', 'null'] }, + occurredDate: { type: ['string', 'null'] }, + occurredTime: { type: ['string', 'null'] }, + suggestedCategoryKind: { + enum: ['normal', 'offset', null] + }, + suggestedCategoryName: { type: ['string', 'null'] }, + suggestedCategoryParentId: { type: ['number', 'null'] }, + suggestedCategoryReason: { type: ['string', 'null'] }, + suggestedCategoryType: { + enum: ['expense', 'income', null] + }, + suggestedVendorName: { type: ['string', 'null'] }, + transactionType: { + enum: ['expense', 'income'] + }, + vendorId: { type: ['number', 'null'] } + }, + required: [ + 'amount', + 'categoryId', + 'confidence', + 'currency', + 'evidence', + 'lineItemSubtotal', + 'lineItems', + 'note', + 'occurredDate', + 'occurredTime', + 'suggestedCategoryKind', + 'suggestedCategoryName', + 'suggestedCategoryParentId', + 'suggestedCategoryReason', + 'suggestedCategoryType', + 'suggestedVendorName', + 'transactionType', + 'vendorId' + ], + type: 'object' + }, + type: 'array' + } + }, + required: [ + 'documentKind', + 'warnings', + 'visibleTotal', + 'visibleTotalCurrency', + 'transactions' + ], + type: 'object' +} as const; + +function stripDataUrl(value: string): string { + const commaIndex = value.indexOf(','); + return value.startsWith('data:') && commaIndex >= 0 + ? value.slice(commaIndex + 1) + : value; +} + +function scanImageBuffer(imageBase64: string): Buffer { + try { + const buffer = Buffer.from(stripDataUrl(imageBase64), 'base64'); + if (buffer.length === 0) { + throw new TransactionScanInputError('Upload a non-empty image.'); + } + if (buffer.length > maxImageBytes) { + throw new TransactionScanInputError( + 'Image must be 10 MB or smaller.' + ); + } + return buffer; + } catch (err) { + if (err instanceof TransactionScanInputError) { + throw err; + } + throw new TransactionScanInputError('Upload a valid image.'); + } +} + +function imageBuffer(body: TransactionScanBody): Buffer { + return scanImageBuffer(body.imageBase64); +} + +function tileCount(length: number): number { + if (length <= tileLongSide) { + return 1; + } + return Math.ceil((length - tileOverlap) / (tileLongSide - tileOverlap)); +} + +function tileRanges( + length: number +): Array<{ readonly start: number; readonly size: number }> { + if (length <= tileLongSide) { + return [{ start: 0, size: length }]; + } + + const ranges: Array<{ readonly start: number; readonly size: number }> = []; + const step = tileLongSide - tileOverlap; + for (let start = 0; start < length; start += step) { + const size = Math.min(tileLongSide, length - start); + const normalizedStart = + size < tileLongSide ? Math.max(0, length - tileLongSide) : start; + const normalizedSize = Math.min(tileLongSide, length - normalizedStart); + if ( + ranges.some( + range => + range.start === normalizedStart && + range.size === normalizedSize + ) + ) { + break; + } + ranges.push({ start: normalizedStart, size: normalizedSize }); + if (normalizedStart + normalizedSize >= length) { + break; + } + } + return ranges; +} + +function resizedDimensions({ + height, + width +}: { + readonly height: number; + readonly width: number; +}) { + const longSide = Math.max(width, height); + const maxLongSide = + tileLongSide + (maxVisionTiles - 1) * (tileLongSide - tileOverlap); + if (tileCount(longSide) <= maxVisionTiles) { + return { height, width, resized: false }; + } + + const scale = maxLongSide / longSide; + return { + height: Math.max(1, Math.round(height * scale)), + width: Math.max(1, Math.round(width * scale)), + resized: true + }; +} + +function originalImageInput( + buffer: Buffer, + mimeType: TransactionScanBody['mimeType'], + width?: number, + height?: number +): PreparedScanImages { + return { + images: [ + { + buffer, + description: 'original image', + height, + mimeType, + width + } + ], + promptContext: { + originalHeight: height, + originalWidth: width, + preprocessing: 'The model received the original uploaded image.', + tiles: [ + { + description: 'original image', + height, + width + } + ] + }, + warnings: [] + }; +} + +function needsVisionTiling(width: number, height: number): boolean { + const longSide = Math.max(width, height); + const shortSide = Math.max(1, Math.min(width, height)); + return ( + longSide > visionLongSideLimit || + longSide / shortSide >= extremeAspectRatio + ); +} + +export async function prepareScanImagesForVision( + buffer: Buffer, + mimeType: TransactionScanBody['mimeType'] +): Promise { + try { + const metadata = await sharp(buffer).metadata(); + const oriented = metadata.autoOrient; + const width = oriented?.width ?? metadata.width; + const height = oriented?.height ?? metadata.height; + if (!width || !height || !needsVisionTiling(width, height)) { + return originalImageInput(buffer, mimeType, width, height); + } + + const nextDimensions = resizedDimensions({ height, width }); + let normalizedPipeline = sharp(buffer).autoOrient(); + if (nextDimensions.resized) { + normalizedPipeline = normalizedPipeline.resize({ + fit: 'fill', + height: nextDimensions.height, + width: nextDimensions.width + }); + } + const normalizedBuffer = await normalizedPipeline.toBuffer(); + + const vertical = nextDimensions.height >= nextDimensions.width; + const ranges = tileRanges( + vertical ? nextDimensions.height : nextDimensions.width + ).slice(0, maxVisionTiles); + const images = await Promise.all( + ranges.map(async (range, index): Promise => { + const extract = vertical + ? { + left: 0, + top: range.start, + width: nextDimensions.width, + height: range.size + } + : { + left: range.start, + top: 0, + width: range.size, + height: nextDimensions.height + }; + const tile = await sharp(normalizedBuffer) + .extract(extract) + .flatten({ background: '#ffffff' }) + .jpeg({ quality: 92 }) + .toBuffer(); + return { + buffer: tile, + description: `tile ${index + 1} of ${ranges.length}`, + height: extract.height, + mimeType: 'image/jpeg', + width: extract.width + }; + }) + ); + const warning = + `Long image was split into ${images.length} ordered tiles for better text reading.` + + (nextDimensions.resized + ? ' It was downscaled before tiling to stay within scan limits.' + : ''); + + return { + images, + promptContext: { + originalHeight: height, + originalWidth: width, + preprocessing: + 'The uploaded image was auto-oriented and split into ordered overlapping tiles. Read tiles in order and treat them as one continuous source.', + tiles: images.map(image => ({ + description: image.description, + height: image.height, + width: image.width + })) + }, + warnings: [warning] + }; + } catch { + return originalImageInput(buffer, mimeType); + } +} + +function oneLine(value: string, maxLength: number): string { + return value.replace(/\s+/g, ' ').trim().slice(0, maxLength); +} + +function stringValue(value: unknown, maxLength = 200): string | null { + return typeof value === 'string' && value.trim() + ? oneLine(value, maxLength) + : null; +} + +function noteValue( + value: unknown, + maxLength = FieldLimits.transactionNote +): string | null { + if (typeof value !== 'string' || !value.trim()) { + return null; + } + + const normalized = value + .replace(/\r\n?/g, '\n') + .split('\n') + .map(line => line.replace(/[ \t]+/g, ' ').trim()) + .join('\n') + .replace(/\n{3,}/g, '\n\n') + .trim(); + return normalized ? normalized.slice(0, maxLength).trimEnd() : null; +} + +function lineItemNote(value: unknown): string | null { + if (!Array.isArray(value)) { + return null; + } + + const lines = value + .map(item => { + if (typeof item !== 'object' || item === null) { + return null; + } + const record = item as Record; + const description = stringValue(record.description, 120); + if (!description) { + return null; + } + const amount = numberValue(record.amount); + const quantity = numberValue(record.quantity); + const amountText = + amount !== null && amount > 0 ? ` - ${amount.toFixed(2)}` : ''; + const quantityText = + quantity !== null && quantity > 1 ? `${quantity} x ` : ''; + return `${quantityText}${description}${amountText}`; + }) + .filter((item): item is string => item !== null); + + return noteValue(lines.join('\n')); +} + +function scanImageContentPart(image: ScanImageInput) { + return { + type: 'input_image' as const, + image_url: `data:${image.mimeType};base64,${image.buffer.toString( + 'base64' + )}`, + detail: 'original' as const + }; +} + +function numberValue(value: unknown): number | null { + if (typeof value !== 'number' || !Number.isFinite(value)) { + return null; + } + return value; +} + +function idValue( + value: unknown, + allowedIds: ReadonlySet +): number | null { + const id = numberValue(value); + return id !== null && allowedIds.has(id) ? id : null; +} + +function confidence(value: unknown): Confidence { + return typeof value === 'string' && confidenceValues.has(value) + ? (value as Confidence) + : 'low'; +} + +function effectiveCategoryType(category: Pick) { + if (category.kind !== 'offset') { + return category.type; + } + return category.type === 'expense' ? 'income' : 'expense'; +} + +function categoryById( + categories: readonly Category[], + categoryId: number | null +): Category | undefined { + return categoryId === null + ? undefined + : categories.find(category => category.id === categoryId); +} + +function transactionType( + value: unknown, + category: Category | undefined +): TransactionType { + if (category) { + return effectiveCategoryType(category); + } + return value === 'income' ? 'income' : 'expense'; +} + +function currencyValue(value: unknown, fallback: string): string { + const currency = stringValue(value, 3)?.toUpperCase(); + return currency && /^[A-Z]{3}$/.test(currency) ? currency : fallback; +} + +function dateValue( + rawDate: unknown, + rawTime: unknown, + timezone: string +): Date | null { + const date = stringValue(rawDate, 10); + if (!date || !/^\d{4}-\d{2}-\d{2}$/.test(date)) { + return null; + } + const time = stringValue(rawTime, 5); + const localTime = time && /^\d{2}:\d{2}$/.test(time) ? time : '12:00'; + return localDateTimeInputToDate(`${date}T${localTime}`, timezone) ?? null; +} + +function suggestedCategory( + raw: RawScannedTransaction, + type: TransactionType, + categories: readonly Category[] +): TransactionScanDraft['suggestedCategory'] { + const name = stringValue( + raw.suggestedCategoryName, + FieldLimits.categoryName + ); + if (!name) { + return null; + } + const suggestedType = + raw.suggestedCategoryType === 'income' || + raw.suggestedCategoryType === 'expense' + ? raw.suggestedCategoryType + : type; + const parentId = idValue( + raw.suggestedCategoryParentId, + new Set( + categories + .filter( + category => + category.parentId === null && + category.type === suggestedType + ) + .map(category => category.id) + ) + ); + + return { + name, + type: suggestedType, + parentId, + kind: + parentId !== null && raw.suggestedCategoryKind === 'offset' + ? 'offset' + : 'normal', + reason: + stringValue(raw.suggestedCategoryReason, 160) ?? + 'No existing category looked like a strong fit.' + }; +} + +function possibleDuplicates({ + categories, + draft, + recentTransactions, + timezone +}: { + readonly categories: readonly Category[]; + readonly draft: Omit; + readonly recentTransactions: readonly Transaction[]; + readonly timezone: string; +}): number[] { + if (!draft.amount || !draft.currency || !draft.occurredAt) { + return []; + } + + const draftDate = dateToLocalDateParam(draft.occurredAt, timezone); + const category = categoryById(categories, draft.categoryId); + const type = category + ? effectiveCategoryType(category) + : draft.transactionType; + + return recentTransactions + .filter(transaction => { + if (Math.abs(transaction.amount - draft.amount!) > 0.005) { + return false; + } + if (transaction.currency !== draft.currency) { + return false; + } + if ( + dateToLocalDateParam(transaction.occurredAt, timezone) !== + draftDate + ) { + return false; + } + if (transaction.type !== type) { + return false; + } + if ( + draft.vendorId !== null && + transaction.vendorId !== draft.vendorId + ) { + return false; + } + return true; + }) + .slice(0, 3) + .map(transaction => transaction.id); +} + +function sanitizeDraft({ + categories, + defaultCurrency, + raw, + recentTransactions, + timezone, + vendors +}: { + readonly categories: readonly Category[]; + readonly defaultCurrency: string; + readonly raw: RawScannedTransaction; + readonly recentTransactions: readonly Transaction[]; + readonly timezone: string; + readonly vendors: readonly Vendor[]; +}): Omit | undefined { + const categoryIds = new Set(categories.map(category => category.id)); + const vendorIds = new Set(vendors.map(vendor => vendor.id)); + const categoryId = idValue(raw.categoryId, categoryIds); + const vendorId = idValue(raw.vendorId, vendorIds); + const category = categoryById(categories, categoryId); + const type = transactionType(raw.transactionType, category); + const amount = numberValue(raw.amount); + const safeAmount = amount !== null && amount > 0 ? amount : null; + const fieldConfidence = raw.confidence ?? {}; + const categoryConfidence = confidence(fieldConfidence.category); + const exactCategoryId = + categoryId !== null && categoryConfidence !== 'low' ? categoryId : null; + const draft = { + amount: safeAmount, + categoryId: exactCategoryId, + suggestedCategory: + exactCategoryId === null + ? suggestedCategory(raw, type, categories) + : null, + currency: currencyValue(raw.currency, defaultCurrency), + occurredAt: dateValue(raw.occurredDate, raw.occurredTime, timezone), + vendorId, + suggestedVendorName: + vendorId === null + ? stringValue(raw.suggestedVendorName, FieldLimits.vendorName) + : null, + transactionType: type, + note: noteValue(raw.note) ?? lineItemNote(raw.lineItems), + evidence: stringValue(raw.evidence, 500) ?? '', + confidence: { + amount: confidence(fieldConfidence.amount), + category: categoryConfidence, + currency: raw.currency + ? confidence(fieldConfidence.currency) + : 'low', + date: raw.occurredDate ? confidence(fieldConfidence.date) : 'low', + overall: confidence(fieldConfidence.overall), + vendor: confidence(fieldConfidence.vendor) + }, + possibleDuplicateTransactionIds: [] + } satisfies Omit; + + if (!draft.evidence && !draft.amount && !draft.note) { + return undefined; + } + + return { + ...draft, + possibleDuplicateTransactionIds: possibleDuplicates({ + categories, + draft, + recentTransactions, + timezone + }) + }; +} + +function parseJson(value: string): unknown { + try { + return JSON.parse(value) as unknown; + } catch { + return null; + } +} + +async function getUser(db: AppDb, userId: number): Promise { + const user = await db.users.find(userId); + if (!user) { + throw new TransactionScanInputError('User was not found.'); + } + return user as UserDb; +} + +async function correctionExamples( + db: AppDb, + userId: number +): Promise { + const rows = await scanItemTable(db).where(item => item.userId, userId); + + return rows + .filter(row => row.decision) + .sort( + (left, right) => + (right.decidedAt?.getTime() ?? 0) - + (left.decidedAt?.getTime() ?? 0) + ) + .slice(0, maxCorrectionExamples) + .map(row => ({ + decision: row.decision ?? '', + draft: parseJson(row.draftJson), + corrected: row.correctedJson ? parseJson(row.correctedJson) : null + })); +} + +function promptCategories(categories: readonly Category[]): PromptCategory[] { + return categories.map(category => ({ + id: category.id, + name: category.name, + displayName: category.displayName, + parentId: category.parentId, + parentName: category.parentName, + type: category.type, + kind: category.kind, + effectiveType: effectiveCategoryType(category) + })); +} + +function promptVendors(vendors: readonly Vendor[]): PromptVendor[] { + return vendors.map(vendor => ({ + id: vendor.id, + name: vendor.name, + displayName: vendor.displayName, + resolvedName: vendor.resolvedName, + domain: vendor.domain, + suggestedCategoryId: vendor.suggestedCategoryId, + suggestedCategoryDisplayName: vendor.suggestedCategoryDisplayName, + transactionCount: vendor.transactionCount + })); +} + +function promptTransactions( + transactions: readonly Transaction[], + timezone: string +) { + return transactions.map(transaction => ({ + id: transaction.id, + amount: transaction.amount, + currency: transaction.currency, + localDate: dateToLocalDateParam(transaction.occurredAt, timezone), + type: transaction.type, + categoryId: transaction.categoryId, + category: transaction.categoryDisplayName, + vendorId: transaction.vendorId, + vendorName: transaction.vendorName, + note: transaction.note + })); +} + +function scanPrompt() { + return [ + 'You extract draft personal-finance transactions from one uploaded image.', + 'The image may be a receipt, invoice, bank app screenshot, or bank statement.', + 'Return one or more transactions. Split one receipt into multiple transactions when visible line items clearly belong to different categories or vendors.', + 'When one receipt or invoice is split, each draft amount must include only the line items assigned to that draft plus a proportional share of visible shared tax, fees, tips, and discounts.', + 'Rounded split drafts should add up to the visible document total when the total is visible. Assign any one-cent rounding remainder to the largest split group.', + 'Use multiline notes to list the line items included in each split draft, one item per line when possible.', + 'Do not split when category or vendor grouping is ambiguous; return a single transaction with an itemized note and lower confidence instead.', + 'Do not create transactions, categories, or vendors. Only choose existing IDs from context or suggest names for the user to create later.', + 'Prefer existing category IDs and vendor IDs only when they are a strong fit.', + 'When no existing category is a strong fit, set categoryId to null and fill suggestedCategoryName, suggestedCategoryType, suggestedCategoryKind, optional suggestedCategoryParentId, and suggestedCategoryReason.', + 'Use positive amounts only. Infer expense/income through transactionType and category fit; do not encode signs in amount.', + 'If a field is not visible or not reliable, return null for that field and low confidence.', + 'Use correction examples as user-specific preferences and avoid repeating prior mistakes.', + 'Use only facts visible in the image and the provided context. Do not invent vendors, dates, currencies, amounts, or line items.' + ].join(' '); +} + +function promptInput({ + categories, + correctionExamples, + imageProcessing, + recentTransactions, + user, + vendors +}: { + readonly categories: readonly Category[]; + readonly correctionExamples: readonly CorrectionExample[]; + readonly imageProcessing: PreparedScanImages['promptContext']; + readonly recentTransactions: readonly Transaction[]; + readonly user: UserDb; + readonly vendors: readonly Vendor[]; +}) { + return { + user: { + defaultCurrency: user.defaultCurrency, + timezone: user.timezone, + localToday: dateToLocalDateParam(new Date(), user.timezone) + }, + categories: promptCategories(categories), + vendors: promptVendors(vendors), + recentTransactions: promptTransactions( + recentTransactions, + user.timezone + ), + correctionExamples, + imageProcessing, + outputRules: { + occurredDate: + 'Use YYYY-MM-DD when visible or inferable from the document; otherwise null.', + occurredTime: 'Use HH:mm when visible; otherwise null.', + categoryId: + 'Use only IDs from categories when the match is strong; otherwise null and provide suggested category fields.', + vendorId: 'Use only IDs from vendors.', + suggestedCategoryParentId: + 'Use only a parent category ID from categories, or null.', + lineItems: + 'For each draft, include only the visible line items that belong to that draft. Leave empty for bank statement/app transaction rows.', + lineItemSubtotal: + 'Subtotal of line items before proportional shared tax, fees, tips, and discounts, or null when not visible.', + visibleTotal: + 'Visible receipt/invoice total after tax, fees, tips, and discounts, or null when not visible.', + splitTotals: + 'When splitting one receipt/invoice, allocate shared adjustments proportionally by line item subtotal and make split amounts sum to the visible total within one cent.', + note: 'Use multiline notes for split drafts and list included line items. Do not include items assigned to another draft.', + maxTransactions: maxDrafts + } + }; +} + +function documentKind(value: unknown): TransactionScanResponse['documentKind'] { + return typeof value === 'string' && documentKinds.has(value) + ? (value as TransactionScanResponse['documentKind']) + : 'other'; +} + +function warnings(value: unknown): string[] { + return Array.isArray(value) + ? value + .map(item => stringValue(item, 200)) + .filter((item): item is string => item !== null) + .slice(0, 8) + : []; +} + +export async function scanTransactionsFromImage( + db: AppDb, + config: Config, + userId: number, + body: TransactionScanBody, + options: ScanProgressOptions = {} +): Promise { + const buffer = imageBuffer(body); + const imageHash = createHash('sha256').update(buffer).digest('hex'); + options.onProgress?.('preparing'); + const user = await getUser(db, userId); + const [categories, vendors, recentTransactions, examples] = + await Promise.all([ + listCategories(db, userId, { activeOnly: true }), + listVendors(db, userId, { limit: maxContextVendors }), + listTransactions(db, userId, { + direction: 'desc', + limit: maxRecentTransactions, + page: 1 + }).then(response => response.items), + correctionExamples(db, userId) + ]); + const preparedImages = await prepareScanImagesForVision( + buffer, + body.mimeType + ); + + options.onProgress?.('analyzing'); + const parsed = await generateStructuredJsonFromContent( + config, + { + content: [ + { + type: 'input_text', + text: JSON.stringify( + promptInput({ + categories, + correctionExamples: examples, + imageProcessing: preparedImages.promptContext, + recentTransactions, + user, + vendors + }) + ) + }, + ...preparedImages.images.map(scanImageContentPart) + ], + model: config.openai.transactionScanModel, + schema: scanResultSchema, + schemaName: 'transaction_image_scan', + system: scanPrompt() + } + ); + + options.onProgress?.('saving'); + const scanWarnings = [ + ...warnings(parsed.warnings), + ...preparedImages.warnings + ].slice(0, 8); + const scan = await db.transactionScans.insert({ + userId, + documentKind: documentKind(parsed.documentKind), + imageHash, + model: config.openai.transactionScanModel, + warningsJson: JSON.stringify(scanWarnings) + }); + + const rawTransactions = Array.isArray(parsed.transactions) + ? (parsed.transactions as RawScannedTransaction[]) + : []; + const drafts: TransactionScanDraft[] = []; + + for (const raw of rawTransactions.slice(0, maxDrafts)) { + const sanitized = sanitizeDraft({ + categories, + defaultCurrency: user.defaultCurrency, + raw, + recentTransactions, + timezone: user.timezone, + vendors + }); + if (!sanitized) { + continue; + } + + const item = await scanItemTable(db).insert({ + scanId: scan.id, + userId, + draftJson: JSON.stringify(sanitized) + }); + drafts.push({ ...sanitized, id: item.id }); + } + + return { + scanId: scan.id, + documentKind: documentKind(parsed.documentKind), + warnings: scanWarnings, + drafts + }; +} + +async function ensureTransactionOwner( + db: AppDb, + userId: number, + transactionId: number +): Promise { + const transaction = await db.transactions + .where(row => row.id, transactionId) + .where(row => row.userId, userId) + .first(); + if (!transaction) { + throw new TransactionScanInputError('Transaction was not found.'); + } +} + +async function storeScanAttachment({ + attachment, + db, + scan, + userId +}: { + readonly attachment: NonNullable; + readonly db: AppDb; + readonly scan: TransactionScanDb; + readonly userId: number; +}): Promise { + const imageBase64 = stripDataUrl(attachment.imageBase64); + const buffer = scanImageBuffer(imageBase64); + const imageHash = createHash('sha256').update(buffer).digest('hex'); + if (imageHash !== scan.imageHash) { + throw new TransactionScanInputError( + 'Confirmed scan image did not match the original scan.' + ); + } + + const existing = await db.transactionScanImages + .where(row => row.scanId, scan.id) + .where(row => row.userId, userId) + .first(); + const values = { + imageHash, + mimeType: attachment.mimeType, + fileName: stringValue(attachment.fileName, 255) ?? null, + sizeBytes: buffer.length, + imageBase64, + updatedAt: new Date() + }; + + if (existing) { + await db.transactionScanImages + .where(row => row.id, existing.id) + .where(row => row.userId, userId) + .update(values as never); + return; + } + + await db.transactionScanImages.insert({ + scanId: scan.id, + userId, + ...values + } as never); +} + +export async function recordTransactionScanDecision( + db: AppDb, + userId: number, + scanId: number, + itemId: number, + body: TransactionScanDecisionBody +): Promise { + const item = await scanItemTable(db) + .where(row => row.id, itemId) + .where(row => row.scanId, scanId) + .where(row => row.userId, userId) + .first(); + if (!item) { + throw new TransactionScanNotFoundError('Scan item was not found.'); + } + const scan = await db.transactionScans + .where(row => row.id, scanId) + .where(row => row.userId, userId) + .first(); + if (!scan) { + throw new TransactionScanNotFoundError('Scan was not found.'); + } + + if (body.decision === 'confirmed') { + if (!body.transactionId || !body.correctedTransaction) { + throw new TransactionScanInputError( + 'Confirmed scan items require a transaction and corrected values.' + ); + } + await ensureTransactionOwner(db, userId, body.transactionId); + if (body.attachment) { + await storeScanAttachment({ + attachment: body.attachment, + db, + scan, + userId + }); + } + } + + await scanItemTable(db) + .where(row => row.id, itemId) + .where(row => row.userId, userId) + .update({ + decision: body.decision, + correctedJson: body.correctedTransaction + ? JSON.stringify(body.correctedTransaction) + : (null as never), + transactionId: (body.transactionId ?? null) as never, + createdCategoryId: (body.createdCategoryId ?? null) as never, + createdVendorId: (body.createdVendorId ?? null) as never, + decidedAt: new Date(), + updatedAt: new Date() + }); +} diff --git a/apps/api/src/application/transactions.test.ts b/apps/api/src/application/transactions.test.ts index 4519270..fb907d3 100644 --- a/apps/api/src/application/transactions.test.ts +++ b/apps/api/src/application/transactions.test.ts @@ -1,10 +1,11 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import type { CategoryDb, VendorDb } from '../db/schemas.js'; import { categoryTrendMaxBuckets, compareTransactionsByOccurrenceAsc, compareTransactionsByOccurrenceDesc, dashboardStatsGroupBy, + getTransactionScanImage, percentChange, resolveCategoryTrendRange, resolveDashboardComparisonRange, @@ -451,6 +452,59 @@ describe('transaction category signs', () => { }); }); +describe('transaction scan images', () => { + it('returns the latest confirmed scan image for a transaction', async () => { + const scanTimestamp = new Date('2026-06-01T12:00:00.000Z'); + const row = { + scanId: 10, + scanItemId: 20, + fileName: 'receipt.png', + mimeType: 'image/png', + sizeBytes: '5', + createdAt: scanTimestamp, + imageBase64: Buffer.from('image').toString('base64') + }; + const query = { + join: vi.fn(() => query), + where: vi.fn(() => query), + orderBy: vi.fn(() => query), + select: vi.fn(() => query), + first: vi.fn(async () => row) + }; + const knex = vi.fn(() => query); + + await expect( + getTransactionScanImage(knex as never, 1, 42) + ).resolves.toEqual({ + scanId: 10, + scanItemId: 20, + fileName: 'receipt.png', + mimeType: 'image/png', + sizeBytes: 5, + createdAt: scanTimestamp, + imageBase64: Buffer.from('image').toString('base64') + }); + expect(knex).toHaveBeenCalledWith('transaction_scan_items as item'); + expect(query.where).toHaveBeenCalledWith('item.user_id', 1); + expect(query.where).toHaveBeenCalledWith('item.transaction_id', 42); + expect(query.where).toHaveBeenCalledWith('item.decision', 'confirmed'); + }); + + it('throws when a transaction has no stored scan image', async () => { + const query = { + join: vi.fn(() => query), + where: vi.fn(() => query), + orderBy: vi.fn(() => query), + select: vi.fn(() => query), + first: vi.fn(async () => undefined) + }; + + await expect( + getTransactionScanImage(vi.fn(() => query) as never, 1, 42) + ).rejects.toBeInstanceOf(TransactionNotFoundError); + }); +}); + describe('stats range resolution', () => { it('compares an in-progress month with the same elapsed previous month', () => { const now = new Date(2026, 4, 10, 12, 34, 0, 0); diff --git a/apps/api/src/application/transactions.ts b/apps/api/src/application/transactions.ts index 435a2d2..26cef41 100644 --- a/apps/api/src/application/transactions.ts +++ b/apps/api/src/application/transactions.ts @@ -10,7 +10,8 @@ import type { StatsQuery, StatsWindowResponse, Transaction, - TransactionListQuery + TransactionListQuery, + TransactionScanImageResponse } from '@xpenser/contracts'; import { addLocalDays, @@ -33,6 +34,7 @@ import { statsBucketKeyInTimeZone, statsBucketLabelInTimeZone } from '@xpenser/timezone'; +import type { Knex } from 'knex'; import type { Config } from '../config.js'; import type { AppDb, @@ -62,6 +64,20 @@ type DashboardPeriod = NonNullable; type DashboardCategory = DashboardSummary['byCategory'][number]; type DashboardVendor = DashboardSummary['topVendors'][number]; +type TransactionScanAttachment = NonNullable; +type TransactionScanAttachmentRow = { + readonly createdAt: Date; + readonly fileName: string | null; + readonly mimeType: 'image/jpeg' | 'image/png' | 'image/webp'; + readonly scanId: number; + readonly scanItemId: number; + readonly sizeBytes: number | string; + readonly transactionId: number; +}; + +type TransactionScanImageRow = TransactionScanAttachmentRow & { + readonly imageBase64: string; +}; type StatsBucket = StatsOverview['trend'][number]; @@ -210,7 +226,8 @@ function categoryForTransaction( function mapTransaction( row: TransactionDb, categoriesById: ReadonlyMap, - vendorsById: ReadonlyMap + vendorsById: ReadonlyMap, + scanAttachments: ReadonlyMap = new Map() ): Transaction { const category = categoryForTransaction(row, categoriesById); const fields = categoryFields(category, row, categoriesById); @@ -236,11 +253,65 @@ function mapTransaction( exchangeRateDate: row.exchangeRateDate, occurredAt: row.occurredAt, note: row.note ?? undefined, + scanAttachment: scanAttachments.get(row.id) ?? null, createdAt: row.createdAt, updatedAt: row.updatedAt }; } +function scanAttachmentFromRow( + row: TransactionScanAttachmentRow +): TransactionScanAttachment { + return { + scanId: Number(row.scanId), + scanItemId: Number(row.scanItemId), + fileName: row.fileName, + mimeType: row.mimeType, + sizeBytes: Number(row.sizeBytes), + createdAt: row.createdAt + }; +} + +async function scanAttachmentsByTransaction( + knex: Knex | undefined, + userId: number, + transactionIds: readonly number[] +): Promise> { + const uniqueIds = [...new Set(transactionIds)]; + if (!knex || uniqueIds.length === 0) { + return new Map(); + } + + const rows = (await knex('transaction_scan_items as item') + .join( + 'transaction_scan_images as image', + 'image.scan_id', + 'item.scan_id' + ) + .where('item.user_id', userId) + .where('image.user_id', userId) + .where('item.decision', 'confirmed') + .whereIn('item.transaction_id', uniqueIds) + .orderBy('item.decided_at', 'desc') + .select({ + transactionId: 'item.transaction_id', + scanId: 'item.scan_id', + scanItemId: 'item.id', + fileName: 'image.file_name', + mimeType: 'image.mime_type', + sizeBytes: 'image.size_bytes', + createdAt: 'image.created_at' + })) as TransactionScanAttachmentRow[]; + + const attachments = new Map(); + for (const row of rows) { + if (!attachments.has(row.transactionId)) { + attachments.set(row.transactionId, scanAttachmentFromRow(row)); + } + } + return attachments; +} + export function compareTransactionsByOccurrenceDesc( left: Pick, right: Pick @@ -302,7 +373,8 @@ async function validateVendor( export async function listTransactions( db: AppDb, userId: number, - query: TransactionListQuery + query: TransactionListQuery, + knex?: Knex ) { const page = Math.max(1, query.page ?? 1); const limit = Math.min(100, Math.max(1, query.limit ?? 50)); @@ -416,13 +488,22 @@ export async function listTransactions( ); }); const offset = (page - 1) * limit; + const pageRows = filtered.slice(offset, offset + limit) as TransactionDb[]; + const scanAttachments = await scanAttachmentsByTransaction( + knex, + userId, + pageRows.map(transaction => transaction.id) + ); return { - items: filtered - .slice(offset, offset + limit) - .map(transaction => - mapTransaction(transaction, categoriesById, vendorsById) - ), + items: pageRows.map(transaction => + mapTransaction( + transaction, + categoriesById, + vendorsById, + scanAttachments + ) + ), total: filtered.length, page, limit @@ -498,6 +579,43 @@ export async function getTransaction( return mapTransaction(row as TransactionDb, categoriesById, vendorsById); } +export async function getTransactionScanImage( + knex: Knex, + userId: number, + transactionId: number +): Promise { + const row = (await knex('transaction_scan_items as item') + .join( + 'transaction_scan_images as image', + 'image.scan_id', + 'item.scan_id' + ) + .where('item.user_id', userId) + .where('image.user_id', userId) + .where('item.transaction_id', transactionId) + .where('item.decision', 'confirmed') + .orderBy('item.decided_at', 'desc') + .select({ + scanId: 'item.scan_id', + scanItemId: 'item.id', + fileName: 'image.file_name', + mimeType: 'image.mime_type', + sizeBytes: 'image.size_bytes', + createdAt: 'image.created_at', + imageBase64: 'image.image_base64' + }) + .first()) as TransactionScanImageRow | undefined; + + if (!row) { + throw new TransactionNotFoundError('Scanned image was not found.'); + } + + return { + ...scanAttachmentFromRow({ ...row, transactionId }), + imageBase64: row.imageBase64 + }; +} + export async function updateTransaction( db: AppDb, config: Config, diff --git a/apps/api/src/config.test.ts b/apps/api/src/config.test.ts index 7873dcb..1d7e5a8 100644 --- a/apps/api/src/config.test.ts +++ b/apps/api/src/config.test.ts @@ -47,6 +47,7 @@ describe('API config', () => { expect(config.emailConfirmation.tokenTtlSeconds).toBe(86_400); expect(config.resend.emailFrom).toBe('Xpenser '); expect(config.openai.reportModel).toBe('gpt-5-mini'); + expect(config.openai.transactionScanModel).toBe('gpt-5.5'); }); it('normalizes vendor enrichment feature flags', async () => { diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index ea6eae6..3ecb935 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -94,6 +94,10 @@ export const config = parseEnv( reportModel: env( 'OPENAI_REPORT_MODEL', string().default('gpt-5-mini') + ), + transactionScanModel: env( + 'OPENAI_TRANSACTION_SCAN_MODEL', + string().default('gpt-5.5') ) }, resend: { diff --git a/apps/api/src/db/migrations/014_transaction_scans.ts b/apps/api/src/db/migrations/014_transaction_scans.ts new file mode 100644 index 0000000..ab9e2e4 --- /dev/null +++ b/apps/api/src/db/migrations/014_transaction_scans.ts @@ -0,0 +1,71 @@ +import type { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.createTable('transaction_scans', table => { + table.increments('id').primary(); + table + .integer('user_id') + .notNullable() + .references('id') + .inTable('users') + .onDelete('CASCADE'); + table.string('document_kind', 32).notNullable(); + table.string('image_hash', 128).notNullable(); + table.string('model', 128).notNullable(); + table.text('warnings_json').notNullable(); + table.timestamp('created_at').notNullable().defaultTo(knex.fn.now()); + table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now()); + + table.index(['user_id'], 'idx_transaction_scans_user_id'); + table.index(['user_id', 'created_at'], 'idx_transaction_scans_recent'); + }); + + await knex.schema.createTable('transaction_scan_items', table => { + table.increments('id').primary(); + table + .integer('scan_id') + .notNullable() + .references('id') + .inTable('transaction_scans') + .onDelete('CASCADE'); + table + .integer('user_id') + .notNullable() + .references('id') + .inTable('users') + .onDelete('CASCADE'); + table.text('draft_json').notNullable(); + table.string('decision', 32); + table.text('corrected_json'); + table + .integer('transaction_id') + .references('id') + .inTable('transactions') + .onDelete('SET NULL'); + table + .integer('created_category_id') + .references('id') + .inTable('categories') + .onDelete('SET NULL'); + table + .integer('created_vendor_id') + .references('id') + .inTable('vendors') + .onDelete('SET NULL'); + table.timestamp('decided_at'); + table.timestamp('created_at').notNullable().defaultTo(knex.fn.now()); + table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now()); + + table.index(['scan_id'], 'idx_transaction_scan_items_scan_id'); + table.index(['user_id'], 'idx_transaction_scan_items_user_id'); + table.index( + ['user_id', 'decided_at'], + 'idx_transaction_scan_items_recent_decisions' + ); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists('transaction_scan_items'); + await knex.schema.dropTableIfExists('transaction_scans'); +} diff --git a/apps/api/src/db/migrations/015_transaction_scan_images.ts b/apps/api/src/db/migrations/015_transaction_scan_images.ts new file mode 100644 index 0000000..343f1ad --- /dev/null +++ b/apps/api/src/db/migrations/015_transaction_scan_images.ts @@ -0,0 +1,35 @@ +import type { Knex } from 'knex'; + +export async function up(knex: Knex): Promise { + await knex.schema.createTable('transaction_scan_images', table => { + table.increments('id').primary(); + table + .integer('scan_id') + .notNullable() + .references('id') + .inTable('transaction_scans') + .onDelete('CASCADE'); + table + .integer('user_id') + .notNullable() + .references('id') + .inTable('users') + .onDelete('CASCADE'); + table.string('image_hash', 128).notNullable(); + table.string('mime_type', 64).notNullable(); + table.text('file_name'); + table.integer('size_bytes').notNullable(); + table.text('image_base64').notNullable(); + table.timestamp('created_at').notNullable().defaultTo(knex.fn.now()); + table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now()); + + table.unique(['scan_id'], { + indexName: 'uq_transaction_scan_images_scan_id' + }); + table.index(['user_id'], 'idx_transaction_scan_images_user_id'); + }); +} + +export async function down(knex: Knex): Promise { + await knex.schema.dropTableIfExists('transaction_scan_images'); +} diff --git a/apps/api/src/db/migrations/016_transaction_note_length.ts b/apps/api/src/db/migrations/016_transaction_note_length.ts new file mode 100644 index 0000000..318d372 --- /dev/null +++ b/apps/api/src/db/migrations/016_transaction_note_length.ts @@ -0,0 +1,66 @@ +import type { Knex } from 'knex'; + +async function dropConstraintIfExists( + knex: Knex, + tableName: string, + constraintName: string +) { + await knex.raw(`alter table ?? drop constraint if exists ??`, [ + tableName, + constraintName + ]); +} + +async function addCheckConstraint( + knex: Knex, + tableName: string, + constraintName: string, + expression: string +) { + await knex.raw(`alter table ?? add constraint ?? check (${expression})`, [ + tableName, + constraintName + ]); +} + +export async function up(knex: Knex): Promise { + if (!(await knex.schema.hasColumn('transactions', 'note'))) { + return; + } + + await dropConstraintIfExists( + knex, + 'transactions', + 'chk_transactions_note_length' + ); + await addCheckConstraint( + knex, + 'transactions', + 'chk_transactions_note_length', + 'char_length(note) <= 2000' + ); +} + +export async function down(knex: Knex): Promise { + if (!(await knex.schema.hasColumn('transactions', 'note'))) { + return; + } + + await knex.raw(` + update transactions + set note = left(note, 500) + where note is not null + and char_length(note) > 500 + `); + await dropConstraintIfExists( + knex, + 'transactions', + 'chk_transactions_note_length' + ); + await addCheckConstraint( + knex, + 'transactions', + 'chk_transactions_note_length', + 'char_length(note) <= 500' + ); +} diff --git a/apps/api/src/db/schemas.ts b/apps/api/src/db/schemas.ts index fc7807a..427828b 100644 --- a/apps/api/src/db/schemas.ts +++ b/apps/api/src/db/schemas.ts @@ -197,6 +197,77 @@ export const TransactionDbSchema = object({ category: CategoryDbSchema.optional() }).hasTableName('transactions'); +export const TransactionScanDbSchema = object({ + id: number().primaryKey(), + userId: number() + .hasColumnName('user_id') + .references('users', 'id') + .onDelete('CASCADE') + .index('idx_transaction_scans_user_id'), + documentKind: string().hasColumnName('document_kind'), + imageHash: string().hasColumnName('image_hash'), + model: string(), + warningsJson: string().hasColumnName('warnings_json'), + createdAt: date().hasColumnName('created_at').defaultTo('now'), + updatedAt: date().hasColumnName('updated_at').defaultTo('now') +}).hasTableName('transaction_scans'); + +export const TransactionScanItemDbSchema = object({ + id: number().primaryKey(), + scanId: number() + .hasColumnName('scan_id') + .references('transaction_scans', 'id') + .onDelete('CASCADE') + .index('idx_transaction_scan_items_scan_id'), + userId: number() + .hasColumnName('user_id') + .references('users', 'id') + .onDelete('CASCADE') + .index('idx_transaction_scan_items_user_id'), + draftJson: string().hasColumnName('draft_json'), + decision: string().optional(), + correctedJson: string().hasColumnName('corrected_json').optional(), + transactionId: number() + .hasColumnName('transaction_id') + .references('transactions', 'id') + .onDelete('SET NULL') + .optional(), + createdCategoryId: number() + .hasColumnName('created_category_id') + .references('categories', 'id') + .onDelete('SET NULL') + .optional(), + createdVendorId: number() + .hasColumnName('created_vendor_id') + .references('vendors', 'id') + .onDelete('SET NULL') + .optional(), + decidedAt: date().hasColumnName('decided_at').optional(), + createdAt: date().hasColumnName('created_at').defaultTo('now'), + updatedAt: date().hasColumnName('updated_at').defaultTo('now') +}).hasTableName('transaction_scan_items'); + +export const TransactionScanImageDbSchema = object({ + id: number().primaryKey(), + scanId: number() + .hasColumnName('scan_id') + .references('transaction_scans', 'id') + .onDelete('CASCADE') + .index('idx_transaction_scan_images_scan_id'), + userId: number() + .hasColumnName('user_id') + .references('users', 'id') + .onDelete('CASCADE') + .index('idx_transaction_scan_images_user_id'), + imageHash: string().hasColumnName('image_hash'), + mimeType: string().hasColumnName('mime_type'), + fileName: string().hasColumnName('file_name').optional(), + sizeBytes: number().hasColumnName('size_bytes'), + imageBase64: string().hasColumnName('image_base64'), + createdAt: date().hasColumnName('created_at').defaultTo('now'), + updatedAt: date().hasColumnName('updated_at').defaultTo('now') +}).hasTableName('transaction_scan_images'); + export const ExchangeRateDbSchema = object({ id: number().primaryKey(), baseCurrency: string().hasColumnName('base_currency'), @@ -233,6 +304,13 @@ export const TransactionEntity = defineEntity(TransactionDbSchema).belongsTo( l => l.categoryId, r => r.id ); +export const TransactionScanEntity = defineEntity(TransactionScanDbSchema); +export const TransactionScanItemEntity = defineEntity( + TransactionScanItemDbSchema +); +export const TransactionScanImageEntity = defineEntity( + TransactionScanImageDbSchema +); export const ExchangeRateEntity = defineEntity(ExchangeRateDbSchema); export const entityMap = { @@ -245,6 +323,9 @@ export const entityMap = { categories: CategoryEntity, vendors: VendorEntity, transactions: TransactionEntity, + transactionScans: TransactionScanEntity, + transactionScanItems: TransactionScanItemEntity, + transactionScanImages: TransactionScanImageEntity, exchangeRates: ExchangeRateEntity }; @@ -355,3 +436,42 @@ export type TransactionDb = { readonly createdAt: Date; readonly updatedAt: Date; }; + +export type TransactionScanDb = { + readonly id: number; + readonly userId: number; + readonly documentKind: string; + readonly imageHash: string; + readonly model: string; + readonly warningsJson: string; + readonly createdAt: Date; + readonly updatedAt: Date; +}; + +export type TransactionScanItemDb = { + readonly id: number; + readonly scanId: number; + readonly userId: number; + readonly draftJson: string; + readonly decision?: string | null; + readonly correctedJson?: string | null; + readonly transactionId?: number | null; + readonly createdCategoryId?: number | null; + readonly createdVendorId?: number | null; + readonly decidedAt?: Date | null; + readonly createdAt: Date; + readonly updatedAt: Date; +}; + +export type TransactionScanImageDb = { + readonly id: number; + readonly scanId: number; + readonly userId: number; + readonly imageHash: string; + readonly mimeType: 'image/jpeg' | 'image/png' | 'image/webp'; + readonly fileName?: string | null; + readonly sizeBytes: number; + readonly imageBase64: string; + readonly createdAt: Date; + readonly updatedAt: Date; +}; diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts index 123a2bd..378305b 100644 --- a/apps/api/src/server.ts +++ b/apps/api/src/server.ts @@ -49,7 +49,7 @@ export function buildServer( }); const server = createServer({ - maxBodySize: 1024 * 1024 + maxBodySize: 20 * 1024 * 1024 }) .use(tracingMiddleware({ excludePaths: ['/health'] })) .use(corsMiddleware(config)) diff --git a/apps/web/app/(app)/capture/page.tsx b/apps/web/app/(app)/capture/page.tsx index d6fa51a..aeac169 100644 --- a/apps/web/app/(app)/capture/page.tsx +++ b/apps/web/app/(app)/capture/page.tsx @@ -1,5 +1,5 @@ import { redirect } from 'next/navigation'; -import { QuickCaptureForm } from '@/components/quick-capture-form'; +import { TransactionCaptureWorkspace } from '@/components/transaction-scan-capture'; import { getApiClient } from '@/lib/api'; import { categoriesByRecentUse } from '@/lib/capture-suggestions'; @@ -24,7 +24,7 @@ export default async function CapturePage() { return (
- ({ + auth: vi.fn(), + createXpenserClient: vi.fn(), + transactionScanStart: vi.fn() +})); + +vi.mock('@/auth', () => ({ + auth: mocks.auth +})); + +vi.mock('@/lib/config', () => ({ + webConfig: { apiBaseUrl: 'https://api.example.test' } +})); + +vi.mock('@xpenser/client', () => ({ + createXpenserClient: mocks.createXpenserClient +})); + +import { POST } from './route'; + +const uploadId = '00000000-0000-4000-8000-000000000001'; +const userId = '1'; + +function scanRequest(body: Record) { + return new Request('https://app.example.test/api/transaction-scans', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body) + }); +} + +function chunkBody(overrides: Partial> = {}) { + const bytes = Buffer.from('receipt bytes'); + return { + chunkBase64: bytes.toString('base64'), + chunkIndex: 0, + fileName: 'receipt.jpg', + fileSize: bytes.length, + mimeType: 'image/jpeg', + totalChunks: 1, + uploadId, + ...overrides + }; +} + +describe('transaction scan route', () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.auth.mockResolvedValue({ + apiToken: 'api-token', + user: { id: userId } + }); + mocks.createXpenserClient.mockReturnValue({ + transactionScans: { start: mocks.transactionScanStart } + }); + mocks.transactionScanStart.mockResolvedValue({ + jobId: '00000000-0000-4000-8000-000000000042', + token: 'scan-token' + }); + }); + + afterEach(async () => { + await deleteScanUpload(userId, uploadId); + }); + + it('forwards a valid image to the scanner API', async () => { + const response = await POST(scanRequest(chunkBody())); + + await expect(response.json()).resolves.toEqual({ + attachment: { + fileName: 'receipt.jpg', + mimeType: 'image/jpeg', + uploadId + }, + job: { + jobId: '00000000-0000-4000-8000-000000000042', + token: 'scan-token' + } + }); + expect(response.status).toBe(200); + expect(mocks.createXpenserClient).toHaveBeenCalledWith({ + baseUrl: 'https://api.example.test', + getToken: expect.any(Function), + retryOnTimeout: false + }); + expect(mocks.transactionScanStart).toHaveBeenCalledWith({ + body: { + imageBase64: Buffer.from('receipt bytes').toString('base64'), + mimeType: 'image/jpeg', + fileName: 'receipt.jpg' + } + }); + }); + + it('rejects oversized images before calling the scanner API', async () => { + const response = await POST( + scanRequest( + chunkBody({ + fileSize: TransactionScanLimits.maxImageBytes + 1 + }) + ) + ); + + await expect(response.json()).resolves.toEqual({ + error: 'Image must be 10 MB or smaller.' + }); + expect(response.status).toBe(413); + expect(mocks.transactionScanStart).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/app/api/transaction-scans/route.ts b/apps/web/app/api/transaction-scans/route.ts new file mode 100644 index 0000000..fb4205d --- /dev/null +++ b/apps/web/app/api/transaction-scans/route.ts @@ -0,0 +1,222 @@ +import { createXpenserClient } from '@xpenser/client'; +import type { TransactionScanJobResponse } from '@xpenser/contracts'; +import { NextResponse } from 'next/server'; +import { auth } from '@/auth'; +import { webConfig } from '@/lib/config'; +import { + assembleScanUploadChunks, + cleanupStaleScanUploads, + createScanUploadId, + isAllowedScanImageType, + isScanUploadId, + type StoredScanAttachment, + scanUploadChunkCountError, + scanUploadFileSizeError, + storeScanUpload, + writeScanUploadChunk +} from '@/lib/transaction-scan-upload-store'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +type ScanRouteResponse = + | { readonly error: string; readonly job?: undefined } + | { readonly error?: undefined; readonly uploaded: true } + | { + readonly attachment: StoredScanAttachment; + readonly error?: undefined; + readonly job: TransactionScanJobResponse; + }; + +type ScanChunkBody = { + readonly chunkBase64: string; + readonly chunkIndex: number; + readonly fileName?: string; + readonly fileSize: number; + readonly mimeType: string; + readonly totalChunks: number; + readonly uploadId: string; +}; + +function apiErrorStatus(err: unknown): number | undefined { + return typeof err === 'object' && + err !== null && + 'status' in err && + typeof err.status === 'number' + ? err.status + : undefined; +} + +function apiErrorMessage(err: unknown): string | undefined { + const body = + typeof err === 'object' && err !== null && 'body' in err + ? (err as { readonly body?: unknown }).body + : undefined; + return typeof body === 'object' && + body !== null && + 'message' in body && + typeof body.message === 'string' + ? body.message + : undefined; +} + +function errorResponse(message: string, status: number) { + return NextResponse.json({ error: message }, { status }); +} + +function scanChunkBody(value: unknown): ScanChunkBody | undefined { + if (typeof value !== 'object' || value === null) { + return undefined; + } + + const record = value as Record; + if ( + typeof record.chunkBase64 !== 'string' || + typeof record.chunkIndex !== 'number' || + typeof record.fileSize !== 'number' || + typeof record.mimeType !== 'string' || + typeof record.totalChunks !== 'number' || + typeof record.uploadId !== 'string' + ) { + return undefined; + } + + if (record.fileName !== undefined && typeof record.fileName !== 'string') { + return undefined; + } + + return { + chunkBase64: record.chunkBase64, + chunkIndex: record.chunkIndex, + fileName: record.fileName, + fileSize: record.fileSize, + mimeType: record.mimeType, + totalChunks: record.totalChunks, + uploadId: record.uploadId + }; +} + +function validChunkIndex(chunkIndex: number, totalChunks: number): boolean { + return ( + Number.isSafeInteger(chunkIndex) && + chunkIndex >= 0 && + chunkIndex < totalChunks + ); +} + +function bodyValidationError(body: ScanChunkBody): string | undefined { + if (!isScanUploadId(body.uploadId)) { + return 'Could not scan the image. Try again.'; + } + if (!isAllowedScanImageType(body.mimeType)) { + return 'Upload a PNG, JPEG, or WebP image.'; + } + if (!validChunkIndex(body.chunkIndex, body.totalChunks)) { + return 'Could not scan the image. Try again.'; + } + return ( + scanUploadFileSizeError(body.fileSize) ?? + scanUploadChunkCountError(body.totalChunks) + ); +} + +export async function POST(request: Request) { + const session = await auth(); + if (!session?.apiToken) { + return errorResponse('Unauthorized', 401); + } + + let body: ScanChunkBody | undefined; + try { + body = scanChunkBody(await request.json()); + } catch { + return errorResponse('Could not scan the image. Try again.', 400); + } + if (!body) { + return errorResponse('Could not scan the image. Try again.', 400); + } + + const bodyError = bodyValidationError(body); + if (bodyError) { + return errorResponse( + bodyError, + bodyError.includes('10 MB') ? 413 : 400 + ); + } + if (!isAllowedScanImageType(body.mimeType)) { + return errorResponse('Upload a PNG, JPEG, or WebP image.', 400); + } + + const userId = session.user.id; + const mimeType = body.mimeType; + await cleanupStaleScanUploads(); + + try { + await writeScanUploadChunk({ + chunkBase64: body.chunkBase64, + chunkIndex: body.chunkIndex, + uploadId: body.uploadId, + userId + }); + } catch { + return errorResponse('Could not scan the image. Try again.', 400); + } + + if (body.chunkIndex < body.totalChunks - 1) { + return NextResponse.json({ uploaded: true }); + } + + let image: Buffer; + try { + image = await assembleScanUploadChunks({ + expectedSize: body.fileSize, + totalChunks: body.totalChunks, + uploadId: body.uploadId, + userId + }); + } catch { + return errorResponse('Could not scan the image. Try again.', 400); + } + + const client = createXpenserClient({ + baseUrl: webConfig.apiBaseUrl, + getToken: () => session.apiToken, + retryOnTimeout: false + }); + + try { + const imageBase64 = image.toString('base64'); + const attachment = await storeScanUpload({ + buffer: image, + fileName: body.fileName, + mimeType, + uploadId: body.uploadId, + userId + }); + const job = await client.transactionScans.start({ + body: { + imageBase64, + mimeType, + fileName: body.fileName + } + }); + return NextResponse.json({ attachment, job }); + } catch (err) { + const status = apiErrorStatus(err); + if (status === 400) { + return errorResponse( + apiErrorMessage(err) ?? + 'Could not scan the image. Try a clearer image.', + 400 + ); + } + if (status === 401) { + return errorResponse('Session expired.', 401); + } + throw err; + } +} + +export async function GET() { + return NextResponse.json({ uploadId: createScanUploadId() }); +} diff --git a/apps/web/app/api/transactions/[transactionId]/scan-image/route.ts b/apps/web/app/api/transactions/[transactionId]/scan-image/route.ts new file mode 100644 index 0000000..4551a1b --- /dev/null +++ b/apps/web/app/api/transactions/[transactionId]/scan-image/route.ts @@ -0,0 +1,88 @@ +import { createXpenserClient } from '@xpenser/client'; +import { type NextRequest, NextResponse } from 'next/server'; +import { auth } from '@/auth'; +import { webConfig } from '@/lib/config'; + +export const dynamic = 'force-dynamic'; +export const runtime = 'nodejs'; + +type RouteContext = { + readonly params: + | { readonly transactionId: string } + | Promise<{ readonly transactionId: string }>; +}; + +function isApiErrorStatus(err: unknown, status: number): boolean { + return ( + typeof err === 'object' && + err !== null && + 'status' in err && + err.status === status + ); +} + +function parseTransactionId(value: string): number | undefined { + const id = Number(value); + return Number.isSafeInteger(id) && id > 0 ? id : undefined; +} + +function contentDisposition(fileName?: string | null): string { + const safeName = + fileName + ?.replace(/[^a-zA-Z0-9._-]+/g, '_') + .replace(/^_+|_+$/g, '') + .slice(0, 120) || 'scanned-transaction-image'; + return `inline; filename="${safeName}"`; +} + +export async function GET(_request: NextRequest, context: RouteContext) { + const { transactionId } = await Promise.resolve(context.params); + const id = parseTransactionId(transactionId); + if (!id) { + return NextResponse.json( + { message: 'Transaction was not found.' }, + { status: 404 } + ); + } + + const session = await auth(); + if (!session?.apiToken) { + return NextResponse.json({ message: 'Unauthorized' }, { status: 401 }); + } + + const client = createXpenserClient({ + baseUrl: webConfig.apiBaseUrl, + getToken: () => session.apiToken, + retryOnTimeout: false, + timeoutMs: 30_000 + }); + + try { + const image = await client.transactions.scanImage({ + params: { id } + }); + const buffer = Buffer.from(image.imageBase64, 'base64'); + return new NextResponse(buffer, { + headers: { + 'Cache-Control': 'private, no-store', + 'Content-Disposition': contentDisposition(image.fileName), + 'Content-Length': String(buffer.byteLength), + 'Content-Type': image.mimeType + } + }); + } catch (err) { + if (isApiErrorStatus(err, 401)) { + return NextResponse.json( + { message: 'Session expired.' }, + { status: 401 } + ); + } + if (isApiErrorStatus(err, 404)) { + return NextResponse.json( + { message: 'Scanned image was not found.' }, + { status: 404 } + ); + } + throw err; + } +} diff --git a/apps/web/components/forms/category-form.tsx b/apps/web/components/forms/category-form.tsx index a8f1dbf..734a973 100644 --- a/apps/web/components/forms/category-form.tsx +++ b/apps/web/components/forms/category-form.tsx @@ -22,6 +22,7 @@ export function CategoryForm({ categories = [], first = false, initialCategory, + initialValues, namePlaceholder, onSaved, submitLabel = initialCategory ? 'Save category' : 'Create category' @@ -29,8 +30,12 @@ export function CategoryForm({ readonly categories?: readonly Category[]; readonly first?: boolean; readonly initialCategory?: Category; + readonly initialValues?: Pick< + Category, + 'kind' | 'name' | 'parentId' | 'type' + >; readonly namePlaceholder?: string; - readonly onSaved?: () => void; + readonly onSaved?: (category?: Category) => void; readonly submitLabel?: string; }) { const form = useSchemaForm(CreateCategoryBodySchema); @@ -59,12 +64,15 @@ export function CategoryForm({ const offsetKindLabel = selectedType === 'expense' ? 'Return' : 'Expense'; useEffect(() => { - const nextType = initialCategory?.type ?? 'expense'; - const nextParentId = initialCategory?.parentId ?? null; - const nextKind = initialCategory?.kind ?? 'normal'; + const nextType = + initialCategory?.type ?? initialValues?.type ?? 'expense'; + const nextParentId = + initialCategory?.parentId ?? initialValues?.parentId ?? null; + const nextKind = + initialCategory?.kind ?? initialValues?.kind ?? 'normal'; form.reset({ - name: initialCategory?.name, + name: initialCategory?.name ?? initialValues?.name, type: nextType, parentId: nextParentId, kind: nextKind @@ -73,7 +81,7 @@ export function CategoryForm({ setSelectedParentId(nextParentId); setSelectedKind(nextKind); setFormVersion(version => version + 1); - }, [form, initialCategory]); + }, [form, initialCategory, initialValues]); async function handleSubmit(event: FormEvent) { event.preventDefault(); @@ -100,9 +108,10 @@ export function CategoryForm({ router.refresh(); onSaved?.(); } else { - await createCategoryAction(formData); + const category = await createCategoryAction(formData); form.reset({ type: 'expense', parentId: null, kind: 'normal' }); router.refresh(); + onSaved?.(category); } } catch (caught) { if (isNextRedirectError(caught)) { diff --git a/apps/web/components/quick-capture-form.test.tsx b/apps/web/components/quick-capture-form.test.tsx index 3f8e1e6..f3710dc 100644 --- a/apps/web/components/quick-capture-form.test.tsx +++ b/apps/web/components/quick-capture-form.test.tsx @@ -141,7 +141,7 @@ describe('QuickCaptureForm', () => { expect(screen.getByRole('combobox', { name: 'Currency' })).toBeTruthy(); expect(screen.getByLabelText('Date and time')).toBeTruthy(); - expect(screen.getByLabelText('Note')).toBeTruthy(); + expect(screen.getByLabelText('Note').tagName).toBe('TEXTAREA'); expect(screen.queryByRole('button', { name: 'Details' })).toBeNull(); expect( screen.queryByRole('combobox', { name: 'All categories' }) @@ -152,7 +152,7 @@ describe('QuickCaptureForm', () => { target: { value: '12.34' } }); fireEvent.change(screen.getByLabelText('Note'), { - target: { value: 'Quick capture note' } + target: { value: 'Quick capture note\nSecond line' } }); fireEvent.click( screen.getByRole('button', { name: 'Save transaction' }) @@ -168,12 +168,12 @@ describe('QuickCaptureForm', () => { expect(formData.get('amount')).toBe('12.34'); expect(formData.get('currency')).toBe('USD'); expect(formData.get('effect')).toBeNull(); - expect(formData.get('note')).toBe('Quick capture note'); + expect(formData.get('note')).toBe('Quick capture note\nSecond line'); expect(formData.get('occurredAt')).toBeTruthy(); expect(refresh).toHaveBeenCalledOnce(); - expect((screen.getByLabelText('Note') as HTMLInputElement).value).toBe( - '' - ); + expect( + (screen.getByLabelText('Note') as HTMLTextAreaElement).value + ).toBe(''); expect(screen.getByText('Saved')).toBeTruthy(); fireEvent.click(screen.getByRole('button', { name: 'Undo' })); diff --git a/apps/web/components/quick-capture-form.tsx b/apps/web/components/quick-capture-form.tsx index 8ae7947..f5181d0 100644 --- a/apps/web/components/quick-capture-form.tsx +++ b/apps/web/components/quick-capture-form.tsx @@ -454,12 +454,14 @@ export function QuickCaptureForm({ field.note} form={form} label="Note" name="note" + variant="textarea" /> {error ? ( diff --git a/apps/web/components/transaction-dialog.test.tsx b/apps/web/components/transaction-dialog.test.tsx index 0f3bcea..004b405 100644 --- a/apps/web/components/transaction-dialog.test.tsx +++ b/apps/web/components/transaction-dialog.test.tsx @@ -71,8 +71,9 @@ describe('TransactionDialog', () => { fireEvent.change(screen.getByLabelText('Amount'), { target: { value: '23.45' } }); + expect(screen.getByLabelText('Note').tagName).toBe('TEXTAREA'); fireEvent.change(screen.getByLabelText('Note'), { - target: { value: 'Updated note' } + target: { value: 'Updated note\nItem line' } }); fireEvent.click(screen.getByRole('button', { name: 'Save changes' })); @@ -83,7 +84,7 @@ describe('TransactionDialog', () => { expect(formData?.get('categoryId')).toBe('7'); expect(formData?.get('amount')).toBe('23.45'); expect(formData?.get('currency')).toBe('USD'); - expect(formData?.get('note')).toBe('Updated note'); + expect(formData?.get('note')).toBe('Updated note\nItem line'); expect(refresh).toHaveBeenCalledOnce(); }); }); diff --git a/apps/web/components/transaction-dialog.tsx b/apps/web/components/transaction-dialog.tsx index b8d4b6c..dfc2b36 100644 --- a/apps/web/components/transaction-dialog.tsx +++ b/apps/web/components/transaction-dialog.tsx @@ -438,12 +438,14 @@ export function TransactionDialog({ /> field.note} form={form} label="Note" name="note" + variant="textarea" /> {error ? ( {error} diff --git a/apps/web/components/transaction-scan-capture.test.tsx b/apps/web/components/transaction-scan-capture.test.tsx new file mode 100644 index 0000000..89fc10a --- /dev/null +++ b/apps/web/components/transaction-scan-capture.test.tsx @@ -0,0 +1,190 @@ +/** + * @vitest-environment jsdom + */ + +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import type { + Category, + Currency, + TransactionScanProgressEvent +} from '@xpenser/contracts'; +import { XpenserFormProvider } from '@xpenser/ui'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { TransactionCaptureWorkspace } from './transaction-scan-capture'; + +const refresh = vi.fn(); +const createCaptureTransactionAction = vi.fn(); +const createVendorAction = vi.fn(); +const recordTransactionScanDecisionAction = vi.fn(); +const progress = vi.fn(); +const originalCreateObjectURL = URL.createObjectURL; +const originalRevokeObjectURL = URL.revokeObjectURL; + +vi.mock('next/navigation', () => ({ + useRouter: () => ({ refresh }) +})); + +vi.mock('@xpenser/client', () => ({ + createXpenserClient: () => ({ + transactionScans: { progress } + }) +})); + +vi.mock('@/lib/actions', () => ({ + createCaptureTransactionAction: (formData: FormData) => + createCaptureTransactionAction(formData), + createVendorAction: (formData: FormData) => createVendorAction(formData), + recordTransactionScanDecisionAction: (body: unknown) => + recordTransactionScanDecisionAction(body) +})); + +const timestamp = new Date('2026-06-01T12:00:00.000Z'); + +function category(overrides: Partial = {}): Category { + return { + id: 7, + name: 'Groceries', + type: 'expense', + parentId: null, + kind: 'normal', + displayName: 'Groceries', + inUse: true, + hasChildren: false, + archivedAt: null, + createdAt: timestamp, + updatedAt: timestamp, + ...overrides + }; +} + +const currencies: Currency[] = [{ code: 'USD', name: 'US dollar' }]; + +function subscription( + releaseComplete: Promise +): AsyncIterable & { close: () => void } { + return { + async *[Symbol.asyncIterator]() { + yield { + jobId: 'job-1', + stage: 'analyzing', + message: 'Reading image details with AI.', + progress: 45, + scan: null, + error: null + }; + await releaseComplete; + yield { + jobId: 'job-1', + stage: 'complete', + message: 'Found 0 transactions for review.', + progress: 100, + scan: { + scanId: 1, + documentKind: 'receipt', + warnings: [], + drafts: [] + }, + error: null + }; + }, + close: vi.fn() + }; +} + +function renderWorkspace() { + return render( + + + + ); +} + +describe('TransactionCaptureWorkspace scan upload', () => { + beforeEach(() => { + Object.defineProperty(URL, 'createObjectURL', { + configurable: true, + value: vi.fn(() => 'blob:receipt') + }); + Object.defineProperty(URL, 'revokeObjectURL', { + configurable: true, + value: vi.fn() + }); + vi.stubGlobal('fetch', vi.fn()); + }); + + afterEach(() => { + Object.defineProperty(URL, 'createObjectURL', { + configurable: true, + value: originalCreateObjectURL + }); + Object.defineProperty(URL, 'revokeObjectURL', { + configurable: true, + value: originalRevokeObjectURL + }); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + createCaptureTransactionAction.mockReset(); + createVendorAction.mockReset(); + recordTransactionScanDecisionAction.mockReset(); + progress.mockReset(); + refresh.mockReset(); + }); + + it('starts scanning as soon as an image is selected', async () => { + let completeScan: () => void = () => {}; + const releaseComplete = new Promise(resolve => { + completeScan = resolve; + }); + progress.mockReturnValue(subscription(releaseComplete)); + vi.mocked(fetch).mockResolvedValue({ + ok: true, + json: () => + Promise.resolve({ + attachment: { + fileName: 'receipt.png', + mimeType: 'image/png', + uploadId: 'upload-1' + }, + job: { + jobId: 'job-1', + token: 'token-1' + } + }) + } as Response); + + renderWorkspace(); + fireEvent.click(screen.getByRole('button', { name: 'Scan' })); + expect(screen.queryByRole('button', { name: 'Scan image' })).toBeNull(); + + fireEvent.change(screen.getByLabelText('Choose image'), { + target: { + files: [ + new File(['receipt'], 'receipt.png', { + type: 'image/png' + }) + ] + } + }); + + await waitFor(() => expect(fetch).toHaveBeenCalledOnce()); + await waitFor(() => + expect(screen.getByRole('progressbar')).toBeTruthy() + ); + expect( + screen.getByText('Reading visible text and totals.') + ).toBeTruthy(); + + completeScan(); + + await waitFor(() => + expect(screen.getByText('No transactions found')).toBeTruthy() + ); + }); +}); diff --git a/apps/web/components/transaction-scan-capture.tsx b/apps/web/components/transaction-scan-capture.tsx new file mode 100644 index 0000000..94579c9 --- /dev/null +++ b/apps/web/components/transaction-scan-capture.tsx @@ -0,0 +1,1401 @@ +'use client'; + +import { createXpenserClient } from '@xpenser/client'; +import { + type Category, + type Currency, + FieldLimits, + type Transaction, + type TransactionScanDecisionBody, + type TransactionScanDraft, + type TransactionScanJobResponse, + TransactionScanLimits, + type TransactionScanProgressEvent, + type TransactionScanResponse, + type Vendor +} from '@xpenser/contracts'; +import { + dateToLocalDateTimeInput, + localDateTimeInputToDate +} from '@xpenser/timezone'; +import { + Button, + Card, + CardContent, + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, + Field, + FieldError, + FieldGroup, + FieldLabel, + Input, + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectTrigger, + SelectValue, + Textarea +} from '@xpenser/ui'; +import { + AlertCircleIcon, + CheckCircle2Icon, + ChevronLeftIcon, + ChevronRightIcon, + ImageUpIcon, + PlusIcon, + Trash2Icon +} from 'lucide-react'; +import { useRouter } from 'next/navigation'; +import { + type ChangeEvent, + type DragEvent, + type FormEvent, + useEffect, + useMemo, + useRef, + useState +} from 'react'; +import { + createCaptureTransactionAction, + createVendorAction, + recordTransactionScanDecisionAction +} from '@/lib/actions'; +import { + categoryEffectiveType, + categoryTypeLabel, + transactionCategoryOptions +} from '@/lib/category-display'; +import { formatDateTime, formatTransactionMoney } from '@/lib/format'; +import { transactionCurrencyOptions } from '@/lib/transaction-currencies'; +import { CategoryForm } from './forms/category-form'; +import { QuickCaptureForm } from './quick-capture-form'; +import { VendorPicker } from './vendor-picker'; + +type CaptureMode = 'manual' | 'scan'; +type Decision = 'confirmed' | 'discarded'; +type ScanAttachment = { + readonly fileName?: string; + readonly mimeType: NonNullable< + TransactionScanDecisionBody['attachment'] + >['mimeType']; + readonly uploadId: string; +}; +type TransactionType = Category['type']; + +const maxImageBytes = TransactionScanLimits.maxImageBytes; +const uploadChunkBytes = TransactionScanLimits.uploadChunkBytes; +const allowedScanImageTypes = ['image/jpeg', 'image/png', 'image/webp']; +const analyzingMessages = [ + 'Reading visible text and totals.', + 'Matching vendors and categories.', + 'Checking whether line items should be split.', + 'Allocating shared tax, fees, and discounts.', + 'Preparing transactions for review.' +]; + +type ScanProgressStage = + | TransactionScanProgressEvent['stage'] + | 'connecting' + | 'uploading'; + +type ScanProgressUpdate = { + readonly message: string; + readonly progress: number; + readonly stage: ScanProgressStage; +}; + +type ScanFileDetails = { + readonly height?: number; + readonly key: string; + readonly name: string; + readonly size: number; + readonly width?: number; +}; + +type ScanResultResponse = + | { readonly error: string; readonly scan?: undefined } + | { + readonly attachment: ScanAttachment; + readonly error?: undefined; + readonly scan: TransactionScanResponse; + }; + +type ScanUploadRouteResponse = + | { readonly error: string; readonly job?: undefined } + | { readonly error?: undefined; readonly uploaded: true } + | { + readonly attachment: ScanAttachment; + readonly error?: undefined; + readonly job: TransactionScanJobResponse; + }; + +function blobBase64(blob: Blob): Promise { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onerror = () => reject(new Error('Could not read image.')); + reader.onload = () => { + const value = reader.result; + if (typeof value !== 'string') { + reject(new Error('Could not read image.')); + return; + } + const commaIndex = value.indexOf(','); + resolve(commaIndex >= 0 ? value.slice(commaIndex + 1) : value); + }; + reader.readAsDataURL(blob); + }); +} + +function browserApiBaseUrl(): string { + const configured = process.env.NEXT_PUBLIC_API_BASE_URL; + if (configured) { + return configured.replace(/\/$/, ''); + } + if ( + window.location.hostname === 'localhost' && + window.location.port === '3000' + ) { + return 'http://localhost:4000'; + } + return new URL('/external-api', window.location.href) + .toString() + .replace(/\/$/, ''); +} + +function scanError(event: TransactionScanProgressEvent): string { + return event.error ?? 'Could not scan the image. Try again.'; +} + +function scanEventProgress(event: TransactionScanProgressEvent): number { + return Math.min(100, Math.round(40 + event.progress * 0.6)); +} + +function fileKey(file: File): string { + return `${file.name}:${file.size}:${file.lastModified}`; +} + +function formatFileSize(bytes: number): string { + return `${(bytes / 1024 / 1024).toFixed(2)} MB`; +} + +function readImageDimensions( + file: File +): Promise<{ readonly height: number; readonly width: number } | null> { + return new Promise(resolve => { + const image = new Image(); + const objectUrl = URL.createObjectURL(file); + image.onload = () => { + URL.revokeObjectURL(objectUrl); + resolve({ + height: image.naturalHeight, + width: image.naturalWidth + }); + }; + image.onerror = () => { + URL.revokeObjectURL(objectUrl); + resolve(null); + }; + image.src = objectUrl; + }); +} + +async function waitForScanJob( + job: TransactionScanJobResponse, + onProgress: (update: ScanProgressUpdate) => void +): Promise { + const client = createXpenserClient({ + baseUrl: browserApiBaseUrl(), + retryOnTimeout: false + }); + const subscription = client.transactionScans.progress({ + query: { jobId: job.jobId, token: job.token }, + reconnect: { maxRetries: 3, backoffLimit: 5_000 } + }); + + try { + for await (const event of subscription) { + onProgress({ + message: event.message, + progress: scanEventProgress(event), + stage: event.stage + }); + if (event.stage === 'failed') { + throw new Error(scanError(event)); + } + if (event.stage === 'complete' && event.scan) { + return event.scan; + } + } + } finally { + subscription.close(); + } + + throw new Error('Could not connect to scan progress. Try again.'); +} + +async function uploadAndScanImageFile( + file: File, + onProgress: (update: ScanProgressUpdate) => void +): Promise { + const uploadId = crypto.randomUUID(); + const totalChunks = Math.ceil(file.size / uploadChunkBytes); + onProgress({ + message: 'Uploading image.', + progress: 4, + stage: 'uploading' + }); + + for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex += 1) { + const start = chunkIndex * uploadChunkBytes; + const response = await fetch('/api/transaction-scans', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + chunkBase64: await blobBase64( + file.slice(start, start + uploadChunkBytes) + ), + chunkIndex, + fileName: file.name, + fileSize: file.size, + mimeType: file.type, + totalChunks, + uploadId + }) + }); + const result = (await response + .json() + .catch(() => null)) as ScanUploadRouteResponse | null; + + if (!response.ok) { + return { + error: + result?.error ?? + (response.status === 413 + ? 'Image must be 10 MB or smaller.' + : 'Could not scan the image. Try again.') + }; + } + if (result?.error) { + return result; + } + if (result && 'attachment' in result && 'job' in result) { + onProgress({ + message: 'Connecting to scan progress.', + progress: 38, + stage: 'connecting' + }); + try { + const scan = await waitForScanJob(result.job, onProgress); + return { attachment: result.attachment, scan }; + } catch (err) { + return { + error: + err instanceof Error + ? err.message + : 'Could not scan the image. Try again.' + }; + } + } + onProgress({ + message: `Uploading image (${Math.round( + ((chunkIndex + 1) / totalChunks) * 100 + )}%).`, + progress: Math.max( + 4, + Math.round(((chunkIndex + 1) / totalChunks) * 35) + ), + stage: 'uploading' + }); + } + + return { error: 'Could not scan the image. Try again.' }; +} + +function parseAmount(value: string): number | undefined { + const normalized = value.trim().replace(',', '.'); + if (!/^\d+(?:\.\d{1,2})?$/.test(normalized)) { + return undefined; + } + + const amount = Number(normalized); + return Number.isFinite(amount) && amount > 0 ? amount : undefined; +} + +function firstCategoryId( + categories: readonly Category[], + type: TransactionType +): number | undefined { + return categories.find(category => categoryEffectiveType(category) === type) + ?.id; +} + +function draftCategoryType( + draft: TransactionScanDraft, + categories: readonly Category[] +): TransactionType { + const category = categories.find(item => item.id === draft.categoryId); + return category ? categoryEffectiveType(category) : draft.transactionType; +} + +function initialValues({ + categories, + defaultCurrency, + draft, + timezone +}: { + readonly categories: readonly Category[]; + readonly defaultCurrency: string; + readonly draft: TransactionScanDraft; + readonly timezone: string; +}) { + const type = draftCategoryType(draft, categories); + return { + amount: draft.amount ? String(draft.amount) : '', + categoryId: draft.categoryId ?? undefined, + currency: draft.currency ?? defaultCurrency, + occurredAtText: dateToLocalDateTimeInput( + draft.occurredAt ?? new Date(), + timezone + ), + note: draft.note ?? '', + type, + vendorId: draft.vendorId + }; +} + +function confidenceLabel(value: string): string { + return `${value.slice(0, 1).toUpperCase()}${value.slice(1)}`; +} + +function savedSummary(transaction: Transaction, timezone: string) { + const vendor = transaction.vendorName ? `${transaction.vendorName} - ` : ''; + return `${vendor}${transaction.categoryDisplayName} - ${formatTransactionMoney( + transaction.amount, + transaction.currency, + transaction.type, + transaction.categoryKind + )} - ${formatDateTime(transaction.occurredAt, timezone)}`; +} + +function nextPendingIndex( + drafts: readonly TransactionScanDraft[], + decisions: Readonly>, + currentIndex: number +): number | undefined { + const afterCurrent = drafts.findIndex( + (draft, index) => index > currentIndex && !decisions[draft.id] + ); + if (afterCurrent >= 0) { + return afterCurrent; + } + const beforeCurrent = drafts.findIndex(draft => !decisions[draft.id]); + return beforeCurrent >= 0 ? beforeCurrent : undefined; +} + +function ScanUpload({ + onScanned +}: { + readonly onScanned: ( + scan: TransactionScanResponse, + attachment: ScanAttachment + ) => void; +}) { + const inputRef = useRef(null); + const [fileDetails, setFileDetails] = useState( + null + ); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + const [progress, setProgress] = useState(null); + const [analysisTick, setAnalysisTick] = useState(0); + + useEffect(() => { + if (progress?.stage !== 'analyzing') { + return; + } + const timer = window.setInterval( + () => setAnalysisTick(value => value + 1), + 5_000 + ); + return () => window.clearInterval(timer); + }, [progress?.stage]); + + function setNextProgress(update: ScanProgressUpdate) { + setProgress(current => { + if (current?.stage !== update.stage) { + setAnalysisTick(0); + } + return update; + }); + } + + async function scanSelectedFile(file: File) { + const key = fileKey(file); + setFileDetails({ + key, + name: file.name, + size: file.size + }); + setError(null); + setProgress(null); + void readImageDimensions(file).then(dimensions => { + if (!dimensions) { + return; + } + setFileDetails(current => + current?.key === key + ? { + ...current, + height: dimensions.height, + width: dimensions.width + } + : current + ); + }); + + if (!allowedScanImageTypes.includes(file.type)) { + setError('Upload a PNG, JPEG, or WebP image.'); + if (inputRef.current) { + inputRef.current.value = ''; + } + return; + } + if (file.size > maxImageBytes) { + setError('Image must be 10 MB or smaller.'); + if (inputRef.current) { + inputRef.current.value = ''; + } + return; + } + + setPending(true); + try { + const result = await uploadAndScanImageFile(file, setNextProgress); + if (result.error) { + setError(result.error); + return; + } + if (!('attachment' in result)) { + setError('Could not scan the image. Try again.'); + return; + } + onScanned(result.scan, result.attachment); + setProgress(null); + } catch { + setError('Could not scan the image. Try again.'); + } finally { + setPending(false); + if (inputRef.current) { + inputRef.current.value = ''; + } + } + } + + function handleFileChange(event: ChangeEvent) { + const selected = event.target.files?.[0]; + if (selected) { + void scanSelectedFile(selected); + } + } + + function handleDrop(event: DragEvent) { + event.preventDefault(); + if (pending) { + return; + } + const selected = event.dataTransfer.files?.[0]; + if (selected) { + void scanSelectedFile(selected); + } + } + + const displayProgress = + progress?.stage === 'analyzing' + ? Math.min(82, progress.progress + analysisTick * 6) + : (progress?.progress ?? 0); + const displayMessage = + progress?.stage === 'analyzing' + ? analyzingMessages[analysisTick % analyzingMessages.length] + : progress?.message; + + return ( + + +
+ event.preventDefault()} + onDrop={handleDrop} + > + + + + + + {fileDetails + ? 'Choose another image' + : 'Choose image'} + + + {fileDetails ? ( +
+

+ {fileDetails.name} +

+

+ {formatFileSize(fileDetails.size)} + {fileDetails.width && fileDetails.height + ? ` - ${fileDetails.width}x${fileDetails.height}` + : ''} +

+
+ ) : null} + {error ? ( + {error} + ) : null} + {pending && progress && displayMessage ? ( +
+
+
+
+ {displayMessage} +
+ ) : null} +
+ + + ); +} + +function SuggestedVendor({ + name, + onCreated +}: { + readonly name: string; + readonly onCreated: (vendor: Vendor) => void; +}) { + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + + async function handleCreate() { + const formData = new FormData(); + formData.set('name', name); + + setPending(true); + setError(null); + try { + onCreated(await createVendorAction(formData)); + } catch { + setError('Could not create vendor.'); + } finally { + setPending(false); + } + } + + return ( +
+
+

Suggested vendor

+

{name}

+
+ {error ? {error} : null} + +
+ ); +} + +function SuggestedCategory({ + categories, + draft, + onCreated +}: { + readonly categories: readonly Category[]; + readonly draft: TransactionScanDraft; + readonly onCreated: (category: Category) => void; +}) { + const [open, setOpen] = useState(false); + const suggestion = draft.suggestedCategory; + if (!suggestion) { + return null; + } + + return ( +
+
+

Suggested category

+

+ {categoryTypeLabel(suggestion.type)} - {suggestion.name} +

+

+ {suggestion.reason} +

+
+ + + + + + + Create category + + The new category will be selected for this scanned + transaction. + + + { + if (category) { + onCreated(category); + } + setOpen(false); + }} + /> + + +
+ ); +} + +function ScanWizard({ + attachment, + categories, + currencies, + defaultCurrency, + onReset, + scan, + setCategories, + setVendors, + timezone, + transactionCurrencies, + vendors +}: { + readonly attachment: ScanAttachment; + readonly categories: readonly Category[]; + readonly currencies: readonly Currency[]; + readonly defaultCurrency: string; + readonly onReset: () => void; + readonly scan: TransactionScanResponse; + readonly setCategories: (categories: readonly Category[]) => void; + readonly setVendors: (vendors: readonly Vendor[]) => void; + readonly timezone: string; + readonly transactionCurrencies: readonly string[]; + readonly vendors: readonly Vendor[]; +}) { + const router = useRouter(); + const transactionCategories = useMemo( + () => transactionCategoryOptions(categories), + [categories] + ); + const currencyOptions = useMemo( + () => + transactionCurrencyOptions( + currencies, + defaultCurrency, + transactionCurrencies + ), + [currencies, defaultCurrency, transactionCurrencies] + ); + const [currentIndex, setCurrentIndex] = useState(0); + const [decisions, setDecisions] = useState>({}); + const [lastSaved, setLastSaved] = useState(null); + const [pending, setPending] = useState(false); + const [error, setError] = useState(null); + + const draft = scan.drafts[currentIndex]; + const values = useMemo( + () => + draft + ? initialValues({ + categories: transactionCategories, + defaultCurrency, + draft, + timezone + }) + : undefined, + [defaultCurrency, draft, timezone, transactionCategories] + ); + const [amount, setAmount] = useState(values?.amount ?? ''); + const [categoryId, setCategoryId] = useState( + values?.categoryId + ); + const [currency, setCurrency] = useState( + values?.currency ?? defaultCurrency + ); + const [occurredAtText, setOccurredAtText] = useState( + values?.occurredAtText ?? dateToLocalDateTimeInput(new Date(), timezone) + ); + const [note, setNote] = useState(values?.note ?? ''); + const [selectedType, setSelectedType] = useState( + values?.type ?? 'expense' + ); + const [vendorId, setVendorId] = useState( + values?.vendorId + ); + const [createdCategoryId, setCreatedCategoryId] = useState( + null + ); + const [createdVendorId, setCreatedVendorId] = useState(null); + const [attachmentSubmitted, setAttachmentSubmitted] = useState(false); + + function loadDraft(nextIndex: number) { + const nextDraft = scan.drafts[nextIndex]; + if (!nextDraft) { + return; + } + const next = initialValues({ + categories: transactionCategories, + defaultCurrency, + draft: nextDraft, + timezone + }); + setCurrentIndex(nextIndex); + setAmount(next.amount); + setCategoryId(next.categoryId); + setCurrency(next.currency); + setOccurredAtText(next.occurredAtText); + setNote(next.note); + setSelectedType(next.type); + setVendorId(next.vendorId); + setCreatedCategoryId(null); + setCreatedVendorId(null); + setError(null); + setLastSaved(null); + } + + function moveAfterDecision(nextDecisions: Record) { + const nextIndex = nextPendingIndex( + scan.drafts, + nextDecisions, + currentIndex + ); + if (nextIndex !== undefined) { + loadDraft(nextIndex); + } + } + + function handleTypeChange(type: TransactionType) { + setSelectedType(type); + const current = transactionCategories.find( + category => category.id === categoryId + ); + if (current && categoryEffectiveType(current) === type) { + return; + } + setCategoryId(firstCategoryId(transactionCategories, type)); + } + + function handleVendorChange(vendor: Vendor | undefined) { + setVendorId(vendor?.id ?? null); + if (!vendor?.suggestedCategoryId) { + return; + } + const suggested = transactionCategories.find( + category => category.id === vendor.suggestedCategoryId + ); + if (!suggested) { + return; + } + setSelectedType(categoryEffectiveType(suggested)); + setCategoryId(suggested.id); + } + + function handleCategoryCreated(category: Category) { + setCategories([ + category, + ...categories.filter(item => item.id !== category.id) + ]); + setSelectedType(categoryEffectiveType(category)); + setCategoryId(category.id); + setCreatedCategoryId(category.id); + } + + function handleVendorCreated(vendor: Vendor) { + setVendors([vendor, ...vendors.filter(item => item.id !== vendor.id)]); + setVendorId(vendor.id); + setCreatedVendorId(vendor.id); + if (vendor.suggestedCategoryId) { + const suggested = transactionCategories.find( + category => category.id === vendor.suggestedCategoryId + ); + if (suggested) { + setSelectedType(categoryEffectiveType(suggested)); + setCategoryId(suggested.id); + } + } + } + + async function handleDiscard() { + if (!draft) { + return; + } + setPending(true); + setError(null); + try { + await recordTransactionScanDecisionAction({ + scanId: scan.scanId, + itemId: draft.id, + body: { decision: 'discarded' } + }); + const nextDecisions = { + ...decisions, + [draft.id]: 'discarded' as const + }; + setDecisions(nextDecisions); + moveAfterDecision(nextDecisions); + } catch { + setError('Could not discard this scanned transaction.'); + } finally { + setPending(false); + } + } + + async function handleConfirm(event: FormEvent) { + event.preventDefault(); + if (!draft) { + return; + } + + const amountValue = parseAmount(amount); + const occurredAt = localDateTimeInputToDate(occurredAtText, timezone); + if (amountValue === undefined) { + setError('Enter a positive amount with up to two decimals.'); + return; + } + if (!categoryId) { + setError('Choose a category.'); + return; + } + if (!currency) { + setError('Choose a currency.'); + return; + } + if (!occurredAt) { + setError('Choose a valid date and time.'); + return; + } + + const formData = new FormData(); + formData.set('amount', String(amountValue)); + formData.set('categoryId', String(categoryId)); + formData.set('currency', currency); + formData.set('occurredAt', occurredAt.toISOString()); + if (vendorId) { + formData.set('vendorId', String(vendorId)); + } + if (note.trim()) { + formData.set('note', note.trim()); + } + + setPending(true); + setError(null); + try { + const transaction = await createCaptureTransactionAction(formData); + const shouldSubmitAttachment = !attachmentSubmitted; + await recordTransactionScanDecisionAction({ + scanId: scan.scanId, + itemId: draft.id, + body: { + decision: 'confirmed', + transactionId: transaction.id, + createdCategoryId, + createdVendorId, + correctedTransaction: { + amount: amountValue, + categoryId, + currency, + occurredAt, + vendorId: vendorId ?? null, + note: note.trim() || null + }, + attachment: shouldSubmitAttachment ? attachment : undefined + } + }); + if (shouldSubmitAttachment) { + setAttachmentSubmitted(true); + } + const nextDecisions = { + ...decisions, + [draft.id]: 'confirmed' as const + }; + setDecisions(nextDecisions); + setLastSaved(transaction); + router.refresh(); + moveAfterDecision(nextDecisions); + } catch { + setError('Could not save this scanned transaction.'); + } finally { + setPending(false); + } + } + + if (scan.drafts.length === 0) { + return ( + + +
+

+ No transactions found +

+

+ Try a clearer receipt, invoice, or banking + screenshot. +

+
+ + + + ); + } + + if (!draft) { + return null; + } + + const completeCount = Object.keys(decisions).length; + const complete = completeCount === scan.drafts.length; + const filteredCategories = transactionCategories.filter( + category => categoryEffectiveType(category) === selectedType + ); + + if (complete) { + return ( + + + +
+

Scan reviewed

+

+ Confirmed{' '} + { + Object.values(decisions).filter( + value => value === 'confirmed' + ).length + }{' '} + and discarded{' '} + { + Object.values(decisions).filter( + value => value === 'discarded' + ).length + } + . +

+
+ + + + ); + } + + return ( + + +
+
+
+

+ Transaction {currentIndex + 1} of{' '} + {scan.drafts.length} +

+

+ {scan.documentKind.replace('_', ' ')} +

+
+
+ + +
+
+ {scan.warnings.map(warning => ( +
+ + {warning} +
+ ))} +
+ +
+ +
+

+ Visible evidence +

+

+ {draft.evidence || + 'No supporting text was returned.'} +

+

+ Confidence:{' '} + {confidenceLabel(draft.confidence.overall)} +

+
+ + {draft.possibleDuplicateTransactionIds.length > 0 ? ( +
+ + + Possible duplicate of transaction{' '} + {draft.possibleDuplicateTransactionIds.join( + ', ' + )} + . + +
+ ) : null} + + + Type +
+ {(['expense', 'income'] as const).map(type => ( + + ))} +
+ + + + Category + + + + {!categoryId ? ( + + ) : null} + + + + {!vendorId && draft.suggestedVendorName ? ( + + ) : null} + + + + Amount + +
+ + setAmount(event.target.value) + } + placeholder="0.00" + step="0.01" + type="text" + value={amount} + /> + +
+ + + + + Date and time + + + setOccurredAtText(event.target.value) + } + type="datetime-local" + value={occurredAtText} + /> + + + + Note +