Add transaction image scanner

apps/api/package.json

@@ -31,7 +31,8 @@
         "@xpenser/contracts": "0.1.0",
         "@xpenser/timezone": "0.1.0",
         "knex": "3.1.0",
-        "pg": "8.13.3"
+        "pg": "8.13.3",
+        "sharp": "0.34.5"
     },
     "devDependencies": {
         "@types/node": "22.15.3",

apps/api/src/api/endpoints.ts

@@ -1,5 +1,5 @@
 import { api, PrincipalSchema } from '@xpenser/contracts';
-import { ConfigToken, DbToken, LoggerToken } from '../di/tokens.js';
+import { ConfigToken, DbToken, KnexToken, LoggerToken } from '../di/tokens.js';
 
 export const RegisterEndpoint = api.auth.register
     .inject({ db: DbToken, config: ConfigToken })
@@ -265,7 +265,7 @@ export const EnrichVendorEndpoint = api.vendors.enrich
 
 export const ListTransactionsEndpoint = api.transactions.list
     .authorize(PrincipalSchema)
-    .inject({ db: DbToken })
+    .inject({ db: DbToken, knex: KnexToken })
     .summary('List transactions')
     .description('Lists transactions owned by the authenticated user.')
     .tags('transactions')
@@ -297,6 +297,54 @@ export const DeleteTransactionEndpoint = api.transactions.delete
     .tags('transactions')
     .operationId('deleteTransaction');
 
+export const GetTransactionScanImageEndpoint = api.transactions.scanImage
+    .authorize(PrincipalSchema)
+    .inject({ knex: KnexToken })
+    .summary('Get scanned transaction image')
+    .description(
+        'Returns the original scanner image attached to a confirmed transaction.'
+    )
+    .tags('transactions')
+    .operationId('getTransactionScanImage');
+
+export const CreateTransactionScanEndpoint = api.transactionScans.create
+    .authorize(PrincipalSchema)
+    .inject({ db: DbToken, config: ConfigToken })
+    .summary('Scan transaction image')
+    .description(
+        'Extracts draft transactions from an uploaded receipt, invoice, bank app screenshot, or statement image.'
+    )
+    .tags('transaction-scans')
+    .operationId('createTransactionScan');
+
+export const StartTransactionScanJobEndpoint = api.transactionScans.start
+    .authorize(PrincipalSchema)
+    .inject({ db: DbToken, config: ConfigToken })
+    .summary('Start transaction image scan')
+    .description(
+        'Starts an asynchronous multimodal scan job and returns a short-lived progress token.'
+    )
+    .tags('transaction-scans')
+    .operationId('startTransactionScanJob');
+
+export const TransactionScanProgressEndpoint = api.transactionScans.progress
+    .summary('Transaction image scan progress')
+    .description(
+        'Streams progress and the final scan result for a short-lived scan job token.'
+    )
+    .tags('transaction-scans')
+    .operationId('transactionScanProgress');
+
+export const DecideTransactionScanItemEndpoint = api.transactionScans.decide
+    .authorize(PrincipalSchema)
+    .inject({ db: DbToken })
+    .summary('Record transaction scan decision')
+    .description(
+        'Records whether a scanned draft was confirmed or discarded, including user corrections for future scans.'
+    )
+    .tags('transaction-scans')
+    .operationId('decideTransactionScanItem');
+
 export const DashboardSummaryEndpoint = api.dashboard.summary
     .authorize(PrincipalSchema)
     .inject({ db: DbToken })
@@ -385,7 +433,14 @@ export const endpoints = {
         list: ListTransactionsEndpoint,
         create: CreateTransactionEndpoint,
         update: UpdateTransactionEndpoint,
-        delete: DeleteTransactionEndpoint
+        delete: DeleteTransactionEndpoint,
+        scanImage: GetTransactionScanImageEndpoint
+    },
+    transactionScans: {
+        create: CreateTransactionScanEndpoint,
+        start: StartTransactionScanJobEndpoint,
+        progress: TransactionScanProgressEndpoint,
+        decide: DecideTransactionScanItemEndpoint
     },
     dashboard: {
         summary: DashboardSummaryEndpoint,

apps/api/src/api/handlers/index.ts

@@ -29,12 +29,19 @@ import {
     telegramStatusHandler,
     telegramTokenHandler
 } from './telegram.js';
+import {
+    createTransactionScanHandler,
+    decideTransactionScanItemHandler,
+    startTransactionScanJobHandler,
+    transactionScanProgressHandler
+} from './transaction-scans.js';
 import {
     categoryTrendHandler,
     createTransactionHandler,
     dashboardSummaryHandler,
     dashboardWindowHandler,
     deleteTransactionHandler,
+    getTransactionScanImageHandler,
     listTransactionsHandler,
     statsOverviewHandler,
     statsWindowHandler,
@@ -98,7 +105,14 @@ export const handlers = {
         list: listTransactionsHandler,
         create: createTransactionHandler,
         update: updateTransactionHandler,
-        delete: deleteTransactionHandler
+        delete: deleteTransactionHandler,
+        scanImage: getTransactionScanImageHandler
+    },
+    transactionScans: {
+        create: createTransactionScanHandler,
+        start: startTransactionScanJobHandler,
+        progress: transactionScanProgressHandler,
+        decide: decideTransactionScanItemHandler
     },
     dashboard: {
         summary: dashboardSummaryHandler,

apps/api/src/api/handlers/transaction-scans.ts

@@ -0,0 +1,79 @@
+import {
+    ActionResult,
+    type Handler,
+    type SubscriptionHandler
+} from '@cleverbrush/server';
+import {
+    startTransactionScanJob,
+    subscribeTransactionScanJob
+} from '../../application/transaction-scan-jobs.js';
+import {
+    recordTransactionScanDecision,
+    scanTransactionsFromImage,
+    TransactionScanInputError,
+    TransactionScanNotFoundError
+} from '../../application/transaction-scans.js';
+import type {
+    CreateTransactionScanEndpoint,
+    DecideTransactionScanItemEndpoint,
+    StartTransactionScanJobEndpoint,
+    TransactionScanProgressEndpoint
+} from '../endpoints.js';
+
+export const createTransactionScanHandler: Handler<
+    typeof CreateTransactionScanEndpoint
+> = async ({ body, principal }, { db, config }) => {
+    try {
+        const scan = await scanTransactionsFromImage(
+            db,
+            config,
+            principal.userId,
+            body
+        );
+        return ActionResult.created(
+            scan,
+            `/api/transaction-scans/${scan.scanId}`
+        );
+    } catch (err) {
+        if (err instanceof TransactionScanInputError) {
+            return ActionResult.badRequest({ message: err.message });
+        }
+        throw err;
+    }
+};
+
+export const startTransactionScanJobHandler: Handler<
+    typeof StartTransactionScanJobEndpoint
+> = async ({ body, principal }, { db, config }) => {
+    const job = startTransactionScanJob(db, config, principal.userId, body);
+    return ActionResult.accepted(job);
+};
+
+export const transactionScanProgressHandler: SubscriptionHandler<
+    typeof TransactionScanProgressEndpoint
+> = async function* ({ query, signal }) {
+    yield* subscribeTransactionScanJob(query, signal);
+};
+
+export const decideTransactionScanItemHandler: Handler<
+    typeof DecideTransactionScanItemEndpoint
+> = async ({ body, params, principal }, { db }) => {
+    try {
+        await recordTransactionScanDecision(
+            db,
+            principal.userId,
+            params.scanId,
+            params.itemId,
+            body
+        );
+        return ActionResult.noContent();
+    } catch (err) {
+        if (err instanceof TransactionScanInputError) {
+            return ActionResult.badRequest({ message: err.message });
+        }
+        if (err instanceof TransactionScanNotFoundError) {
+            return ActionResult.notFound({ message: err.message });
+        }
+        throw err;
+    }
+};

apps/api/src/api/handlers/transactions.ts

@@ -5,6 +5,7 @@ import {
     dashboardSummary,
     dashboardWindow,
     deleteTransaction,
+    getTransactionScanImage,
     listTransactions,
     statsOverview,
     statsWindow,
@@ -19,6 +20,7 @@ import type {
     DashboardSummaryEndpoint,
     DashboardWindowEndpoint,
     DeleteTransactionEndpoint,
+    GetTransactionScanImageEndpoint,
     ListTransactionsEndpoint,
     StatsOverviewEndpoint,
     StatsWindowEndpoint,
@@ -27,8 +29,8 @@ import type {
 
 export const listTransactionsHandler: Handler<
     typeof ListTransactionsEndpoint
-> = async ({ query, principal }, { db }) => {
-    return listTransactions(db, principal.userId, query);
+> = async ({ query, principal }, { db, knex }) => {
+    return listTransactions(db, principal.userId, query, knex);
 };
 
 export const createTransactionHandler: Handler<
@@ -93,6 +95,19 @@ export const deleteTransactionHandler: Handler<
     }
 };
 
+export const getTransactionScanImageHandler: Handler<
+    typeof GetTransactionScanImageEndpoint
+> = async ({ params, principal }, { knex }) => {
+    try {
+        return await getTransactionScanImage(knex, principal.userId, params.id);
+    } catch (err) {
+        if (err instanceof TransactionNotFoundError) {
+            return ActionResult.notFound({ message: err.message });
+        }
+        throw err;
+    }
+};
+
 export const dashboardSummaryHandler: Handler<
     typeof DashboardSummaryEndpoint
 > = async ({ query, principal }, { db }) => {

apps/api/src/application/openai.test.ts

@@ -1,11 +1,16 @@
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import type { Config } from '../config.js';
-import { generateStructuredJson, OpenAIConfigError } from './openai.js';
+import {
+    generateStructuredJson,
+    generateStructuredJsonFromContent,
+    OpenAIConfigError
+} from './openai.js';
 
 const config = {
     openai: {
         apiKey: 'sk-test',
-        reportModel: 'gpt-5-mini'
+        reportModel: 'gpt-5-mini',
+        transactionScanModel: 'gpt-5.5'
     }
 } as Config;
 
@@ -47,6 +52,69 @@ describe('generateStructuredJson', () => {
                 })
             })
         );
+        expect(
+            JSON.parse(String(fetchSpy.mock.calls[0]?.[1]?.body))
+        ).toMatchObject({
+            store: false,
+            text: {
+                format: {
+                    name: 'test_schema',
+                    strict: true,
+                    type: 'json_schema'
+                }
+            }
+        });
+    });
+
+    it('requests structured JSON with image input content', async () => {
+        const fetchSpy = vi.spyOn(globalThis, 'fetch').mockResolvedValue({
+            ok: true,
+            json: () =>
+                Promise.resolve({
+                    output_text: JSON.stringify({ count: 2 })
+                })
+        } as Response);
+
+        await expect(
+            generateStructuredJsonFromContent<{ count: number }>(config, {
+                content: [
+                    { type: 'input_text', text: 'Extract transactions.' },
+                    {
+                        type: 'input_image',
+                        image_url: 'data:image/png;base64,abc',
+                        detail: 'original'
+                    }
+                ],
+                model: config.openai.reportModel,
+                schema: {
+                    type: 'object',
+                    properties: { count: { type: 'number' } },
+                    required: ['count'],
+                    additionalProperties: false
+                },
+                schemaName: 'image_schema',
+                system: 'Write JSON.'
+            })
+        ).resolves.toEqual({ count: 2 });
+
+        expect(
+            JSON.parse(String(fetchSpy.mock.calls[0]?.[1]?.body))
+        ).toMatchObject({
+            input: [
+                expect.any(Object),
+                {
+                    role: 'user',
+                    content: [
+                        { type: 'input_text', text: 'Extract transactions.' },
+                        {
+                            type: 'input_image',
+                            image_url: 'data:image/png;base64,abc',
+                            detail: 'original'
+                        }
+                    ]
+                }
+            ]
+        });
     });
 
     it('throws when OpenAI is not configured', async () => {