fix: add application-level retry for SQLite init under concurrent access (#1154)

## Summary

- Adds exponential backoff retry (up to 5 attempts) around `PRAGMA
journal_mode=WAL` and schema creation in both `CatalogStore` and
`ExtensionCatalogStore`
- Fixes "database is locked" crashes when multiple `swamp workflow run`
processes start concurrently and race to initialize the same SQLite
database
- The existing `busy_timeout=5000` pragma doesn't reliably cover the
journal mode switch in Deno's `node:sqlite`, so application-level retry
fills the gap

## Test Plan

- [x] Existing `CatalogStore: constructor retries under write lock
contention` test passes
- [x] Full test suite passes (4249 tests)
- [x] `deno check`, `deno lint`, `deno fmt` all clean
- [ ] UAT adversarial test `swamp workflow run concurrently produces
distinct data versions` should now pass

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: stack72

src/infrastructure/persistence/catalog_store.ts

@@ -75,18 +75,50 @@ export class CatalogStore {
     ensureDirSync(dirname(dbPath));
     this.db = new DatabaseSync(dbPath);
     this.db.exec("PRAGMA busy_timeout=5000");
-    this.db.exec("PRAGMA journal_mode=WAL");
+    this.initializeWithRetry();
+  }
 
-    // Ensure catalog_meta exists so migrateIfNeeded() can read schema_version.
-    // Must run before createSchema() because v2-only indexes fail on a v1 table.
-    this.db.exec(`
-      CREATE TABLE IF NOT EXISTS catalog_meta (
-        key   TEXT PRIMARY KEY,
-        value TEXT NOT NULL
-      );
-    `);
-    this.migrateIfNeeded();
-    this.createSchema();
+  /**
+   * Initializes WAL mode and schema with retry logic.
+   * PRAGMA journal_mode=WAL requires an exclusive lock to switch modes.
+   * When multiple processes open the database simultaneously, the SQLite
+   * busy handler may not cover the mode switch reliably, so we retry at
+   * the application level with exponential backoff.
+   */
+  private initializeWithRetry(): void {
+    const MAX_RETRIES = 5;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      try {
+        this.db.exec("PRAGMA journal_mode=WAL");
+
+        // Ensure catalog_meta exists so migrateIfNeeded() can read schema_version.
+        // Must run before createSchema() because v2-only indexes fail on a v1 table.
+        this.db.exec(`
+          CREATE TABLE IF NOT EXISTS catalog_meta (
+            key   TEXT PRIMARY KEY,
+            value TEXT NOT NULL
+          );
+        `);
+        this.migrateIfNeeded();
+        this.createSchema();
+        return;
+      } catch (error: unknown) {
+        const isLock = error instanceof Error &&
+          /database is (locked|busy)/i.test(error.message);
+        if (isLock && attempt < MAX_RETRIES) {
+          const delay = 100 * Math.pow(2, attempt) +
+            Math.floor(Math.random() * 50);
+          Atomics.wait(
+            new Int32Array(new SharedArrayBuffer(4)),
+            0,
+            0,
+            delay,
+          );
+          continue;
+        }
+        throw error;
+      }
+    }
   }
 
   private createSchema(): void {

src/infrastructure/persistence/extension_catalog_store.ts

@@ -67,11 +67,41 @@ export class ExtensionCatalogStore {
   constructor(dbPath: string) {
     ensureDirSync(dirname(dbPath));
     this.db = new DatabaseSync(dbPath);
-    // busy_timeout must be set BEFORE journal_mode=WAL so SQLite retries
-    // instead of failing immediately when another process holds the lock.
     this.db.exec("PRAGMA busy_timeout=5000");
-    this.db.exec("PRAGMA journal_mode=WAL");
-    this.createSchema();
+    this.initializeWithRetry();
+  }
+
+  /**
+   * Initializes WAL mode and schema with retry logic.
+   * PRAGMA journal_mode=WAL requires an exclusive lock to switch modes.
+   * When multiple processes open the database simultaneously, the SQLite
+   * busy handler may not cover the mode switch reliably, so we retry at
+   * the application level with exponential backoff.
+   */
+  private initializeWithRetry(): void {
+    const MAX_RETRIES = 5;
+    for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
+      try {
+        this.db.exec("PRAGMA journal_mode=WAL");
+        this.createSchema();
+        return;
+      } catch (error: unknown) {
+        const isLock = error instanceof Error &&
+          /database is (locked|busy)/i.test(error.message);
+        if (isLock && attempt < MAX_RETRIES) {
+          const delay = 100 * Math.pow(2, attempt) +
+            Math.floor(Math.random() * 50);
+          Atomics.wait(
+            new Int32Array(new SharedArrayBuffer(4)),
+            0,
+            0,
+            delay,
+          );
+          continue;
+        }
+        throw error;
+      }
+    }
   }
 
   private createSchema(): void {