From 54dda17dfaf166e76ee5fd36e6603fde20fe6eb2 Mon Sep 17 00:00:00 2001 From: Rinjani Analytics Date: Wed, 17 Jun 2026 07:48:59 +0700 Subject: [PATCH 1/2] feat(feeds): OFAC SDN sanctioned crypto-address feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one free, authoritative on-chain attribution source. Every digital- currency address on the US Treasury OFAC SDN list is ground-truth "this wallet belongs to a sanctioned entity" — the signal we'd otherwise pay Arkham for ($1K/yr post-trial), but free and structured. Custom connector (apps/worker/src/feeds/ofac.ts): the crypto addresses live only in sdn.xml (the CSV export drops the digital-currency entries) and the declarative feed-engine is JSON/CSV/text only, so this parses XML via fast-xml-parser. Dual-sink in one pass: - iocs — type `crypto-address`, tagged `sanctioned` → surfaces in the Feeds "Landscape shift" tag band + Indicators + graph. - wallets — entityType `sanctioned`, confidence 100, attributionSource `ofac` → the free on-chain attribution layer. Wired through additionalFeeds → feedRegistry (`ofac`) and scheduled daily at 03:00 UTC in JOB_REGISTRY (OFAC republishes only on designation actions; the upsert is idempotent on iocs.value / wallets.ref_id). Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/api/src/queues/scheduler.ts | 14 + .../src/services/feedSync/additionalFeeds.ts | 6 + .../api/src/services/feedSync/feedRegistry.ts | 6 +- apps/worker/package.json | 1 + apps/worker/src/feeds/ofac.ts | 260 ++++++++++++++++++ pnpm-lock.yaml | 16 ++ 6 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 apps/worker/src/feeds/ofac.ts diff --git a/apps/api/src/queues/scheduler.ts b/apps/api/src/queues/scheduler.ts index 9cbd469..cc918f9 100644 --- a/apps/api/src/queues/scheduler.ts +++ b/apps/api/src/queues/scheduler.ts @@ -149,6 +149,20 @@ export const JOB_REGISTRY: ScheduledJobRegistration[] = [ queue: feedSyncQueue, payload: { source: 'openphish' }, }, + { + // OFAC SDN sanctioned crypto addresses (US Treasury). The free, + // authoritative on-chain attribution feed — dual-sinks to iocs + // (tag `sanctioned`, drives Landscape shift) + wallets. OFAC only + // republishes on a designation action (days–weeks apart), so daily + // at 03:00 UTC is ample; the upsert is idempotent. + key: 'ofacSync', + jobId: 'scheduled-ofac-sync', + name: 'ofac-sync', + description: 'Sync OFAC SDN sanctioned cryptocurrency addresses', + defaultCron: '0 3 * * *', + queue: feedSyncQueue, + payload: { source: 'ofac' }, + }, // --- Knowledge base syncs ------------------------------------------ { diff --git a/apps/api/src/services/feedSync/additionalFeeds.ts b/apps/api/src/services/feedSync/additionalFeeds.ts index aee33ca..261db60 100644 --- a/apps/api/src/services/feedSync/additionalFeeds.ts +++ b/apps/api/src/services/feedSync/additionalFeeds.ts @@ -62,6 +62,12 @@ export async function syncOpenPhishFeed(): Promise { return normalise(await syncOpenPhish()); } +export async function syncOFACFeed(): Promise { + // @ts-ignore — worker scripts outside rootDir, resolved at runtime + const { syncOFAC } = await import('../../../../worker/src/feeds/ofac'); + return normalise(await syncOFAC()); +} + export async function syncMITREFeed(): Promise { try { // @ts-ignore diff --git a/apps/api/src/services/feedSync/feedRegistry.ts b/apps/api/src/services/feedSync/feedRegistry.ts index 7bfa1d9..1a7e51f 100644 --- a/apps/api/src/services/feedSync/feedRegistry.ts +++ b/apps/api/src/services/feedSync/feedRegistry.ts @@ -20,7 +20,7 @@ import { syncCveOrgFeed } from './cveOrgSync'; import { syncAbuseSSLFeed, syncThreatFoxFeed, syncURLhausFeed, syncMalwareBazaarFeed, syncOpenPhishFeed, syncMITREFeed, syncMISPGalaxyFeed, - syncEPSSFeed, + syncEPSSFeed, syncOFACFeed, } from './additionalFeeds'; import { syncHibpBreaches } from './hibpSync'; import { FeedManifest as FeedManifestSchema } from '@rinjani/feed-engine'; @@ -47,6 +47,10 @@ const FEED_REGISTRY: Record = { urlhaus: () => syncURLhausFeed(), malwarebazaar: () => syncMalwareBazaarFeed(), openphish: () => syncOpenPhishFeed(), + // OFAC SDN sanctioned crypto addresses — the one free, authoritative + // on-chain attribution source. Dual-sinks to iocs (tag `sanctioned`, + // surfaces in Landscape shift) + wallets (entityType `sanctioned`). + ofac: () => syncOFACFeed(), mitre: () => syncMITREFeed(), mispgalaxy: () => syncMISPGalaxyFeed(), // EPSS — FIRST.org's daily exploit-prediction score. Pairs with the diff --git a/apps/worker/package.json b/apps/worker/package.json index 0049140..347e131 100644 --- a/apps/worker/package.json +++ b/apps/worker/package.json @@ -19,6 +19,7 @@ "bullmq": "^5.77.3", "dotenv": "^17.2.3", "drizzle-orm": "^0.45.2", + "fast-xml-parser": "^4.5.0", "ioredis": "^5.10.1" }, "devDependencies": { diff --git a/apps/worker/src/feeds/ofac.ts b/apps/worker/src/feeds/ofac.ts new file mode 100644 index 0000000..706a278 --- /dev/null +++ b/apps/worker/src/feeds/ofac.ts @@ -0,0 +1,260 @@ +/** + * OFAC SDN — sanctioned cryptocurrency addresses (US Treasury) + * + * The one free, authoritative on-chain attribution source. Every digital- + * currency address on the OFAC Specially Designated Nationals (SDN) list is a + * ground-truth "this wallet belongs to a sanctioned entity" fact — exactly the + * signal we'd otherwise pay Arkham for, but free and structured. + * + * Source: https://www.treasury.gov/ofac/downloads/sdn.xml (no key, ~15 MB) + * Format: XML. The crypto addresses live ONLY in the XML (the sdn.csv export + * drops the digital-currency entries), so this is a custom + * connector — the declarative feed-engine is JSON/CSV/text only. + * + * Each carries the entity name + sanctions programs + an ; + * we keep the rows whose is "Digital Currency Address - ". + * + * DUAL SINK (one pass): + * 1. iocs — type `crypto-address`, tagged `sanctioned` → feeds the Feeds + * page "Landscape shift" tag band + Indicators search + graph. + * 2. wallets — entityType `sanctioned`, confidence 100, attributionSource + * `ofac` → the on-chain attribution layer (free Arkham stand-in). + * + * Updates: OFAC republishes on each designation action (irregular, days–weeks). + * Daily is plenty; the upsert is idempotent on iocs.value / wallets.ref_id. + */ + +import { XMLParser } from 'fast-xml-parser'; +import { db, sql } from '@rinjani/db'; +import { iocs, wallets } from '@rinjani/db/schema'; +import type { NewWallet } from '@rinjani/db/schema'; +import { createLogger } from '../lib/logger'; + +const log = createLogger('OFAC'); + +const OFAC_SDN_XML_URL = process.env.OFAC_SDN_XML_URL + ?? 'https://www.treasury.gov/ofac/downloads/sdn.xml'; +const BATCH_SIZE = 200; + +interface SyncResult { + processed: number; + failed: number; + errors: string[]; +} + +/** + * Map an OFAC digital-currency ticker to our internal chain code. OFAC uses + * `XBT` for Bitcoin; everything else is the common ticker. Unknowns fall + * through to the lower-cased ticker so a newly-added asset still ingests. + */ +const CHAIN_BY_TICKER: Record = { + XBT: 'btc', BTC: 'btc', ETH: 'eth', USDT: 'usdt', USDC: 'usdc', + XMR: 'xmr', LTC: 'ltc', ZEC: 'zec', DASH: 'dash', BTG: 'btg', + ETC: 'etc', BSV: 'bsv', BCH: 'bch', XVG: 'xvg', XRP: 'xrp', + TRX: 'trx', ARB: 'arb', BNB: 'bnb', SOL: 'sol', +}; + +/** Normalise fast-xml-parser's "string | object | array | undefined" to an array. */ +function asArray(v: T | T[] | undefined | null): T[] { + if (v === undefined || v === null) return []; + return Array.isArray(v) ? v : [v]; +} + +interface SanctionedAddress { + address: string; + chain: string; + entityName: string; + sdnType: string; // Entity | Individual | Vessel | … + programs: string[]; // e.g. ["CYBER2", "DPRK3"] + uid: string; // OFAC SDN UID — stable external ref +} + +const DCA_RE = /Digital Currency Address\s*-\s*([A-Za-z0-9]+)/i; + +/** Pull every digital-currency address out of the parsed SDN document. */ +function extractAddresses(doc: unknown): SanctionedAddress[] { + // … ; fast-xml-parser strips the default namespace so + // the local names come through verbatim. + const root = (doc as { sdnList?: { sdnEntry?: unknown } })?.sdnList; + const entries = asArray(root?.sdnEntry as Record | Record[]); + const out: SanctionedAddress[] = []; + + for (const entry of entries) { + const ids = asArray((entry.idList as { id?: unknown } | undefined)?.id as + Record | Record[]); + const crypto = ids.filter((id) => DCA_RE.test(String(id?.idType ?? ''))); + if (crypto.length === 0) continue; + + // Entity name: OFAC puts an org's full name in with an empty + // ; individuals split across both. + const first = String(entry.firstName ?? '').trim(); + const last = String(entry.lastName ?? '').trim(); + const entityName = [first, last].filter(Boolean).join(' ') || `SDN ${String(entry.uid ?? '')}`; + const sdnType = String(entry.sdnType ?? 'Entity').trim(); + const programs = asArray((entry.programList as { program?: unknown } | undefined)?.program) + .map((p) => String(p).trim()) + .filter(Boolean); + const uid = String(entry.uid ?? '').trim(); + + for (const id of crypto) { + const ticker = (String(id.idType).match(DCA_RE)?.[1] ?? '').toUpperCase(); + const address = String(id.idNumber ?? '').trim(); + if (!address) continue; + out.push({ + address, + chain: CHAIN_BY_TICKER[ticker] ?? ticker.toLowerCase(), + entityName, + sdnType, + programs, + uid, + }); + } + } + return out; +} + +async function fetchSdn(): Promise { + const res = await fetch(OFAC_SDN_XML_URL, { headers: { Accept: 'application/xml' } }); + if (!res.ok) { + throw new Error(`OFAC SDN fetch failed: ${res.status} ${res.statusText}`); + } + const xml = await res.text(); + const parser = new XMLParser({ ignoreAttributes: true, parseTagValue: false, trimValues: true }); + const doc = parser.parse(xml); + return extractAddresses(doc); +} + +export async function syncOFAC(): Promise { + log.info('Starting sync', { feedUrl: OFAC_SDN_XML_URL }); + const result: SyncResult = { processed: 0, failed: 0, errors: [] }; + + let addresses: SanctionedAddress[]; + try { + addresses = await fetchSdn(); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + log.error('Fetch/parse failed', err as Error); + result.errors.push(`Fetch error: ${msg}`); + result.failed = 1; + return result; + } + log.info('Extracted sanctioned crypto addresses', { count: addresses.length }); + + // De-dupe on chain:address — the same wallet can be listed under multiple + // SDN entries; keep the first (programs differ but attribution is the same + // "sanctioned" fact). + const seen = new Set(); + const unique = addresses.filter((a) => { + const key = `${a.chain}:${a.address}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + + for (let i = 0; i < unique.length; i += BATCH_SIZE) { + const slice = unique.slice(i, i + BATCH_SIZE); + try { + await writeBatch(slice); + result.processed += slice.length; + } catch (err) { + result.failed += slice.length; + const msg = err instanceof Error ? err.message : String(err); + if (result.errors.length < 10) result.errors.push(`Batch insert failed: ${msg}`); + log.error('Batch insert error', err as Error); + } + } + + log.info('Sync completed', { processed: result.processed, failed: result.failed }); + return result; +} + +async function writeBatch(batch: SanctionedAddress[]): Promise { + const now = new Date(); + + // --- IOC sink: surfaces in Landscape shift via the `sanctioned` tag --- + const iocRows = batch.map((a) => ({ + type: 'crypto-address', + value: a.address, + source: 'ofac', + threatType: 'sanctioned', + confidence: 100, // OFAC SDN is authoritative ground truth + severity: 'high', + firstSeen: now, + lastSeen: now, + tags: [ + 'ofac', + 'sanctioned', + 'ofac-sdn', + a.chain, + ...a.programs.map((p) => `ofac:${p.toLowerCase()}`), + ], + metadata: { + chain: a.chain, + entity: a.entityName, + sdn_type: a.sdnType, + programs: a.programs, + sdn_uid: a.uid, + feed_source: 'ofac', + }, + })); + + await db.insert(iocs) + .values(iocRows) + .onConflictDoUpdate({ + target: iocs.value, + set: { lastSeen: now, updatedAt: now }, + }); + + // --- Wallet sink: the free on-chain attribution layer (Arkham stand-in) --- + const walletRows: NewWallet[] = batch.map((a) => ({ + refId: `${a.chain}:${a.address}`, + address: a.address, + chain: a.chain, + name: a.entityName, + entityLabel: a.entityName, + entityType: 'sanctioned', + confidence: 100, + attributionSource: 'ofac', + riskTags: ['sanctioned', 'ofac-sdn', ...a.programs.map((p) => p.toLowerCase())], + externalReferences: [{ + source_name: 'OFAC SDN', + url: 'https://sanctionssearch.ofac.treas.gov/', + external_id: a.uid, + }], + })); + + await db.insert(wallets) + .values(walletRows) + .onConflictDoUpdate({ + target: wallets.refId, + set: { + name: sql`excluded.name`, + entityLabel: sql`excluded.entity_label`, + entityType: sql`excluded.entity_type`, + confidence: sql`excluded.confidence`, + attributionSource: sql`excluded.attribution_source`, + riskTags: sql`excluded.risk_tags`, + updatedAt: now, + }, + }); +} + +/** Standalone runner — `tsx apps/worker/src/feeds/ofac.ts`. */ +export async function runOFACSync(): Promise { + log.info('Starting full sync'); + try { + const result = await syncOFAC(); + log.info('Full sync completed', { processed: result.processed, failed: result.failed }); + } catch (error) { + log.error('Sync failed', error as Error); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + runOFACSync() + .then(() => process.exit(0)) + .catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b22ec7b..d107013 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -249,6 +249,9 @@ importers: drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@opentelemetry/api@1.9.0)(@types/pg@8.16.0)(pg@8.18.0)(postgres@3.4.8) + fast-xml-parser: + specifier: ^4.5.0 + version: 4.5.6 ioredis: specifier: ^5.10.1 version: 5.10.1 @@ -3240,6 +3243,10 @@ packages: fast-uri@3.1.2: resolution: {integrity: sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==} + fast-xml-parser@4.5.6: + resolution: {integrity: sha512-Yd4vkROfJf8AuJrDIVMVmYfULKmIJszVsMv7Vo71aocsKgFxpdlpSHXSaInvyYfgw2PRuObQSW2GFpVMUjxu9A==} + hasBin: true + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -4230,6 +4237,9 @@ packages: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} + strnum@1.1.2: + resolution: {integrity: sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==} + sucrase@3.35.1: resolution: {integrity: sha512-DhuTmvZWux4H1UOnWMB3sk0sbaCVOoQZjv8u1rDoTV0HTdGem9hkAZtl4JZy8P2z4Bg0nT+YMeOFyVr4zcG5Tw==} engines: {node: '>=16 || 14 >=14.17'} @@ -7712,6 +7722,10 @@ snapshots: fast-uri@3.1.2: {} + fast-xml-parser@4.5.6: + dependencies: + strnum: 1.1.2 + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 @@ -8674,6 +8688,8 @@ snapshots: strip-json-comments@2.0.1: {} + strnum@1.1.2: {} + sucrase@3.35.1: dependencies: '@jridgewell/gen-mapping': 0.3.13 From 2eae4c9d9f43d23422a5695e99cf1ae1ebf02103 Mon Sep 17 00:00:00 2001 From: Rinjani Analytics Date: Wed, 17 Jun 2026 08:16:25 +0700 Subject: [PATCH 2/2] test(feeds): add ofac to feed-registry expected keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The feed-registry test pins the exact FEED_REGISTRY key set; adding the ofac feed took it 13 → 14. Keep the list in lock-step. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/api/src/__tests__/feed-registry.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/api/src/__tests__/feed-registry.test.ts b/apps/api/src/__tests__/feed-registry.test.ts index 60a5cc2..2b59f15 100644 --- a/apps/api/src/__tests__/feed-registry.test.ts +++ b/apps/api/src/__tests__/feed-registry.test.ts @@ -15,9 +15,11 @@ describe('Feed Registry', () => { // `epss` joined as Phase 1's exploit-prediction enrichment. // `hibp` joined as Phase 5 #3's HIBP breach catalog (free-tier // /breaches sync only — no paid /breachedaccount). + // `ofac` joined as the free, authoritative on-chain attribution feed + // (OFAC SDN sanctioned crypto addresses; dual-sinks iocs + wallets). const EXPECTED_FEEDS = [ 'otx', 'cisa', 'cveorg', 'nvd', 'abusessl', 'threatfox', - 'urlhaus', 'malwarebazaar', 'openphish', 'mitre', 'mispgalaxy', + 'urlhaus', 'malwarebazaar', 'openphish', 'ofac', 'mitre', 'mispgalaxy', 'epss', 'hibp', ];