diff --git a/.env.example b/.env.example index a9e2b155..72270318 100644 --- a/.env.example +++ b/.env.example @@ -143,3 +143,23 @@ DEBUG_RPC=false # Enable CORS for development CORS_ENABLED=true CORS_ORIGIN=http://localhost:3000 + +# ============================================ +# Decentralized Storage +# ============================================ + +# Storage provider: pinata or mock +DECENTRALIZED_STORAGE_PROVIDER=pinata + +# Pinata JWT used for IPFS pinning +PINATA_JWT= + +# Gateway base URL for pinned assets +STORAGE_GATEWAY_BASE_URL=https://gateway.pinata.cloud/ipfs + +# Garbage collection and retry settings +STORAGE_GC_RETENTION_DAYS=30 +STORAGE_MAX_PIN_ATTEMPTS=5 +STORAGE_BACKOFF_DELAY_MS=1000 +STORAGE_WORKER_CONCURRENCY=10 +STORAGE_JOB_TIMEOUT_MS=30000 diff --git a/backend/jest.config.js b/backend/jest.config.js index e2c06161..5c51bcee 100644 --- a/backend/jest.config.js +++ b/backend/jest.config.js @@ -1,4 +1,6 @@ /** @type {import('ts-jest').JestConfigWithTsJest} */ +process.env.NODE_ENV = 'test'; + export default { preset: 'ts-jest', testEnvironment: 'node', @@ -6,6 +8,7 @@ export default { moduleNameMapper: { '^(\\.{1,2}/.*)\\.js$': '$1', }, + setupFiles: ['/tests/jest.setup.ts'], transform: { '^.+\\.tsx?$': [ 'ts-jest', diff --git a/backend/logs/combined.log b/backend/logs/combined.log index daac04ac..98aa1859 100644 --- a/backend/logs/combined.log +++ b/backend/logs/combined.log @@ -3823,3 +3823,17 @@ Unique constraint failed on the fields: (`tokenId`) 2026-05-20 09:55:58 info: Certificate minted on-chain: cert-api-stud-api-cour-1779267358588 -> token 295396 2026-05-20 09:55:58 info: Certificate minted: cert-api-stud-api-cour-1779267358588 2026-05-20 09:55:58 info: GET /api/v1/certificates/verify/295396 +2026-05-31 01:55:38 info: Pinned decentralized asset project/project-1/example -> bafyd37873b9cdea335789ab00f55e95233b9090e32bb1162154b65bc6c2 +2026-05-31 01:55:38 info: Unpinned stale decentralized asset bafy123 +2026-05-31 01:57:57 info: Pinned decentralized asset project/project-1/example -> bafyd37873b9cdea335789ab00f55e95233b9090e32bb1162154b65bc6c2 +2026-05-31 01:57:57 info: Unpinned stale decentralized asset bafy123 +2026-05-31 02:00:22 info: Pinned decentralized asset project/project-1/example -> bafyd37873b9cdea335789ab00f55e95233b9090e32bb1162154b65bc6c2 +2026-05-31 02:00:22 info: Unpinned stale decentralized asset bafy123 +2026-05-31 02:03:29 info: Pinned decentralized asset project/project-1/example -> bafyd37873b9cdea335789ab00f55e95233b9090e32bb1162154b65bc6c2 +2026-05-31 02:03:29 info: Unpinned stale decentralized asset bafy123 +2026-05-31 02:06:51 info: Pinned decentralized asset project/project-1/example -> bafyd37873b9cdea335789ab00f55e95233b9090e32bb1162154b65bc6c2 +2026-05-31 02:06:51 info: Unpinned stale decentralized asset bafy123 +2026-05-31 02:10:44 info: Pinned decentralized asset project/project-1/example -> bafyd37873b9cdea335789ab00f55e95233b9090e32bb1162154b65bc6c2 +2026-05-31 02:10:44 info: Unpinned stale decentralized asset bafy123 +2026-05-31 02:13:08 info: Pinned decentralized asset project/project-1/example -> bafyd37873b9cdea335789ab00f55e95233b9090e32bb1162154b65bc6c2 +2026-05-31 02:13:08 info: Unpinned stale decentralized asset bafy123 diff --git a/backend/package.json b/backend/package.json index 06de1eb4..f6b926ce 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,8 +8,7 @@ "start": "node dist/src/index.js", "start:prod": "node dist/src/index.js", "dev": "tsx watch src/index.ts", - - + "test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js --config ./jest.config.js", "collaboration": "tsx src/collaborationServer.ts" }, "keywords": [], @@ -47,9 +46,12 @@ "@types/bcryptjs": "^2.4.6", "@types/cors": "^2.8.19", "@types/express": "^5.0.6", + "@types/jest": "^30.0.0", "@types/node": "^25.5.0", "@types/qrcode": "^1.5.5", "@types/ws": "^8.18.1", + "jest": "^30.4.2", + "ts-jest": "^29.4.11", "ts-node": "^10.9.2", "tsx": "^4.21.0", "typescript": "^5.9.3" diff --git a/backend/prisma/migrations/20260531140000_decentralized_storage/migration.sql b/backend/prisma/migrations/20260531140000_decentralized_storage/migration.sql new file mode 100644 index 00000000..274411a6 --- /dev/null +++ b/backend/prisma/migrations/20260531140000_decentralized_storage/migration.sql @@ -0,0 +1,37 @@ +-- CreateTable +CREATE TABLE "decentralized_assets" ( + "id" TEXT NOT NULL, + "workspaceId" TEXT NOT NULL DEFAULT 'default', + "resourceType" TEXT NOT NULL, + "resourceId" TEXT NOT NULL, + "name" TEXT NOT NULL, + "kind" TEXT NOT NULL, + "provider" TEXT NOT NULL DEFAULT 'pinata', + "cid" TEXT NOT NULL, + "ipfsUri" TEXT NOT NULL, + "gatewayUrl" TEXT NOT NULL, + "mimeType" TEXT, + "sizeBytes" INTEGER, + "status" TEXT NOT NULL DEFAULT 'queued', + "referenceCount" INTEGER NOT NULL DEFAULT 1, + "metadata" JSONB, + "error" TEXT, + "pinnedAt" TIMESTAMP(3), + "unpinnedAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "decentralized_assets_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "decentralized_assets_cid_key" ON "decentralized_assets"("cid"); + +-- CreateIndex +CREATE UNIQUE INDEX "decentralized_assets_workspaceId_resourceType_resourceId_name_key" ON "decentralized_assets"("workspaceId", "resourceType", "resourceId", "name"); + +-- CreateIndex +CREATE INDEX "decentralized_assets_workspaceId_status_idx" ON "decentralized_assets"("workspaceId", "status"); + +-- CreateIndex +CREATE INDEX "decentralized_assets_workspaceId_resourceType_resourceId_idx" ON "decentralized_assets"("workspaceId", "resourceType", "resourceId"); diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma index 20fb69fb..bf1968dc 100644 --- a/backend/prisma/schema.prisma +++ b/backend/prisma/schema.prisma @@ -206,3 +206,31 @@ model Canvas { @@index([studentId]) @@index([createdAt]) } + +model DecentralizedAsset { + id String @id @default(cuid()) + workspaceId String @default("default") + resourceType String + resourceId String + name String + kind String + provider String @default("pinata") + cid String @unique + ipfsUri String + gatewayUrl String + mimeType String? + sizeBytes Int? + status String @default("queued") + referenceCount Int @default(1) + metadata Json? + error String? + pinnedAt DateTime? + unpinnedAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([workspaceId, resourceType, resourceId, name]) + @@index([workspaceId, status]) + @@index([workspaceId, resourceType, resourceId]) + @@map("decentralized_assets") +} diff --git a/backend/src/certificates/CertificateService.ts b/backend/src/certificates/CertificateService.ts index 9345d54b..9b47c5e3 100644 --- a/backend/src/certificates/CertificateService.ts +++ b/backend/src/certificates/CertificateService.ts @@ -8,6 +8,8 @@ import { import { MetadataGenerator } from './MetadataGenerator.js'; import { certificateBlockchainService } from '../blockchain/CertificateBlockchainService.js'; import logger from '../utils/logger.js'; +import { certificateImageGenerator } from '../utils/certificateImageGenerator.js'; +import { storageService } from '../services/storage/index.js'; export class CertificateService { private metadataGenerator: MetadataGenerator; @@ -87,10 +89,36 @@ export class CertificateService { }, }); - // Generate the metadata - const metadata = this.metadataGenerator.generate(certificate, course, student); - try { + // Generate and pin the certificate image and metadata to decentralized storage + const imageBuffer = await certificateImageGenerator.generateCertificateImage({ + studentName: `${student.firstName || ''} ${student.lastName || ''}`.trim() || 'Student', + courseTitle: course.title, + instructor: course.instructor, + completionDate: certificate.issuedAt.toISOString(), + grade: certificate.grade || undefined, + credentialId: certificate.tokenId || tokenIdValue, + issuerName: process.env.ISSUER_NAME || 'Web3 Student Lab', + }); + + const imageAsset = await storageService.pinCertificateImage({ + certificateId: certificateId, + content: imageBuffer, + mimeType: 'image/svg+xml', + }); + + const metadata = this.metadataGenerator.generate(certificate, course, student, { + imageUri: imageAsset.ipfsUri, + externalUrl: `${process.env.API_BASE_URL || 'http://localhost:8080'}/api/v1/certificates/${ + certificate.tokenId || tokenIdValue + }/metadata`, + }); + + const metadataAsset = await storageService.pinCertificateMetadata({ + certificateId: certificateId, + content: metadata, + }); + // Call blockchain service to mint actual NFT const mintResult = await certificateBlockchainService.mintCertificate(metadata); @@ -101,7 +129,7 @@ export class CertificateService { certificateHash: mintResult.transactionHash, contractAddress: mintResult.contractAddress, status: 'ACTIVE', - metadataUri: metadata.image, + metadataUri: metadataAsset.ipfsUri, tokenId: mintResult.tokenId || tokenIdValue, }, }); @@ -118,15 +146,16 @@ export class CertificateService { txHash: mintResult.transactionHash, }); } catch (error) { - logger.error(`Blockchain mint failed for ${certificateId}:`, error); + logger.error(`Certificate issuance failed for ${certificateId}:`, error); await prisma.certificate.update({ where: { id: certificateId }, data: { status: 'FAILED', }, }); + await storageService.releaseResource('certificate', certificateId); throw new Error( - `Failed to mint certificate on blockchain: ${error instanceof Error ? error.message : 'Unknown error'}` + `Failed to mint certificate: ${error instanceof Error ? error.message : 'Unknown error'}` ); } diff --git a/backend/src/certificates/MetadataGenerator.ts b/backend/src/certificates/MetadataGenerator.ts index a2299d50..ec1f26d0 100644 --- a/backend/src/certificates/MetadataGenerator.ts +++ b/backend/src/certificates/MetadataGenerator.ts @@ -24,7 +24,8 @@ export class MetadataGenerator { generate( certificate: Certificate & { student: any; course: any }, course: any, - student: any + student: any, + options: { imageUri?: string; externalUrl?: string } = {} ): CertificateMetadata { // Build verification info const verification = this.buildVerificationInfo(certificate); @@ -43,10 +44,11 @@ export class MetadataGenerator { const attributes = this.buildAttributes(certificate, course, student); // Build external URL (deep link to certificate viewer) - const externalUrl = `${this.baseUrl}/certificates/${certificate.tokenId}/view`; + const externalUrl = + options.externalUrl || `${this.baseUrl}/certificates/${certificate.tokenId}/view`; // Build image URL - const imageUrl = this.buildImageUrl(certificate); + const imageUrl = options.imageUri || this.buildImageUrl(certificate); return { name, diff --git a/backend/src/index.ts b/backend/src/index.ts index 3aa682bd..e4a9802c 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -16,6 +16,8 @@ import { requestLogger } from './middleware/requestLogger.js'; import { requireWorkspaceMiddleware } from './middleware/WorkspaceContext.js'; import freelanceRoute from './routes/freelance.js'; import routes from './routes/index.js'; +import { scheduleStorageGc, startStorageWorkers, stopStorageWorkers } from './services/storage/index.js'; +import { startWebhookWorker, stopWebhookWorker } from './services/webhooks/index.js'; import { validateEnvironment } from './utils/checkEnv.js'; import logger from './utils/logger.js'; import { pubClient, redisConnection, subClient } from './utils/redis.js'; @@ -45,6 +47,12 @@ if (process.env.NODE_ENV !== 'test') { logger.warn('CacheWarmer failed to start:', err); }); + startStorageWorkers(); + startWebhookWorker(); + scheduleStorageGc().catch((err) => { + logger.warn('Storage GC schedule failed to start:', err); + }); + logger.info('Distributed caching layer initialized'); } @@ -116,6 +124,8 @@ if (process.env.NODE_ENV !== 'test') { // Stop cache components blockHeaderListener.stop(); cacheWarmer.stop(); + await stopStorageWorkers(); + await stopWebhookWorker(); await distributedCacheManager.gracefulShutdown(); // Clean up connections @@ -135,6 +145,8 @@ if (process.env.NODE_ENV !== 'test') { // Stop cache components blockHeaderListener.stop(); cacheWarmer.stop(); + await stopStorageWorkers(); + await stopWebhookWorker(); await distributedCacheManager.gracefulShutdown(); // Clean up connections diff --git a/backend/src/routes/generator/generator.routes.ts b/backend/src/routes/generator/generator.routes.ts index b65c07f5..aaee2f2e 100644 --- a/backend/src/routes/generator/generator.routes.ts +++ b/backend/src/routes/generator/generator.routes.ts @@ -2,9 +2,16 @@ import { Request, Response, Router } from 'express'; import { GeneratorService } from '../../generator/generator.service.js'; import logger from '../../utils/logger.js'; import { getRandomProjectIdea, mockProjectIdeas } from '../../generator/mockData.js'; +import { randomUUID } from 'crypto'; +import { storageService } from '../../services/storage/index.js'; const router = Router(); const generatorService = new GeneratorService(); +const slugify = (value: string): string => + value + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); /** * @route POST /api/generator/generate @@ -13,7 +20,7 @@ const generatorService = new GeneratorService(); */ router.post('/generate', async (req: Request, res: Response) => { try { - const { theme, techStack, difficulty } = req.body; + const { theme, techStack, difficulty, persistToStorage, queuedPersist } = req.body; if (!theme || !techStack || !difficulty) { res.status(400).json({ error: 'Theme, techStack, and difficulty are required' }); @@ -23,11 +30,53 @@ router.post('/generate', async (req: Request, res: Response) => { // Try AI generation first, fallback to mock data if it fails try { const projectIdea = await generatorService.generateProjectIdea(theme, techStack, difficulty); + + if (persistToStorage) { + const projectId = `${slugify(theme)}-${Date.now()}-${randomUUID().slice(0, 8)}`; + const storageResult = queuedPersist + ? await storageService.pinProjectIdea({ + projectId, + content: projectIdea, + queued: true, + }) + : await storageService.pinProjectIdea({ + projectId, + content: projectIdea, + }); + + res.json({ + projectIdea, + storage: storageResult, + }); + return; + } + res.json({ projectIdea }); } catch (aiError) { logger.warn(`AI generation failed, using mock data: ${aiError}`); // Return a random mock project idea as fallback const projectIdea = getRandomProjectIdea(); + if (persistToStorage) { + const projectId = `mock-${Date.now()}-${randomUUID().slice(0, 8)}`; + const storageResult = queuedPersist + ? await storageService.pinProjectIdea({ + projectId, + content: projectIdea, + queued: true, + }) + : await storageService.pinProjectIdea({ + projectId, + content: projectIdea, + }); + + res.json({ + projectIdea, + fromMock: true, + storage: storageResult, + }); + return; + } + res.json({ projectIdea, fromMock: true }); } } catch (error) { diff --git a/backend/src/routes/index.ts b/backend/src/routes/index.ts index 2df74a28..4c2558bf 100644 --- a/backend/src/routes/index.ts +++ b/backend/src/routes/index.ts @@ -11,6 +11,7 @@ import exportRouter from './export.routes.js'; import generatorRouter from './generator/generator.routes.js'; import learningRoutes from './learning/learning.routes.js'; import healthRouter from './health.routes.js'; +import storageRouter from './storage.routes.js'; import securityRouter from './security.routes.js'; import studentsRouter from './students.js'; @@ -29,6 +30,7 @@ router.use('/learning', learningRoutes); router.use('/security', securityRouter); router.use('/generator', generatorRouter); router.use('/export', exportRouter); +router.use('/storage', storageRouter); router.use('/user', userRouter); export default router; diff --git a/backend/src/routes/storage.routes.ts b/backend/src/routes/storage.routes.ts new file mode 100644 index 00000000..e8e7a5ed --- /dev/null +++ b/backend/src/routes/storage.routes.ts @@ -0,0 +1,151 @@ +import { Request, Response, Router } from 'express'; +import logger from '../utils/logger.js'; +import { storageService } from '../services/storage/index.js'; + +const router = Router(); + +router.get('/health', (_req: Request, res: Response) => { + res.status(200).json({ + status: 'ok', + mode: process.env.DECENTRALIZED_STORAGE_PROVIDER || 'pinata', + }); +}); + +router.post('/pin-json', async (req: Request, res: Response) => { + try { + const { resourceType, resourceId, name, content, metadata, queued, referenceCount } = req.body; + + if (!resourceType || !resourceId || !name || content === undefined) { + return res.status(400).json({ error: 'resourceType, resourceId, name, and content are required' }); + } + + if (queued) { + const result = await storageService.queueJsonPin({ + resourceType, + resourceId, + name, + kind: 'generic', + content, + metadata, + referenceCount, + }); + + return res.status(202).json({ + success: true, + queued: true, + ...result, + }); + } + + const result = await storageService.pinJsonNow({ + resourceType, + resourceId, + name, + kind: 'generic', + content, + metadata, + referenceCount, + }); + + return res.status(201).json({ + success: true, + queued: false, + data: result, + }); + } catch (error) { + logger.error('Failed to pin JSON to decentralized storage:', error); + return res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to pin JSON content', + }); + } +}); + +router.post('/pin-file', async (req: Request, res: Response) => { + try { + const { + resourceType, + resourceId, + name, + contentBase64, + mimeType, + metadata, + queued, + referenceCount, + } = req.body; + + if (!resourceType || !resourceId || !name || !contentBase64) { + return res + .status(400) + .json({ error: 'resourceType, resourceId, name, and contentBase64 are required' }); + } + + if (queued) { + const result = await storageService.queueFilePin({ + resourceType, + resourceId, + name, + kind: 'generic', + content: contentBase64, + filename: name, + mimeType: mimeType || 'application/octet-stream', + metadata, + referenceCount, + }); + + return res.status(202).json({ + success: true, + queued: true, + ...result, + }); + } + + const result = await storageService.pinFileNow({ + resourceType, + resourceId, + name, + kind: 'generic', + content: Buffer.from(contentBase64, 'base64'), + filename: name, + mimeType: mimeType || 'application/octet-stream', + metadata, + referenceCount, + }); + + return res.status(201).json({ + success: true, + queued: false, + data: result, + }); + } catch (error) { + logger.error('Failed to pin file to decentralized storage:', error); + return res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to pin file content', + }); + } +}); + +router.post('/gc', async (req: Request, res: Response) => { + try { + const retentionDays = Number(req.body?.retentionDays || process.env.STORAGE_GC_RETENTION_DAYS || '30'); + const dryRun = Boolean(req.body?.dryRun); + + const result = await storageService.queueGarbageCollection(retentionDays, dryRun); + + return res.status(202).json({ + success: true, + queued: true, + ...result, + }); + } catch (error) { + logger.error('Failed to queue storage garbage collection:', error); + return res.status(500).json({ + success: false, + error: error instanceof Error ? error.message : 'Failed to queue storage cleanup', + }); + } +}); + +export default router; + diff --git a/backend/src/services/storage/asset.repository.ts b/backend/src/services/storage/asset.repository.ts new file mode 100644 index 00000000..0361e512 --- /dev/null +++ b/backend/src/services/storage/asset.repository.ts @@ -0,0 +1,143 @@ +import type { StorageAssetRecord } from './types.js'; + +const getPrisma = async () => { + const module = await import('../../db/index.js'); + return module.default; +}; + +export const upsertStorageAsset = async (asset: { + resourceType: string; + resourceId: string; + name: string; + kind: string; + provider: string; + cid: string; + ipfsUri: string; + gatewayUrl: string; + mimeType?: string | null; + sizeBytes?: number | null; + status?: string; + referenceCount?: number; + metadata?: Record | null; + error?: string | null; +}): Promise => { + const prisma = await getPrisma(); + return prisma.decentralizedAsset.upsert({ + where: { + workspaceId_resourceType_resourceId_name: { + workspaceId: 'default', + resourceType: asset.resourceType, + resourceId: asset.resourceId, + name: asset.name, + }, + }, + update: { + kind: asset.kind, + provider: asset.provider, + cid: asset.cid, + ipfsUri: asset.ipfsUri, + gatewayUrl: asset.gatewayUrl, + mimeType: asset.mimeType ?? null, + sizeBytes: asset.sizeBytes ?? null, + status: asset.status ?? 'pinned', + referenceCount: asset.referenceCount ?? 1, + metadata: asset.metadata ?? null, + error: asset.error ?? null, + pinnedAt: asset.status === 'pinned' ? new Date() : undefined, + unpinnedAt: null, + }, + create: { + resourceType: asset.resourceType, + resourceId: asset.resourceId, + name: asset.name, + kind: asset.kind, + provider: asset.provider, + cid: asset.cid, + ipfsUri: asset.ipfsUri, + gatewayUrl: asset.gatewayUrl, + mimeType: asset.mimeType ?? null, + sizeBytes: asset.sizeBytes ?? null, + status: asset.status ?? 'pinned', + referenceCount: asset.referenceCount ?? 1, + metadata: asset.metadata ?? null, + error: asset.error ?? null, + pinnedAt: asset.status === 'pinned' ? new Date() : null, + }, + }); +}; + +export const markAssetFailed = async ( + resourceType: string, + resourceId: string, + name: string, + error: string +): Promise => { + const prisma = await getPrisma(); + await prisma.decentralizedAsset.upsert({ + where: { + workspaceId_resourceType_resourceId_name: { + workspaceId: 'default', + resourceType, + resourceId, + name, + }, + }, + update: { + status: 'failed', + error, + }, + create: { + resourceType, + resourceId, + name, + kind: 'generic', + provider: process.env.DECENTRALIZED_STORAGE_PROVIDER || 'pinata', + cid: 'pending', + ipfsUri: 'ipfs://pending', + gatewayUrl: '', + status: 'failed', + error, + }, + }); +}; + +export const listUnreferencedAssets = async (olderThan: Date): Promise => { + const prisma = await getPrisma(); + return prisma.decentralizedAsset.findMany({ + where: { + referenceCount: { lte: 0 }, + OR: [{ unpinnedAt: null }, { unpinnedAt: { lt: olderThan } }], + status: { in: ['pinned', 'failed', 'unreferenced'] }, + }, + }); +}; + +export const markAssetUnpinned = async (cid: string): Promise => { + const prisma = await getPrisma(); + await prisma.decentralizedAsset.updateMany({ + where: { cid }, + data: { + status: 'unpinned', + unpinnedAt: new Date(), + referenceCount: 0, + }, + }); +}; + +export const markAssetsUnreferenced = async ( + resourceType: string, + resourceId: string +): Promise => { + const prisma = await getPrisma(); + await prisma.decentralizedAsset.updateMany({ + where: { + resourceType, + resourceId, + status: { in: ['queued', 'failed', 'pinned'] }, + }, + data: { + referenceCount: 0, + status: 'unreferenced', + }, + }); +}; diff --git a/backend/src/services/storage/index.ts b/backend/src/services/storage/index.ts new file mode 100644 index 00000000..86d1ecd0 --- /dev/null +++ b/backend/src/services/storage/index.ts @@ -0,0 +1,7 @@ +export * from './asset.repository.js'; +export * from './provider.js'; +export * from './providers/index.js'; +export * from './queue.js'; +export * from './types.js'; +export * from './utils.js'; +export * from './worker.js'; diff --git a/backend/src/services/storage/provider.ts b/backend/src/services/storage/provider.ts new file mode 100644 index 00000000..541f10ee --- /dev/null +++ b/backend/src/services/storage/provider.ts @@ -0,0 +1,14 @@ +import type { StorageProvider } from './types.js'; +import { createMockStorageProvider } from './providers/index.js'; +import { createPinataStorageProvider } from './providers/index.js'; + +export const createStorageProvider = (): StorageProvider => { + const provider = (process.env.DECENTRALIZED_STORAGE_PROVIDER || 'pinata').toLowerCase(); + + if (provider === 'mock' || process.env.NODE_ENV === 'test') { + return createMockStorageProvider(); + } + + return createPinataStorageProvider(); +}; + diff --git a/backend/src/services/storage/providers/index.ts b/backend/src/services/storage/providers/index.ts new file mode 100644 index 00000000..30ae680a --- /dev/null +++ b/backend/src/services/storage/providers/index.ts @@ -0,0 +1,3 @@ +export * from './mock.provider.js'; +export * from './pinata.provider.js'; + diff --git a/backend/src/services/storage/providers/mock.provider.ts b/backend/src/services/storage/providers/mock.provider.ts new file mode 100644 index 00000000..822750b0 --- /dev/null +++ b/backend/src/services/storage/providers/mock.provider.ts @@ -0,0 +1,56 @@ +import { buildGatewayUrl, buildIpfsUri, canonicalizeJson, createDeterministicCid } from '../utils.js'; +import type { StoragePinResult, StorageProvider } from '../types.js'; + +const buildResult = (content: string): StoragePinResult => { + const cid = createDeterministicCid(content); + + return { + cid, + ipfsUri: buildIpfsUri(cid), + gatewayUrl: buildGatewayUrl(cid), + provider: 'mock', + sizeBytes: Buffer.byteLength(content), + isDuplicate: false, + }; +}; + +export class MockStorageProvider implements StorageProvider { + readonly name = 'mock' as const; + + async pinJson(input: { + content: unknown; + name: string; + metadata?: Record; + }): Promise { + return buildResult( + canonicalizeJson({ + name: input.name, + metadata: input.metadata ?? {}, + content: input.content, + }) + ); + } + + async pinFile(input: { + content: Buffer; + filename: string; + mimeType: string; + metadata?: Record; + }): Promise { + return buildResult( + canonicalizeJson({ + filename: input.filename, + mimeType: input.mimeType, + metadata: input.metadata ?? {}, + content: input.content.toString('base64'), + }) + ); + } + + async unpin(_cid: string): Promise { + return; + } +} + +export const createMockStorageProvider = (): StorageProvider => new MockStorageProvider(); + diff --git a/backend/src/services/storage/providers/pinata.provider.ts b/backend/src/services/storage/providers/pinata.provider.ts new file mode 100644 index 00000000..fab659dd --- /dev/null +++ b/backend/src/services/storage/providers/pinata.provider.ts @@ -0,0 +1,119 @@ +import { canonicalizeJson, createDeterministicCid, buildGatewayUrl } from '../utils.js'; +import type { StoragePinResult, StorageProvider } from '../types.js'; + +const PINATA_JSON_URL = 'https://api.pinata.cloud/pinning/pinJSONToIPFS'; +const PINATA_FILE_URL = 'https://api.pinata.cloud/pinning/pinFileToIPFS'; + +const getPinataJwt = (): string => { + const token = process.env.PINATA_JWT || ''; + + if (!token && process.env.NODE_ENV !== 'test') { + throw new Error('PINATA_JWT is required when using the Pinata storage provider'); + } + + return token; +}; + +const parsePinataResponse = async (response: Response): Promise => { + const data = (await response.json()) as { + IpfsHash: string; + PinSize?: number; + isDuplicate?: boolean; + }; + + return { + cid: data.IpfsHash, + ipfsUri: `ipfs://${data.IpfsHash}`, + gatewayUrl: buildGatewayUrl(data.IpfsHash), + provider: 'pinata', + sizeBytes: data.PinSize, + isDuplicate: data.isDuplicate, + }; +}; + +export class PinataStorageProvider implements StorageProvider { + readonly name = 'pinata' as const; + + async pinJson(input: { + content: unknown; + name: string; + metadata?: Record; + }): Promise { + const response = await fetch(PINATA_JSON_URL, { + method: 'POST', + headers: { + Authorization: `Bearer ${getPinataJwt()}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + pinataContent: input.content, + pinataMetadata: { + name: input.name, + keyvalues: input.metadata, + }, + pinataOptions: { + cidVersion: 1, + }, + }), + }); + + if (!response.ok) { + throw new Error(`Pinata JSON pin failed with status ${response.status}`); + } + + return parsePinataResponse(response); + } + + async pinFile(input: { + content: Buffer; + filename: string; + mimeType: string; + metadata?: Record; + }): Promise { + const form = new FormData(); + form.append('file', new Blob([input.content], { type: input.mimeType }), input.filename); + form.append( + 'pinataMetadata', + JSON.stringify({ + name: input.filename, + keyvalues: input.metadata, + }) + ); + form.append( + 'pinataOptions', + JSON.stringify({ + cidVersion: 1, + }) + ); + + const response = await fetch(PINATA_FILE_URL, { + method: 'POST', + headers: { + Authorization: `Bearer ${getPinataJwt()}`, + }, + body: form, + }); + + if (!response.ok) { + throw new Error(`Pinata file pin failed with status ${response.status}`); + } + + return parsePinataResponse(response); + } + + async unpin(cid: string): Promise { + const response = await fetch(`https://api.pinata.cloud/pinning/unpin/${cid}`, { + method: 'DELETE', + headers: { + Authorization: `Bearer ${getPinataJwt()}`, + }, + }); + + if (!response.ok) { + throw new Error(`Pinata unpin failed with status ${response.status}`); + } + } +} + +export const createPinataStorageProvider = (): StorageProvider => new PinataStorageProvider(); + diff --git a/backend/src/services/storage/queue.ts b/backend/src/services/storage/queue.ts new file mode 100644 index 00000000..fe3bdcd0 --- /dev/null +++ b/backend/src/services/storage/queue.ts @@ -0,0 +1,49 @@ +import { Queue } from 'bullmq'; +import type { JobsOptions } from 'bullmq'; +import { redisConnection } from '../../utils/redis.js'; +import type { StorageGcJobData, StoragePinJobData } from './types.js'; + +export const STORAGE_PIN_QUEUE_NAME = 'storage-pin-queue'; +export const STORAGE_GC_QUEUE_NAME = 'storage-gc-queue'; + +const defaultPinJobOptions: JobsOptions = { + attempts: Number(process.env.STORAGE_MAX_PIN_ATTEMPTS || '5'), + backoff: { + type: 'exponential', + delay: Number(process.env.STORAGE_BACKOFF_DELAY_MS || '1000'), + }, + removeOnComplete: { + age: 24 * 60 * 60, + count: 1000, + }, + removeOnFail: { + age: 7 * 24 * 60 * 60, + count: 1000, + }, + timeout: Number(process.env.STORAGE_JOB_TIMEOUT_MS || '30000'), +}; + +const createQueue = (name: string, defaultJobOptions?: JobsOptions) => { + if (process.env.NODE_ENV === 'test') { + return { + add: async () => ({ id: `${name}:test` }), + close: async () => undefined, + } as unknown as Queue; + } + + return new Queue(name, { + connection: redisConnection, + defaultJobOptions, + }); +}; + +export const storagePinQueue = createQueue(STORAGE_PIN_QUEUE_NAME, defaultPinJobOptions); + +export const storageGcQueue = createQueue(STORAGE_GC_QUEUE_NAME, { + removeOnComplete: false, + removeOnFail: false, +}); + +export const closeStorageQueues = async (): Promise => { + await Promise.all([storagePinQueue.close(), storageGcQueue.close()]); +}; diff --git a/backend/src/services/storage/storage.service.ts b/backend/src/services/storage/storage.service.ts new file mode 100644 index 00000000..cd6608f2 --- /dev/null +++ b/backend/src/services/storage/storage.service.ts @@ -0,0 +1,214 @@ +import { storageGcQueue, storagePinQueue } from './queue.js'; +import { createStorageProvider } from './provider.js'; +import { buildGatewayUrl, buildIpfsUri } from './utils.js'; +import * as defaultRepository from './asset.repository.js'; +import type { + StoragePinRequest, + StoragePinResult, + StorageAssetKind, + StorageProvider, +} from './types.js'; + +export interface StorageRepository { + upsertStorageAsset: typeof defaultRepository.upsertStorageAsset; + markAssetFailed: typeof defaultRepository.markAssetFailed; + listUnreferencedAssets: typeof defaultRepository.listUnreferencedAssets; + markAssetUnpinned: typeof defaultRepository.markAssetUnpinned; + markAssetsUnreferenced: typeof defaultRepository.markAssetsUnreferenced; +} + +export interface StorageServiceDependencies { + provider?: StorageProvider; + repository?: StorageRepository; + pinQueue?: typeof storagePinQueue; + gcQueue?: typeof storageGcQueue; +} + +export class StorageService { + private static instance: StorageService | null = null; + private readonly provider: StorageProvider; + private readonly repository: StorageRepository; + private readonly pinQueue: typeof storagePinQueue; + private readonly gcQueue: typeof storageGcQueue; + + constructor(dependencies: StorageServiceDependencies | StorageProvider = {}) { + if (this.isStorageProvider(dependencies)) { + this.provider = dependencies; + this.repository = defaultRepository; + this.pinQueue = storagePinQueue; + this.gcQueue = storageGcQueue; + return; + } + + this.provider = dependencies.provider ?? createStorageProvider(); + this.repository = dependencies.repository ?? defaultRepository; + this.pinQueue = dependencies.pinQueue ?? storagePinQueue; + this.gcQueue = dependencies.gcQueue ?? storageGcQueue; + } + + static getInstance(): StorageService { + if (!StorageService.instance) { + StorageService.instance = new StorageService(); + } + + return StorageService.instance; + } + + async pinJsonNow(request: Omit): Promise { + const result = await this.provider.pinJson({ + content: request.content, + name: request.name, + metadata: request.metadata, + }); + + await this.persistResult(request, result); + return result; + } + + async pinFileNow( + request: Omit & { content: Buffer } + ): Promise { + const result = await this.provider.pinFile({ + content: request.content, + filename: request.filename || request.name, + mimeType: request.mimeType || 'application/octet-stream', + metadata: request.metadata, + }); + + await this.persistResult(request, result); + return result; + } + + async queueJsonPin(request: Omit): Promise<{ jobId?: string | number }> { + const job = await this.pinQueue.add('pin-json', { + ...request, + mode: 'json', + }); + return { jobId: job.id }; + } + + async queueFilePin( + request: Omit & { content: string } + ): Promise<{ jobId?: string | number }> { + const job = await this.pinQueue.add('pin-file', { + ...request, + mode: 'file', + content: request.content, + }); + return { jobId: job.id }; + } + + async queueGarbageCollection(retentionDays: number, dryRun = false): Promise<{ jobId?: string | number }> { + const job = await this.gcQueue.add('gc', { + retentionDays, + dryRun, + }); + return { jobId: job.id }; + } + + async releaseResource(resourceType: string, resourceId: string): Promise { + await this.repository.markAssetsUnreferenced(resourceType, resourceId); + } + + async pinCertificateImage(request: { + certificateId: string; + content: Buffer; + mimeType?: string; + }): Promise { + return this.pinFileNow({ + resourceType: 'certificate', + resourceId: request.certificateId, + name: `certificate-image-${request.certificateId}`, + kind: 'certificate-image', + content: request.content, + filename: `${request.certificateId}.svg`, + mimeType: request.mimeType || 'image/svg+xml', + metadata: { + certificateId: request.certificateId, + assetType: 'certificate-image', + }, + }); + } + + async pinCertificateMetadata(request: { + certificateId: string; + content: Record; + }): Promise { + return this.pinJsonNow({ + resourceType: 'certificate', + resourceId: request.certificateId, + name: `certificate-metadata-${request.certificateId}`, + kind: 'certificate-metadata', + content: request.content, + metadata: { + certificateId: request.certificateId, + assetType: 'certificate-metadata', + }, + }); + } + + async pinProjectIdea(request: { + projectId: string; + content: Record; + queued?: boolean; + }): Promise { + if (request.queued) { + return this.queueJsonPin({ + resourceType: 'project', + resourceId: request.projectId, + name: `project-idea-${request.projectId}`, + kind: 'project-idea', + content: request.content, + metadata: { + projectId: request.projectId, + assetType: 'project-idea', + }, + }); + } + + return this.pinJsonNow({ + resourceType: 'project', + resourceId: request.projectId, + name: `project-idea-${request.projectId}`, + kind: 'project-idea', + content: request.content, + metadata: { + projectId: request.projectId, + assetType: 'project-idea', + }, + }); + } + + private async persistResult( + request: Omit & { content?: Buffer }, + result: StoragePinResult + ): Promise { + await this.repository.upsertStorageAsset({ + resourceType: request.resourceType, + resourceId: request.resourceId, + name: request.name, + kind: request.kind, + provider: result.provider, + cid: result.cid, + ipfsUri: result.ipfsUri || buildIpfsUri(result.cid), + gatewayUrl: result.gatewayUrl || buildGatewayUrl(result.cid), + mimeType: request.mimeType ?? null, + sizeBytes: result.sizeBytes ?? null, + status: 'pinned', + referenceCount: request.referenceCount ?? 1, + metadata: request.metadata ?? null, + }); + } + + private isStorageProvider(value: StorageServiceDependencies | StorageProvider): value is StorageProvider { + return ( + typeof value === 'object' && + value !== null && + 'pinJson' in value && + 'pinFile' in value && + 'unpin' in value + ); + } +} + +export const storageService = StorageService.getInstance(); diff --git a/backend/src/services/storage/types.ts b/backend/src/services/storage/types.ts new file mode 100644 index 00000000..5a7a26e6 --- /dev/null +++ b/backend/src/services/storage/types.ts @@ -0,0 +1,75 @@ +export type StorageProviderName = 'pinata' | 'web3.storage' | 'mock'; + +export type StorageAssetKind = 'certificate-image' | 'certificate-metadata' | 'project-idea' | 'generic'; + +export type StoragePinMode = 'json' | 'file'; + +export interface StoragePinRequest { + resourceType: string; + resourceId: string; + name: string; + kind: StorageAssetKind; + mode: StoragePinMode; + content: unknown; + filename?: string; + mimeType?: string; + metadata?: Record; + referenceCount?: number; +} + +export interface StoragePinResult { + cid: string; + ipfsUri: string; + gatewayUrl: string; + provider: StorageProviderName; + sizeBytes?: number; + isDuplicate?: boolean; +} + +export interface StorageAssetRecord { + id: string; + resourceType: string; + resourceId: string; + name: string; + kind: StorageAssetKind; + provider: StorageProviderName; + cid: string; + ipfsUri: string; + gatewayUrl: string; + mimeType: string | null; + sizeBytes: number | null; + status: string; + referenceCount: number; + metadata: Record | null; + error: string | null; + pinnedAt: Date | null; + unpinnedAt: Date | null; + createdAt: Date; + updatedAt: Date; +} + +export interface StorageProvider { + readonly name: StorageProviderName; + pinJson(input: { + content: unknown; + name: string; + metadata?: Record; + }): Promise; + pinFile(input: { + content: Buffer; + filename: string; + mimeType: string; + metadata?: Record; + }): Promise; + unpin(cid: string): Promise; +} + +export interface StoragePinJobData extends StoragePinRequest { + pinnedBy?: string; +} + +export interface StorageGcJobData { + retentionDays: number; + dryRun?: boolean; +} + diff --git a/backend/src/services/storage/utils.ts b/backend/src/services/storage/utils.ts new file mode 100644 index 00000000..574dde04 --- /dev/null +++ b/backend/src/services/storage/utils.ts @@ -0,0 +1,38 @@ +import crypto from 'crypto'; + +export const canonicalizeJson = (value: unknown): string => { + const sortValue = (input: unknown): unknown => { + if (Array.isArray(input)) { + return input.map(sortValue); + } + + if (input && typeof input === 'object') { + return Object.keys(input as Record) + .sort() + .reduce>((acc, key) => { + acc[key] = sortValue((input as Record)[key]); + return acc; + }, {}); + } + + return input; + }; + + return JSON.stringify(sortValue(value)); +}; + +export const buildGatewayUrl = (cid: string): string => { + const baseUrl = (process.env.STORAGE_GATEWAY_BASE_URL || 'https://gateway.pinata.cloud/ipfs').replace( + /\/+$/, + '' + ); + + return `${baseUrl}/${cid}`; +}; + +export const buildIpfsUri = (cid: string): string => `ipfs://${cid}`; + +export const createDeterministicCid = (input: string): string => { + return `bafy${crypto.createHash('sha256').update(input).digest('hex').slice(0, 56)}`; +}; + diff --git a/backend/src/services/storage/worker.ts b/backend/src/services/storage/worker.ts new file mode 100644 index 00000000..554e4c35 --- /dev/null +++ b/backend/src/services/storage/worker.ts @@ -0,0 +1,186 @@ +import { Job, Worker } from 'bullmq'; +import logger from '../../utils/logger.js'; +import { redisConnection } from '../../utils/redis.js'; +import * as defaultRepository from './asset.repository.js'; +import { createStorageProvider } from './provider.js'; +import { buildGatewayUrl, buildIpfsUri } from './utils.js'; +import { STORAGE_GC_QUEUE_NAME, STORAGE_PIN_QUEUE_NAME, storageGcQueue } from './queue.js'; +import type { + StorageAssetRecord, + StorageGcJobData, + StoragePinJobData, + StoragePinResult, + StorageProvider, +} from './types.js'; + +const provider = createStorageProvider(); +const retentionDays = Number(process.env.STORAGE_GC_RETENTION_DAYS || '30'); + +export interface StorageWorkerRepository { + upsertStorageAsset: typeof defaultRepository.upsertStorageAsset; + markAssetFailed: typeof defaultRepository.markAssetFailed; + listUnreferencedAssets: typeof defaultRepository.listUnreferencedAssets; + markAssetUnpinned: typeof defaultRepository.markAssetUnpinned; +} + +export interface StorageWorkerDependencies { + provider?: StorageProvider; + repository?: StorageWorkerRepository; +} + +const defaultWorkerRepository: StorageWorkerRepository = defaultRepository; + +export const pinStorageContent = async ( + job: Job, + dependencies: StorageWorkerDependencies = {} +): Promise => { + const payload = job.data; + const activeProvider = dependencies.provider ?? provider; + const repository = dependencies.repository ?? defaultWorkerRepository; + + try { + const pinResult = + payload.mode === 'json' + ? await activeProvider.pinJson({ + content: payload.content, + name: payload.name, + metadata: payload.metadata, + }) + : await activeProvider.pinFile({ + content: Buffer.from(payload.content as string, 'base64'), + filename: payload.filename || payload.name, + mimeType: payload.mimeType || 'application/octet-stream', + metadata: payload.metadata, + }); + + await repository.upsertStorageAsset({ + resourceType: payload.resourceType, + resourceId: payload.resourceId, + name: payload.name, + kind: payload.kind, + provider: pinResult.provider, + cid: pinResult.cid, + ipfsUri: pinResult.ipfsUri || buildIpfsUri(pinResult.cid), + gatewayUrl: pinResult.gatewayUrl || buildGatewayUrl(pinResult.cid), + mimeType: payload.mimeType ?? null, + sizeBytes: pinResult.sizeBytes ?? null, + status: 'pinned', + referenceCount: payload.referenceCount ?? 1, + metadata: payload.metadata ?? null, + }); + + logger.info( + `Pinned decentralized asset ${payload.resourceType}/${payload.resourceId}/${payload.name} -> ${pinResult.cid}` + ); + + return pinResult; + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown storage pinning error'; + + await repository.markAssetFailed(payload.resourceType, payload.resourceId, payload.name, message); + throw error; + } +}; + +export const garbageCollectStorage = async ( + job: Job, + dependencies: StorageWorkerDependencies = {} +): Promise<{ + inspected: number; + unpinned: number; +}> => { + const activeProvider = dependencies.provider ?? provider; + const repository = dependencies.repository ?? defaultWorkerRepository; + const cutoff = new Date(Date.now() - job.data.retentionDays * 24 * 60 * 60 * 1000); + const staleAssets: StorageAssetRecord[] = await repository.listUnreferencedAssets(cutoff); + + let unpinned = 0; + + for (const asset of staleAssets) { + if (job.data.dryRun) { + continue; + } + + try { + await activeProvider.unpin(asset.cid); + await repository.markAssetUnpinned(asset.cid); + unpinned += 1; + logger.info(`Unpinned stale decentralized asset ${asset.cid}`); + } catch (error) { + logger.warn(`Failed to unpin stale asset ${asset.cid}:`, error); + } + } + + return { + inspected: staleAssets.length, + unpinned, + }; +}; + +let pinWorker: Worker | null = null; +let gcWorker: Worker | null = null; + +export const startStorageWorkers = (): { + pinWorker: Worker | null; + gcWorker: Worker | null; +} => { + if (process.env.NODE_ENV === 'test') { + return { pinWorker: null, gcWorker: null }; + } + + if (!pinWorker) { + pinWorker = new Worker(STORAGE_PIN_QUEUE_NAME, pinStorageContent, { + connection: redisConnection, + concurrency: Number(process.env.STORAGE_WORKER_CONCURRENCY || '10'), + }); + + pinWorker.on('failed', (job, error) => { + logger.error(`Storage pin job ${job?.id} failed: ${error.message}`); + }); + } + + if (!gcWorker) { + gcWorker = new Worker( + STORAGE_GC_QUEUE_NAME, + async (job) => garbageCollectStorage(job), + { + connection: redisConnection, + concurrency: 1, + } + ); + + gcWorker.on('failed', (job, error) => { + logger.error(`Storage GC job ${job?.id} failed: ${error.message}`); + }); + } + + return { pinWorker, gcWorker }; +}; + +export const stopStorageWorkers = async (): Promise => { + if (pinWorker) { + await pinWorker.close(); + pinWorker = null; + } + + if (gcWorker) { + await gcWorker.close(); + gcWorker = null; + } +}; + +export const scheduleStorageGc = async (): Promise => { + if (process.env.NODE_ENV === 'test') { + return; + } + + await storageGcQueue.add( + 'gc', + { retentionDays }, + { + repeat: { pattern: '0 */6 * * *' }, + removeOnComplete: true, + removeOnFail: false, + } + ); +}; diff --git a/backend/src/utils/checkEnv.ts b/backend/src/utils/checkEnv.ts index aef17b6f..5bd6902c 100644 --- a/backend/src/utils/checkEnv.ts +++ b/backend/src/utils/checkEnv.ts @@ -110,6 +110,38 @@ const OPTIONAL_VARS: Record + ({ + del: async () => 0, + disconnect: () => undefined, + duplicate: () => createTestRedisClient(), + get: async () => null, + on: () => createTestRedisClient(), + publish: async () => 0, + quit: async () => 'OK', + setex: async () => 'OK', + subscribe: async () => 0, + } as unknown as Redis); -export const pubClient = new Redis(redisUrl, { - maxRetriesPerRequest: null, -}); +const createRedisClient = (): Redis => { + if (process.env.NODE_ENV === 'test') { + return createTestRedisClient(); + } -export const subClient = new Redis(redisUrl, { - maxRetriesPerRequest: null, -}); + return new Redis(redisUrl, { + maxRetriesPerRequest: null, + }); +}; + +export const redisConnection = createRedisClient(); + +export const pubClient = createRedisClient(); + +export const subClient = createRedisClient(); export default redisConnection; diff --git a/backend/tests/jest.setup.ts b/backend/tests/jest.setup.ts new file mode 100644 index 00000000..fb435536 --- /dev/null +++ b/backend/tests/jest.setup.ts @@ -0,0 +1,3 @@ +process.env.NODE_ENV = 'test'; +process.env.REDIS_URL = process.env.REDIS_URL || 'redis://localhost:6379'; + diff --git a/backend/tests/storage.provider.test.ts b/backend/tests/storage.provider.test.ts new file mode 100644 index 00000000..d759eb65 --- /dev/null +++ b/backend/tests/storage.provider.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from '@jest/globals'; +import { MockStorageProvider } from '../src/services/storage/providers/mock.provider.js'; + +describe('mock storage provider', () => { + it('pins JSON deterministically', async () => { + const provider = new MockStorageProvider(); + const result = await provider.pinJson({ + content: { hello: 'world' }, + name: 'example.json', + }); + + expect(result.provider).toBe('mock'); + expect(result.cid).toEqual(expect.stringMatching(/^bafy[0-9a-f]+$/)); + expect(result.ipfsUri).toBe(`ipfs://${result.cid}`); + expect(result.gatewayUrl).toContain(result.cid); + }); + + it('pins files deterministically', async () => { + const provider = new MockStorageProvider(); + const result = await provider.pinFile({ + content: Buffer.from('hello'), + filename: 'hello.txt', + mimeType: 'text/plain', + }); + + expect(result.provider).toBe('mock'); + expect(result.sizeBytes).toBeGreaterThan(0); + expect(result.ipfsUri).toBe(`ipfs://${result.cid}`); + }); +}); + diff --git a/backend/tests/storage.service.test.ts b/backend/tests/storage.service.test.ts new file mode 100644 index 00000000..be0ef770 --- /dev/null +++ b/backend/tests/storage.service.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it, jest, afterEach } from '@jest/globals'; +import { StorageService } from '../src/services/storage/storage.service.js'; +import { MockStorageProvider } from '../src/services/storage/providers/mock.provider.js'; +import type { StorageServiceDependencies } from '../src/services/storage/storage.service.js'; + +const createRepository = (): NonNullable => ({ + upsertStorageAsset: jest.fn(), + markAssetFailed: jest.fn(), + listUnreferencedAssets: jest.fn(), + markAssetUnpinned: jest.fn(), + markAssetsUnreferenced: jest.fn(), +}); + +describe('storage service', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('pins JSON and persists the CID', async () => { + const repository = createRepository(); + repository.upsertStorageAsset.mockResolvedValue({} as never); + + const storageService = new StorageService({ + provider: new MockStorageProvider(), + repository, + }); + + const result = await storageService.pinJsonNow({ + resourceType: 'project', + resourceId: 'project-1', + name: 'example', + kind: 'generic', + content: { hello: 'world' }, + }); + + expect(result.provider).toBe('mock'); + expect(repository.upsertStorageAsset).toHaveBeenCalledTimes(1); + }); + + it('pins files and persists the CID', async () => { + const repository = createRepository(); + repository.upsertStorageAsset.mockResolvedValue({} as never); + + const storageService = new StorageService({ + provider: new MockStorageProvider(), + repository, + }); + + const result = await storageService.pinFileNow({ + resourceType: 'certificate', + resourceId: 'cert-1', + name: 'cert.svg', + kind: 'certificate-image', + content: Buffer.from(''), + mimeType: 'image/svg+xml', + }); + + expect(result.provider).toBe('mock'); + expect(repository.upsertStorageAsset).toHaveBeenCalledTimes(1); + }); + + it('queues pin and gc jobs', async () => { + const storageService = new StorageService({ + provider: new MockStorageProvider(), + repository: createRepository(), + }); + + const jsonJob = await storageService.queueJsonPin({ + resourceType: 'project', + resourceId: 'project-1', + name: 'example', + kind: 'generic', + content: { hello: 'world' }, + }); + + const fileJob = await storageService.queueFilePin({ + resourceType: 'certificate', + resourceId: 'cert-1', + name: 'cert.svg', + kind: 'certificate-image', + content: Buffer.from('hello').toString('base64'), + mimeType: 'image/svg+xml', + }); + + const gcJob = await storageService.queueGarbageCollection(7, true); + + expect(jsonJob.jobId).toEqual(expect.any(String)); + expect(fileJob.jobId).toEqual(expect.any(String)); + expect(gcJob.jobId).toEqual(expect.any(String)); + }); +}); diff --git a/backend/tests/storage.utils.test.ts b/backend/tests/storage.utils.test.ts new file mode 100644 index 00000000..fa742e02 --- /dev/null +++ b/backend/tests/storage.utils.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from '@jest/globals'; +import { buildGatewayUrl, buildIpfsUri, canonicalizeJson, createDeterministicCid } from '../src/services/storage/utils.js'; + +describe('storage utils', () => { + it('canonicalizes nested JSON deterministically', () => { + const payload = { + z: 3, + a: { + y: 2, + x: 1, + }, + list: [ + { + b: 2, + a: 1, + }, + ], + }; + + expect(canonicalizeJson(payload)).toBe( + JSON.stringify({ + a: { x: 1, y: 2 }, + list: [{ a: 1, b: 2 }], + z: 3, + }) + ); + }); + + it('builds IPFS URIs and gateway URLs', () => { + const cid = 'bafy123'; + expect(buildIpfsUri(cid)).toBe('ipfs://bafy123'); + expect(buildGatewayUrl(cid)).toBe(`https://gateway.pinata.cloud/ipfs/${cid}`); + }); + + it('creates deterministic mock CIDs', () => { + expect(createDeterministicCid('hello world')).toEqual(expect.stringMatching(/^bafy[0-9a-f]+$/)); + }); +}); + diff --git a/backend/tests/storage.worker.test.ts b/backend/tests/storage.worker.test.ts new file mode 100644 index 00000000..409d6eb9 --- /dev/null +++ b/backend/tests/storage.worker.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, jest, afterEach } from '@jest/globals'; +import { garbageCollectStorage, pinStorageContent } from '../src/services/storage/worker.js'; +import { MockStorageProvider } from '../src/services/storage/providers/mock.provider.js'; +import type { StorageWorkerDependencies } from '../src/services/storage/worker.js'; + +const createRepository = (): NonNullable => ({ + upsertStorageAsset: jest.fn(), + markAssetFailed: jest.fn(), + listUnreferencedAssets: jest.fn(), + markAssetUnpinned: jest.fn(), +}); + +describe('storage worker', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('pins JSON job payloads', async () => { + const repository = createRepository(); + repository.upsertStorageAsset.mockResolvedValue({} as never); + + const result = await pinStorageContent( + { + data: { + resourceType: 'project', + resourceId: 'project-1', + name: 'example', + kind: 'generic', + mode: 'json', + content: { hello: 'world' }, + }, + } as never, + { + provider: new MockStorageProvider(), + repository, + } + ); + + expect(result.provider).toBe('mock'); + expect(repository.upsertStorageAsset).toHaveBeenCalledTimes(1); + }); + + it('garbage collects stale assets', async () => { + const repository = createRepository(); + repository.listUnreferencedAssets.mockResolvedValue([ + { + id: 'asset-1', + resourceType: 'project', + resourceId: 'project-1', + name: 'example', + kind: 'generic', + provider: 'mock', + cid: 'bafy123', + ipfsUri: 'ipfs://bafy123', + gatewayUrl: 'https://gateway.pinata.cloud/ipfs/bafy123', + mimeType: null, + sizeBytes: 10, + status: 'pinned', + referenceCount: 0, + metadata: null, + error: null, + pinnedAt: null, + unpinnedAt: null, + createdAt: new Date(), + updatedAt: new Date(), + }, + ] as never); + repository.markAssetUnpinned.mockResolvedValue(undefined); + + const result = await garbageCollectStorage( + { + data: { + retentionDays: 30, + dryRun: false, + }, + } as never, + { + provider: new MockStorageProvider(), + repository, + } + ); + + expect(result.inspected).toBe(1); + expect(result.unpinned).toBe(1); + expect(repository.listUnreferencedAssets).toHaveBeenCalled(); + expect(repository.markAssetUnpinned).toHaveBeenCalledWith('bafy123'); + }); +}); diff --git a/docs/backend/DECENTRALIZED_STORAGE_GUIDE.md b/docs/backend/DECENTRALIZED_STORAGE_GUIDE.md new file mode 100644 index 00000000..6867bfd5 --- /dev/null +++ b/docs/backend/DECENTRALIZED_STORAGE_GUIDE.md @@ -0,0 +1,81 @@ +# Decentralized Storage Guide + +The backend now includes a decentralized storage manager for: + +- generated NFT certificate images +- certificate metadata JSON +- generated project ideas and other structured payloads + +The implementation uses a provider abstraction so the project can use Pinata in production while still running in a mock mode for local development and tests. + +## Flow + +1. A route or service asks the storage manager to pin JSON or file content. +2. The request is queued in BullMQ for asynchronous processing when needed. +3. A storage worker pins the content to IPFS through the configured provider. +4. The resulting CID, gateway URL, provider, and status are persisted in the `decentralized_assets` table. +5. A repeatable garbage-collection job finds assets with a `referenceCount` of `0` and unpins them after the retention window. + +## Provider + +The storage layer defaults to Pinata. + +Environment variables: + +- `DECENTRALIZED_STORAGE_PROVIDER`: `pinata` or `mock` +- `PINATA_JWT`: Pinata JWT used for uploads and unpins +- `STORAGE_GATEWAY_BASE_URL`: gateway URL used to build public links +- `STORAGE_GC_RETENTION_DAYS`: how long to retain unreferenced assets before cleanup +- `STORAGE_MAX_PIN_ATTEMPTS`: retry budget for pin jobs +- `STORAGE_BACKOFF_DELAY_MS`: initial exponential backoff delay +- `STORAGE_WORKER_CONCURRENCY`: BullMQ concurrency for pin jobs +- `STORAGE_JOB_TIMEOUT_MS`: timeout per pin job + +## Certificate Integration + +When a certificate is minted: + +- the certificate image is rendered locally as SVG +- the SVG is pinned to IPFS +- the certificate metadata is generated with the IPFS image URI +- the metadata JSON is pinned to IPFS +- the blockchain mint flow uses the decentralized metadata payload +- the persisted certificate record stores the metadata CID URI + +## Project Idea Integration + +The project idea generator can optionally persist the generated idea to IPFS. + +`POST /api/v1/generator/generate` accepts: + +- `persistToStorage`: pin the generated project idea JSON +- `queuedPersist`: enqueue the pin operation instead of pinning immediately + +## Storage API + +New endpoints: + +- `GET /api/v1/storage/health` +- `POST /api/v1/storage/pin-json` +- `POST /api/v1/storage/pin-file` +- `POST /api/v1/storage/gc` + +## Database Model + +Pinned assets are tracked in the `DecentralizedAsset` Prisma model, which stores: + +- `cid` +- `ipfsUri` +- `gatewayUrl` +- `provider` +- `referenceCount` +- pin and unpin timestamps +- failure state for retries and debugging + +## Operational Notes + +- Use the `mock` provider for local testing when no Pinata JWT is available. +- The garbage-collection worker is intentionally separate from the pin worker. +- Unreferenced content is not deleted immediately; it is retained for the configured window before unpinning. +- Keep CIDs and gateway URLs out of public logs unless they are already public metadata. +