feat: add per-SP piece cleanup job to bound storage growth

apps/backend/src/common/logging.ts

@@ -168,3 +168,16 @@ export type JobLogContext = {
   providerId?: bigint;
   providerName?: string;
 };
+
+/**
+ * Structured logging context for piece cleanup operations
+ */
+export type PieceCleanupLogContext = {
+  jobId?: string;
+  providerAddress: string;
+  providerId?: bigint;
+  providerName?: string;
+  storedBytes?: number;
+  thresholdBytes?: number;
+  targetBytes?: number;
+};

apps/backend/src/config/app.config.ts

@@ -95,6 +95,30 @@ export const configValidationSchema = Joi.object({
   DATA_SET_CREATION_JOB_TIMEOUT_SECONDS: Joi.number().min(60).default(300), // 5 minutes max runtime for dataset creation jobs
   IPFS_BLOCK_FETCH_CONCURRENCY: Joi.number().integer().min(1).max(32).default(6),
 
+  // Piece Cleanup
+  MAX_DATASET_STORAGE_SIZE_BYTES: Joi.number()
+    .integer()
+    .min(1)
+    .default(24 * 1024 * 1024 * 1024), // 24 GiB per SP
+  TARGET_DATASET_STORAGE_SIZE_BYTES: Joi.number()
+    .integer()
+    .min(1)
+    .default(20 * 1024 * 1024 * 1024) // 20 GiB per SP
+    .custom((value, helpers) => {
+      const max = helpers.state.ancestors?.[0]?.MAX_DATASET_STORAGE_SIZE_BYTES;
+      if (max != null && value >= max) {
+        return helpers.error("any.invalid", {
+          message: `TARGET_DATASET_STORAGE_SIZE_BYTES (${value}) must be less than MAX_DATASET_STORAGE_SIZE_BYTES (${max})`,
+        });
+      }
+      return value;
+    }, "target < max validation"),
+  JOB_PIECE_CLEANUP_PER_SP_PER_HOUR: Joi.number()
+    .min(0.001)
+    .max(20)
+    .default(1 / 24), // ~once per day
+  MAX_PIECE_CLEANUP_RUNTIME_SECONDS: Joi.number().min(60).default(300), // 5 minutes max runtime for cleanup jobs
+
   // Dataset
   DEALBOT_LOCAL_DATASETS_PATH: Joi.string().default(DEFAULT_LOCAL_DATASETS_PATH),
   RANDOM_PIECE_SIZES: Joi.string().default("10485760"), // 10 MiB
@@ -239,6 +263,20 @@ export interface IJobsConfig {
    * Uses AbortController to actively cancel job execution.
    */
   retrievalJobTimeoutSeconds: number;
+  /**
+   * Target number of piece cleanup runs per storage provider per hour.
+   *
+   * Increasing this makes cleanup more aggressive at the cost of more SP API calls.
+   * Only used when `DEALBOT_JOBS_MODE=pgboss`.
+   */
+  pieceCleanupPerSpPerHour: number;
+  /**
+   * Maximum runtime (seconds) for piece cleanup jobs before forced abort.
+   *
+   * Uses AbortController to actively cancel job execution.
+   * Only used when `DEALBOT_JOBS_MODE=pgboss`.
+   */
+  maxPieceCleanupRuntimeSeconds: number;
 }
 
 export interface IDatasetConfig {
@@ -258,6 +296,11 @@ export interface IRetrievalConfig {
   ipfsBlockFetchConcurrency: number;
 }
 
+export interface IPieceCleanupConfig {
+  maxDatasetStorageSizeBytes: number;
+  targetDatasetStorageSizeBytes: number;
+}
+
 export interface ISpBlocklistConfig {
   /** Provider numeric IDs to block from all scheduled checks. */
   ids: Set<string>;
@@ -274,6 +317,7 @@ export interface IConfig {
   dataset: IDatasetConfig;
   timeouts: ITimeoutConfig;
   retrieval: IRetrievalConfig;
+  pieceCleanup: IPieceCleanupConfig;
   spBlocklists: ISpBlocklistConfig;
 }
 
@@ -346,6 +390,8 @@ export function loadConfig(): IConfig {
       dealJobTimeoutSeconds: Number.parseInt(process.env.DEAL_JOB_TIMEOUT_SECONDS || "360", 10),
       retrievalJobTimeoutSeconds: Number.parseInt(process.env.RETRIEVAL_JOB_TIMEOUT_SECONDS || "60", 10),
       dataSetCreationJobTimeoutSeconds: Number.parseInt(process.env.DATA_SET_CREATION_JOB_TIMEOUT_SECONDS || "300", 10),
+      pieceCleanupPerSpPerHour: Number.parseFloat(process.env.JOB_PIECE_CLEANUP_PER_SP_PER_HOUR || String(1 / 24)),
+      maxPieceCleanupRuntimeSeconds: Number.parseInt(process.env.MAX_PIECE_CLEANUP_RUNTIME_SECONDS || "300", 10),
     },
     dataset: {
       localDatasetsPath: process.env.DEALBOT_LOCAL_DATASETS_PATH || DEFAULT_LOCAL_DATASETS_PATH,
@@ -375,6 +421,16 @@ export function loadConfig(): IConfig {
     retrieval: {
       ipfsBlockFetchConcurrency: Number.parseInt(process.env.IPFS_BLOCK_FETCH_CONCURRENCY || "6", 10),
     },
+    pieceCleanup: {
+      maxDatasetStorageSizeBytes: Number.parseInt(
+        process.env.MAX_DATASET_STORAGE_SIZE_BYTES || String(24 * 1024 * 1024 * 1024),
+        10,
+      ),
+      targetDatasetStorageSizeBytes: Number.parseInt(
+        process.env.TARGET_DATASET_STORAGE_SIZE_BYTES || String(20 * 1024 * 1024 * 1024),
+        10,
+      ),
+    },
     spBlocklists: {
       ids: parseIdList(process.env.BLOCKED_SP_IDS),
       addresses: parseAddressList(process.env.BLOCKED_SP_ADDRESSES),

apps/backend/src/database/entities/deal.entity.ts

@@ -132,6 +132,13 @@ export class Deal {
   @Column({ name: "retry_count", default: 0 })
   retryCount: number;
 
+  // Piece cleanup tracking
+  @Column({ name: "cleaned_up", default: false })
+  cleanedUp: boolean;
+
+  @Column({ name: "cleaned_up_at", type: "timestamptz", nullable: true })
+  cleanedUpAt: Date | null;
+
   @CreateDateColumn({ name: "created_at", type: "timestamptz" })
   createdAt: Date;
 

apps/backend/src/database/entities/job-schedule-state.entity.ts

@@ -7,7 +7,8 @@ export type JobType =
   | "metrics"
   | "metrics_cleanup"
   | "providers_refresh"
-  | "data_retention_poll";
+  | "data_retention_poll"
+  | "piece_cleanup";
 
 @Entity("job_schedule_state")
 @Index("job_schedule_state_job_type_sp_unique", ["jobType", "spAddress"], { unique: true })

apps/backend/src/database/migrations/1761500000002-AddPieceCleanupColumns.ts

@@ -0,0 +1,18 @@
+import type { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddPieceCleanupColumns1761500000002 implements MigrationInterface {
+  name = "AddPieceCleanupColumns1761500000002";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+      ALTER TABLE deals
+        ADD COLUMN IF NOT EXISTS cleaned_up BOOLEAN NOT NULL DEFAULT false,
+        ADD COLUMN IF NOT EXISTS cleaned_up_at TIMESTAMPTZ NULL
+    `);
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`ALTER TABLE deals DROP COLUMN IF EXISTS cleaned_up_at`);
+    await queryRunner.query(`ALTER TABLE deals DROP COLUMN IF EXISTS cleaned_up`);
+  }
+}

apps/backend/src/jobs/jobs.module.ts

@@ -7,6 +7,7 @@ import { StorageProvider } from "../database/entities/storage-provider.entity.js
 import { DealModule } from "../deal/deal.module.js";
 import { MetricsModule } from "../metrics/metrics.module.js";
 import { MetricsWorkerModule } from "../metrics/metrics-worker.module.js";
+import { PieceCleanupModule } from "../piece-cleanup/piece-cleanup.module.js";
 import { RetrievalModule } from "../retrieval/retrieval.module.js";
 import { WalletSdkModule } from "../wallet-sdk/wallet-sdk.module.js";
 import { JobsService } from "./jobs.service.js";
@@ -25,6 +26,7 @@ const metricsModule = runMode === "worker" ? MetricsWorkerModule : MetricsModule
     metricsModule,
     WalletSdkModule,
     DataRetentionModule,
+    PieceCleanupModule,
   ],
   providers: [JobsService, JobScheduleRepository],
 })