feat(db): add automatic purge of expired terminated clusters

db/sql/cron.sql

@@ -34,6 +34,14 @@ SELECT cron.schedule_in_database(
   'clusteriq'
 );
 
+-- pg_cron task for purging expired terminated clusters daily at midnight
+SELECT cron.schedule_in_database(
+  'purge_expired_clusters',
+  '0 0 * * *',
+  $$SELECT purge_expired_clusters();$$,
+  'clusteriq'
+);
+
 -- Function to check easier how the pg_cron tasks went
 CREATE OR REPLACE FUNCTION pg_cron_history(p_limit int DEFAULT 20)
 RETURNS TABLE(

db/sql/init.sql

@@ -773,3 +773,27 @@ BEGIN
   REFRESH MATERIALIZED VIEW m_instances_full_view_with_tags;
 END;
 $$ LANGUAGE plpgsql;
+
+-- Purges terminated clusters whose last_scan_ts is older than retention_days.
+-- Cascading FKs handle deletion of instances, tags, expenses, targets, schedules, and action_runs.
+-- Triggers handle deletion of associated events.
+CREATE OR REPLACE FUNCTION purge_expired_clusters(retention_days INTEGER DEFAULT 365)
+RETURNS INTEGER AS $$
+DECLARE
+  deleted_count INTEGER;
+BEGIN
+  WITH deleted AS (
+    DELETE FROM clusters
+    WHERE status = 'Terminated'
+      AND last_scan_ts < NOW() - (retention_days || ' days')::INTERVAL
+    RETURNING id
+  )
+  SELECT COUNT(*) INTO deleted_count FROM deleted;
+
+  IF deleted_count > 0 THEN
+    PERFORM refresh_materialized_views();
+  END IF;
+
+  RETURN deleted_count;
+END;
+$$ LANGUAGE plpgsql;

deployments/compose/compose-devel.yaml

@@ -142,6 +142,9 @@ services:
         echo "Creating PG_CRON tasks"
         psql postgresql://postgres:admin@pgsql:5432/postgres < /cron.sql && { echo "Ok"; } || { echo "Cron Tasks creation Failed"; exit 1; }
 
+        echo "Configuring purge retention (${CIQ_MAX_DATA_AGE:-365} days)"
+        psql postgresql://postgres:admin@pgsql:5432/postgres -c "UPDATE cron.job SET command = \$\$SELECT purge_expired_clusters(${CIQ_MAX_DATA_AGE:-365})\$\$ WHERE jobname = \$\$purge_expired_clusters\$\$;"
+
         if [[ $CIQ_DB_PRELOAD_DATA == "true" ]]; then
           echo "Initializing DB with fake data"
           psql postgresql://user:password@pgsql:5432/clusteriq < /load_example_data.sql && { echo "Ok"; } || { echo "Initialization Failed"; exit 1; }
@@ -150,6 +153,7 @@ services:
         echo "Done!"
       '
     environment:
+      CIQ_MAX_DATA_AGE: "365"
       CIQ_DB_PRELOAD_DATA: "false"
     volumes:
       - ./../../db/sql/init.sql:/init.sql:ro,Z

deployments/helm/cluster-iq/templates/database/configmap-init.yaml

@@ -43,6 +43,14 @@ data:
       'clusteriq'
     );
 
+    -- pg_cron task for purging expired terminated clusters daily at midnight
+    SELECT cron.schedule_in_database(
+      'purge_expired_clusters',
+      '0 0 * * *',
+      $$SELECT purge_expired_clusters();$$,
+      'clusteriq'
+    );
+
     -- Function to check easier how the pg_cron tasks went
     CREATE OR REPLACE FUNCTION pg_cron_history(p_limit int DEFAULT 20)
     RETURNS TABLE(
@@ -738,3 +746,27 @@ data:
       REFRESH MATERIALIZED VIEW m_instances_full_view_with_tags;
     END;
     $$ LANGUAGE plpgsql;
+
+    -- Purges terminated clusters whose last_scan_ts is older than retention_days.
+    -- Cascading FKs handle deletion of instances, tags, expenses, targets, schedules, and action_runs.
+    -- Triggers handle deletion of associated events.
+    CREATE OR REPLACE FUNCTION purge_expired_clusters(retention_days INTEGER DEFAULT 365)
+    RETURNS INTEGER AS $$
+    DECLARE
+      deleted_count INTEGER;
+    BEGIN
+      WITH deleted AS (
+        DELETE FROM clusters
+        WHERE status = 'Terminated'
+          AND last_scan_ts < NOW() - (retention_days || ' days')::INTERVAL
+        RETURNING id
+      )
+      SELECT COUNT(*) INTO deleted_count FROM deleted;
+
+      IF deleted_count > 0 THEN
+        PERFORM refresh_materialized_views();
+      END IF;
+
+      RETURN deleted_count;
+    END;
+    $$ LANGUAGE plpgsql;

deployments/helm/cluster-iq/templates/database/job.yaml

@@ -39,6 +39,9 @@ spec:
             - name: cron
               mountPath: /var/lib/pgsql/cron.sql
               subPath: cron.sql
+          env:
+            - name: CIQ_MAX_DATA_AGE
+              value: "{{ .Values.database.maxDataAge }}"
           envFrom:
             - secretRef:
                 name: postgresql
@@ -56,6 +59,9 @@ spec:
               echo "Creating PG_CRON tasks"
               psql postgresql://postgres:$POSTGRESQL_ADMIN_PASSWORD@{{ $dbHost }}:{{ .Values.database.service.port }}/postgres < /var/lib/pgsql/cron.sql && { echo "Ok"; } || { echo "Cron Tasks creation Failed"; exit 1; }
 
+              echo "Configuring purge retention (${CIQ_MAX_DATA_AGE:-365} days)"
+              psql postgresql://postgres:$POSTGRESQL_ADMIN_PASSWORD@{{ $dbHost }}:{{ .Values.database.service.port }}/postgres -c "UPDATE cron.job SET command = \$\$SELECT purge_expired_clusters(${CIQ_MAX_DATA_AGE:-365})\$\$ WHERE jobname = \$\$purge_expired_clusters\$\$;"
+
               echo "Done!"
       restartPolicy: OnFailure
 {{- end }}

deployments/helm/cluster-iq/values.yaml

@@ -412,6 +412,9 @@ database:
 # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/
   replicaCount: 1
 
+  # Maximum age in days for terminated clusters before they are purged from the database
+  maxDataAge: "365"
+
   # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/
   image:
     repository: quay.io/ecosystem-appeng/cluster-iq-pgsql