diff --git a/db/sql/cron.sql b/db/sql/cron.sql index fe841b3a..74ec6bb5 100644 --- a/db/sql/cron.sql +++ b/db/sql/cron.sql @@ -34,6 +34,14 @@ SELECT cron.schedule_in_database( 'clusteriq' ); +-- pg_cron task for purging expired terminated clusters daily at midnight +SELECT cron.schedule_in_database( + 'purge_expired_clusters', + '0 0 * * *', + $$SELECT purge_expired_clusters();$$, + 'clusteriq' +); + -- Function to check easier how the pg_cron tasks went CREATE OR REPLACE FUNCTION pg_cron_history(p_limit int DEFAULT 20) RETURNS TABLE( diff --git a/db/sql/init.sql b/db/sql/init.sql index e99d9524..046f2e40 100644 --- a/db/sql/init.sql +++ b/db/sql/init.sql @@ -773,3 +773,27 @@ BEGIN REFRESH MATERIALIZED VIEW m_instances_full_view_with_tags; END; $$ LANGUAGE plpgsql; + +-- Purges terminated clusters whose last_scan_ts is older than retention_days. +-- Cascading FKs handle deletion of instances, tags, expenses, targets, schedules, and action_runs. +-- Triggers handle deletion of associated events. +CREATE OR REPLACE FUNCTION purge_expired_clusters(retention_days INTEGER DEFAULT 365) +RETURNS INTEGER AS $$ +DECLARE + deleted_count INTEGER; +BEGIN + WITH deleted AS ( + DELETE FROM clusters + WHERE status = 'Terminated' + AND last_scan_ts < NOW() - (retention_days || ' days')::INTERVAL + RETURNING id + ) + SELECT COUNT(*) INTO deleted_count FROM deleted; + + IF deleted_count > 0 THEN + PERFORM refresh_materialized_views(); + END IF; + + RETURN deleted_count; +END; +$$ LANGUAGE plpgsql; diff --git a/deployments/compose/compose-devel.yaml b/deployments/compose/compose-devel.yaml index d3f86112..c8014c9a 100644 --- a/deployments/compose/compose-devel.yaml +++ b/deployments/compose/compose-devel.yaml @@ -142,6 +142,9 @@ services: echo "Creating PG_CRON tasks" psql postgresql://postgres:admin@pgsql:5432/postgres < /cron.sql && { echo "Ok"; } || { echo "Cron Tasks creation Failed"; exit 1; } + echo "Configuring purge retention (${CIQ_MAX_DATA_AGE:-365} days)" + psql postgresql://postgres:admin@pgsql:5432/postgres -c "UPDATE cron.job SET command = \$\$SELECT purge_expired_clusters(${CIQ_MAX_DATA_AGE:-365})\$\$ WHERE jobname = \$\$purge_expired_clusters\$\$;" + if [[ $CIQ_DB_PRELOAD_DATA == "true" ]]; then echo "Initializing DB with fake data" psql postgresql://user:password@pgsql:5432/clusteriq < /load_example_data.sql && { echo "Ok"; } || { echo "Initialization Failed"; exit 1; } @@ -150,6 +153,7 @@ services: echo "Done!" ' environment: + CIQ_MAX_DATA_AGE: "365" CIQ_DB_PRELOAD_DATA: "false" volumes: - ./../../db/sql/init.sql:/init.sql:ro,Z diff --git a/deployments/helm/cluster-iq/templates/database/configmap-init.yaml b/deployments/helm/cluster-iq/templates/database/configmap-init.yaml index e8282432..53a0f3ba 100644 --- a/deployments/helm/cluster-iq/templates/database/configmap-init.yaml +++ b/deployments/helm/cluster-iq/templates/database/configmap-init.yaml @@ -43,6 +43,14 @@ data: 'clusteriq' ); + -- pg_cron task for purging expired terminated clusters daily at midnight + SELECT cron.schedule_in_database( + 'purge_expired_clusters', + '0 0 * * *', + $$SELECT purge_expired_clusters();$$, + 'clusteriq' + ); + -- Function to check easier how the pg_cron tasks went CREATE OR REPLACE FUNCTION pg_cron_history(p_limit int DEFAULT 20) RETURNS TABLE( @@ -738,3 +746,27 @@ data: REFRESH MATERIALIZED VIEW m_instances_full_view_with_tags; END; $$ LANGUAGE plpgsql; + + -- Purges terminated clusters whose last_scan_ts is older than retention_days. + -- Cascading FKs handle deletion of instances, tags, expenses, targets, schedules, and action_runs. + -- Triggers handle deletion of associated events. + CREATE OR REPLACE FUNCTION purge_expired_clusters(retention_days INTEGER DEFAULT 365) + RETURNS INTEGER AS $$ + DECLARE + deleted_count INTEGER; + BEGIN + WITH deleted AS ( + DELETE FROM clusters + WHERE status = 'Terminated' + AND last_scan_ts < NOW() - (retention_days || ' days')::INTERVAL + RETURNING id + ) + SELECT COUNT(*) INTO deleted_count FROM deleted; + + IF deleted_count > 0 THEN + PERFORM refresh_materialized_views(); + END IF; + + RETURN deleted_count; + END; + $$ LANGUAGE plpgsql; diff --git a/deployments/helm/cluster-iq/templates/database/job.yaml b/deployments/helm/cluster-iq/templates/database/job.yaml index 0d478b3a..35f168d9 100644 --- a/deployments/helm/cluster-iq/templates/database/job.yaml +++ b/deployments/helm/cluster-iq/templates/database/job.yaml @@ -39,6 +39,9 @@ spec: - name: cron mountPath: /var/lib/pgsql/cron.sql subPath: cron.sql + env: + - name: CIQ_MAX_DATA_AGE + value: "{{ .Values.database.maxDataAge }}" envFrom: - secretRef: name: postgresql @@ -56,6 +59,9 @@ spec: echo "Creating PG_CRON tasks" psql postgresql://postgres:$POSTGRESQL_ADMIN_PASSWORD@{{ $dbHost }}:{{ .Values.database.service.port }}/postgres < /var/lib/pgsql/cron.sql && { echo "Ok"; } || { echo "Cron Tasks creation Failed"; exit 1; } + echo "Configuring purge retention (${CIQ_MAX_DATA_AGE:-365} days)" + psql postgresql://postgres:$POSTGRESQL_ADMIN_PASSWORD@{{ $dbHost }}:{{ .Values.database.service.port }}/postgres -c "UPDATE cron.job SET command = \$\$SELECT purge_expired_clusters(${CIQ_MAX_DATA_AGE:-365})\$\$ WHERE jobname = \$\$purge_expired_clusters\$\$;" + echo "Done!" restartPolicy: OnFailure {{- end }} diff --git a/deployments/helm/cluster-iq/values.yaml b/deployments/helm/cluster-iq/values.yaml index c8e42b39..be3a4750 100644 --- a/deployments/helm/cluster-iq/values.yaml +++ b/deployments/helm/cluster-iq/values.yaml @@ -412,6 +412,9 @@ database: # This will set the replicaset count more information can be found here: https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/ replicaCount: 1 + # Maximum age in days for terminated clusters before they are purged from the database + maxDataAge: "365" + # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: repository: quay.io/ecosystem-appeng/cluster-iq-pgsql