From b896ddb62a904d877bd2e4583e59461eac1187c7 Mon Sep 17 00:00:00 2001 From: iryabov Date: Wed, 8 Apr 2026 14:00:37 +0200 Subject: [PATCH] refactor: update job triggers and clean up obsolete jobs in scheduler configuration --- .../epam/drill/admin/DrillAdminApplication.kt | 6 ++-- .../drill/admin/config/SchedulerConfig.kt | 7 ++--- .../admin/common/scheduler/CommonJobs.kt | 2 +- .../drill/admin/metrics/config/MetricsJobs.kt | 4 +-- .../job/MetricsDataRetentionPolicyJob.kt | 12 +++----- .../metrics/repository/MetricsRepository.kt | 2 +- .../repository/impl/MetricsRepositoryImpl.kt | 30 +++++++++++++------ .../rawdata/config/RawDataWriterModule.kt | 4 +-- .../rawdata/job/DataRetentionPolicyJob.kt | 4 +-- .../migration/V31__Delete_obsolete_jobs.sql | 3 ++ 10 files changed, 42 insertions(+), 32 deletions(-) create mode 100644 admin-writer/src/main/resources/raw_data/db/migration/V31__Delete_obsolete_jobs.sql diff --git a/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt b/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt index b2f69d730..7218c0f79 100644 --- a/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt +++ b/admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt @@ -30,7 +30,7 @@ import com.epam.drill.admin.metrics.route.metricsManagementRoutes import com.epam.drill.admin.route.rootRoute import com.epam.drill.admin.route.uiConfigRoute import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig -import com.epam.drill.admin.writer.rawdata.config.dataRetentionPolicyJob +import com.epam.drill.admin.writer.rawdata.config.rawDataRetentionPolicyJob import com.epam.drill.admin.writer.rawdata.config.rawDataDIModule import com.epam.drill.admin.writer.rawdata.route.* import io.ktor.http.* @@ -211,8 +211,8 @@ private fun Application.initScheduler() { } scheduler.start() scheduler.scheduleJob(updateMetricsEtlJob, schedulerConfig.etlTrigger) - scheduler.scheduleJob(dataRetentionPolicyJob, schedulerConfig.retentionPoliciesTrigger) - scheduler.scheduleJob(metricsDataRetentionPolicyJob, schedulerConfig.retentionPoliciesTrigger) + scheduler.scheduleJob(rawDataRetentionPolicyJob, schedulerConfig.getRetentionPoliciesTrigger("rawDataRetentionPolicyTrigger")) + scheduler.scheduleJob(metricsDataRetentionPolicyJob, schedulerConfig.getRetentionPoliciesTrigger("metricsRetentionPolicyTrigger")) scheduler.addJob(deleteMetricsDataJob) } diff --git a/admin-app/src/main/kotlin/com/epam/drill/admin/config/SchedulerConfig.kt b/admin-app/src/main/kotlin/com/epam/drill/admin/config/SchedulerConfig.kt index a9acd96f0..146eb169b 100644 --- a/admin-app/src/main/kotlin/com/epam/drill/admin/config/SchedulerConfig.kt +++ b/admin-app/src/main/kotlin/com/epam/drill/admin/config/SchedulerConfig.kt @@ -34,16 +34,15 @@ class SchedulerConfig(private val config: ApplicationConfig) { val etlTrigger: CronTrigger get() = TriggerBuilder.newTrigger() - .withIdentity("etlTrigger", "refreshViews") + .withIdentity("etlTrigger", "drill") .startNow() .withSchedule( CronScheduleBuilder.cronSchedule(etlJobCron) ) .build() - val retentionPoliciesTrigger: CronTrigger - get() = TriggerBuilder.newTrigger() - .withIdentity("retentionPolicyTrigger", "retentionPolicies") + fun getRetentionPoliciesTrigger(triggerName: String): CronTrigger = TriggerBuilder.newTrigger() + .withIdentity(triggerName, "drill") .startNow() .withSchedule( CronScheduleBuilder.cronSchedule(dataRetentionJobCron) diff --git a/admin-common/src/main/kotlin/com/epam/drill/admin/common/scheduler/CommonJobs.kt b/admin-common/src/main/kotlin/com/epam/drill/admin/common/scheduler/CommonJobs.kt index 8721290d1..640cc8dc1 100644 --- a/admin-common/src/main/kotlin/com/epam/drill/admin/common/scheduler/CommonJobs.kt +++ b/admin-common/src/main/kotlin/com/epam/drill/admin/common/scheduler/CommonJobs.kt @@ -32,4 +32,4 @@ fun getTestDataDeletionDataMap(groupId: String, testSessionId: String) = JobData } val deleteMetricsDataJobKey: JobKey - get() = JobKey.jobKey("deleteMetricsData", "metricsJobs") \ No newline at end of file + get() = JobKey.jobKey("metricsDeletionDataJob", "drill") \ No newline at end of file diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsJobs.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsJobs.kt index c9d2dcc75..5fe0cf167 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsJobs.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsJobs.kt @@ -26,7 +26,7 @@ fun getUpdateMetricsEtlDataMap(groupId: String?, reset: Boolean) = JobDataMap(). put("reset", reset) } val updateMetricsEtlJobKey: JobKey - get() = JobKey.jobKey("metricsEtl", "metricsJobs") + get() = JobKey.jobKey("metricsEtl", "drill") val updateMetricsEtlJob: JobDetail get() = JobBuilder.newJob(UpdateMetricsEtlJob::class.java) .storeDurably() @@ -38,7 +38,7 @@ val metricsDataRetentionPolicyJob: JobDetail get() = JobBuilder.newJob(MetricsDataRetentionPolicyJob::class.java) .storeDurably() .withDescription("Job for deleting metrics data older than the retention period.") - .withIdentity("metricsRetentionPolicyJob", "metricsJobs") + .withIdentity("metricsRetentionPolicyJob", "drill") .build() val deleteMetricsDataJob: JobDetail get() = JobBuilder.newJob(DeleteMetricsDataJob::class.java) diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/job/MetricsDataRetentionPolicyJob.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/job/MetricsDataRetentionPolicyJob.kt index d98228e55..f513c43fc 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/job/MetricsDataRetentionPolicyJob.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/job/MetricsDataRetentionPolicyJob.kt @@ -36,23 +36,19 @@ class MetricsDataRetentionPolicyJob( val groupId: String? = context.mergedJobDataMap.getString("groupId") runBlocking { transaction { - metricsRepository.getMetricsPeriodDays().let { - if (groupId != null) mapOf(groupId to (it[groupId] ?: Instant.EPOCH)) else it - }.map { (groupId, initTimestamp) -> + metricsRepository.getMetricsPeriodDays().let { periodDays -> + if (groupId != null) mapOf(groupId to (periodDays[groupId] ?: Instant.EPOCH)) else periodDays + }.filterValues { it > Instant.EPOCH }.map { (groupId, initTimestamp) -> async { logger.info { "Deleting all metrics data for groupId [$groupId] older than $initTimestamp..." } metricsRepository.deleteAllBuildDataCreatedBefore(groupId, initTimestamp) metricsRepository.deleteAllTestDataCreatedBefore(groupId, initTimestamp) metricsRepository.deleteAllDailyDataCreatedBefore(groupId, initTimestamp) + metricsRepository.deleteAllOrphanReferences(groupId, initTimestamp) logger.info { "Metrics data for groupId $groupId older than $initTimestamp deleted successfully." } } }.awaitAll() } - transaction { - logger.info { "Deleting orphan references..." } - metricsRepository.deleteAllOrphanReferences() - logger.info { "Orphan references deleted successfully." } - } } } diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt index 0fd544e79..7a5b5d08b 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt @@ -152,7 +152,7 @@ interface MetricsRepository { suspend fun deleteAllBuildDataCreatedBefore(groupId: String, timestamp: Instant) suspend fun deleteAllTestDataCreatedBefore(groupId: String, timestamp: Instant) suspend fun deleteAllDailyDataCreatedBefore(groupId: String, timestamp: Instant) - suspend fun deleteAllOrphanReferences() + suspend fun deleteAllOrphanReferences(groupId: String, timestamp: Instant) suspend fun deleteAllBuildDataByBuildId(groupId: String, appId: String, buildId: String) suspend fun deleteAllTestDataByTestSessionId(groupId: String, testSessionId: String) diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt index b57e53a37..d57486b4d 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt @@ -326,7 +326,8 @@ class MetricsRepositoryImpl : MetricsRepository { ), """.trimIndent(), buildId, baselineBuildId ) - append(""" + append( + """ ImpactedTestsWithResults AS ( SELECT COUNT(*) AS impacted_tests, @@ -335,7 +336,8 @@ class MetricsRepositoryImpl : MetricsRepository { FROM ImpactedTests it LEFT JOIN TestLaunches tl ON tl.test_definition_id = it.test_definition_id ) - """.trimIndent()) + """.trimIndent() + ) append( """ SELECT @@ -796,39 +798,49 @@ class MetricsRepositoryImpl : MetricsRepository { ) } - override suspend fun deleteAllOrphanReferences() = transaction { + override suspend fun deleteAllOrphanReferences(groupId: String, timestamp: Instant) = transaction { + val timestamp = Timestamp.from(timestamp) executeUpdate( """ DELETE FROM metrics.methods m - WHERE NOT EXISTS (SELECT 1 + WHERE m.group_id = ? + AND m.created_at_day < ? + AND NOT EXISTS (SELECT 1 FROM metrics.build_methods bm WHERE bm.group_id = m.group_id AND bm.app_id = m.app_id AND bm.method_id = m.method_id ) - """.trimIndent() + """.trimIndent(), + groupId, timestamp ) executeUpdate( """ DELETE FROM metrics.method_daily_coverage c - WHERE NOT EXISTS (SELECT 1 + WHERE c.group_id = ? + AND c.created_at_day < ? + AND NOT EXISTS (SELECT 1 FROM metrics.methods m WHERE m.group_id = c.group_id AND m.app_id = c.app_id AND m.method_id = c.method_id ) - """.trimIndent() + """.trimIndent(), + groupId, timestamp ) executeUpdate( """ DELETE FROM metrics.test_to_code_mapping tcm - WHERE NOT EXISTS (SELECT 1 + WHERE tcm.group_id = ? + AND tcm.created_at_day < ? + AND NOT EXISTS (SELECT 1 FROM metrics.methods m WHERE m.group_id = tcm.group_id AND m.app_id = tcm.app_id AND m.signature = tcm.signature ) - """.trimIndent() + """.trimIndent(), + groupId, timestamp ) } } diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt index 001f08e2c..05a706f7c 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt @@ -94,9 +94,9 @@ val settingsServicesDIModule bind() with singleton { SettingsServiceImpl(groupSettingsRepository = instance()) } } -val dataRetentionPolicyJob: JobDetail +val rawDataRetentionPolicyJob: JobDetail get() = JobBuilder.newJob(DataRetentionPolicyJob::class.java) .storeDurably() .withDescription("Job for deleting raw data older than the retention period.") - .withIdentity("rawDataRetentionPolicyJob", "retentionPolicies") + .withIdentity("rawDataRetentionPolicyJob", "drill") .build() \ No newline at end of file diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/job/DataRetentionPolicyJob.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/job/DataRetentionPolicyJob.kt index 357570cb9..e7714cc1f 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/job/DataRetentionPolicyJob.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/job/DataRetentionPolicyJob.kt @@ -47,14 +47,14 @@ class DataRetentionPolicyJob( val retentionPeriodDays = settings.retentionPeriodDays ?: return@forEach val createdBefore: LocalDate = LocalDate.now(ZoneId.systemDefault()).minusDays(retentionPeriodDays.toLong()) transaction { - logger.debug { "Deleting all data of $groupId older than $createdBefore..." } + logger.info { "Deleting all data for group [$groupId] older than $createdBefore..." } coverageRepository.deleteAllCreatedBefore(groupId, createdBefore) instanceRepository.deleteAllCreatedBefore(groupId, createdBefore) testLaunchRepository.deleteAllCreatedBefore(groupId, createdBefore) testSessionRepository.deleteAllCreatedBefore(groupId, createdBefore) methodRepository.deleteAllCreatedBefore(groupId, createdBefore) buildRepository.deleteAllCreatedBefore(groupId, createdBefore) - logger.debug { "Data of $groupId older than $createdBefore deleted successfully." } + logger.info { "Data for group [$groupId] older than $createdBefore deleted successfully." } } } } diff --git a/admin-writer/src/main/resources/raw_data/db/migration/V31__Delete_obsolete_jobs.sql b/admin-writer/src/main/resources/raw_data/db/migration/V31__Delete_obsolete_jobs.sql new file mode 100644 index 000000000..f61fbf78f --- /dev/null +++ b/admin-writer/src/main/resources/raw_data/db/migration/V31__Delete_obsolete_jobs.sql @@ -0,0 +1,3 @@ +DELETE FROM quartz.cron_triggers WHERE trigger_group in ('retentionPolicies', 'refreshViews'); +DELETE FROM quartz.triggers WHERE trigger_group in ('retentionPolicies', 'refreshViews'); +DELETE FROM quartz.job_details WHERE job_group in ('retentionPolicies', 'metricsJobs');