refactor: update job triggers and clean up obsolete jobs in scheduler configuration

admin-app/src/main/kotlin/com/epam/drill/admin/DrillAdminApplication.kt

@@ -30,7 +30,7 @@ import com.epam.drill.admin.metrics.route.metricsManagementRoutes
 import com.epam.drill.admin.route.rootRoute
 import com.epam.drill.admin.route.uiConfigRoute
 import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig
-import com.epam.drill.admin.writer.rawdata.config.dataRetentionPolicyJob
+import com.epam.drill.admin.writer.rawdata.config.rawDataRetentionPolicyJob
 import com.epam.drill.admin.writer.rawdata.config.rawDataDIModule
 import com.epam.drill.admin.writer.rawdata.route.*
 import io.ktor.http.*
@@ -211,8 +211,8 @@ private fun Application.initScheduler() {
     }
     scheduler.start()
     scheduler.scheduleJob(updateMetricsEtlJob, schedulerConfig.etlTrigger)
-    scheduler.scheduleJob(dataRetentionPolicyJob, schedulerConfig.retentionPoliciesTrigger)
-    scheduler.scheduleJob(metricsDataRetentionPolicyJob, schedulerConfig.retentionPoliciesTrigger)
+    scheduler.scheduleJob(rawDataRetentionPolicyJob, schedulerConfig.getRetentionPoliciesTrigger("rawDataRetentionPolicyTrigger"))
+    scheduler.scheduleJob(metricsDataRetentionPolicyJob, schedulerConfig.getRetentionPoliciesTrigger("metricsRetentionPolicyTrigger"))
     scheduler.addJob(deleteMetricsDataJob)
 }
 

admin-app/src/main/kotlin/com/epam/drill/admin/config/SchedulerConfig.kt

@@ -34,16 +34,15 @@ class SchedulerConfig(private val config: ApplicationConfig) {
 
     val etlTrigger: CronTrigger
         get() = TriggerBuilder.newTrigger()
-            .withIdentity("etlTrigger", "refreshViews")
+            .withIdentity("etlTrigger", "drill")
             .startNow()
             .withSchedule(
                 CronScheduleBuilder.cronSchedule(etlJobCron)
             )
             .build()
 
-    val retentionPoliciesTrigger: CronTrigger
-        get() = TriggerBuilder.newTrigger()
-            .withIdentity("retentionPolicyTrigger", "retentionPolicies")
+    fun getRetentionPoliciesTrigger(triggerName: String): CronTrigger = TriggerBuilder.newTrigger()
+            .withIdentity(triggerName, "drill")
             .startNow()
             .withSchedule(
                 CronScheduleBuilder.cronSchedule(dataRetentionJobCron)

admin-common/src/main/kotlin/com/epam/drill/admin/common/scheduler/CommonJobs.kt

@@ -32,4 +32,4 @@ fun getTestDataDeletionDataMap(groupId: String, testSessionId: String) = JobData
 }
 
 val deleteMetricsDataJobKey: JobKey
-    get() = JobKey.jobKey("deleteMetricsData", "metricsJobs")
+    get() = JobKey.jobKey("metricsDeletionDataJob", "drill")

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsJobs.kt

@@ -26,7 +26,7 @@ fun getUpdateMetricsEtlDataMap(groupId: String?, reset: Boolean) = JobDataMap().
     put("reset", reset)
 }
 val updateMetricsEtlJobKey: JobKey
-    get() = JobKey.jobKey("metricsEtl", "metricsJobs")
+    get() = JobKey.jobKey("metricsEtl", "drill")
 val updateMetricsEtlJob: JobDetail
     get() = JobBuilder.newJob(UpdateMetricsEtlJob::class.java)
         .storeDurably()
@@ -38,7 +38,7 @@ val metricsDataRetentionPolicyJob: JobDetail
     get() = JobBuilder.newJob(MetricsDataRetentionPolicyJob::class.java)
         .storeDurably()
         .withDescription("Job for deleting metrics data older than the retention period.")
-        .withIdentity("metricsRetentionPolicyJob", "metricsJobs")
+        .withIdentity("metricsRetentionPolicyJob", "drill")
         .build()
 val deleteMetricsDataJob: JobDetail
     get() = JobBuilder.newJob(DeleteMetricsDataJob::class.java)

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/job/MetricsDataRetentionPolicyJob.kt

@@ -36,23 +36,19 @@ class MetricsDataRetentionPolicyJob(
         val groupId: String? = context.mergedJobDataMap.getString("groupId")
         runBlocking {
             transaction {
-                metricsRepository.getMetricsPeriodDays().let {
-                    if (groupId != null) mapOf(groupId to (it[groupId] ?: Instant.EPOCH)) else it
-                }.map { (groupId, initTimestamp) ->
+                metricsRepository.getMetricsPeriodDays().let { periodDays ->
+                    if (groupId != null) mapOf(groupId to (periodDays[groupId] ?: Instant.EPOCH)) else periodDays
+                }.filterValues { it > Instant.EPOCH }.map { (groupId, initTimestamp) ->
                     async {
                         logger.info { "Deleting all metrics data for groupId [$groupId] older than $initTimestamp..." }
                         metricsRepository.deleteAllBuildDataCreatedBefore(groupId, initTimestamp)
                         metricsRepository.deleteAllTestDataCreatedBefore(groupId, initTimestamp)
                         metricsRepository.deleteAllDailyDataCreatedBefore(groupId, initTimestamp)
+                        metricsRepository.deleteAllOrphanReferences(groupId, initTimestamp)
                         logger.info { "Metrics data for groupId $groupId older than $initTimestamp deleted successfully." }
                     }
                 }.awaitAll()
             }
-            transaction {
-                logger.info { "Deleting orphan references..." }
-                metricsRepository.deleteAllOrphanReferences()
-                logger.info { "Orphan references deleted successfully." }
-            }
         }
     }
 

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt

@@ -152,7 +152,7 @@ interface MetricsRepository {
     suspend fun deleteAllBuildDataCreatedBefore(groupId: String, timestamp: Instant)
     suspend fun deleteAllTestDataCreatedBefore(groupId: String, timestamp: Instant)
     suspend fun deleteAllDailyDataCreatedBefore(groupId: String, timestamp: Instant)
-    suspend fun deleteAllOrphanReferences()
+    suspend fun deleteAllOrphanReferences(groupId: String, timestamp: Instant)
 
     suspend fun deleteAllBuildDataByBuildId(groupId: String, appId: String, buildId: String)
     suspend fun deleteAllTestDataByTestSessionId(groupId: String, testSessionId: String)

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt

@@ -326,7 +326,8 @@ class MetricsRepositoryImpl : MetricsRepository {
                     ),  
             """.trimIndent(), buildId, baselineBuildId
             )
-            append("""
+            append(
+                """
                     ImpactedTestsWithResults AS (
                         SELECT  
                             COUNT(*) AS impacted_tests,
@@ -335,7 +336,8 @@ class MetricsRepositoryImpl : MetricsRepository {
                         FROM ImpactedTests it	
                         LEFT JOIN TestLaunches tl ON tl.test_definition_id = it.test_definition_id	
                     ) 
-            """.trimIndent())
+            """.trimIndent()
+            )
             append(
                 """                    
                    SELECT 
@@ -796,39 +798,49 @@ class MetricsRepositoryImpl : MetricsRepository {
         )
     }
 
-    override suspend fun deleteAllOrphanReferences() = transaction {
+    override suspend fun deleteAllOrphanReferences(groupId: String, timestamp: Instant) = transaction {
+        val timestamp = Timestamp.from(timestamp)
         executeUpdate(
             """
                 DELETE FROM metrics.methods m
-                WHERE NOT EXISTS (SELECT 1 
+                WHERE m.group_id = ?
+                    AND m.created_at_day < ?
+                    AND NOT EXISTS (SELECT 1 
                     FROM metrics.build_methods bm 
                     WHERE bm.group_id = m.group_id 
                         AND bm.app_id = m.app_id
                         AND bm.method_id = m.method_id
                 )
-                """.trimIndent()
+                """.trimIndent(),
+            groupId, timestamp
         )
         executeUpdate(
             """
                 DELETE FROM metrics.method_daily_coverage c
-                WHERE NOT EXISTS (SELECT 1
+                WHERE c.group_id = ?
+                    AND c.created_at_day < ?
+                    AND NOT EXISTS (SELECT 1
                     FROM metrics.methods m
                     WHERE m.group_id = c.group_id
                         AND m.app_id = c.app_id
                         AND m.method_id = c.method_id
                 )
-                """.trimIndent()
+                """.trimIndent(),
+            groupId, timestamp
         )
         executeUpdate(
             """
                 DELETE FROM metrics.test_to_code_mapping tcm
-                WHERE NOT EXISTS (SELECT 1
+                WHERE tcm.group_id = ?
+                    AND tcm.created_at_day < ?
+                    AND NOT EXISTS (SELECT 1
                     FROM metrics.methods m
                     WHERE m.group_id = tcm.group_id
                         AND m.app_id = tcm.app_id
                         AND m.signature = tcm.signature
                 )
-                """.trimIndent()
+                """.trimIndent(),
+            groupId, timestamp
         )
     }
 }

admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/config/RawDataWriterModule.kt

@@ -94,9 +94,9 @@ val settingsServicesDIModule
         bind<SettingsService>() with singleton { SettingsServiceImpl(groupSettingsRepository = instance()) }
     }
 
-val dataRetentionPolicyJob: JobDetail
+val rawDataRetentionPolicyJob: JobDetail
     get() = JobBuilder.newJob(DataRetentionPolicyJob::class.java)
         .storeDurably()
         .withDescription("Job for deleting raw data older than the retention period.")
-        .withIdentity("rawDataRetentionPolicyJob", "retentionPolicies")
+        .withIdentity("rawDataRetentionPolicyJob", "drill")
         .build()

admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/job/DataRetentionPolicyJob.kt

@@ -47,14 +47,14 @@ class DataRetentionPolicyJob(
                 val retentionPeriodDays = settings.retentionPeriodDays ?: return@forEach
                 val createdBefore: LocalDate = LocalDate.now(ZoneId.systemDefault()).minusDays(retentionPeriodDays.toLong())
                 transaction {
-                    logger.debug { "Deleting all data of $groupId older than $createdBefore..." }
+                    logger.info { "Deleting all data for group [$groupId] older than $createdBefore..." }
                     coverageRepository.deleteAllCreatedBefore(groupId, createdBefore)
                     instanceRepository.deleteAllCreatedBefore(groupId, createdBefore)
                     testLaunchRepository.deleteAllCreatedBefore(groupId, createdBefore)
                     testSessionRepository.deleteAllCreatedBefore(groupId, createdBefore)
                     methodRepository.deleteAllCreatedBefore(groupId, createdBefore)
                     buildRepository.deleteAllCreatedBefore(groupId, createdBefore)
-                    logger.debug { "Data of $groupId older than $createdBefore deleted successfully." }
+                    logger.info { "Data for group [$groupId] older than $createdBefore deleted successfully." }
                 }
             }
         }

admin-writer/src/main/resources/raw_data/db/migration/V31__Delete_obsolete_jobs.sql

@@ -0,0 +1,3 @@
+DELETE FROM quartz.cron_triggers WHERE trigger_group in ('retentionPolicies', 'refreshViews');
+DELETE FROM quartz.triggers WHERE trigger_group in ('retentionPolicies', 'refreshViews');
+DELETE FROM quartz.job_details WHERE job_group in ('retentionPolicies', 'metricsJobs');