Merge pull request #463 from Drill4J/feature/etl-optimization

RomanDavlyatshin · web-flow · commit ae1150fa2181 · 2026-02-16T16:44:43.000+04:00
Feature: test definition &amp; test launches dedicated API endpoints
diff --git a/admin-app/src/main/resources/openapi.yml b/admin-app/src/main/resources/openapi.yml
@@ -189,6 +189,54 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/MessageResponse'
+  /api/data-ingest/test-definitions:
+    post:
+      summary: Save test definitions
+      operationId: postTestDefinitions
+      tags:
+        - data-ingest
+      security:
+        - apiKeyAuth: [ ]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/AddTestDefinitionsPayload'
+          application/protobuf:
+            schema:
+              $ref: '#/components/schemas/AddTestDefinitionsPayload'
+      responses:
+        '200':
+          description: Test definitions saved
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MessageResponse'
+  /api/data-ingest/test-launches:
+    post:
+      summary: Save test launches
+      operationId: postTestLaunches
+      tags:
+        - data-ingest
+      security:
+        - apiKeyAuth: [ ]
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/AddTestLaunchesPayload'
+          application/protobuf:
+            schema:
+              $ref: '#/components/schemas/AddTestLaunchesPayload'
+      responses:
+        '200':
+          description: Test launches saved
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/MessageResponse'
   /api/data-ingest/sessions:
     put:
       summary: Save test sessions
@@ -1644,6 +1692,78 @@ components:
         - SKIPPED
         - SMART_SKIPPED
         - UNKNOWN
+    AddTestDefinitionsPayload:
+      type: object
+      required:
+        - groupId
+        - definitions
+      properties:
+        groupId:
+          type: string
+        definitions:
+          type: array
+          items:
+            $ref: '#/components/schemas/TestDefinitionPayload'
+    TestDefinitionPayload:
+      type: object
+      required:
+        - id
+        - runner
+        - name
+      properties:
+        id:
+          type: string
+        runner:
+          type: string
+        name:
+          type: string
+        type:
+          type: string
+          nullable: true
+        path:
+          type: string
+          nullable: true
+        tags:
+          type: array
+          items:
+            type: string
+          default: [ ]
+        metadata:
+          type: object
+          nullable: true
+          additionalProperties: true
+    AddTestLaunchesPayload:
+      type: object
+      required:
+        - groupId
+        - testSessionId
+        - launches
+      properties:
+        groupId:
+          type: string
+        testSessionId:
+          type: string
+        launches:
+          type: array
+          items:
+            $ref: '#/components/schemas/TestLaunchPayload'
+    TestLaunchPayload:
+      type: object
+      required:
+        - id
+        - testDefinitionId
+      properties:
+        id:
+          type: string
+        testDefinitionId:
+          type: string
+        result:
+          type: string
+          nullable: true
+        duration:
+          type: integer
+          format: int32
+          nullable: true
     SessionPayload:
       type: object
       properties:
diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/RawDataWriterRoutes.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/RawDataWriterRoutes.kt
@@ -16,21 +16,22 @@
 package com.epam.drill.admin.writer.rawdata.route
 
 import com.epam.drill.admin.common.principal.User
+import com.epam.drill.admin.common.route.ok
 import com.epam.drill.admin.writer.rawdata.service.RawDataWriter
 import io.ktor.client.*
 import io.ktor.client.engine.apache.*
 import io.ktor.client.plugins.contentnegotiation.*
 import io.ktor.client.request.*
 import io.ktor.http.*
 import io.ktor.resources.*
-import io.ktor.server.resources.put
-import io.ktor.server.resources.post
-import io.ktor.server.resources.get
-import io.ktor.server.resources.delete
 import io.ktor.serialization.kotlinx.json.*
 import io.ktor.server.application.*
+import io.ktor.server.auth.*
 import io.ktor.server.plugins.*
 import io.ktor.server.request.*
+import io.ktor.server.resources.*
+import io.ktor.server.resources.post
+import io.ktor.server.resources.put
 import io.ktor.server.routing.*
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
@@ -42,8 +43,6 @@ import org.kodein.di.instance
 import org.kodein.di.ktor.closestDI
 import java.io.InputStream
 import java.util.zip.GZIPInputStream
-import com.epam.drill.admin.common.route.ok
-import io.ktor.server.auth.principal
 
 private val logger = KotlinLogging.logger {}
 
@@ -65,6 +64,12 @@ class TestMetadataRoute()
 @Resource("sessions")
 class TestSessionRoute()
 
+@Resource("test-definitions")
+class TestDefinitionsRoute()
+
+@Resource("test-launches")
+class TestLaunchesRoute()
+
 @Resource("method-ignore-rules")
 class MethodIgnoreRulesRoute() {
     @Resource("/{id}")
@@ -81,6 +86,8 @@ fun Route.dataIngestRoutes() {
         putMethods()
         postTestMetadata()
         putTestSessions()
+        postTestDefinitions()
+        postTestLaunches()
         postMethodIgnoreRules()
         getMethodIgnoreRules()
         deleteMethodIgnoreRule()
@@ -142,6 +149,24 @@ fun Route.putTestSessions() {
     }
 }
 
+fun Route.postTestDefinitions() {
+    val rawDataWriter by closestDI().instance<RawDataWriter>()
+
+    post<TestDefinitionsRoute> {
+        rawDataWriter.saveTestDefinitions(call.decompressAndReceive())
+        call.ok("Test definitions saved")
+    }
+}
+
+fun Route.postTestLaunches() {
+    val rawDataWriter by closestDI().instance<RawDataWriter>()
+
+    post<TestLaunchesRoute> {
+        rawDataWriter.saveTestLaunches(call.decompressAndReceive())
+        call.ok("Test launches saved")
+    }
+}
+
 fun Route.postMethodIgnoreRules() {
     val rawDataWriter by closestDI().instance<RawDataWriter>()
 
diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt
@@ -19,7 +19,7 @@ import kotlinx.datetime.Instant
 import kotlinx.serialization.Serializable
 import kotlinx.serialization.json.JsonElement
 
-// TODO rework alongside with Java Autotest Agent
+// TODO update test agent
 @Serializable
 class AddTestsPayload(
     val groupId: String,
@@ -70,4 +70,41 @@ class SessionPayload(
     val testTaskId: String,
     val startedAt: Instant,
     val builds: List<SingleSessionBuildPayload> = emptyList(),
+)
+
+@Serializable
+class AddTestLaunchesPayload(
+    val groupId: String,
+    val testSessionId: String,
+    val launches: List<TestLaunchPayload>,
+)
+
+@Serializable
+class TestLaunchPayload (
+    val id: String,
+    val testDefinitionId: String,
+    val result: String?,
+    val duration: Int? = null,
+)
+
+
+@Serializable
+class AddTestDefinitionsPayload(
+    val groupId: String,
+    val definitions: List<TestDefinitionPayload>
+)
+
+// TODO: update test agent
+//  Order of fields, and field definitions changed compared to original TestDefinition class:
+//       - name and runner are no longer nullable
+//       - type field is moved and became nullable
+@Serializable
+class TestDefinitionPayload(
+    val id: String,
+    val runner: String,
+    val name: String,
+    val type: String?,
+    val path: String?,
+    val tags: List<String> = emptyList(),
+    val metadata: JsonElement? = null,
 )
diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/RawDataWriter.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/RawDataWriter.kt
@@ -25,6 +25,8 @@ interface RawDataWriter {
     suspend fun saveMethods(methodsPayload: MethodsPayload)
     suspend fun saveCoverage(coveragePayload: CoveragePayload)
     suspend fun saveTestMetadata(testsPayload: AddTestsPayload)
+    suspend fun saveTestDefinitions(testDefinitionsPayload: AddTestDefinitionsPayload)
+    suspend fun saveTestLaunches(testLaunchesPayload: AddTestLaunchesPayload)
     suspend fun saveTestSession(sessionPayload: SessionPayload, user: User?)
     suspend fun saveMethodIgnoreRule(rulePayload: MethodIgnoreRulePayload)
     suspend fun getAllMethodIgnoreRules(): List<MethodIgnoreRuleView>
diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt
@@ -177,6 +177,7 @@ class RawDataServiceImpl(
             }
     }
 
+    // WARNING: keep this method for backward compatibility with existing adoptions
     override suspend fun saveTestMetadata(testsPayload: AddTestsPayload) = transaction {
         testsPayload.tests.map { test ->
             TestLaunch(
@@ -203,6 +204,34 @@ class RawDataServiceImpl(
         }.let { testDefinitionRepository.createMany(it) }
     }
 
+    override suspend fun saveTestDefinitions(testDefinitionsPayload: AddTestDefinitionsPayload) = transaction {
+        testDefinitionsPayload.definitions.map { definition ->
+            TestDefinition(
+                groupId = testDefinitionsPayload.groupId,
+                id = definition.id,
+                type = definition.type,
+                runner = definition.runner,
+                name = definition.name,
+                path = definition.path,
+                tags = definition.tags,
+                metadata = definition.metadata
+            )
+        }.let { testDefinitionRepository.createMany(it) }
+    }
+
+    override suspend fun saveTestLaunches(testLaunchesPayload: AddTestLaunchesPayload) = transaction {
+        testLaunchesPayload.launches.map { launch ->
+            TestLaunch(
+                groupId = testLaunchesPayload.groupId,
+                id = launch.id,
+                testDefinitionId = launch.testDefinitionId,
+                testSessionId = testLaunchesPayload.testSessionId,
+                result = launch.result.toString(),
+                duration = launch.duration
+            )
+        }.let { testLaunchRepository.createMany(it) }
+    }
+
     override suspend fun saveTestSession(sessionPayload: SessionPayload, user: User?) {
         val testSession = TestSession(
             id = sessionPayload.id,
diff --git a/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestMetadataApiTest.kt b/admin-writer/src/test/kotlin/com/epam/drill/admin/writer/rawdata/TestMetadataApiTest.kt
