Merge pull request #411 from Drill4J/feature/test-definitions-add-met…

…adata-key-value-EPMDJ-10972

feature: raw_data.test_definitions - add tags and metadata

.github/workflows/check.yml

@@ -42,7 +42,7 @@ jobs:
         with:
           arguments: check
       - if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-reports-${{ matrix.config.preset }}
           path: ${{ github.workspace }}/*/build/reports/*

.github/workflows/release.yml

@@ -27,7 +27,7 @@ jobs:
         with:
           arguments: check
       - if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: test-reports-${{ matrix.config.preset }}
           path: ${{ github.workspace }}/*/build/reports/*

admin-app/src/main/resources/openapi.yml

@@ -348,58 +348,13 @@ paths:
           required: true
           schema:
             type: string
-        - name: instanceId
-          in: query
-          schema:
-            type: string
-        - name: commitSha
-          in: query
-          schema:
-            type: string
-        - name: buildVersion
-          in: query
-          schema:
-            type: string
-        - name: baselineInstanceId
-          in: query
-          schema:
-            type: string
-        - name: baselineCommitSha
+        - name: testsToSkip
           in: query
           schema:
-            type: string
-        - name: baselineBuildVersion
-          in: query
-          schema:
-            type: string
-      responses:
-        '200':
-          description: Recommended tests
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/DataResponse'
-  /api/metrics/tests-to-skip:
-    get:
-      summary: Get tests to skip
-      tags:
-        - metrics
-      security:
-        - apiKeyAuth: [ ]
-      parameters:
-        - name: groupId
-          in: query
-          required: true
-          schema:
-            type: string
+            type: boolean
+            default: false
         - name: testTaskId
           in: query
-          required: true
-          schema:
-            type: string
-        - name: targetAppId
-          in: query
-          required: true
           schema:
             type: string
         - name: targetInstanceId
@@ -432,11 +387,11 @@ paths:
             type: integer
       responses:
         '200':
-          description: Tests to skip
+          description: Recommended tests
           content:
             application/json:
               schema:
-                $ref: '#/components/schemas/ListDataResponse'
+                $ref: '#/components/schemas/DataResponse'
   # User Authentication Routes
   /api/sign-in:
     post:
@@ -922,6 +877,8 @@ components:
     AddTestsPayload:
       type: object
       properties:
+        groupId:
+          type: string
         sessionId:
           type: string
         tests:
@@ -931,9 +888,7 @@ components:
     TestInfo:
       type: object
       properties:
-        groupId:
-          type: string
-        id:
+        testLaunchId:
           type: string
         testDefinitionId:
           type: string
@@ -948,31 +903,24 @@ components:
     TestDetails:
       type: object
       properties:
-        engine:
+        runner:
           type: string
         path:
           type: string
         testName:
           type: string
-        params:
-          type: object
-          additionalProperties:
+        testParams:
+          type: array
+          items:
             type: string
         metadata:
           type: object
           additionalProperties:
             type: string
-        labels:
+        tags:
           type: array
           items:
-            $ref: '#/components/schemas/Label'
-    Label:
-      type: object
-      properties:
-        name:
-          type: string
-        value:
-          type: string
+            type: string
     TestResult:
       type: string
       enum:

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt

@@ -17,13 +17,20 @@ package com.epam.drill.admin.metrics.config
 
 import kotlinx.coroutines.CoroutineDispatcher
 import kotlinx.coroutines.Dispatchers
+import kotlinx.serialization.builtins.ListSerializer
+import kotlinx.serialization.builtins.MapSerializer
+import kotlinx.serialization.builtins.serializer
+import kotlinx.serialization.json.*
 import org.jetbrains.exposed.sql.Database
 import org.jetbrains.exposed.sql.IColumnType
+import org.jetbrains.exposed.sql.TextColumnType
 import org.jetbrains.exposed.sql.Transaction
 import org.jetbrains.exposed.sql.statements.Statement
 import org.jetbrains.exposed.sql.statements.StatementType
 import org.jetbrains.exposed.sql.statements.api.PreparedStatementApi
 import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction
+import org.postgresql.jdbc.PgArray
+import org.postgresql.util.PGobject
 import java.sql.ResultSet
 import javax.sql.DataSource
 
@@ -51,24 +58,28 @@ fun Transaction.executeQueryReturnMap(sqlQuery: String, vararg params: Any?): Li
             val rowObject = mutableMapOf<String, Any?>()
 
             for (i in 1..columnCount) {
-                val columnName = metaData.getColumnName(i)
-                val columnType = metaData.getColumnType(i)
-
-                val columnValue = when (columnType) {
-                    java.sql.Types.INTEGER -> resultSet.getInt(i)
-                    java.sql.Types.BIGINT -> resultSet.getLong(i)
-                    java.sql.Types.FLOAT -> resultSet.getFloat(i)
-                    java.sql.Types.DOUBLE -> resultSet.getDouble(i)
-                    java.sql.Types.DECIMAL, java.sql.Types.NUMERIC -> resultSet.getBigDecimal(i)
-                    java.sql.Types.BOOLEAN -> resultSet.getBoolean(i)
-                    java.sql.Types.VARCHAR, java.sql.Types.CHAR, java.sql.Types.LONGVARCHAR -> resultSet.getString(i)
-                    java.sql.Types.DATE -> resultSet.getDate(i)
-                    java.sql.Types.TIMESTAMP -> resultSet.getTimestamp(i)
-                    java.sql.Types.TIME -> resultSet.getTime(i)
-                    java.sql.Types.BINARY, java.sql.Types.VARBINARY, java.sql.Types.LONGVARBINARY -> resultSet.getBytes(i)
-                    else -> resultSet.getObject(i) // Fallback to generic Object type
+                val name = metaData.getColumnName(i)
+                val value = when (val dbValue = resultSet.getObject(i)) {
+                    is PgArray -> {
+                        val array = dbValue.array
+                        array?.let { it as Array<*> }?.toList() ?: emptyList<String>()
+                    }
+
+                    is PGobject -> {
+                        val json = dbValue.value
+                        if (json != null) {
+                            val jsonObject = Json.parseToJsonElement(json).jsonObject
+                            jsonObject.mapValues { (_, value) -> value.jsonPrimitiveOrElement() }
+                        } else {
+                            emptyMap()
+                        }
+                    }
+
+                    is java.sql.Timestamp -> dbValue.toLocalDateTime()
+                    else -> dbValue
                 }
-                rowObject[columnName] = if (resultSet.wasNull()) null else columnValue
+
+                rowObject[name] = if (resultSet.wasNull()) null else value
             }
             result.add(rowObject)
         }
@@ -88,12 +99,25 @@ fun Transaction.executeUpdate(sql: String, vararg params: Any?) {
     })
 }
 
-private fun <T : Any> Transaction.executePreparedStatement(stmt: String, vararg params: Any?, transform: (ResultSet) -> T): T? {
+private fun <T : Any> Transaction.executePreparedStatement(
+    stmt: String,
+    vararg params: Any?,
+    transform: (ResultSet) -> T
+): T? {
     if (stmt.isEmpty()) return null
 
     return exec(object : Statement<T>(StatementType.SELECT, emptyList()) {
         override fun PreparedStatementApi.executeInternal(transaction: Transaction): T? {
-            params.forEachIndexed { idx, value -> set(idx + 1, value ?: "NULL") }
+            params.forEachIndexed { idx, value ->
+                if (value != null) {
+                    set(idx + 1, value)
+                } else {
+                    // WORKAROUND: TextColumnType is employed to trick expose to write null values
+                    // Possible issues with: BinaryColumnType, BlobColumnType
+                    // see setNull implementation for more details
+                    setNull(idx + 1, TextColumnType())
+                }
+            }
             executeQuery()
             return resultSet?.use { transform(it) }
         }
@@ -102,3 +126,11 @@ private fun <T : Any> Transaction.executePreparedStatement(stmt: String, vararg
         override fun arguments(): Iterable<Iterable<Pair<IColumnType<*>, Any?>>> = emptyList()
     })
 }
+
+private fun JsonElement.jsonPrimitiveOrElement(): Any {
+    return when (this) {
+        is JsonPrimitive -> this.contentOrNull ?: this.booleanOrNull ?: this.longOrNull ?: this.doubleOrNull ?: ""
+        is JsonObject -> this.mapValues { (_, value) -> value.jsonPrimitiveOrElement() }
+        is JsonArray -> this.map { it.jsonPrimitiveOrElement() }
+    }
+}

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt

@@ -29,19 +29,14 @@ interface MetricsRepository {
         coverageThreshold: Double
     ): Map<String, String>
 
-    suspend fun getRecommendedTests(
-        buildId: String,
-        baselineBuildId: String
-    ): List<Map<String, Any>>
-
     suspend fun refreshMaterializedView(viewName: String)
 
     suspend fun getRecommendedTests(
         groupId: String,
         targetBuildId: String,
+        baselineBuildId: String? = null,
         testsToSkip: Boolean = false,
         testTaskId: String? = null,
-        baselineBuildId: String? = null,
         coveragePeriodFrom: LocalDateTime
     ): List<Map<String, Any?>>
 }

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt

@@ -23,7 +23,10 @@ import kotlinx.serialization.descriptors.SerialDescriptor
 import kotlinx.serialization.descriptors.buildClassSerialDescriptor
 import kotlinx.serialization.encoding.Decoder
 import kotlinx.serialization.encoding.Encoder
+import kotlinx.serialization.json.Json
+import org.postgresql.util.PGobject
 import java.sql.Timestamp
+import org.postgresql.jdbc.PgArray
 
 object AnySerializer : KSerializer<Any?> {
     override val descriptor: SerialDescriptor = buildClassSerialDescriptor("Any")

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt

@@ -60,7 +60,7 @@ class MetricsRepositoryImpl : MetricsRepository {
     }
 
     override suspend fun getBuildDiffReport(
-        buildId: String,
+        targetBuildId: String,
         baselineBuildId: String,
         coverageThreshold: Double
     ) = transaction {
@@ -73,7 +73,7 @@ class MetricsRepositoryImpl : MetricsRepository {
                     ),
                     RecommendedTests AS (
                         SELECT *
-                        FROM raw_data.get_recommended_tests(?, ?)
+                        FROM raw_data.get_recommended_tests_v2(?, ?)
                     )	
                     SELECT 
                         (SELECT count(*) FROM Risks WHERE __risk_type = 'new') as changes_new_methods,
@@ -83,30 +83,16 @@ class MetricsRepositoryImpl : MetricsRepository {
                         (SELECT CAST(SUM(__covered_probes) AS FLOAT) / SUM(__probes_count) FROM Risks) as coverage,
                         (SELECT count(*) FROM RecommendedTests) as recommended_tests
                 """.trimIndent(),
-            buildId,
+            targetBuildId,
             baselineBuildId,
-            buildId,
+            targetBuildId,
             baselineBuildId
             //,coverageThreshold
         ).first() as Map<String, String>
 
 
     }
 
-    override suspend fun getRecommendedTests(
-        buildId: String,
-        baselineBuildId: String
-    ): List<Map<String, Any>> = transaction {
-        executeQueryReturnMap(
-            """
-            SELECT *
-            FROM raw_data.get_recommended_tests(?, ?)
-            """.trimIndent(),
-            buildId,
-            baselineBuildId,
-        ) as List<Map<String, Any>>
-    }
-
     override suspend fun refreshMaterializedView(viewName: String) = transaction {
         executeUpdate(
             """             
@@ -118,17 +104,23 @@ class MetricsRepositoryImpl : MetricsRepository {
     override suspend fun getRecommendedTests(
         groupId: String,
         targetBuildId: String,
+        baselineBuildId: String?,
         testsToSkip: Boolean,
         testTaskId: String?,
-        baselineBuildId: String?,
         coveragePeriodFrom: LocalDateTime
     ): List<Map<String, Any?>> = transaction {
         executeQueryReturnMap(
             """            
                 SELECT 
-                    __test_runner_id AS test_engine,
+                    __group_id AS group_id,
+                    __test_definition_id AS test_definition_id,
+                    __test_runner AS test_runner,
                     __test_path AS test_path,
-                    __test_name AS test_name
+                    __test_name AS test_name,
+                    __test_type AS test_type,
+                    __tags AS tags,
+                    __metadata AS metadata,
+                    __created_at AS created_at
                 FROM raw_data.get_recommended_tests_v2(
                     ?,	   -- group_id                    
                     ?, 	   -- target_build_id
@@ -146,4 +138,4 @@ class MetricsRepositoryImpl : MetricsRepository {
             coveragePeriodFrom
         )
     }
-}
+}