From ef2c59c9afdb9cb5c30635f2a2eb1e895929d00f Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Tue, 21 Jan 2025 14:04:43 +0400 Subject: [PATCH 01/17] feat: test definitions add metadata keyvalue + make tags varchar array --- .../metrics/repository/impl/AnySerializer.kt | 23 +++++++++++++++++++ admin-writer/build.gradle.kts | 2 +- .../writer/rawdata/entity/TestMetadata.kt | 1 + .../impl/TestDefinitionRepositoryImpl.kt | 1 + .../service/impl/RawDataServiceImpl.kt | 5 ++-- .../rawdata/table/TestDefinitionTable.kt | 5 ++++ .../raw_data/db/migration/R__3_Functions.sql | 3 ++- .../V10_Test_Definitions_Metadata.sql | 2 ++ .../V11_Test_Definitions_Tags_As_Array.sql | 2 ++ 9 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 admin-writer/src/main/resources/raw_data/db/migration/V10_Test_Definitions_Metadata.sql create mode 100644 admin-writer/src/main/resources/raw_data/db/migration/V11_Test_Definitions_Tags_As_Array.sql diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt index 827554ade..f401e083e 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt @@ -23,7 +23,10 @@ import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.descriptors.buildClassSerialDescriptor import kotlinx.serialization.encoding.Decoder import kotlinx.serialization.encoding.Encoder +import kotlinx.serialization.json.Json +import org.postgresql.util.PGobject import java.sql.Timestamp +import org.postgresql.jdbc.PgArray object AnySerializer : KSerializer { override val descriptor: SerialDescriptor = buildClassSerialDescriptor("Any") @@ -41,10 +44,30 @@ object AnySerializer : KSerializer { val mapSerializer = MapSerializer(String.serializer(), this) encoder.encodeSerializableValue(mapSerializer, value as Map) } + is PgArray -> { + val array = value.array + if (array != null) { + val list = (array as Array<*>).map { it as String } + val listSerializer = ListSerializer(String.serializer()) + encoder.encodeSerializableValue(listSerializer, list) + } else { + encoder.encodeNull() + } + } is List<*> -> { val listSerializer = ListSerializer(this) encoder.encodeSerializableValue(listSerializer, value) } + is PGobject -> { + val json = value.value + if (json == null) { + val parsedValue = Json.decodeFromString>(json) + val mapSerializer = MapSerializer(String.serializer(), this) + encoder.encodeSerializableValue(mapSerializer, parsedValue) + } else { + encoder.encodeNull() + } + } else -> { val serializer = value::class.serializerOrNull() as KSerializer? if (serializer != null) diff --git a/admin-writer/build.gradle.kts b/admin-writer/build.gradle.kts index e506e9c08..4bc44e0b0 100644 --- a/admin-writer/build.gradle.kts +++ b/admin-writer/build.gradle.kts @@ -1,4 +1,3 @@ -import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import java.net.URI plugins { @@ -56,6 +55,7 @@ dependencies { implementation("org.jetbrains.exposed:exposed-core:$exposedVersion") implementation("org.jetbrains.exposed:exposed-jdbc:$exposedVersion") implementation("org.jetbrains.exposed:exposed-java-time:$exposedVersion") + implementation("org.jetbrains.exposed:exposed-json:$exposedVersion") implementation("org.quartz-scheduler:quartz:$quartzVersion") api("org.flywaydb:flyway-core:$flywaydbVersion") compileOnly("org.postgresql:postgresql:$postgresSqlVersion") diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt index 4685415ac..fa79c7232 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt @@ -38,6 +38,7 @@ class TestDefinition( val name: String?, val path: String?, val tags: String?, + val metadata: Map? ) class TestSession ( diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/repository/impl/TestDefinitionRepositoryImpl.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/repository/impl/TestDefinitionRepositoryImpl.kt index aa3c72cf2..ee829bbb6 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/repository/impl/TestDefinitionRepositoryImpl.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/repository/impl/TestDefinitionRepositoryImpl.kt @@ -34,6 +34,7 @@ class TestDefinitionRepositoryImpl: TestDefinitionRepository { this[TestDefinitionTable.runner] = it.runner this[TestDefinitionTable.name] = it.name this[TestDefinitionTable.path] = it.path + this[TestDefinitionTable.metadata] = it.metadata this[TestDefinitionTable.tags] = it.tags } } diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt index 0e703499f..e36c5e15e 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt @@ -15,13 +15,13 @@ */ package com.epam.drill.admin.writer.rawdata.service.impl +import com.epam.drill.admin.common.service.generateBuildId import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig.transaction import com.epam.drill.admin.writer.rawdata.entity.* import com.epam.drill.admin.writer.rawdata.repository.* import com.epam.drill.admin.writer.rawdata.route.payload.* import com.epam.drill.admin.writer.rawdata.service.RawDataWriter import com.epam.drill.admin.writer.rawdata.views.MethodIgnoreRuleView -import com.epam.drill.admin.common.service.generateBuildId import kotlinx.datetime.TimeZone import kotlinx.datetime.toJavaLocalDateTime import kotlinx.datetime.toLocalDateTime @@ -181,7 +181,8 @@ class RawDataServiceImpl( runner = test.details.engine, name = test.details.testName, path = test.details.path, - tags = test.details.labels.map { x -> x.value }.joinToString(",") + tags = test.details.labels.map { x -> x.value }.joinToString(","), + metadata = test.details.metadata ) }.let(testDefinitionRepository::createMany) } diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt index 467336e10..f974a9be6 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt @@ -15,7 +15,11 @@ */ package com.epam.drill.admin.writer.rawdata.table +import kotlinx.serialization.builtins.MapSerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.Json import org.jetbrains.exposed.sql.javatime.datetime +import org.jetbrains.exposed.sql.json.json object TestDefinitionTable : StringIdTable("raw_data.test_definitions") { val groupId = varchar("group_id", SHORT_TEXT_LENGTH) @@ -24,5 +28,6 @@ object TestDefinitionTable : StringIdTable("raw_data.test_definitions") { val name = varchar("name", MEDIUM_TEXT_LENGTH).nullable() val path = varchar("path", MEDIUM_TEXT_LENGTH).nullable() val tags = varchar("tags", MEDIUM_TEXT_LENGTH).nullable() + val metadata = json("metadata", Json, MapSerializer(String.serializer(), String.serializer())).nullable() val createdAt = datetime("created_at").nullable() } \ No newline at end of file diff --git a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql index e6e50bb98..afc647de9 100644 --- a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql +++ b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql @@ -1311,7 +1311,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_recommended_tests( __name VARCHAR, __path VARCHAR, __created_at TIMESTAMP, - __tags VARCHAR + __tags VARCHAR[], + __metadata JSON ) AS $$ BEGIN RETURN QUERY diff --git a/admin-writer/src/main/resources/raw_data/db/migration/V10_Test_Definitions_Metadata.sql b/admin-writer/src/main/resources/raw_data/db/migration/V10_Test_Definitions_Metadata.sql new file mode 100644 index 000000000..0a4621bc4 --- /dev/null +++ b/admin-writer/src/main/resources/raw_data/db/migration/V10_Test_Definitions_Metadata.sql @@ -0,0 +1,2 @@ +ALTER TABLE raw_data.test_definitions +ADD COLUMN IF NOT EXISTS metadata JSON NULL; diff --git a/admin-writer/src/main/resources/raw_data/db/migration/V11_Test_Definitions_Tags_As_Array.sql b/admin-writer/src/main/resources/raw_data/db/migration/V11_Test_Definitions_Tags_As_Array.sql new file mode 100644 index 000000000..c9d49e799 --- /dev/null +++ b/admin-writer/src/main/resources/raw_data/db/migration/V11_Test_Definitions_Tags_As_Array.sql @@ -0,0 +1,2 @@ +ALTER TABLE raw_data.test_definitions +ALTER COLUMN tags TYPE varchar[] USING string_to_array(tags, ','); From 29ceba9201af945c7b46018fbeb34ec08348cbaa Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Tue, 21 Jan 2025 15:48:17 +0400 Subject: [PATCH 02/17] fix: null check typo --- .../epam/drill/admin/metrics/repository/impl/AnySerializer.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt index f401e083e..34a7ffb60 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt @@ -60,7 +60,7 @@ object AnySerializer : KSerializer { } is PGobject -> { val json = value.value - if (json == null) { + if (json != null) { val parsedValue = Json.decodeFromString>(json) val mapSerializer = MapSerializer(String.serializer(), this) encoder.encodeSerializableValue(mapSerializer, parsedValue) From d791bff8b43602fb40f33e02ad0a9ea894d80214 Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Wed, 22 Jan 2025 19:17:09 +0400 Subject: [PATCH 03/17] feat: merge API endpoint for recommended-tests and test-to-skip BUGS: - if no testTaskId supplied the result will always be empty - testsToSkip=true / false / not specified works inconsistently TODO: - test_engine / engine rename to test_runner - build-diff-report add parameters - update openapi.yml --- admin-app/src/main/resources/openapi.yml | 2 +- .../metrics/config/MetricsDatabaseConfig.kt | 22 +-- .../metrics/repository/MetricsRepository.kt | 7 +- .../repository/impl/MetricsRepositoryImpl.kt | 36 ++--- .../drill/admin/metrics/route/MetricRoutes.kt | 42 +---- .../metrics/route/response/TestToSkipView.kt | 2 +- .../admin/metrics/service/MetricsService.kt | 18 +-- .../service/impl/MetricsServiceImpl.kt | 127 ++++++--------- .../raw_data/db/migration/R__3_Functions.sql | 148 +++--------------- 9 files changed, 98 insertions(+), 306 deletions(-) diff --git a/admin-app/src/main/resources/openapi.yml b/admin-app/src/main/resources/openapi.yml index d81b09c95..9a0944c27 100644 --- a/admin-app/src/main/resources/openapi.yml +++ b/admin-app/src/main/resources/openapi.yml @@ -397,7 +397,7 @@ paths: required: true schema: type: string - - name: targetAppId + - name: appId in: query required: true schema: diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt index 1f6b1f8a0..395297752 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt @@ -51,24 +51,14 @@ fun Transaction.executeQueryReturnMap(sqlQuery: String, vararg params: Any?): Li val rowObject = mutableMapOf() for (i in 1..columnCount) { - val columnName = metaData.getColumnName(i) - val columnType = metaData.getColumnType(i) + val name = metaData.getColumnName(i) + val columnValue = resultSet.getObject(i) - val columnValue = when (columnType) { - java.sql.Types.INTEGER -> resultSet.getInt(i) - java.sql.Types.BIGINT -> resultSet.getLong(i) - java.sql.Types.FLOAT -> resultSet.getFloat(i) - java.sql.Types.DOUBLE -> resultSet.getDouble(i) - java.sql.Types.DECIMAL, java.sql.Types.NUMERIC -> resultSet.getBigDecimal(i) - java.sql.Types.BOOLEAN -> resultSet.getBoolean(i) - java.sql.Types.VARCHAR, java.sql.Types.CHAR, java.sql.Types.LONGVARCHAR -> resultSet.getString(i) - java.sql.Types.DATE -> resultSet.getDate(i) - java.sql.Types.TIMESTAMP -> resultSet.getTimestamp(i) - java.sql.Types.TIME -> resultSet.getTime(i) - java.sql.Types.BINARY, java.sql.Types.VARBINARY, java.sql.Types.LONGVARBINARY -> resultSet.getBytes(i) - else -> resultSet.getObject(i) // Fallback to generic Object type + when (columnValue) { + is java.sql.Timestamp -> columnValue.toLocalDateTime() } - rowObject[columnName] = if (resultSet.wasNull()) null else columnValue + + rowObject[name] = if (resultSet.wasNull()) null else columnValue } result.add(rowObject) } diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt index 81fb2f879..9b10948e3 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/MetricsRepository.kt @@ -29,19 +29,14 @@ interface MetricsRepository { coverageThreshold: Double ): Map - suspend fun getRecommendedTests( - buildId: String, - baselineBuildId: String - ): List> - suspend fun refreshMaterializedView(viewName: String) suspend fun getRecommendedTests( groupId: String, targetBuildId: String, + baselineBuildId: String? = null, testsToSkip: Boolean = false, testTaskId: String? = null, - baselineBuildId: String? = null, coveragePeriodFrom: LocalDateTime ): List> } \ No newline at end of file diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt index b1cb7c494..be9538b0f 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/MetricsRepositoryImpl.kt @@ -60,7 +60,7 @@ class MetricsRepositoryImpl : MetricsRepository { } override suspend fun getBuildDiffReport( - buildId: String, + targetBuildId: String, baselineBuildId: String, coverageThreshold: Double ) = transaction { @@ -73,7 +73,7 @@ class MetricsRepositoryImpl : MetricsRepository { ), RecommendedTests AS ( SELECT * - FROM raw_data.get_recommended_tests(?, ?) + FROM raw_data.get_recommended_tests_v2(?, ?) ) SELECT (SELECT count(*) FROM Risks WHERE __risk_type = 'new') as changes_new_methods, @@ -83,9 +83,9 @@ class MetricsRepositoryImpl : MetricsRepository { (SELECT CAST(SUM(__covered_probes) AS FLOAT) / SUM(__probes_count) FROM Risks) as coverage, (SELECT count(*) FROM RecommendedTests) as recommended_tests """.trimIndent(), - buildId, + targetBuildId, baselineBuildId, - buildId, + targetBuildId, baselineBuildId //,coverageThreshold ).first() as Map @@ -93,20 +93,6 @@ class MetricsRepositoryImpl : MetricsRepository { } - override suspend fun getRecommendedTests( - buildId: String, - baselineBuildId: String - ): List> = transaction { - executeQueryReturnMap( - """ - SELECT * - FROM raw_data.get_recommended_tests(?, ?) - """.trimIndent(), - buildId, - baselineBuildId, - ) as List> - } - override suspend fun refreshMaterializedView(viewName: String) = transaction { executeUpdate( """ @@ -118,17 +104,23 @@ class MetricsRepositoryImpl : MetricsRepository { override suspend fun getRecommendedTests( groupId: String, targetBuildId: String, + baselineBuildId: String?, testsToSkip: Boolean, testTaskId: String?, - baselineBuildId: String?, coveragePeriodFrom: LocalDateTime ): List> = transaction { executeQueryReturnMap( """ SELECT - __test_runner_id AS test_engine, + __group_id AS group_id, + __test_definition_id AS test_definition_id, + __test_runner AS test_runner, __test_path AS test_path, - __test_name AS test_name + __test_name AS test_name, + __test_type AS test_type, + __tags AS tags, + __metadata AS metadata, + __created_at AS created_at FROM raw_data.get_recommended_tests_v2( ?, -- group_id ?, -- target_build_id @@ -146,4 +138,4 @@ class MetricsRepositoryImpl : MetricsRepository { coveragePeriodFrom ) } -} \ No newline at end of file +} diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt index 7820500c6..b6e47a977 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt @@ -62,21 +62,8 @@ class Metrics { val groupId: String, val appId: String, - val instanceId: String? = null, - val commitSha: String? = null, - val buildVersion: String? = null, - val baselineInstanceId: String? = null, - val baselineCommitSha: String? = null, - val baselineBuildVersion: String? = null, - ) - - @Resource("/tests-to-skip") - class TestsToSkip( - val parent: Metrics, - - val groupId: String, - val testTaskId: String, - val targetAppId: String, + val testsToSkip: Boolean = true, + val testTaskId: String? = null, val targetInstanceId: String? = null, val targetCommitSha: String? = null, val targetBuildVersion: String? = null, @@ -85,14 +72,12 @@ class Metrics { val baselineBuildVersion: String? = null, val coveragePeriodDays: Int? = null, ) - } fun Route.metricsRoutes() { getBuilds() getBuildDiffReport() getRecommendedTests() - getTestsToSkip() } fun Route.getBuilds() { @@ -132,27 +117,10 @@ fun Route.getRecommendedTests() { get { params -> val report = metricsService.getRecommendedTests( - params.groupId, - params.appId, - params.instanceId, - params.commitSha, - params.buildVersion, - params.baselineInstanceId, - params.baselineCommitSha, - params.baselineBuildVersion - ) - this.call.respond(HttpStatusCode.OK, ApiResponse(report)) - } -} - -fun Route.getTestsToSkip() { - val metricsService by closestDI().instance() - - get { params -> - val testsToSkip = metricsService.getTestsToSkip( groupId = params.groupId, + appId = params.appId, + testsToSkip = params.testsToSkip, testTaskId = params.testTaskId, - targetAppId = params.targetAppId, coveragePeriodDays = params.coveragePeriodDays, targetInstanceId = params.targetInstanceId, targetCommitSha = params.targetCommitSha, @@ -161,6 +129,6 @@ fun Route.getTestsToSkip() { baselineCommitSha = params.baselineCommitSha, baselineBuildVersion = params.baselineBuildVersion ) - this.call.respond(HttpStatusCode.OK, ApiResponse(testsToSkip)) + this.call.respond(HttpStatusCode.OK, ApiResponse(report)) } } diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/TestToSkipView.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/TestToSkipView.kt index 2173265ef..44b3fc1e3 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/TestToSkipView.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/TestToSkipView.kt @@ -19,7 +19,7 @@ import kotlinx.serialization.Serializable @Serializable class TestToSkipView( - val engine: String, + val testRunner: String, val path: String, val testName: String, val params: Map = emptyMap(), diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/MetricsService.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/MetricsService.kt index 61edf7e9a..9ab49b9ef 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/MetricsService.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/MetricsService.kt @@ -15,7 +15,6 @@ */ package com.epam.drill.admin.metrics.service -import com.epam.drill.admin.metrics.route.response.TestToSkipView import com.epam.drill.admin.metrics.views.BuildView interface MetricsService { @@ -40,18 +39,8 @@ interface MetricsService { suspend fun getRecommendedTests( groupId: String, appId: String, - instanceId: String?, - commitSha: String?, - buildVersion: String?, - baselineInstanceId: String?, - baselineCommitSha: String?, - baselineBuildVersion: String? - ): Map - - suspend fun getTestsToSkip( - groupId: String, - testTaskId: String, - targetAppId: String, + testsToSkip: Boolean = false, + testTaskId: String? = null, coveragePeriodDays: Int? = null, targetInstanceId: String? = null, targetCommitSha: String? = null, @@ -59,5 +48,6 @@ interface MetricsService { baselineInstanceId: String? = null, baselineCommitSha: String? = null, baselineBuildVersion: String? = null - ): List + ): Map + } \ No newline at end of file diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt index c67eb708c..7de521af1 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt @@ -149,16 +149,20 @@ class MetricsServiceImpl( override suspend fun getRecommendedTests( groupId: String, appId: String, - instanceId: String?, - commitSha: String?, - buildVersion: String?, + testsToSkip: Boolean, + testTaskId: String?, + coveragePeriodDays: Int?, + targetInstanceId: String?, + targetCommitSha: String?, + targetBuildVersion: String?, baselineInstanceId: String?, baselineCommitSha: String?, baselineBuildVersion: String? - ): Map { - return transaction { + ): Map = transaction { + val hasBaselineBuild = listOf(baselineInstanceId, baselineCommitSha, baselineBuildVersion).any { it != null } - val baselineBuildId = generateBuildId( + val baselineBuildId = takeIf { hasBaselineBuild }?.let { + generateBuildId( groupId, appId, baselineInstanceId, @@ -167,96 +171,59 @@ class MetricsServiceImpl( """ Provide at least one the following: baselineInstanceId, baselineCommitSha, baselineBuildVersion """.trimIndent() - ) - - if (!metricsRepository.buildExists(baselineBuildId)) { - throw BuildNotFound("Baseline build info not found for $baselineBuildId") - } - - val buildId = generateBuildId(groupId, appId, instanceId, commitSha, buildVersion) - if (!metricsRepository.buildExists(buildId)) { - throw BuildNotFound("Build info not found for $buildId") + ).also { buildId -> + if (!metricsRepository.buildExists(buildId)) { + throw BuildNotFound("Baseline build info not found for $buildId") + } } - - val recommendedTests = metricsRepository.getRecommendedTests(buildId, baselineBuildId) - - // TODO add recommended tests UI link - // val recommendedTestsReportPath = metricsServiceUiLinksConfig.recommendedTestsReportPath - mapOf( - "inputParameters" to mapOf( - "groupId" to groupId, - "appId" to appId, - "instanceId" to instanceId, - "commitSha" to commitSha, - "buildVersion" to buildVersion, - "baselineInstanceId" to baselineInstanceId, - "baselineCommitSha" to baselineCommitSha, - "baselineBuildVersion" to baselineBuildVersion - ), - "inferredValues" to mapOf( - "build" to buildId, - "baselineBuild" to baselineBuildId, - ), - "recommendedTests" to recommendedTests, - ) } - } - override suspend fun getTestsToSkip( - groupId: String, - testTaskId: String, - targetAppId: String, - coveragePeriodDays: Int?, - targetInstanceId: String?, - targetCommitSha: String?, - targetBuildVersion: String?, - baselineInstanceId: String?, - baselineCommitSha: String?, - baselineBuildVersion: String? - ): List = transaction { val targetBuildId = generateBuildId( groupId, - targetAppId, + appId, targetInstanceId, targetCommitSha, - targetBuildVersion - ).also { buildId -> - if (!metricsRepository.buildExists(buildId)) { - throw BuildNotFound("Target build info not found for $buildId") - } + targetBuildVersion, + """ + Provide at least one the following: targetInstanceId, targetCommitSha, targetBuildVersion + """.trimIndent() + ) + if (!metricsRepository.buildExists(targetBuildId)) { + throw BuildNotFound("Target build info not found for $targetBuildId") } - val hasBaselineBuild = listOf(baselineInstanceId, baselineCommitSha, baselineBuildVersion).any { it != null } - val baselineBuildId = takeIf { hasBaselineBuild }?.let { - generateBuildId( - groupId, - targetAppId, - baselineInstanceId, - baselineCommitSha, - baselineBuildVersion - ).also { buildId -> - if (!metricsRepository.buildExists(buildId)) { - throw BuildNotFound("Baseline build info not found for $buildId") - } - } - } val coveragePeriodFrom = (coveragePeriodDays ?: testRecommendationsConfig.coveragePeriodDays).let { LocalDateTime.now().minusDays(it.toLong()) } - metricsRepository.getRecommendedTests( + + val recommendedTests = metricsRepository.getRecommendedTests( groupId = groupId, targetBuildId = targetBuildId, - testsToSkip = true, - testTaskId = testTaskId, baselineBuildId = baselineBuildId, + testsToSkip = testsToSkip, + testTaskId = testTaskId, coveragePeriodFrom = coveragePeriodFrom - ).map { - TestToSkipView( - engine = it["test_engine"] as String, - testName = it["test_name"] as String, - path = it["test_path"] as String - ) - } + ) + + // TODO add recommended tests UI link + // val recommendedTestsReportPath = metricsServiceUiLinksConfig.recommendedTestsReportPath + mapOf( + "inputParameters" to mapOf( + "groupId" to groupId, + "appId" to appId, + "targetInstanceId" to targetInstanceId, + "targetCommitSha" to targetCommitSha, + "targetBuildVersion" to targetBuildVersion, + "baselineInstanceId" to baselineInstanceId, + "baselineCommitSha" to baselineCommitSha, + "baselineBuildVersion" to baselineBuildVersion, + ), + "inferredValues" to mapOf( + "build" to targetBuildId, + "baselineBuild" to baselineBuildId, + ), + "data" to recommendedTests, + ) } // TODO good candidate to be moved to common functions (probably) diff --git a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql index afc647de9..efa5b547c 100644 --- a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql +++ b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql @@ -1298,131 +1298,6 @@ $$; ----------------------------------------------------------------- ------------------------------------------------------------------ -CREATE OR REPLACE FUNCTION raw_data.get_recommended_tests( - input_build_id VARCHAR, - input_baseline_build_id VARCHAR, - only_modified_risks BOOLEAN DEFAULT FALSE -) RETURNS TABLE ( - __id VARCHAR, - __group_id VARCHAR, - __type VARCHAR, - __runner VARCHAR, - __name VARCHAR, - __path VARCHAR, - __created_at TIMESTAMP, - __tags VARCHAR[], - __metadata JSON -) AS $$ -BEGIN - RETURN QUERY - WITH - Risks AS ( - SELECT * - FROM raw_data.get_risks(input_build_id, input_baseline_build_id) - WHERE (only_modified_risks IS TRUE AND __risk_type = 'modified') - OR (only_modified_risks IS FALSE) - ), - RelatedBuilds AS ( - WITH - OriginalBuild AS ( - SELECT * - FROM raw_data.builds builds - WHERE builds.id = input_build_id - ) - SELECT * - FROM raw_data.builds builds - WHERE builds.id <> (SELECT id FROM OriginalBuild) - AND builds.group_id = (SELECT group_id FROM OriginalBuild) - AND builds.app_id = (SELECT app_id FROM OriginalBuild) - ), - RelatedMethods AS ( - WITH - RelatedBuildMethods AS ( - SELECT * - FROM raw_data.methods methods - WHERE - methods.build_id IN (SELECT DISTINCT id FROM RelatedBuilds) - AND probes_count > 0 - AND signature IN (SELECT DISTINCT __signature FROM Risks) - ) - SELECT - Risks.__signature, - related.classname, - related.build_id, - related.body_checksum, - related.probe_start_pos, - related.probes_count - FROM Risks - LEFT JOIN RelatedBuildMethods related ON Risks.__signature = related.signature - ), - RelatedCoverage AS ( - WITH - Instances AS ( - SELECT DISTINCT - instances.build_id, - instances.id AS instance_id, - classname - FROM RelatedMethods - LEFT JOIN raw_data.instances instances - ON instances.build_id = RelatedMethods.build_id - ), - Coverage AS ( - SELECT DISTINCT - Instances.build_id, - coverage.instance_id, - coverage.classname, - def.id as test_definition_id, - def.name as test_name, - coverage.probes - FROM Instances - JOIN raw_data.coverage coverage - ON coverage.instance_id = Instances.instance_id - AND coverage.classname = Instances.classname - JOIN raw_data.test_launches launches ON coverage.test_id = launches.id - JOIN raw_data.test_definitions def ON launches.test_definition_id = def.id - ) - SELECT - methods.*, - Coverage.test_definition_id, - Coverage.test_name, - Coverage.probes, - SUBSTRING(Coverage.probes FROM methods.probe_start_pos + 1 FOR methods.probes_count) AS method_probes - FROM RelatedMethods methods - JOIN Coverage - ON methods.build_id = Coverage.build_id - AND methods.classname = Coverage.classname - AND BIT_COUNT(SUBSTRING(Coverage.probes FROM methods.probe_start_pos + 1 FOR methods.probes_count)) > 0 - ), - RisksToTests AS ( - SELECT - Risks.__risk_type AS risk_type, - Risks.__name AS risk_name, - Risks.__signature AS risk_signature, - Risks.__classname AS risk_classname, - Risks.__body_checksum AS risk_body_checksum, - -- can prioritize by - RelatedCoverage.body_checksum, -- matching body - it's literally the same code which was already covered - RelatedCoverage.build_id AS related_build_id, -- more recent builds - RelatedCoverage.test_name, - RelatedCoverage.test_definition_id, - RelatedCoverage.probes, -- more coverage for class overall? - RelatedCoverage.method_probes -- more coverage (BIT_COUNT()) for _that specific method_ - FROM Risks - LEFT JOIN RelatedCoverage ON Risks.__signature = RelatedCoverage.__signature - ), - TestDefinitionIds AS ( - SELECT DISTINCT test_definition_id AS id - FROM RisksToTests - ) - SELECT def.* - FROM raw_data.test_definitions def - JOIN TestDefinitionIds ON TestDefinitionIds.id = def.id; -END; -$$ LANGUAGE plpgsql; - ------------------------------------------------------------------ - ----------------------------------------------------------------- CREATE OR REPLACE FUNCTION raw_data.get_coverage_by_methods_list_v2( @@ -2100,9 +1975,14 @@ CREATE OR REPLACE FUNCTION raw_data.get_recommended_tests_v2( input_coverage_period_from TIMESTAMP DEFAULT NULL ) RETURNS TABLE( __test_definition_id VARCHAR, - __test_runner_id VARCHAR, + __test_runner VARCHAR, __test_path VARCHAR, - __test_name VARCHAR + __test_name VARCHAR, + __test_type VARCHAR, + __group_id VARCHAR, + __created_at TIMESTAMP, + __tags VARCHAR[], + __metadata JSON ) AS $$ BEGIN RETURN QUERY @@ -2160,7 +2040,12 @@ BEGIN def.id as test_definition_id, def.name as test_name, def.runner as test_runner, - def.path as test_path + def.path as test_path, + def.type as test_type, + def.group_id, + def.created_at, + def.tags, + def.metadata FROM raw_data.test_definitions def WHERE def.group_id = input_group_id AND def.id IN ( @@ -2185,7 +2070,12 @@ BEGIN test_definition_id, test_runner, test_path, - test_name + test_name, + test_type, + group_id, + created_at, + tags, + metadata FROM Tests; END; $$ LANGUAGE plpgsql; From 6e537600a5a642f2f198f3f5ce6004d200075c74 Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Thu, 23 Jan 2025 13:08:31 +0400 Subject: [PATCH 04/17] fix: endpoint recommended-tests testsToSkip set default to false --- .../kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt index b6e47a977..894f5bb7d 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/MetricRoutes.kt @@ -62,7 +62,7 @@ class Metrics { val groupId: String, val appId: String, - val testsToSkip: Boolean = true, + val testsToSkip: Boolean = false, val testTaskId: String? = null, val targetInstanceId: String? = null, val targetCommitSha: String? = null, From e71fb7de544e65f42c79846fcfc8bc11ea5e73ee Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Thu, 23 Jan 2025 13:34:40 +0400 Subject: [PATCH 05/17] fix: null-values to sql handling --- .../admin/metrics/config/MetricsDatabaseConfig.kt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt index 395297752..64eec70a8 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt @@ -19,6 +19,7 @@ import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.IColumnType +import org.jetbrains.exposed.sql.TextColumnType import org.jetbrains.exposed.sql.Transaction import org.jetbrains.exposed.sql.statements.Statement import org.jetbrains.exposed.sql.statements.StatementType @@ -83,7 +84,16 @@ private fun Transaction.executePreparedStatement(stmt: String, vararg return exec(object : Statement(StatementType.SELECT, emptyList()) { override fun PreparedStatementApi.executeInternal(transaction: Transaction): T? { - params.forEachIndexed { idx, value -> set(idx + 1, value ?: "NULL") } + params.forEachIndexed { idx, value -> + if (value != null) { + set(idx + 1, value) + } else { + // WORKAROUND: TextColumnType is employed to trick expose to write null values + // Possible issues with: BinaryColumnType, BlobColumnType + // see setNull implementation for more details + setNull(idx + 1, TextColumnType()) + } + } executeQuery() return resultSet?.use { transform(it) } } From 00163c90c663bc9dcdb133d6ee34b62b8c8818c8 Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Thu, 23 Jan 2025 15:58:24 +0400 Subject: [PATCH 06/17] fix: save test tags as array --- .../com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt | 2 +- .../admin/writer/rawdata/service/impl/RawDataServiceImpl.kt | 2 +- .../drill/admin/writer/rawdata/table/TestDefinitionTable.kt | 2 +- gradle.properties | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt index fa79c7232..550171110 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/TestMetadata.kt @@ -37,7 +37,7 @@ class TestDefinition( val runner: String?, val name: String?, val path: String?, - val tags: String?, + val tags: List?, val metadata: Map? ) diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt index e36c5e15e..512bdeb7f 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/service/impl/RawDataServiceImpl.kt @@ -181,7 +181,7 @@ class RawDataServiceImpl( runner = test.details.engine, name = test.details.testName, path = test.details.path, - tags = test.details.labels.map { x -> x.value }.joinToString(","), + tags = test.details.labels.map { label -> label.value }, metadata = test.details.metadata ) }.let(testDefinitionRepository::createMany) diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt index f974a9be6..43b44a7c0 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/table/TestDefinitionTable.kt @@ -27,7 +27,7 @@ object TestDefinitionTable : StringIdTable("raw_data.test_definitions") { val runner = varchar("runner", SHORT_TEXT_LENGTH).nullable() val name = varchar("name", MEDIUM_TEXT_LENGTH).nullable() val path = varchar("path", MEDIUM_TEXT_LENGTH).nullable() - val tags = varchar("tags", MEDIUM_TEXT_LENGTH).nullable() + val tags = array("tags", MEDIUM_TEXT_LENGTH).nullable() val metadata = json("metadata", Json, MapSerializer(String.serializer(), String.serializer())).nullable() val createdAt = datetime("created_at").nullable() } \ No newline at end of file diff --git a/gradle.properties b/gradle.properties index d60b7d60b..1bced1365 100644 --- a/gradle.properties +++ b/gradle.properties @@ -3,7 +3,7 @@ licenseVersion = 0.16.1 kotlinxCoroutinesVersion = 1.7.3 kotlinxSerializationVersion = 1.6.2 kotlinxDatetimeVersion = 0.4.1 -exposedVersion = 0.50.1 +exposedVersion = 0.51.1 grgitVersion = 4.1.1 ktorVersion = 2.3.11 kodeinVersion = 7.20.2 From 871053374bc64ab35fc7cebb920d8569567dba8f Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Thu, 23 Jan 2025 16:40:30 +0400 Subject: [PATCH 07/17] feat: filter reports by test_tags --- .../raw_data/db/migration/R__3_Functions.sql | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql index efa5b547c..d365a2df9 100644 --- a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql +++ b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql @@ -1643,7 +1643,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_materialized_coverage_by_methods_list_v2 test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __signature VARCHAR, @@ -1694,6 +1695,12 @@ BEGIN AND (coverage_created_at_start IS NULL OR coverage.created_at >= coverage_created_at_start) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test_tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY target.signature ) SELECT @@ -1722,7 +1729,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_materialized_aggregate_coverage_by_metho test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, coverage_branches VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __signature VARCHAR, @@ -1776,6 +1784,12 @@ BEGIN AND (coverage_branches IS NULL OR coverage.branch = ANY(coverage_branches)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test_tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY target.signature ) SELECT @@ -1807,7 +1821,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_materialized_aggregate_coverage_by_risks test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, coverage_branches VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __risk_type VARCHAR, @@ -1862,6 +1877,12 @@ BEGIN AND (coverage_branches IS NULL OR coverage.branch = ANY(coverage_branches)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test_tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY risks.signature ) SELECT @@ -1893,7 +1914,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_materialized_coverage_by_risks_v2( coverage_created_at_start TIMESTAMP DEFAULT NULL, test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __risk_type VARCHAR, @@ -1945,6 +1967,12 @@ BEGIN AND (test_results IS NULL OR coverage.test_result = ANY(test_results)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test_tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY risks.signature ) SELECT From db18e1dcc8de70aa6a8b03ca14301a2e22a4bb1f Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Thu, 23 Jan 2025 19:21:09 +0400 Subject: [PATCH 08/17] feat: filter reports with view_methods_coverage by test_tags --- .../raw_data/db/migration/R__3_Functions.sql | 44 +++++++++++++++---- 1 file changed, 36 insertions(+), 8 deletions(-) diff --git a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql index d365a2df9..773fb02d3 100644 --- a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql +++ b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql @@ -1310,7 +1310,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_coverage_by_methods_list_v2( test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __signature VARCHAR, @@ -1361,6 +1362,12 @@ BEGIN AND (coverage_created_at_start IS NULL OR coverage.created_at >= coverage_created_at_start) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY target.signature ) SELECT @@ -1389,7 +1396,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_aggregate_coverage_by_methods_list_v2( test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, coverage_branches VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __signature VARCHAR, @@ -1443,6 +1451,12 @@ BEGIN AND (coverage_branches IS NULL OR coverage.branch = ANY(coverage_branches)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY target.signature ) SELECT @@ -1474,7 +1488,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_aggregate_coverage_by_risks_v2( test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, coverage_branches VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __risk_type VARCHAR, @@ -1529,6 +1544,12 @@ BEGIN AND (coverage_branches IS NULL OR coverage.branch = ANY(coverage_branches)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY risks.signature ) SELECT @@ -1560,7 +1581,8 @@ CREATE OR REPLACE FUNCTION raw_data.get_coverage_by_risks_v2( coverage_created_at_start TIMESTAMP DEFAULT NULL, test_task_ids VARCHAR[] DEFAULT NULL, test_results VARCHAR[] DEFAULT NULL, - coverage_env_ids VARCHAR[] DEFAULT NULL + coverage_env_ids VARCHAR[] DEFAULT NULL, + test_tags VARCHAR DEFAULT NULL ) RETURNS TABLE ( __risk_type VARCHAR, @@ -1612,6 +1634,12 @@ BEGIN AND (test_results IS NULL OR coverage.test_result = ANY(test_results)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) + --filter by test tags + AND (test_tags IS NULL OR coverage.test_definition_id IN ( + SELECT id + FROM raw_data.test_definitions def + WHERE (test_tags = ANY(def.tags)) + )) GROUP BY risks.signature ) SELECT @@ -1695,7 +1723,7 @@ BEGIN AND (coverage_created_at_start IS NULL OR coverage.created_at >= coverage_created_at_start) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) - --filter by test_tags + --filter by test tags AND (test_tags IS NULL OR coverage.test_definition_id IN ( SELECT id FROM raw_data.test_definitions def @@ -1784,7 +1812,7 @@ BEGIN AND (coverage_branches IS NULL OR coverage.branch = ANY(coverage_branches)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) - --filter by test_tags + --filter by test tags AND (test_tags IS NULL OR coverage.test_definition_id IN ( SELECT id FROM raw_data.test_definitions def @@ -1877,7 +1905,7 @@ BEGIN AND (coverage_branches IS NULL OR coverage.branch = ANY(coverage_branches)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) - --filter by test_tags + --filter by test tags AND (test_tags IS NULL OR coverage.test_definition_id IN ( SELECT id FROM raw_data.test_definitions def @@ -1967,7 +1995,7 @@ BEGIN AND (test_results IS NULL OR coverage.test_result = ANY(test_results)) --filter by envs AND (coverage_env_ids IS NULL OR coverage.env_id = ANY(coverage_env_ids)) - --filter by test_tags + --filter by test tags AND (test_tags IS NULL OR coverage.test_definition_id IN ( SELECT id FROM raw_data.test_definitions def From db4dfb6b7f46b76f903684c41784fff8839f8cfc Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Mon, 27 Jan 2025 17:15:46 +0400 Subject: [PATCH 09/17] fix: migration file naming --- ...efinitions_Metadata.sql => V10__Test_Definitions_Metadata.sql} | 0 ..._Tags_As_Array.sql => V11__Test_Definitions_Tags_As_Array.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename admin-writer/src/main/resources/raw_data/db/migration/{V10_Test_Definitions_Metadata.sql => V10__Test_Definitions_Metadata.sql} (100%) rename admin-writer/src/main/resources/raw_data/db/migration/{V11_Test_Definitions_Tags_As_Array.sql => V11__Test_Definitions_Tags_As_Array.sql} (100%) diff --git a/admin-writer/src/main/resources/raw_data/db/migration/V10_Test_Definitions_Metadata.sql b/admin-writer/src/main/resources/raw_data/db/migration/V10__Test_Definitions_Metadata.sql similarity index 100% rename from admin-writer/src/main/resources/raw_data/db/migration/V10_Test_Definitions_Metadata.sql rename to admin-writer/src/main/resources/raw_data/db/migration/V10__Test_Definitions_Metadata.sql diff --git a/admin-writer/src/main/resources/raw_data/db/migration/V11_Test_Definitions_Tags_As_Array.sql b/admin-writer/src/main/resources/raw_data/db/migration/V11__Test_Definitions_Tags_As_Array.sql similarity index 100% rename from admin-writer/src/main/resources/raw_data/db/migration/V11_Test_Definitions_Tags_As_Array.sql rename to admin-writer/src/main/resources/raw_data/db/migration/V11__Test_Definitions_Tags_As_Array.sql From 77661085ddc81a5f091309b99db87998e3aef5c5 Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Mon, 27 Jan 2025 19:25:55 +0400 Subject: [PATCH 10/17] feat: raw_data.get_test_launches filter by group_id --- .../resources/raw_data/db/migration/R__3_Functions.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql index 773fb02d3..c762646f4 100644 --- a/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql +++ b/admin-writer/src/main/resources/raw_data/db/migration/R__3_Functions.sql @@ -934,7 +934,10 @@ $$ LANGUAGE plpgsql; ----------------------------------------------------------------- ----------------------------------------------------------------- -CREATE OR REPLACE FUNCTION raw_data.get_test_launches(input_build_id VARCHAR) +CREATE OR REPLACE FUNCTION raw_data.get_test_launches( + input_build_id VARCHAR, + input_group_id VARCHAR +) RETURNS TABLE ( __id VARCHAR, __group_id VARCHAR, @@ -980,6 +983,9 @@ BEGIN JOIN Coverage ON Coverage.test_id = launch.id JOIN raw_data.test_definitions definitions on definitions.id = launch.test_definition_id JOIN raw_data.test_sessions sessions ON launch.test_session_id = sessions.id + WHERE launch.group_id = input_group_id + AND definitions.group_id = input_group_id + AND sessions.group_id = input_group_id ) SELECT * FROM TestLaunches; From 7f7a9651a748ba844898369db8661df13257517b Mon Sep 17 00:00:00 2001 From: RomanDavlyatshin Date: Mon, 27 Jan 2025 19:46:19 +0400 Subject: [PATCH 11/17] feat: update swagger definitions / openapi.yml --- admin-app/src/main/resources/openapi.yml | 55 +++--------------------- 1 file changed, 5 insertions(+), 50 deletions(-) diff --git a/admin-app/src/main/resources/openapi.yml b/admin-app/src/main/resources/openapi.yml index 9a0944c27..c556f0a81 100644 --- a/admin-app/src/main/resources/openapi.yml +++ b/admin-app/src/main/resources/openapi.yml @@ -348,58 +348,13 @@ paths: required: true schema: type: string - - name: instanceId - in: query - schema: - type: string - - name: commitSha - in: query - schema: - type: string - - name: buildVersion - in: query - schema: - type: string - - name: baselineInstanceId - in: query - schema: - type: string - - name: baselineCommitSha - in: query - schema: - type: string - - name: baselineBuildVersion - in: query - schema: - type: string - responses: - '200': - description: Recommended tests - content: - application/json: - schema: - $ref: '#/components/schemas/DataResponse' - /api/metrics/tests-to-skip: - get: - summary: Get tests to skip - tags: - - metrics - security: - - apiKeyAuth: [ ] - parameters: - - name: groupId + - name: testsToSkip in: query - required: true schema: - type: string + type: boolean + default: false - name: testTaskId in: query - required: true - schema: - type: string - - name: appId - in: query - required: true schema: type: string - name: targetInstanceId @@ -432,11 +387,11 @@ paths: type: integer responses: '200': - description: Tests to skip + description: Recommended tests content: application/json: schema: - $ref: '#/components/schemas/ListDataResponse' + $ref: '#/components/schemas/DataResponse' # User Authentication Routes /api/sign-in: post: From d29020acbc2edb0f36a3ab0a44c79a3e875e6988 Mon Sep 17 00:00:00 2001 From: iryabov Date: Fri, 31 Jan 2025 08:59:49 +0100 Subject: [PATCH 12/17] refactor: move `PgArray` and `PGobject` types to `MetricsDatabaseConfig` EPMDJ-10972 --- .../metrics/config/MetricsDatabaseConfig.kt | 42 ++++++++++++++++--- .../metrics/repository/impl/AnySerializer.kt | 20 --------- 2 files changed, 37 insertions(+), 25 deletions(-) diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt index 64eec70a8..ef3b5f840 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/config/MetricsDatabaseConfig.kt @@ -17,6 +17,10 @@ package com.epam.drill.admin.metrics.config import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.Dispatchers +import kotlinx.serialization.builtins.ListSerializer +import kotlinx.serialization.builtins.MapSerializer +import kotlinx.serialization.builtins.serializer +import kotlinx.serialization.json.* import org.jetbrains.exposed.sql.Database import org.jetbrains.exposed.sql.IColumnType import org.jetbrains.exposed.sql.TextColumnType @@ -25,6 +29,8 @@ import org.jetbrains.exposed.sql.statements.Statement import org.jetbrains.exposed.sql.statements.StatementType import org.jetbrains.exposed.sql.statements.api.PreparedStatementApi import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction +import org.postgresql.jdbc.PgArray +import org.postgresql.util.PGobject import java.sql.ResultSet import javax.sql.DataSource @@ -53,13 +59,27 @@ fun Transaction.executeQueryReturnMap(sqlQuery: String, vararg params: Any?): Li for (i in 1..columnCount) { val name = metaData.getColumnName(i) - val columnValue = resultSet.getObject(i) + val value = when (val dbValue = resultSet.getObject(i)) { + is PgArray -> { + val array = dbValue.array + array?.let { it as Array<*> }?.toList() ?: emptyList() + } - when (columnValue) { - is java.sql.Timestamp -> columnValue.toLocalDateTime() + is PGobject -> { + val json = dbValue.value + if (json != null) { + val jsonObject = Json.parseToJsonElement(json).jsonObject + jsonObject.mapValues { (_, value) -> value.jsonPrimitiveOrElement() } + } else { + emptyMap() + } + } + + is java.sql.Timestamp -> dbValue.toLocalDateTime() + else -> dbValue } - rowObject[name] = if (resultSet.wasNull()) null else columnValue + rowObject[name] = if (resultSet.wasNull()) null else value } result.add(rowObject) } @@ -79,7 +99,11 @@ fun Transaction.executeUpdate(sql: String, vararg params: Any?) { }) } -private fun Transaction.executePreparedStatement(stmt: String, vararg params: Any?, transform: (ResultSet) -> T): T? { +private fun Transaction.executePreparedStatement( + stmt: String, + vararg params: Any?, + transform: (ResultSet) -> T +): T? { if (stmt.isEmpty()) return null return exec(object : Statement(StatementType.SELECT, emptyList()) { @@ -102,3 +126,11 @@ private fun Transaction.executePreparedStatement(stmt: String, vararg override fun arguments(): Iterable, Any?>>> = emptyList() }) } + +private fun JsonElement.jsonPrimitiveOrElement(): Any { + return when (this) { + is JsonPrimitive -> this.contentOrNull ?: this.booleanOrNull ?: this.longOrNull ?: this.doubleOrNull ?: "" + is JsonObject -> this.mapValues { (_, value) -> value.jsonPrimitiveOrElement() } + is JsonArray -> this.map { it.jsonPrimitiveOrElement() } + } +} diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt index 34a7ffb60..52029d33b 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/repository/impl/AnySerializer.kt @@ -44,30 +44,10 @@ object AnySerializer : KSerializer { val mapSerializer = MapSerializer(String.serializer(), this) encoder.encodeSerializableValue(mapSerializer, value as Map) } - is PgArray -> { - val array = value.array - if (array != null) { - val list = (array as Array<*>).map { it as String } - val listSerializer = ListSerializer(String.serializer()) - encoder.encodeSerializableValue(listSerializer, list) - } else { - encoder.encodeNull() - } - } is List<*> -> { val listSerializer = ListSerializer(this) encoder.encodeSerializableValue(listSerializer, value) } - is PGobject -> { - val json = value.value - if (json != null) { - val parsedValue = Json.decodeFromString>(json) - val mapSerializer = MapSerializer(String.serializer(), this) - encoder.encodeSerializableValue(mapSerializer, parsedValue) - } else { - encoder.encodeNull() - } - } else -> { val serializer = value::class.serializerOrNull() as KSerializer? if (serializer != null) From 59d0bf405c549856386af3c93a9c87b59d3980ba Mon Sep 17 00:00:00 2001 From: iryabov Date: Fri, 31 Jan 2025 12:24:59 +0100 Subject: [PATCH 13/17] refactor: remove `createdAt` from `RecommendedTestsView` EPMDJ-10972 --- ...TestToSkipView.kt => RecommendedTestsView.kt} | 11 +++++++---- .../metrics/service/impl/MetricsServiceImpl.kt | 16 +++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) rename admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/{TestToSkipView.kt => RecommendedTestsView.kt} (77%) diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/TestToSkipView.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/RecommendedTestsView.kt similarity index 77% rename from admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/TestToSkipView.kt rename to admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/RecommendedTestsView.kt index 44b3fc1e3..60e36f8cf 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/TestToSkipView.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/route/response/RecommendedTestsView.kt @@ -15,13 +15,16 @@ */ package com.epam.drill.admin.metrics.route.response +import kotlinx.datetime.LocalDateTime import kotlinx.serialization.Serializable @Serializable -class TestToSkipView( +class RecommendedTestsView( + val testDefinitionId: String, val testRunner: String, - val path: String, + val testPath: String, val testName: String, - val params: Map = emptyMap(), - val metadata: Map = emptyMap(), + val testType: String, + val tags: List, + val metadata: Map, ) \ No newline at end of file diff --git a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt index 7de521af1..df4e391a1 100644 --- a/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt +++ b/admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/service/impl/MetricsServiceImpl.kt @@ -20,7 +20,7 @@ import com.epam.drill.admin.metrics.config.MetricsServiceUiLinksConfig import com.epam.drill.admin.metrics.config.TestRecommendationsConfig import com.epam.drill.admin.metrics.exception.BuildNotFound import com.epam.drill.admin.metrics.repository.MetricsRepository -import com.epam.drill.admin.metrics.route.response.TestToSkipView +import com.epam.drill.admin.metrics.route.response.RecommendedTestsView import com.epam.drill.admin.metrics.service.MetricsService import com.epam.drill.admin.common.service.generateBuildId import com.epam.drill.admin.metrics.views.BuildView @@ -203,7 +203,17 @@ class MetricsServiceImpl( testsToSkip = testsToSkip, testTaskId = testTaskId, coveragePeriodFrom = coveragePeriodFrom - ) + ).map { data -> + RecommendedTestsView( + testDefinitionId = data["test_definition_id"] as String, + testRunner = data["test_runner"] as String, + testPath = data["test_path"] as String, + testName = data["test_name"] as String, + testType = data["test_type"] as String, + tags = data["tags"]?.let { it as List } ?: emptyList(), + metadata = data["metadata"]?.let { it as Map } ?: emptyMap() + ) + } // TODO add recommended tests UI link // val recommendedTestsReportPath = metricsServiceUiLinksConfig.recommendedTestsReportPath @@ -222,7 +232,7 @@ class MetricsServiceImpl( "build" to targetBuildId, "baselineBuild" to baselineBuildId, ), - "data" to recommendedTests, + "recommendedTests" to recommendedTests, ) } From 445f02309aed119cce9b3154c992211061967355 Mon Sep 17 00:00:00 2001 From: iryabov Date: Fri, 31 Jan 2025 12:37:07 +0100 Subject: [PATCH 14/17] refactor: rename `TestInfo` into `TestLaunchInfo` feat: rename `engine` to `runner` in /test-metadata endpoint feat: rename `labels` to `tags` in /test-metadata endpoint feat: move `groupId` from `TestLaunchInfo` to `AddTestsPayload` EPMDJ-10972 --- admin-app/src/main/resources/openapi.yml | 12 ++++++------ .../route/payload/TestMetadataPayload.kt | 18 ++++++------------ .../rawdata/service/impl/RawDataServiceImpl.kt | 10 +++++----- 3 files changed, 17 insertions(+), 23 deletions(-) diff --git a/admin-app/src/main/resources/openapi.yml b/admin-app/src/main/resources/openapi.yml index c556f0a81..2893927b8 100644 --- a/admin-app/src/main/resources/openapi.yml +++ b/admin-app/src/main/resources/openapi.yml @@ -877,6 +877,8 @@ components: AddTestsPayload: type: object properties: + groupId: + type: string sessionId: type: string tests: @@ -886,9 +888,7 @@ components: TestInfo: type: object properties: - groupId: - type: string - id: + testLaunchId: type: string testDefinitionId: type: string @@ -903,7 +903,7 @@ components: TestDetails: type: object properties: - engine: + runner: type: string path: type: string @@ -917,10 +917,10 @@ components: type: object additionalProperties: type: string - labels: + tags: type: array items: - $ref: '#/components/schemas/Label' + type: string Label: type: object properties: diff --git a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt index 23f69f0e5..ee856f478 100644 --- a/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt +++ b/admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/route/payload/TestMetadataPayload.kt @@ -21,14 +21,14 @@ import kotlinx.serialization.Serializable // TODO rework alongside with Java Autotest Agent @Serializable class AddTestsPayload( + val groupId: String, val sessionId: String, - val tests: List = emptyList(), + val tests: List = emptyList(), ) @Serializable -class TestInfo( - val groupId: String, - val id: String, +class TestLaunchInfo( + val testLaunchId: String, val testDefinitionId: String, val result: TestResult, val startedAt: Long, @@ -36,20 +36,14 @@ class TestInfo( val details: TestDetails, ) -@Serializable -class Label( - val name: String, - val value: String, -) - @Serializable class TestDetails @JvmOverloads constructor( - val engine: String = "", + val runner: String = "", val path: String = "", val testName: String = "", val params: Map = emptyMap(), val metadata: Map = emptyMap(), - val labels: Set