From 45bde978cbd0edfb4de1bbe2429d566337e2912a Mon Sep 17 00:00:00 2001 From: Sergei Kharitontcev-Beglov Date: Tue, 30 Jul 2024 13:00:16 +0200 Subject: [PATCH 1/4] Added dumping JaCoCo Coverage Information into the JSON file --- .../research/kex/jacoco/CoverageInfo.kt | 20 ++- .../research/kex/jacoco/CoverageReporter.kt | 11 +- .../vorpal/research/kex/jacoco/Serializers.kt | 153 ++++++++++++++++++ 3 files changed, 174 insertions(+), 10 deletions(-) create mode 100644 kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/Serializers.kt diff --git a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageInfo.kt b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageInfo.kt index e2ae42837..bcb0dd4f9 100644 --- a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageInfo.kt +++ b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageInfo.kt @@ -1,15 +1,18 @@ package org.vorpal.research.kex.jacoco +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable import org.vorpal.research.kthelper.assert.unreachable import org.vorpal.research.kthelper.logging.log - -interface CoverageInfo { +@Serializable +sealed interface CoverageInfo { val covered: Int val total: Int val ratio: Double } +@Serializable enum class CoverageUnit(unit: String) { INSTRUCTION("instructions"), BRANCH("branches"), @@ -33,6 +36,7 @@ enum class CoverageUnit(unit: String) { } } +@Serializable enum class AnalysisUnit(unit: String) { METHOD("method"), CLASS("class"), @@ -54,6 +58,8 @@ enum class AnalysisUnit(unit: String) { } } +@Serializable +@SerialName("genericCoverage") data class GenericCoverageInfo( override val covered: Int, override val total: Int, @@ -74,7 +80,8 @@ data class GenericCoverageInfo( } } -abstract class CommonCoverageInfo( +@Serializable +sealed class CommonCoverageInfo( val name: String, val level: AnalysisUnit, val instructionCoverage: CoverageInfo, @@ -103,7 +110,8 @@ abstract class CommonCoverageInfo( return name.hashCode() } } - +@Serializable(with = MethodCoverageInfoSerializer::class) +@SerialName("method") class MethodCoverageInfo( name: String, instructionCoverage: CoverageInfo, @@ -119,6 +127,8 @@ class MethodCoverageInfo( complexityCoverage ) +@Serializable(with = ClassCoverageInfoSerializer::class) +@SerialName("class") class ClassCoverageInfo( name: String, instructionCoverage: CoverageInfo, @@ -146,6 +156,8 @@ class ClassCoverageInfo( } } +@Serializable(with = PackageCoverageInfoSerializer::class) +@SerialName("package") class PackageCoverageInfo( name: String, instructionCoverage: CoverageInfo, diff --git a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt index 992eb2133..e3f09f52b 100644 --- a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt +++ b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt @@ -2,6 +2,8 @@ package org.vorpal.research.kex.jacoco +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json import org.jacoco.core.analysis.Analyzer import org.jacoco.core.analysis.CoverageBuilder import org.jacoco.core.analysis.ICounter @@ -44,12 +46,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.attribute.BasicFileAttributes import java.util.* -import kotlin.io.path.inputStream -import kotlin.io.path.name -import kotlin.io.path.readBytes -import kotlin.io.path.relativeTo -import kotlin.io.path.writeBytes -import kotlin.streams.toList +import kotlin.io.path.* import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds @@ -394,6 +391,8 @@ fun reportCoverage( else -> coverageReporter.computeCoverage(analysisLevel, testClasses) } + kexConfig.getPathValue("kex", "coverageJsonLocation") + ?.writeText(Json.encodeToString(coverageInfo)) log.info( coverageInfo.print(kexConfig.getBooleanValue("kex", "printDetailedCoverage", false)) diff --git a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/Serializers.kt b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/Serializers.kt new file mode 100644 index 000000000..040070294 --- /dev/null +++ b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/Serializers.kt @@ -0,0 +1,153 @@ +package org.vorpal.research.kex.jacoco + +import kotlinx.serialization.KSerializer +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.descriptors.SerialDescriptor +import kotlinx.serialization.encoding.Decoder +import kotlinx.serialization.encoding.Encoder + +/** + * A surrogate class used to simplify the code of the [SurrogateCoverageInfo] + * by providing basic functionality of the decoding/encoding of the [CommonCoverageInfo]. The trick is found in + * [kotlinx serialization guide](https://github.com/Kotlin/kotlinx.serialization/blob/master/docs/serializers.md#composite-serializer-via-surrogate). + * + * + * The abstract class cannot be used in the first way to provide the serializer to abstract class + */ +@Serializable +private sealed class SurrogateCoverageInfo { + abstract val name: String + abstract val level: AnalysisUnit + abstract val instructionCoverage: CoverageInfo + abstract val branchCoverage: CoverageInfo + abstract val linesCoverage: CoverageInfo + abstract val complexityCoverage: CoverageInfo +} + +@Serializable +@SerialName("method") +private data class SurrogateMethodCoverageInfo( + override val name: String, + override val level: AnalysisUnit, + override val instructionCoverage: CoverageInfo, + override val branchCoverage: CoverageInfo, + override val linesCoverage: CoverageInfo, + override val complexityCoverage: CoverageInfo +) : SurrogateCoverageInfo() { + val value: MethodCoverageInfo + get() = MethodCoverageInfo(name, instructionCoverage, branchCoverage, linesCoverage, complexityCoverage) + + companion object { + fun fromValue(value: MethodCoverageInfo) = SurrogateMethodCoverageInfo( + value.name, + value.level, + value.instructionCoverage, + value.branchCoverage, + value.linesCoverage, + value.complexityCoverage, + ) + } +} + +@Serializable +@SerialName("class") +private data class SurrogateClassCoverageInfo( + override val name: String, + override val level: AnalysisUnit, + override val instructionCoverage: CoverageInfo, + override val branchCoverage: CoverageInfo, + override val linesCoverage: CoverageInfo, + override val complexityCoverage: CoverageInfo, + val methods: List +) : SurrogateCoverageInfo() { + val value: ClassCoverageInfo + get() { + val result = ClassCoverageInfo(name, instructionCoverage, branchCoverage, linesCoverage, complexityCoverage) + result.methods.addAll(methods.map { it.value }) + return result + } + + companion object { + fun fromValue(value: ClassCoverageInfo) = SurrogateClassCoverageInfo( + value.name, + value.level, + value.instructionCoverage, + value.branchCoverage, + value.linesCoverage, + value.complexityCoverage, + value.methods.map { SurrogateMethodCoverageInfo.fromValue(it) } + ) + } +} + +@Serializable +@SerialName("package") +private data class SurrogatePackageCoverageInfo( + override val name: String, + override val level: AnalysisUnit, + override val instructionCoverage: CoverageInfo, + override val branchCoverage: CoverageInfo, + override val linesCoverage: CoverageInfo, + override val complexityCoverage: CoverageInfo, + val classes: List, +) : SurrogateCoverageInfo() { + val value: PackageCoverageInfo + get() { + val result = + PackageCoverageInfo(name, instructionCoverage, branchCoverage, linesCoverage, complexityCoverage) + result.classes.addAll(classes.map { it.value }) + return result + } + + companion object { + fun fromValue(value: PackageCoverageInfo) = SurrogatePackageCoverageInfo( + value.name, + value.level, + value.instructionCoverage, + value.branchCoverage, + value.linesCoverage, + value.complexityCoverage, + value.classes.map { SurrogateClassCoverageInfo.fromValue(it) } + ) + } +} + +object MethodCoverageInfoSerializer : KSerializer { + override val descriptor: SerialDescriptor = SurrogateMethodCoverageInfo.serializer().descriptor + override fun serialize(encoder: Encoder, value: MethodCoverageInfo) { + val surrogate = SurrogateMethodCoverageInfo.fromValue(value) + encoder.encodeSerializableValue(SurrogateMethodCoverageInfo.serializer(), surrogate) + } + + override fun deserialize(decoder: Decoder): MethodCoverageInfo { + val surrogate = decoder.decodeSerializableValue(SurrogateMethodCoverageInfo.serializer()) + return surrogate.value + } +} + +object ClassCoverageInfoSerializer : KSerializer { + override val descriptor: SerialDescriptor = SurrogateClassCoverageInfo.serializer().descriptor + override fun serialize(encoder: Encoder, value: ClassCoverageInfo) { + val surrogate = SurrogateClassCoverageInfo.fromValue(value) + encoder.encodeSerializableValue(SurrogateClassCoverageInfo.serializer(), surrogate) + } + override fun deserialize(decoder: Decoder): ClassCoverageInfo { + val surrogate = decoder.decodeSerializableValue(SurrogateClassCoverageInfo.serializer()) + return surrogate.value + } +} + +object PackageCoverageInfoSerializer : KSerializer { + override val descriptor: SerialDescriptor = SurrogatePackageCoverageInfo.serializer().descriptor + override fun serialize(encoder: Encoder, value: PackageCoverageInfo) { + val surrogate = SurrogatePackageCoverageInfo.fromValue(value) + encoder.encodeSerializableValue(SurrogatePackageCoverageInfo.serializer(), surrogate) + } + + override fun deserialize(decoder: Decoder): PackageCoverageInfo { + val surrogate = decoder.decodeSerializableValue(SurrogatePackageCoverageInfo.serializer()) + return surrogate.value + } +} + From 0f7376b41f81c6b3752ca2c42e748930ed8ee31b Mon Sep 17 00:00:00 2001 From: Sergei Kharitontcev-Beglov Date: Wed, 31 Jul 2024 11:50:46 +0200 Subject: [PATCH 2/4] Added more coverage collection to the Kex In the previous commits only target class are considered to fetch coverage. In this commit I introduced the collection of the coverage for all classes --- .../research/kex/jacoco/CoverageReporter.kt | 30 +++++++++++++------ .../research/kex/concolic/ConcolicTest.kt | 4 +-- .../research/kex/symbolic/SymbolicTest.kt | 4 +-- 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt index e3f09f52b..39745be96 100644 --- a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt +++ b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt @@ -105,7 +105,7 @@ open class CoverageReporter( open fun computeCoverage( analysisLevel: AnalysisLevel, testClasses: Set = this.allTestClasses - ): CommonCoverageInfo { + ): Set { ktassert(this.allTestClasses.containsAll(testClasses)) { log.error("Unexpected set of test classes") } @@ -114,9 +114,9 @@ open class CoverageReporter( val coverageBuilder = getCoverageBuilderAndCleanup(classes, testClasses) return when (analysisLevel) { - is PackageLevel -> getPackageCoverage(analysisLevel.pkg, cm, coverageBuilder) - is ClassLevel -> getClassCoverage(cm, coverageBuilder).first() - is MethodLevel -> getMethodCoverage(coverageBuilder, analysisLevel.method)!! + is PackageLevel -> setOf(getPackageCoverage(analysisLevel.pkg, cm, coverageBuilder)) + is ClassLevel -> getClassCoverage(cm, coverageBuilder) + is MethodLevel -> setOf(getMethodCoverage(coverageBuilder, analysisLevel.method)!!) } } @@ -156,8 +156,18 @@ open class CoverageReporter( .toList() is ClassLevel -> { - val klass = analysisLevel.klass.fullName.replace(Package.SEPARATOR, File.separatorChar) - listOf(jacocoInstrumentedDir.resolve("$klass.class")) + if (kexConfig.getBooleanValue("kex", "collectWholeCoverage", false)) { + Files.walk(jacocoInstrumentedDir) + .filter { it.isClass } + // filter stdlib-specific things + .filter { !it.fullyQualifiedName(jacocoInstrumentedDir).asmString.startsWith("java")} + .filter { !it.fullyQualifiedName(jacocoInstrumentedDir).asmString.startsWith("kotlin")} + .filter { !it.fullyQualifiedName(jacocoInstrumentedDir).asmString.startsWith("kex")} + .toList() + } else { + val klass = analysisLevel.klass.fullName.replace(Package.SEPARATOR, File.separatorChar) + listOf(jacocoInstrumentedDir.resolve("$klass.class")) + } } is MethodLevel -> { @@ -386,7 +396,7 @@ fun reportCoverage( coverageSaturation.toList() ) PermanentSaturationCoverageInfo.emit() - coverageSaturation[coverageSaturation.lastKey()]!! + setOf(coverageSaturation[coverageSaturation.lastKey()]!!) } else -> coverageReporter.computeCoverage(analysisLevel, testClasses) @@ -395,10 +405,12 @@ fun reportCoverage( ?.writeText(Json.encodeToString(coverageInfo)) log.info( - coverageInfo.print(kexConfig.getBooleanValue("kex", "printDetailedCoverage", false)) + coverageInfo.joinToString(System.lineSeparator()) { + it.print(kexConfig.getBooleanValue("kex", "printDetailedCoverage", false)) + } ) - PermanentCoverageInfo.putNewInfo(mode, analysisLevel.toString(), coverageInfo) + PermanentCoverageInfo.putNewInfo(mode, analysisLevel.toString(), coverageInfo.first()) PermanentCoverageInfo.emit() } } diff --git a/kex-runner/src/test/kotlin/org/vorpal/research/kex/concolic/ConcolicTest.kt b/kex-runner/src/test/kotlin/org/vorpal/research/kex/concolic/ConcolicTest.kt index c90ed0af3..77cb89fbf 100644 --- a/kex-runner/src/test/kotlin/org/vorpal/research/kex/concolic/ConcolicTest.kt +++ b/kex-runner/src/test/kotlin/org/vorpal/research/kex/concolic/ConcolicTest.kt @@ -34,8 +34,8 @@ abstract class ConcolicTest(testDirectoryName: String) : KexRunnerTest(testDirec } val coverage = CoverageReporter(klass.cm, listOf(jar)).computeCoverage(ClassLevel(klass)) - log.debug(coverage.print(true)) - assertEquals(expectedCoverage, coverage.instructionCoverage.ratio, eps) + log.debug(coverage.first().print(true)) + assertEquals(expectedCoverage, coverage.first().instructionCoverage.ratio, eps) } } } diff --git a/kex-runner/src/test/kotlin/org/vorpal/research/kex/symbolic/SymbolicTest.kt b/kex-runner/src/test/kotlin/org/vorpal/research/kex/symbolic/SymbolicTest.kt index aec99e3a5..3c92e02d7 100644 --- a/kex-runner/src/test/kotlin/org/vorpal/research/kex/symbolic/SymbolicTest.kt +++ b/kex-runner/src/test/kotlin/org/vorpal/research/kex/symbolic/SymbolicTest.kt @@ -33,7 +33,7 @@ abstract class SymbolicTest( } val coverage = CoverageReporter(klass.cm, listOf(jar)).computeCoverage(ClassLevel(klass)) - log.debug(coverage.print(true)) - assertEquals(expectedCoverage, coverage.instructionCoverage.ratio, eps) + log.debug(coverage.first().print(true)) + assertEquals(expectedCoverage, coverage.first().instructionCoverage.ratio, eps) } } From 3078d405a44d430f92bd715776d4140f888d6242 Mon Sep 17 00:00:00 2001 From: Sergei Kharitontcev-Beglov Date: Wed, 31 Jul 2024 16:42:12 +0200 Subject: [PATCH 3/4] Changed the way to add additional packages to the class level analysis --- .../research/kex/jacoco/CoverageReporter.kt | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt index 39745be96..ce49ddc0f 100644 --- a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt +++ b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt @@ -46,6 +46,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.attribute.BasicFileAttributes import java.util.* +import java.util.stream.Collectors import kotlin.io.path.* import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds @@ -156,18 +157,19 @@ open class CoverageReporter( .toList() is ClassLevel -> { - if (kexConfig.getBooleanValue("kex", "collectWholeCoverage", false)) { - Files.walk(jacocoInstrumentedDir) - .filter { it.isClass } - // filter stdlib-specific things - .filter { !it.fullyQualifiedName(jacocoInstrumentedDir).asmString.startsWith("java")} - .filter { !it.fullyQualifiedName(jacocoInstrumentedDir).asmString.startsWith("kotlin")} - .filter { !it.fullyQualifiedName(jacocoInstrumentedDir).asmString.startsWith("kex")} - .toList() - } else { - val klass = analysisLevel.klass.fullName.replace(Package.SEPARATOR, File.separatorChar) - listOf(jacocoInstrumentedDir.resolve("$klass.class")) - } + val additionalValues = kexConfig + .getStringValue("kex", "collectAdditionalCoverage") + ?.split(",") + ?.map { Package.parse(it.trim().asmString) } + ?.toSet() + ?: emptySet() + val targetKlass = analysisLevel.klass.fullName.replace(Package.SEPARATOR, File.separatorChar) + val targetFiles = listOf(jacocoInstrumentedDir.resolve("$targetKlass.class")) + val additionalFiles = Files.walk(jacocoInstrumentedDir) + .filter { it.isClass } + .filter { additionalValues.any { value -> value.isParent(it.fullyQualifiedName(jacocoInstrumentedDir).asmString) } } + .collect(Collectors.toList()) + targetFiles + additionalFiles } is MethodLevel -> { From 293b36ff7b1ab60485344d4e68dfef1ee28996b2 Mon Sep 17 00:00:00 2001 From: Sergei Kharitontcev-Beglov Date: Wed, 31 Jul 2024 16:43:33 +0200 Subject: [PATCH 4/4] Fixed the Java 11 compilation issues Change Java 16+ toList() to Collectors.toList() from Java 11 --- .../kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt index ce49ddc0f..f87c82e20 100644 --- a/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt +++ b/kex-runner/src/main/kotlin/org/vorpal/research/kex/jacoco/CoverageReporter.kt @@ -90,7 +90,7 @@ open class CoverageReporter( }.also { deleteOnExit(it) } - val allTestClasses: Set by lazy { Files.walk(compileDir).filter { it.isClass }.toList().toSet() } + val allTestClasses: Set by lazy { Files.walk(compileDir).filter { it.isClass }.collect(Collectors.toSet()) } protected val compileDir = kexConfig.compiledCodeDirectory protected lateinit var coverageContext: CoverageContext protected lateinit var executionData: Map @@ -154,7 +154,7 @@ open class CoverageReporter( is PackageLevel -> Files.walk(jacocoInstrumentedDir) .filter { it.isClass } .filter { analysisLevel.pkg.isParent(it.fullyQualifiedName(jacocoInstrumentedDir).asmString) } - .toList() + .collect(Collectors.toList()) is ClassLevel -> { val additionalValues = kexConfig