Promote mutation count, fix mutation shuffling.

cs125-illinois · Sep 30, 2022 · 3cb46f1 · 3cb46f1
1 parent 255d945
commit 3cb46f1
Show file tree

Hide file tree

Showing 12 changed files with 74 additions and 24 deletions.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -8,7 +8,7 @@ plugins {
 }
 subprojects {
     group = "com.github.cs125-illinois.questioner"
-    version = "2022.9.10"
+    version = "2022.9.11"
     tasks.withType<KotlinCompile> {
         kotlinOptions {
             jvmTarget = JavaVersion.VERSION_17.toString()

diff --git a/lib/src/main/kotlin/Annotations.kt b/lib/src/main/kotlin/Annotations.kt
@@ -23,7 +23,7 @@ annotation class Correct(
     val maxTimeout: Int = Question.TestingControl.DEFAULT_MAX_TIMEOUT,
     val timeoutMultiplier: Int = Question.TestingControl.DEFAULT_TIMEOUT_MULTIPLIER,
     val minMutationCount: Int = Question.TestingControl.DEFAULT_MIN_MUTATION_COUNT,
-    val maxMutationCount: Int = Question.TestingControl.DEFAULT_MAX_MUTATION_COUNT,
+    val maxMutationCount: Int = -1,
     val outputMultiplier: Int = Question.TestingControl.DEFAULT_OUTPUT_MULTIPLIER,
     val maxExtraComplexity: Int = Question.TestingControl.DEFAULT_MAX_EXTRA_COMPLEXITY,
     val maxDeadCode: Int = Question.TestingControl.DEFAULT_MAX_DEAD_CODE,
@@ -78,7 +78,6 @@ annotation class CheckFeatures {
                 "@$name methods must return List<String>!"
             }
             (method.genericReturnType as ParameterizedType).also { collectionType ->
-                @Suppress("PLATFORM_CLASS_MAPPED_TO_KOTLIN")
                 check(collectionType.rawType == java.util.List::class.java) {
                     "@$name methods must return List<String>"
                 }

diff --git a/lib/src/main/kotlin/Question.kt b/lib/src/main/kotlin/Question.kt
@@ -191,7 +191,6 @@ data class Question(
             const val DEFAULT_MAX_TIMEOUT = 2048
             const val DEFAULT_TIMEOUT_MULTIPLIER = 32
             const val DEFAULT_MIN_MUTATION_COUNT = 0
-            const val DEFAULT_MAX_MUTATION_COUNT = 32
             const val DEFAULT_OUTPUT_MULTIPLIER = 8
             const val DEFAULT_MAX_EXTRA_COMPLEXITY = 2
             const val DEFAULT_MAX_DEAD_CODE = 0
@@ -212,7 +211,7 @@ data class Question(
                 DEFAULT_MAX_TIMEOUT,
                 DEFAULT_TIMEOUT_MULTIPLIER,
                 DEFAULT_MIN_MUTATION_COUNT,
-                DEFAULT_MAX_MUTATION_COUNT,
+                null,
                 DEFAULT_OUTPUT_MULTIPLIER,
                 DEFAULT_MAX_EXTRA_COMPLEXITY,
                 DEFAULT_MAX_DEAD_CODE,

diff --git a/lib/src/main/kotlin/QuestionHelpers.kt b/lib/src/main/kotlin/QuestionHelpers.kt
@@ -24,6 +24,7 @@ import edu.illinois.cs.cs125.jeed.core.fromTemplates
 import edu.illinois.cs.cs125.jeed.core.kompile
 import edu.illinois.cs.cs125.jeed.core.ktLint
 import edu.illinois.cs.cs125.jeed.core.moshi.CompiledSourceResult
+import edu.illinois.cs.cs125.jeed.core.stripComments
 import edu.illinois.cs.cs125.jenisol.core.CaptureOutputControlInput
 import edu.illinois.cs.cs125.jenisol.core.CapturedResult
 import edu.illinois.cs.cs125.jenisol.core.unwrap
@@ -238,6 +239,10 @@ fun Question.mutations(seed: Int, count: Int) = templateSubmission(
     }
     // Templated questions sometimes will mutate the template
     .filter { (contents, _) -> contents != correct.contents }
+    .distinctBy { (contents, _) ->
+        contents.stripComments(Source.FileType.JAVA).hashCode()
+    }
+    .shuffled(random = Random(seed))
     .take(count)
     .map { (contents, source) ->
         Question.IncorrectFile(

diff --git a/lib/src/main/kotlin/Validation.kt b/lib/src/main/kotlin/Validation.kt
@@ -17,7 +17,7 @@ data class CorrectResults(val incorrect: Question.FlatFile, val results: TestRes
 data class IncorrectResults(val incorrect: Question.IncorrectFile, val results: TestResults)
 
 @Suppress("LongMethod", "ComplexMethod")
-suspend fun Question.validate(defaultSeed: Int): ValidationReport {
+suspend fun Question.validate(defaultSeed: Int, maxMutationCount: Int): ValidationReport {
 
     val seed = if (control.seed!! != -1) {
         control.seed!!
@@ -267,7 +267,7 @@ suspend fun Question.validate(defaultSeed: Int): ValidationReport {
     val bootstrapLength = Instant.now().toEpochMilli() - bootStrapStart.toEpochMilli()
 
     val mutationStart = Instant.now()
-    val mutations = mutations(seed, control.maxMutationCount!!).also {
+    val mutations = mutations(seed, control.maxMutationCount ?: maxMutationCount).also {
         if (it.size < control.minMutationCount!!) {
             throw TooFewMutations(correct, it.size, control.minMutationCount!!)
         }

diff --git a/lib/src/main/kotlin/Validator.kt b/lib/src/main/kotlin/Validator.kt
@@ -3,14 +3,25 @@ package edu.illinois.cs.cs125.questioner.lib
 import java.io.File
 
 @Suppress("unused")
-class Validator(questionsFile: File, private val sourceDir: String, private val seed: Int) {
+class Validator(
+    questionsFile: File,
+    private val sourceDir: String,
+    private val seed: Int,
+    private val maxMutationCount: Int
+) {
     private val unvalidatedQuestions = loadFromPath(questionsFile, sourceDir).also {
         assert(it.isNotEmpty())
     }
     private val questions = loadFromPath(questionsFile, sourceDir).also {
         assert(it.isNotEmpty())
     }
-    suspend fun validate(name: String, verbose: Boolean = false, force: Boolean = false, testing: Boolean = false): Pair<Question, ValidationReport?> {
+
+    suspend fun validate(
+        name: String,
+        verbose: Boolean = false,
+        force: Boolean = false,
+        testing: Boolean = false
+    ): Pair<Question, ValidationReport?> {
         val question = if (force) {
             unvalidatedQuestions[name]
         } else {
@@ -27,7 +38,7 @@ class Validator(questionsFile: File, private val sourceDir: String, private val
             question.reportFile(sourceDir).delete()
         }
         try {
-            question.validate(seed).also { report ->
+            question.validate(seed, maxMutationCount).also { report ->
                 if (!testing) {
                     println("$name: ${report.summary}")
                     question.validationFile(sourceDir)

diff --git a/lib/src/test/kotlin/TestTestTesting.kt b/lib/src/test/kotlin/TestTestTesting.kt
@@ -12,7 +12,8 @@ import kotlin.system.measureTimeMillis
 private val validator = Validator(
     Path.of(object {}::class.java.getResource("/questions.json")!!.toURI()).toFile(),
     "",
-    seed = 124
+    seed = 124,
+    maxMutationCount = 64
 )
 
 const val JAVA_EMPTY_SUITE_CLASS = """

diff --git a/lib/src/test/kotlin/TestValidation.kt b/lib/src/test/kotlin/TestValidation.kt
@@ -9,7 +9,8 @@ import java.nio.file.Path
 private val validator = Validator(
     Path.of(object {}::class.java.getResource("/questions.json")!!.toURI()).toFile(),
     "",
-    seed = 124
+    seed = 124,
+    maxMutationCount = 64
 )
 
 class TestValidation : StringSpec({

diff --git a/plugin/src/main/kotlin/GenerateMetatests.kt b/plugin/src/main/kotlin/GenerateMetatests.kt
@@ -20,6 +20,9 @@ abstract class GenerateMetatests : DefaultTask() {
     @get:Input
     abstract var seed: Int
 
+    @get:Input
+    abstract var maxMutationCount: Int
+
     @InputFiles
     val inputFiles = project.extensions.getByType(JavaPluginExtension::class.java)
         .sourceSets.getByName("main").allSource.filter { it.name == ".validation.json" }
@@ -51,7 +54,15 @@ abstract class GenerateMetatests : DefaultTask() {
         tests.first().let { (packageName, questions) ->
             project.file("src/test/kotlin/TestAllQuestions.kt").also {
                 it.parentFile.mkdirs()
-                it.writeText(questions.generateTest(packageName, "TestAllQuestions", sourceRoot, seed))
+                it.writeText(
+                    questions.generateTest(
+                        packageName,
+                        "TestAllQuestions",
+                        sourceRoot,
+                        seed,
+                        maxMutationCount
+                    )
+                )
             }
             project.file("src/test/kotlin/TestUnvalidatedQuestions.kt").also {
                 it.parentFile.mkdirs()
@@ -61,6 +72,7 @@ abstract class GenerateMetatests : DefaultTask() {
                         "TestUnvalidatedQuestions",
                         sourceRoot,
                         seed,
+                        maxMutationCount,
                         onlyNotValidated = true
                     )
                 )
@@ -73,6 +85,7 @@ abstract class GenerateMetatests : DefaultTask() {
                         "TestFocusedQuestions",
                         sourceRoot,
                         seed,
+                        maxMutationCount,
                         onlyFocused = true
                     )
                 )
@@ -107,6 +120,7 @@ fun List<Question>.generateTest(
     klass: String,
     sourceRoot: File,
     seed: Int,
+    maxMutationCount: Int,
     onlyNotValidated: Boolean = false,
     onlyFocused: Boolean = false
 ): String {
@@ -149,39 +163,56 @@ fun List<Question>.generateTest(
     }
     return """$packageNameBlock@file:Suppress("SpellCheckingInspection", "UnusedImport", "unused")
 
-${if (!isBlank) {
+${
+    if (!isBlank) {
         """import edu.illinois.cs.cs125.jeed.core.warm
 import edu.illinois.cs.cs125.questioner.lib.Validator
 """
-    } else {""}}import io.kotest.core.spec.style.StringSpec
-${if (!isBlank) {
+    } else {
+        ""
+    }
+    }import io.kotest.core.spec.style.StringSpec
+${
+    if (!isBlank) {
         """import kotlinx.coroutines.runBlocking
 import java.nio.file.Path
 """
-    } else {""}}
+    } else {
+        ""
+    }
+    }
 /*
  * THIS FILE IS AUTOGENERATED. DO NOT EDIT BY HAND.
  */
 
-/* ktlint-disable max-line-length */${if (!isBlank) {
+/* ktlint-disable max-line-length */${
+    if (!isBlank) {
         """
 
 private val validator = Validator(
   Path.of(object {}::class.java.getResource("/questions.json")!!.toURI()).toFile(),
-  ${"\"\"\"" + sourceRoot.path.replace("/", System.getProperty("file.separator")) + "\"\"\"" },
-  seed = $seed
+  ${"\"\"\"" + sourceRoot.path.replace("/", System.getProperty("file.separator")) + "\"\"\""},
+  seed = $seed,
+  maxMutationCount = $maxMutationCount
 )
 """
-    } else {""}}
+    } else {
+        ""
+    }
+    }
 @Suppress("MaxLineLength", "LargeClass")
-class $klass : StringSpec({${if (!isBlank) {
+class $klass : StringSpec({${
+    if (!isBlank) {
         """
   beforeSpec {
     runBlocking {
       warm(failLint = false, quiet = true, useDocker = false)
     }
   }"""
-    } else {""}}
+    } else {
+        ""
+    }
+    }
 $testBlock
 })
 /* ktlint-enable max-line-length */

diff --git a/plugin/src/main/kotlin/QuestionerConfigExtension.kt b/plugin/src/main/kotlin/QuestionerConfigExtension.kt
@@ -2,4 +2,5 @@ package edu.illinois.cs.cs125.questioner.plugin
 
 open class QuestionerConfigExtension {
     var seed: Int = 124
+    var maxMutationCount: Int = 32
 }
diff --git a/plugin/src/main/kotlin/QuestionerPlugin.kt b/plugin/src/main/kotlin/QuestionerPlugin.kt
@@ -50,7 +50,9 @@ class QuestionerPlugin : Plugin<Project> {
         generateMetatests.dependsOn(project.tasks.getByName("processResources"))
         project.afterEvaluate {
             project.tasks.getByName("processResources").dependsOn(saveQuestions)
+
             generateMetatests.seed = config.seed
+            generateMetatests.maxMutationCount = config.maxMutationCount
 
             project.configurations.getByName("runtimeClasspath") { conf ->
                 val agentJarPath = conf.resolvedConfiguration.resolvedArtifacts.find {

diff --git a/server/src/main/resources/edu.illinois.cs.cs125.questioner.server.version b/server/src/main/resources/edu.illinois.cs.cs125.questioner.server.version
@@ -1 +1 @@
-version=2022.9.10
+version=2022.9.11