Add complexity and line count limits.

build.gradle.kts

@@ -8,7 +8,7 @@ plugins {
 }
 subprojects {
     group = "com.github.cs125-illinois.questioner"
-    version = "2022.10.6"
+    version = "2022.10.7"
     tasks.withType<KotlinCompile> {
         kotlinOptions {
             jvmTarget = JavaVersion.VERSION_17.toString()

lib/src/main/kotlin/Annotations.kt

@@ -34,7 +34,9 @@ annotation class Correct(
     val allocationLimitMultiplier: Int = Question.TestingControl.DEFAULT_ALLOCATION_LIMIT_MULTIPLIER,
     val minExtraSourceLines: Int = Question.TestingControl.DEFAULT_MIN_EXTRA_SOURCE_LINES,
     val sourceLinesMultiplier: Double = Question.TestingControl.DEFAULT_SOURCE_LINES_MULTIPLIER,
-    val seed: Int = Question.TestingControl.DEFAULT_SEED
+    val seed: Int = Question.TestingControl.DEFAULT_SEED,
+    val maxComplexityMultiplier: Int = Question.TestingControl.DEFAULT_MAX_COMPLEXITY_MULTIPLIER,
+    val maxLineCountMultiplier: Int = Question.TestingControl.DEFAULT_MAX_LINECOUNT_MULTIPLIER
 )
 
 @Target(AnnotationTarget.CLASS, AnnotationTarget.FILE)

lib/src/main/kotlin/Question.kt

@@ -182,7 +182,9 @@ data class Question(
         val allocationLimitMultiplier: Int?,
         val minExtraSourceLines: Int?,
         val sourceLinesMultiplier: Double?,
-        val seed: Int?
+        val seed: Int?,
+        val maxComplexityMultiplier: Int?,
+        val maxLineCountMultiplier: Int?
     ) {
         companion object {
             const val DEFAULT_SOLUTION_THROWS = false
@@ -203,6 +205,8 @@ data class Question(
             const val DEFAULT_MIN_EXTRA_SOURCE_LINES = 2
             const val DEFAULT_SOURCE_LINES_MULTIPLIER = 1.5
             const val DEFAULT_SEED = -1
+            const val DEFAULT_MAX_COMPLEXITY_MULTIPLIER = 8
+            const val DEFAULT_MAX_LINECOUNT_MULTIPLIER = 8
 
             val DEFAULTS = TestingControl(
                 DEFAULT_SOLUTION_THROWS,
@@ -223,7 +227,9 @@ data class Question(
                 DEFAULT_ALLOCATION_LIMIT_MULTIPLIER,
                 DEFAULT_MIN_EXTRA_SOURCE_LINES,
                 DEFAULT_SOURCE_LINES_MULTIPLIER,
-                DEFAULT_SEED
+                DEFAULT_SEED,
+                DEFAULT_MAX_COMPLEXITY_MULTIPLIER,
+                DEFAULT_MAX_LINECOUNT_MULTIPLIER
             )
         }
     }
@@ -309,7 +315,7 @@ data class Question(
     ) {
         @Suppress("SpellCheckingInspection")
         enum class Reason {
-            DESIGN, COMPILE, TEST, CHECKSTYLE, TIMEOUT, DEADCODE, LINECOUNT, TOOLONG, MEMORYLIMIT, RECURSION, COMPLEXITY, FEATURES, TOOMUCHOUTPUT
+            DESIGN, COMPILE, TEST, CHECKSTYLE, TIMEOUT, DEADCODE, LINECOUNT, TOOLONG, MEMORYLIMIT, RECURSION, COMPLEXITY, FEATURES, TOOMUCHOUTPUT, MEMOIZATION
         }
     }
 
@@ -594,6 +600,7 @@ fun String.toReason() = when (uppercase(Locale.getDefault())) {
     "COMPLEXITY" -> Question.IncorrectFile.Reason.COMPLEXITY
     "FEATURES" -> Question.IncorrectFile.Reason.FEATURES
     "TOOMUCHOUTPUT" -> Question.IncorrectFile.Reason.TOOMUCHOUTPUT
+    "MEMOIZATION" -> Question.IncorrectFile.Reason.MEMOIZATION
     else -> error("Invalid incorrect reason: $this")
 }
 

lib/src/main/kotlin/QuestionHelpers.kt

@@ -263,10 +263,14 @@ fun Question.mutations(seed: Int, count: Int) = templateSubmission(
         )
     }
 
+class MaxComplexityExceeded(message: String) : RuntimeException(message)
+
 fun Question.computeComplexity(contents: String, language: Question.Language): TestResults.ComplexityComparison {
     val solutionComplexity = published.complexity[language]
     check(solutionComplexity != null) { "Solution complexity not available" }
 
+    val maxComplexity = (control.maxComplexityMultiplier!! * solutionComplexity)
+
     val submissionComplexity = when {
         type == Question.Type.SNIPPET && contents.isBlank() -> 0
         language == Question.Language.java -> {
@@ -307,6 +311,9 @@ $contents
 
         else -> error("Shouldn't get here")
     }
+    if (submissionComplexity > maxComplexity) {
+        throw MaxComplexityExceeded("Submission complexity $submissionComplexity exceeds maximum of $maxComplexity")
+    }
     return TestResults.ComplexityComparison(solutionComplexity, submissionComplexity, control.maxExtraComplexity!!)
 }
 
@@ -379,14 +386,22 @@ ${contents.lines().joinToString("\n") { "  $it" }}
     return TestResults.FeaturesComparison(errors)
 }
 
+class MaxLineCountExceeded(message: String) : RuntimeException(message)
+
 fun Question.computeLineCounts(contents: String, language: Question.Language): TestResults.LineCountComparison {
     val solutionLineCount = published.lineCounts[language]
     check(solutionLineCount != null) { "Solution line count not available" }
+
+    val maxLineCount = (control.maxLineCountMultiplier!! * solutionLineCount.source)
+
     val type = when (language) {
         Question.Language.java -> Source.FileType.JAVA
         Question.Language.kotlin -> Source.FileType.KOTLIN
     }
     val submissionLineCount = contents.countLines(type)
+    if (submissionLineCount.source > maxLineCount) {
+        throw MaxLineCountExceeded("Submission line count ${submissionLineCount.source} exceeds maximum of $maxLineCount")
+    }
     return TestResults.LineCountComparison(
         solutionLineCount,
         submissionLineCount,

lib/src/main/kotlin/TestQuestion.kt

@@ -74,9 +74,15 @@ suspend fun Question.test(
         // Special case when snippet transformation fails
         results.failed.checkCompiledSubmission = "Do not use return statements for this problem"
         results.failedSteps.add(TestResults.Step.checkCompiledSubmission)
+        return results
+    } catch (e: MaxComplexityExceeded) {
+        results.failed.complexity = e.message
+        results.failedSteps.add(TestResults.Step.complexity)
+        return results
     } catch (e: ComplexityFailed) {
-        results.failed.complexity = e
+        results.failed.complexity = "Unable to compute complexity for this submission:\n" + e.errors.joinToString(", ")
         results.failedSteps.add(TestResults.Step.complexity)
+        return results
     }
 
     // features
@@ -90,11 +96,22 @@ suspend fun Question.test(
     } catch (e: Exception) {
         results.failed.features = e.message ?: "Unknown features failure"
         results.failedSteps.add(TestResults.Step.features)
+        return results
     }
 
     // linecount
-    results.complete.lineCount = computeLineCounts(contents, language)
-    results.completedSteps.add(TestResults.Step.lineCount)
+    try {
+        results.complete.lineCount = computeLineCounts(contents, language)
+        results.completedSteps.add(TestResults.Step.lineCount)
+    } catch (e: MaxLineCountExceeded) {
+        results.failed.lineCount = e.message!!
+        results.failedSteps.add(TestResults.Step.lineCount)
+        return results
+    } catch (e: Exception) {
+        results.failed.lineCount = e.message ?: "Unknown line count failure"
+        results.failedSteps.add(TestResults.Step.lineCount)
+        return results
+    }
 
     // execution
     val classLoaderConfiguration = when (language) {

lib/src/main/kotlin/TestResults.kt

@@ -4,7 +4,6 @@ import com.squareup.moshi.JsonClass
 import edu.illinois.cs.cs125.jeed.core.CheckstyleFailed
 import edu.illinois.cs.cs125.jeed.core.CheckstyleResults
 import edu.illinois.cs.cs125.jeed.core.CompilationFailed
-import edu.illinois.cs.cs125.jeed.core.ComplexityFailed
 import edu.illinois.cs.cs125.jeed.core.KtLintFailed
 import edu.illinois.cs.cs125.jeed.core.KtLintResults
 import edu.illinois.cs.cs125.jeed.core.LineCounts
@@ -98,9 +97,9 @@ data class TestResults(
         var checkstyle: CheckstyleFailed? = null,
         var ktlint: KtLintFailed? = null,
         var checkCompiledSubmission: String? = null,
-        var complexity: ComplexityFailed? = null,
+        var complexity: String? = null,
         var features: String? = null,
-        // lineCount doesn't fail
+        var lineCount: String? = null,
         // execution
         var checkExecutedSubmission: String? = null
         // executionCount doesn't fail
@@ -220,7 +219,7 @@ data class TestResults(
         } else if (failed.ktlint != null) {
             "Ktlint failed:${failed.ktlint?.let { ": $it" } ?: ""}"
         } else if (failed.complexity != null) {
-            "Computing complexity failed: ${failed.complexity!!.message ?: "unknown error"}"
+            "Computing complexity failed: ${failed.complexity ?: "unknown error"}"
         } else if (failed.checkCompiledSubmission != null) {
             "Checking submission failed: ${failed.checkCompiledSubmission}"
         } else if (failed.checkExecutedSubmission != null) {

lib/src/main/kotlin/Validation.kt

@@ -484,6 +484,9 @@ private fun TestResults.validate(reason: Question.IncorrectFile.Reason) {
         Question.IncorrectFile.Reason.FEATURES -> require(failed.features != null) {
             "Expected submission to fail feature check"
         }
+        Question.IncorrectFile.Reason.MEMOIZATION -> require(failed.complexity != null && failed.complexity!!.contains("exceeds maximum")) {
+            "Expected submission to be so complex as to suggest memoization"
+        }
         else -> require(complete.testing?.passed == false) {
             "Expected submission to fail tests"
         }

lib/src/test/resources/questions.json

@@ -1123,7 +1123,7 @@
     "type": "KLASS",
     "klass": "Question",
     "metadata": {
-      "contentHash": "0f5d87fda490b597fa6cf22e0d1d96d1",
+      "contentHash": "85cad84d1b50bd029f2262d40c3c9d1c",
       "packageName": "com.examples.testing.withconstructornotnull",
       "version": "2022.10.0",
       "author": "challen@illinois.edu",
@@ -1135,7 +1135,7 @@
     "annotatedControls": {},
     "question": {
       "klass": "Question",
-      "contents": "import edu.illinois.cs.cs125.jenisol.core.NotNull;\nimport edu.illinois.cs.cs125.questioner.lib.Correct;\n\n/*\n * Testing @NotNull annotation on constructor parameters.\n */\n\n@Correct(name = \"Test Constructor NotNull\", version = \"2022.10.0\", author = \"challen@illinois.edu\", focused = true)\npublic class Question {\n  private final int stringLength;\n\n  public Question(@NotNull String value) {\n    stringLength = value.length();\n  }\n\n  public int getStringLength() {\n    return stringLength;\n  }\n}",
+      "contents": "import edu.illinois.cs.cs125.jenisol.core.NotNull;\nimport edu.illinois.cs.cs125.questioner.lib.Correct;\n\n/*\n * Testing @NotNull annotation on constructor parameters.\n */\n\n@Correct(\n    name = \"Test Constructor NotNull\",\n    version = \"2022.10.0\",\n    author = \"challen@illinois.edu\",\n    focused = true)\npublic class Question {\n  private final int stringLength;\n\n  public Question(@NotNull String value) {\n    stringLength = value.length();\n  }\n\n  public int getStringLength() {\n    return stringLength;\n  }\n}",
       "language": "java",
       "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/testing/withconstructornotnull/Question.java"
     },
@@ -3334,13 +3334,14 @@
     "type": "METHOD",
     "klass": "Question",
     "metadata": {
-      "contentHash": "1de3e62a1da5371e08f407b53e2861c9",
+      "contentHash": "cf69028309acfa6fae8cc88f3063f219",
       "packageName": "com.examples.addone",
       "version": "2021.6.0",
       "author": "challen@illinois.edu",
       "javaDescription": "<p>Write a method <code>addOne</code> that returns its <code>int</code> argument plus one.</p>",
       "kotlinDescription": "<p>Write a method <code>addOne</code> that returns its <code>Int</code> argument plus one.</p>",
       "usedFiles": [
+        "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/memoization/Question.java",
         "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/timeout/Question.java",
         "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/toolong/Question.java",
         "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/memorylimit/Question.java",
@@ -3354,13 +3355,13 @@
     "annotatedControls": {},
     "question": {
       "klass": "Question",
-      "contents": "import edu.illinois.cs.cs125.questioner.lib.Correct;\nimport edu.illinois.cs.cs125.questioner.lib.Wrap;\n\n/*\n * Write a method `addOne` that returns its `int` argument plus one.\n */\n\n@Correct(name = \"Add One\", author = \"challen@illinois.edu\", version = \"2021.6.0\")\n@Wrap\npublic class Question {\n  int addOne(int value) {\n    return value + 1;\n  }\n}",
+      "contents": "import edu.illinois.cs.cs125.jenisol.core.FixedParameters;\nimport edu.illinois.cs.cs125.questioner.lib.Correct;\nimport edu.illinois.cs.cs125.questioner.lib.Wrap;\nimport java.util.Arrays;\nimport java.util.List;\n\n/*\n * Write a method `addOne` that returns its `int` argument plus one.\n */\n\n@Correct(name = \"Add One\", author = \"challen@illinois.edu\", version = \"2021.6.0\")\n@Wrap\npublic class Question {\n  // Here to avoid dead code errors in the memoization test\n  @FixedParameters\n  private static final List<Integer> FIXED = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);\n\n  int addOne(int value) {\n    return value + 1;\n  }\n}",
       "language": "java",
       "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/Question.java"
     },
     "correct": {
       "klass": "Question",
-      "contents": "int addOne(int value) {\n  return value + 1;\n}",
+      "contents": "// Here to avoid dead code errors in the memoization test\n\nint addOne(int value) {\n  return value + 1;\n}",
       "language": "java",
       "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/Question.java",
       "complexity": 1,
@@ -3379,8 +3380,8 @@
       },
       "lineCount": {
         "source": 3,
-        "comment": 0,
-        "blank": 0
+        "comment": 1,
+        "blank": 1
       },
       "expectedDeadCount": 1
     },
@@ -3434,6 +3435,15 @@
       }
     ],
     "incorrect": [
+      {
+        "klass": "Question",
+        "contents": "int addOne(int value) {\n  if (value == 1) {\n    return 2;\n  } else if (value == 2) {\n    return 3;\n  } else if (value == 3) {\n    return 4;\n  } else if (value == 4) {\n    return 5;\n  } else if (value == 5) {\n    return 6;\n  } else if (value == 6) {\n    return 7;\n  } else if (value == 7) {\n    return 8;\n  } else if (value == 8) {\n    return 9;\n  } else if (value == 9) {\n    return 10;\n  } else if (value == 10) {\n    return 11;\n  } else if (value == 11) {\n    return 12;\n  } else {\n    return value + 1;\n  }\n}",
+        "reason": "MEMOIZATION",
+        "language": "java",
+        "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/memoization/Question.java",
+        "starter": false,
+        "needed": true
+      },
       {
         "klass": "Question",
         "contents": "int addOne(int value) {\n  int j = 0;\n  for (int i = 0; i < 1024; i++) {\n    j++;\n  }\n  return value + 1;\n}",
@@ -3546,8 +3556,8 @@
       "lineCounts": {
         "java": {
           "source": 3,
-          "comment": 0,
-          "blank": 0
+          "comment": 1,
+          "blank": 1
         },
         "kotlin": {
           "source": 3,

plugin/src/main/kotlin/save/ParseJava.kt

@@ -184,6 +184,8 @@ data class ParsedJavaFile(val path: String, val contents: String) {
                 val minExtraSourceLines = parameters["minExtraSourceLines"]?.toInt()
                 val sourceLinesMultiplier = parameters["sourceLinesMultiplier"]?.toDouble()
                 val seed = parameters["seed"]?.toInt()
+                val maxComplexityMultiplier = parameters["maxComplexityMultiplier"]?.toInt()
+                val maxLineCountMultiplier = parameters["maxLineCountMultiplier"]?.toInt()
 
                 Question.CorrectData(
                     path,
@@ -211,7 +213,9 @@ data class ParsedJavaFile(val path: String, val contents: String) {
                         allocationLimitMultiplier,
                         minExtraSourceLines,
                         sourceLinesMultiplier,
-                        seed
+                        seed,
+                        maxComplexityMultiplier,
+                        maxLineCountMultiplier
                     )
                 )
             }

server/src/main/resources/edu.illinois.cs.cs125.questioner.server.version

@@ -1 +1 @@
-version=2022.10.6
+version=2022.10.7