From 1592d2d393b32b93cc7b3e014b13a2ee27c73e3f Mon Sep 17 00:00:00 2001 From: Geoffrey Challen Date: Sun, 30 Oct 2022 22:22:43 -0500 Subject: [PATCH] Add complexity and line count limits. --- build.gradle.kts | 2 +- lib/src/main/kotlin/Annotations.kt | 4 ++- lib/src/main/kotlin/Question.kt | 13 +++++++-- lib/src/main/kotlin/QuestionHelpers.kt | 15 ++++++++++ lib/src/main/kotlin/TestQuestion.kt | 23 +++++++++++++-- lib/src/main/kotlin/TestResults.kt | 7 ++--- lib/src/main/kotlin/Validation.kt | 3 ++ lib/src/test/resources/questions.json | 28 +++++++++++++------ plugin/src/main/kotlin/save/ParseJava.kt | 6 +++- ...llinois.cs.cs125.questioner.server.version | 2 +- 10 files changed, 80 insertions(+), 23 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b2fd9ca..10a7975 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -8,7 +8,7 @@ plugins { } subprojects { group = "com.github.cs125-illinois.questioner" - version = "2022.10.6" + version = "2022.10.7" tasks.withType { kotlinOptions { jvmTarget = JavaVersion.VERSION_17.toString() diff --git a/lib/src/main/kotlin/Annotations.kt b/lib/src/main/kotlin/Annotations.kt index 0701e8f..9f9e643 100644 --- a/lib/src/main/kotlin/Annotations.kt +++ b/lib/src/main/kotlin/Annotations.kt @@ -34,7 +34,9 @@ annotation class Correct( val allocationLimitMultiplier: Int = Question.TestingControl.DEFAULT_ALLOCATION_LIMIT_MULTIPLIER, val minExtraSourceLines: Int = Question.TestingControl.DEFAULT_MIN_EXTRA_SOURCE_LINES, val sourceLinesMultiplier: Double = Question.TestingControl.DEFAULT_SOURCE_LINES_MULTIPLIER, - val seed: Int = Question.TestingControl.DEFAULT_SEED + val seed: Int = Question.TestingControl.DEFAULT_SEED, + val maxComplexityMultiplier: Int = Question.TestingControl.DEFAULT_MAX_COMPLEXITY_MULTIPLIER, + val maxLineCountMultiplier: Int = Question.TestingControl.DEFAULT_MAX_LINECOUNT_MULTIPLIER ) @Target(AnnotationTarget.CLASS, AnnotationTarget.FILE) diff --git a/lib/src/main/kotlin/Question.kt b/lib/src/main/kotlin/Question.kt index 8015c2e..c2e0171 100644 --- a/lib/src/main/kotlin/Question.kt +++ b/lib/src/main/kotlin/Question.kt @@ -182,7 +182,9 @@ data class Question( val allocationLimitMultiplier: Int?, val minExtraSourceLines: Int?, val sourceLinesMultiplier: Double?, - val seed: Int? + val seed: Int?, + val maxComplexityMultiplier: Int?, + val maxLineCountMultiplier: Int? ) { companion object { const val DEFAULT_SOLUTION_THROWS = false @@ -203,6 +205,8 @@ data class Question( const val DEFAULT_MIN_EXTRA_SOURCE_LINES = 2 const val DEFAULT_SOURCE_LINES_MULTIPLIER = 1.5 const val DEFAULT_SEED = -1 + const val DEFAULT_MAX_COMPLEXITY_MULTIPLIER = 8 + const val DEFAULT_MAX_LINECOUNT_MULTIPLIER = 8 val DEFAULTS = TestingControl( DEFAULT_SOLUTION_THROWS, @@ -223,7 +227,9 @@ data class Question( DEFAULT_ALLOCATION_LIMIT_MULTIPLIER, DEFAULT_MIN_EXTRA_SOURCE_LINES, DEFAULT_SOURCE_LINES_MULTIPLIER, - DEFAULT_SEED + DEFAULT_SEED, + DEFAULT_MAX_COMPLEXITY_MULTIPLIER, + DEFAULT_MAX_LINECOUNT_MULTIPLIER ) } } @@ -309,7 +315,7 @@ data class Question( ) { @Suppress("SpellCheckingInspection") enum class Reason { - DESIGN, COMPILE, TEST, CHECKSTYLE, TIMEOUT, DEADCODE, LINECOUNT, TOOLONG, MEMORYLIMIT, RECURSION, COMPLEXITY, FEATURES, TOOMUCHOUTPUT + DESIGN, COMPILE, TEST, CHECKSTYLE, TIMEOUT, DEADCODE, LINECOUNT, TOOLONG, MEMORYLIMIT, RECURSION, COMPLEXITY, FEATURES, TOOMUCHOUTPUT, MEMOIZATION } } @@ -594,6 +600,7 @@ fun String.toReason() = when (uppercase(Locale.getDefault())) { "COMPLEXITY" -> Question.IncorrectFile.Reason.COMPLEXITY "FEATURES" -> Question.IncorrectFile.Reason.FEATURES "TOOMUCHOUTPUT" -> Question.IncorrectFile.Reason.TOOMUCHOUTPUT + "MEMOIZATION" -> Question.IncorrectFile.Reason.MEMOIZATION else -> error("Invalid incorrect reason: $this") } diff --git a/lib/src/main/kotlin/QuestionHelpers.kt b/lib/src/main/kotlin/QuestionHelpers.kt index 7d47e63..fc70c4d 100644 --- a/lib/src/main/kotlin/QuestionHelpers.kt +++ b/lib/src/main/kotlin/QuestionHelpers.kt @@ -263,10 +263,14 @@ fun Question.mutations(seed: Int, count: Int) = templateSubmission( ) } +class MaxComplexityExceeded(message: String) : RuntimeException(message) + fun Question.computeComplexity(contents: String, language: Question.Language): TestResults.ComplexityComparison { val solutionComplexity = published.complexity[language] check(solutionComplexity != null) { "Solution complexity not available" } + val maxComplexity = (control.maxComplexityMultiplier!! * solutionComplexity) + val submissionComplexity = when { type == Question.Type.SNIPPET && contents.isBlank() -> 0 language == Question.Language.java -> { @@ -307,6 +311,9 @@ $contents else -> error("Shouldn't get here") } + if (submissionComplexity > maxComplexity) { + throw MaxComplexityExceeded("Submission complexity $submissionComplexity exceeds maximum of $maxComplexity") + } return TestResults.ComplexityComparison(solutionComplexity, submissionComplexity, control.maxExtraComplexity!!) } @@ -379,14 +386,22 @@ ${contents.lines().joinToString("\n") { " $it" }} return TestResults.FeaturesComparison(errors) } +class MaxLineCountExceeded(message: String) : RuntimeException(message) + fun Question.computeLineCounts(contents: String, language: Question.Language): TestResults.LineCountComparison { val solutionLineCount = published.lineCounts[language] check(solutionLineCount != null) { "Solution line count not available" } + + val maxLineCount = (control.maxLineCountMultiplier!! * solutionLineCount.source) + val type = when (language) { Question.Language.java -> Source.FileType.JAVA Question.Language.kotlin -> Source.FileType.KOTLIN } val submissionLineCount = contents.countLines(type) + if (submissionLineCount.source > maxLineCount) { + throw MaxLineCountExceeded("Submission line count ${submissionLineCount.source} exceeds maximum of $maxLineCount") + } return TestResults.LineCountComparison( solutionLineCount, submissionLineCount, diff --git a/lib/src/main/kotlin/TestQuestion.kt b/lib/src/main/kotlin/TestQuestion.kt index 5cb63a3..56eace5 100644 --- a/lib/src/main/kotlin/TestQuestion.kt +++ b/lib/src/main/kotlin/TestQuestion.kt @@ -74,9 +74,15 @@ suspend fun Question.test( // Special case when snippet transformation fails results.failed.checkCompiledSubmission = "Do not use return statements for this problem" results.failedSteps.add(TestResults.Step.checkCompiledSubmission) + return results + } catch (e: MaxComplexityExceeded) { + results.failed.complexity = e.message + results.failedSteps.add(TestResults.Step.complexity) + return results } catch (e: ComplexityFailed) { - results.failed.complexity = e + results.failed.complexity = "Unable to compute complexity for this submission:\n" + e.errors.joinToString(", ") results.failedSteps.add(TestResults.Step.complexity) + return results } // features @@ -90,11 +96,22 @@ suspend fun Question.test( } catch (e: Exception) { results.failed.features = e.message ?: "Unknown features failure" results.failedSteps.add(TestResults.Step.features) + return results } // linecount - results.complete.lineCount = computeLineCounts(contents, language) - results.completedSteps.add(TestResults.Step.lineCount) + try { + results.complete.lineCount = computeLineCounts(contents, language) + results.completedSteps.add(TestResults.Step.lineCount) + } catch (e: MaxLineCountExceeded) { + results.failed.lineCount = e.message!! + results.failedSteps.add(TestResults.Step.lineCount) + return results + } catch (e: Exception) { + results.failed.lineCount = e.message ?: "Unknown line count failure" + results.failedSteps.add(TestResults.Step.lineCount) + return results + } // execution val classLoaderConfiguration = when (language) { diff --git a/lib/src/main/kotlin/TestResults.kt b/lib/src/main/kotlin/TestResults.kt index 910fda1..d3c1837 100644 --- a/lib/src/main/kotlin/TestResults.kt +++ b/lib/src/main/kotlin/TestResults.kt @@ -4,7 +4,6 @@ import com.squareup.moshi.JsonClass import edu.illinois.cs.cs125.jeed.core.CheckstyleFailed import edu.illinois.cs.cs125.jeed.core.CheckstyleResults import edu.illinois.cs.cs125.jeed.core.CompilationFailed -import edu.illinois.cs.cs125.jeed.core.ComplexityFailed import edu.illinois.cs.cs125.jeed.core.KtLintFailed import edu.illinois.cs.cs125.jeed.core.KtLintResults import edu.illinois.cs.cs125.jeed.core.LineCounts @@ -98,9 +97,9 @@ data class TestResults( var checkstyle: CheckstyleFailed? = null, var ktlint: KtLintFailed? = null, var checkCompiledSubmission: String? = null, - var complexity: ComplexityFailed? = null, + var complexity: String? = null, var features: String? = null, - // lineCount doesn't fail + var lineCount: String? = null, // execution var checkExecutedSubmission: String? = null // executionCount doesn't fail @@ -220,7 +219,7 @@ data class TestResults( } else if (failed.ktlint != null) { "Ktlint failed:${failed.ktlint?.let { ": $it" } ?: ""}" } else if (failed.complexity != null) { - "Computing complexity failed: ${failed.complexity!!.message ?: "unknown error"}" + "Computing complexity failed: ${failed.complexity ?: "unknown error"}" } else if (failed.checkCompiledSubmission != null) { "Checking submission failed: ${failed.checkCompiledSubmission}" } else if (failed.checkExecutedSubmission != null) { diff --git a/lib/src/main/kotlin/Validation.kt b/lib/src/main/kotlin/Validation.kt index 6c7aea8..b1867d7 100644 --- a/lib/src/main/kotlin/Validation.kt +++ b/lib/src/main/kotlin/Validation.kt @@ -484,6 +484,9 @@ private fun TestResults.validate(reason: Question.IncorrectFile.Reason) { Question.IncorrectFile.Reason.FEATURES -> require(failed.features != null) { "Expected submission to fail feature check" } + Question.IncorrectFile.Reason.MEMOIZATION -> require(failed.complexity != null && failed.complexity!!.contains("exceeds maximum")) { + "Expected submission to be so complex as to suggest memoization" + } else -> require(complete.testing?.passed == false) { "Expected submission to fail tests" } diff --git a/lib/src/test/resources/questions.json b/lib/src/test/resources/questions.json index eb83287..efac570 100644 --- a/lib/src/test/resources/questions.json +++ b/lib/src/test/resources/questions.json @@ -1123,7 +1123,7 @@ "type": "KLASS", "klass": "Question", "metadata": { - "contentHash": "0f5d87fda490b597fa6cf22e0d1d96d1", + "contentHash": "85cad84d1b50bd029f2262d40c3c9d1c", "packageName": "com.examples.testing.withconstructornotnull", "version": "2022.10.0", "author": "challen@illinois.edu", @@ -1135,7 +1135,7 @@ "annotatedControls": {}, "question": { "klass": "Question", - "contents": "import edu.illinois.cs.cs125.jenisol.core.NotNull;\nimport edu.illinois.cs.cs125.questioner.lib.Correct;\n\n/*\n * Testing @NotNull annotation on constructor parameters.\n */\n\n@Correct(name = \"Test Constructor NotNull\", version = \"2022.10.0\", author = \"challen@illinois.edu\", focused = true)\npublic class Question {\n private final int stringLength;\n\n public Question(@NotNull String value) {\n stringLength = value.length();\n }\n\n public int getStringLength() {\n return stringLength;\n }\n}", + "contents": "import edu.illinois.cs.cs125.jenisol.core.NotNull;\nimport edu.illinois.cs.cs125.questioner.lib.Correct;\n\n/*\n * Testing @NotNull annotation on constructor parameters.\n */\n\n@Correct(\n name = \"Test Constructor NotNull\",\n version = \"2022.10.0\",\n author = \"challen@illinois.edu\",\n focused = true)\npublic class Question {\n private final int stringLength;\n\n public Question(@NotNull String value) {\n stringLength = value.length();\n }\n\n public int getStringLength() {\n return stringLength;\n }\n}", "language": "java", "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/testing/withconstructornotnull/Question.java" }, @@ -3334,13 +3334,14 @@ "type": "METHOD", "klass": "Question", "metadata": { - "contentHash": "1de3e62a1da5371e08f407b53e2861c9", + "contentHash": "cf69028309acfa6fae8cc88f3063f219", "packageName": "com.examples.addone", "version": "2021.6.0", "author": "challen@illinois.edu", "javaDescription": "

Write a method addOne that returns its int argument plus one.

", "kotlinDescription": "

Write a method addOne that returns its Int argument plus one.

", "usedFiles": [ + "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/memoization/Question.java", "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/timeout/Question.java", "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/toolong/Question.java", "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/memorylimit/Question.java", @@ -3354,13 +3355,13 @@ "annotatedControls": {}, "question": { "klass": "Question", - "contents": "import edu.illinois.cs.cs125.questioner.lib.Correct;\nimport edu.illinois.cs.cs125.questioner.lib.Wrap;\n\n/*\n * Write a method `addOne` that returns its `int` argument plus one.\n */\n\n@Correct(name = \"Add One\", author = \"challen@illinois.edu\", version = \"2021.6.0\")\n@Wrap\npublic class Question {\n int addOne(int value) {\n return value + 1;\n }\n}", + "contents": "import edu.illinois.cs.cs125.jenisol.core.FixedParameters;\nimport edu.illinois.cs.cs125.questioner.lib.Correct;\nimport edu.illinois.cs.cs125.questioner.lib.Wrap;\nimport java.util.Arrays;\nimport java.util.List;\n\n/*\n * Write a method `addOne` that returns its `int` argument plus one.\n */\n\n@Correct(name = \"Add One\", author = \"challen@illinois.edu\", version = \"2021.6.0\")\n@Wrap\npublic class Question {\n // Here to avoid dead code errors in the memoization test\n @FixedParameters\n private static final List FIXED = Arrays.asList(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12);\n\n int addOne(int value) {\n return value + 1;\n }\n}", "language": "java", "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/Question.java" }, "correct": { "klass": "Question", - "contents": "int addOne(int value) {\n return value + 1;\n}", + "contents": "// Here to avoid dead code errors in the memoization test\n\nint addOne(int value) {\n return value + 1;\n}", "language": "java", "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/Question.java", "complexity": 1, @@ -3379,8 +3380,8 @@ }, "lineCount": { "source": 3, - "comment": 0, - "blank": 0 + "comment": 1, + "blank": 1 }, "expectedDeadCount": 1 }, @@ -3434,6 +3435,15 @@ } ], "incorrect": [ + { + "klass": "Question", + "contents": "int addOne(int value) {\n if (value == 1) {\n return 2;\n } else if (value == 2) {\n return 3;\n } else if (value == 3) {\n return 4;\n } else if (value == 4) {\n return 5;\n } else if (value == 5) {\n return 6;\n } else if (value == 6) {\n return 7;\n } else if (value == 7) {\n return 8;\n } else if (value == 8) {\n return 9;\n } else if (value == 9) {\n return 10;\n } else if (value == 10) {\n return 11;\n } else if (value == 11) {\n return 12;\n } else {\n return value + 1;\n }\n}", + "reason": "MEMOIZATION", + "language": "java", + "path": "/Users/challen/code/questioner-problems/src/main/java/com/examples/addone/incorrect/java/memoization/Question.java", + "starter": false, + "needed": true + }, { "klass": "Question", "contents": "int addOne(int value) {\n int j = 0;\n for (int i = 0; i < 1024; i++) {\n j++;\n }\n return value + 1;\n}", @@ -3546,8 +3556,8 @@ "lineCounts": { "java": { "source": 3, - "comment": 0, - "blank": 0 + "comment": 1, + "blank": 1 }, "kotlin": { "source": 3, diff --git a/plugin/src/main/kotlin/save/ParseJava.kt b/plugin/src/main/kotlin/save/ParseJava.kt index c5087b0..0174373 100644 --- a/plugin/src/main/kotlin/save/ParseJava.kt +++ b/plugin/src/main/kotlin/save/ParseJava.kt @@ -184,6 +184,8 @@ data class ParsedJavaFile(val path: String, val contents: String) { val minExtraSourceLines = parameters["minExtraSourceLines"]?.toInt() val sourceLinesMultiplier = parameters["sourceLinesMultiplier"]?.toDouble() val seed = parameters["seed"]?.toInt() + val maxComplexityMultiplier = parameters["maxComplexityMultiplier"]?.toInt() + val maxLineCountMultiplier = parameters["maxLineCountMultiplier"]?.toInt() Question.CorrectData( path, @@ -211,7 +213,9 @@ data class ParsedJavaFile(val path: String, val contents: String) { allocationLimitMultiplier, minExtraSourceLines, sourceLinesMultiplier, - seed + seed, + maxComplexityMultiplier, + maxLineCountMultiplier ) ) } diff --git a/server/src/main/resources/edu.illinois.cs.cs125.questioner.server.version b/server/src/main/resources/edu.illinois.cs.cs125.questioner.server.version index ab8ba50..88a7816 100644 --- a/server/src/main/resources/edu.illinois.cs.cs125.questioner.server.version +++ b/server/src/main/resources/edu.illinois.cs.cs125.questioner.server.version @@ -1 +1 @@ -version=2022.10.6 \ No newline at end of file +version=2022.10.7 \ No newline at end of file