#85 Override executed script

README.md

@@ -114,15 +114,17 @@ Then, you may add the dependencies to ```datamaintain-core``` and the driver mod
 
 | Key | Description | Default value | Mandatory? | Values examples |
 |---|---|---|---|---|
+| default.script.action | The default script action | ```RUN``` | no | ```RUN``` or ```MARK_AS_EXECUTED``` |
 | scan.path | Path to the folder containing all your scripts | ```./scripts/``` | yes |  |
 | scan.identifier.regex | Regex that will be used to determine an identifier for each file. It has to contain a capturing group. Identifiers are then used to sort the scripts before running them. | ```(.*)``` (with this regex, the script's whole name will be its identifier) | no | With the regex ```(.*?)_.*```, a script named ```1.23_my-script.js``` will have ```1.23``` as its identifier  |
 | scan.tags.createFromFolder | If true, scripts will have their parent folders names as tags. Relative path to ```scan.path``` is used.  | ```false``` | no | ```false``` or ```true``` |
 | tag.*your_tag* | Glob paths to your scripts that you want to apply the tag "your_tag" on. To declare multiple tags, you will have to add multiple properties in your settings. A tag ```my_tag``` will have as as property name ```tag.my_tag``` **WARNING:** ALWAYS declare your tags using absolute paths. Relative paths and even using a tilde (~) won't do the trick. |  | no | ```[data/*, script1.js, old/old_script1.js]``` |
 | filter.tags.whitelisted | Scripts that have these tags will be considered | None | no | ```DATA,tag``` |
 | filter.tags.blacklisted | Scripts that have these tags will be ignored. A script having a whitelisted tag and a blacklisted tag will be ignored | None | no | ```DATA,tag``` |
-| execution.mode | Execution mode. Possible values:<br />- ```NORMAL```: Regular execution: your scripts will be run on your database.<br />- ```DRY```: Scripts will not be executed. A full report of what would happen is you ran Datamaintain normally will be logged.<br />- ```FORCE_AS_EXECUTED```: Scripts will not be executed but their execution will be remembered by Datamaintain for later executions. | ```NORMAL``` | no | ```NORMAL```, ```DRY``` or ```FORCE_MARK_AS_EXECUTED``` |
+| execution.mode | Execution mode. Possible values:<br />- ```NORMAL```: Regular execution: the action for each script will be done.<br />- ```DRY```: No action will be done on script. A full report of what would happen is you ran Datamaintain normally will be logged.<br />- ```FORCE_AS_EXECUTED```: **Deprecated (will be removed in 2.0, use action) !** Scripts will not be executed but their execution will be remembered by Datamaintain for later executions. | ```NORMAL``` | no | ```NORMAL```, ```DRY``` ~~or FORCE_MARK_AS_EXECUTED~~ |
 | verbose | If true, more logs will be printed | ```false``` | no | ```true``` or ```false``` |
 | prune.tags.to.run.again | Scripts that have these tags will be run, even they were already executed  | None | no | ```tag,again``` |
+| prune.scripts.override.executed | Allow datamaintain to override a script if it detect a checksum change on a script already runned (assuming its filename) | ```false``` | no | ```true``` or ```false``` |
 ### Mongo driver configuration
 
 | Key | Description | Default value | Mandatory? | Values examples |

modules/cli/src/main/kotlin/datamaintain/cli/App.kt

@@ -9,6 +9,7 @@ import com.github.ajalt.clikt.parameters.types.choice
 import datamaintain.core.Datamaintain
 import datamaintain.core.config.CoreConfigKey
 import datamaintain.core.config.DatamaintainConfig
+import datamaintain.core.script.ScriptAction
 import datamaintain.core.step.check.allCheckRuleNames
 import datamaintain.core.step.executor.ExecutionMode
 import datamaintain.db.driver.mongo.MongoConfigKey
@@ -71,6 +72,10 @@ class UpdateDb(val runner: (DatamaintainConfig) -> Unit = ::defaultUpdateDbRunne
 
     private val executionMode by option(help = "execution mode").choice(ExecutionMode.values().map { it.name }.map { it to it }.toMap())
 
+    private val action by option(help = "script action").choice(ScriptAction.values().map { it.name }.map { it to it }.toMap())
+
+    private val allowAutoOverride: Boolean? by option(help = "Allow datamaintain to automaticaly override scripts").flag()
+
     private val verbose: Boolean? by option(help = "verbose").flag()
 
     private val mongoSaveOutput: Boolean? by option(help = "save mongo output").flag()
@@ -121,10 +126,12 @@ class UpdateDb(val runner: (DatamaintainConfig) -> Unit = ::defaultUpdateDbRunne
         mongoSaveOutput?.let { props.put(MongoConfigKey.DB_MONGO_SAVE_OUTPUT.key, it.toString()) }
         mongoPrintOutput?.let { props.put(MongoConfigKey.DB_MONGO_PRINT_OUTPUT.key, it.toString()) }
         executionMode?.let { props.put(CoreConfigKey.EXECUTION_MODE.key, it) }
+        action?.let { props.put(CoreConfigKey.DEFAULT_SCRIPT_ACTION.key, it) }
         tagsMatchers?.forEach {
             props.put("${CoreConfigKey.TAG.key}.${it.first}", it.second)
         }
         checkRules?.let { props.put(CoreConfigKey.CHECK_RULES.key, it.joinToString(",")) }
+        allowAutoOverride?.let { props.put(CoreConfigKey.PRUNE_OVERRIDE_UPDATED_SCRIPTS.key, it.toString()) }
     }
 }
 

modules/cli/src/test/kotlin/datamaintain/cli/AppTest.kt

@@ -68,6 +68,56 @@ internal class AppTest {
         }
     }
 
+    @Test
+    fun `should build default config without auto override`() {
+        // Given
+        val configWrapper = ConfigWrapper()
+
+        fun runner(config: DatamaintainConfig) {
+            configWrapper.datamaintainConfig = config
+        }
+
+        val argv = getSmallestArgs()
+
+        // When
+        App().subcommands(UpdateDb(runner = ::runner), ListExecutedScripts()).main(argv)
+
+        // Then
+        defaultChecks(configWrapper)
+        expectThat(configWrapper) {
+            get { datamaintainConfig }.isNotNull()
+        }
+        expectThat(configWrapper.datamaintainConfig!!.overrideExecutedScripts) {
+            isFalse()
+        }
+    }
+
+    @Test
+    fun `should build config with auto override`() {
+        // Given
+        val configWrapper = ConfigWrapper()
+
+        fun runner(config: DatamaintainConfig) {
+            configWrapper.datamaintainConfig = config
+        }
+
+        val argv = getSmallestArgs().plus(listOf(
+                "--allow-auto-override"
+        ))
+
+        // When
+        App().subcommands(UpdateDb(runner = ::runner), ListExecutedScripts()).main(argv)
+
+        // Then
+        defaultChecks(configWrapper)
+        expectThat(configWrapper) {
+            get { datamaintainConfig }.isNotNull()
+        }
+        expectThat(configWrapper.datamaintainConfig!!.overrideExecutedScripts) {
+            isTrue()
+        }
+    }
+
     private fun defaultChecks(configWrapper: ConfigWrapper) {
         expectThat(configWrapper) {
             get { datamaintainConfig }

modules/core/src/main/kotlin/datamaintain/core/config/DatamaintainConfig.kt

@@ -3,6 +3,7 @@ package datamaintain.core.config
 import datamaintain.core.config.ConfigKey.Companion.overrideBySystemProperties
 import datamaintain.core.config.CoreConfigKey.*
 import datamaintain.core.db.driver.DatamaintainDriverConfig
+import datamaintain.core.script.ScriptAction
 import datamaintain.core.script.Tag
 import datamaintain.core.script.TagMatcher
 import datamaintain.core.step.executor.ExecutionMode
@@ -20,14 +21,17 @@ data class DatamaintainConfig @JvmOverloads constructor(val path: Path = Paths.g
                                                         val whitelistedTags: Set<Tag> = setOf(),
                                                         val blacklistedTags: Set<Tag> = setOf(),
                                                         val tagsToPlayAgain: Set<Tag> = setOf(),
+                                                        val overrideExecutedScripts: Boolean = PRUNE_OVERRIDE_UPDATED_SCRIPTS.default!!.toBoolean(),
                                                         val tagsMatchers: Set<TagMatcher> = setOf(),
                                                         val checkRules: Sequence<String> = emptySequence(),
                                                         val executionMode: ExecutionMode = defaultExecutionMode,
+                                                        val defaultScriptAction: ScriptAction = defaultAction,
                                                         val driverConfig: DatamaintainDriverConfig,
                                                         val verbose: Boolean = VERBOSE.default!!.toBoolean()) {
 
     companion object {
         private val defaultExecutionMode = ExecutionMode.NORMAL
+        public val defaultAction = ScriptAction.RUN
 
         @JvmStatic
         fun buildConfig(configInputStream: InputStream, driverConfig: DatamaintainDriverConfig): DatamaintainConfig {
@@ -40,18 +44,32 @@ data class DatamaintainConfig @JvmOverloads constructor(val path: Path = Paths.g
         @JvmOverloads
         fun buildConfig(driverConfig: DatamaintainDriverConfig, props: Properties = Properties()): DatamaintainConfig {
             overrideBySystemProperties(props, values().asList())
+
+            var executionMode = ExecutionMode.fromNullable(props.getNullableProperty(EXECUTION_MODE), defaultExecutionMode)
+
+            val scriptAction = if (ExecutionMode.FORCE_MARK_AS_EXECUTED == executionMode) {
+                // To be compliant with previous version (and avoid breaking changes)
+                // we set script action from ExecutionMode.FORCE_MARK_AS_EXECUTED
+                executionMode = ExecutionMode.NORMAL
+                ScriptAction.MARK_AS_EXECUTED
+            } else {
+                ScriptAction.fromNullable(props.getNullableProperty(DEFAULT_SCRIPT_ACTION), defaultAction)
+            }
+
             return DatamaintainConfig(
                     Paths.get(props.getProperty(SCAN_PATH)),
                     Regex(props.getProperty(SCAN_IDENTIFIER_REGEX)),
                     props.getProperty(CREATE_TAGS_FROM_FOLDER).toBoolean(),
                     extractTags(props.getNullableProperty(TAGS_WHITELISTED)),
                     extractTags(props.getNullableProperty(TAGS_BLACKLISTED)),
                     extractTags(props.getNullableProperty(PRUNE_TAGS_TO_RUN_AGAIN)),
+                    props.getProperty(PRUNE_OVERRIDE_UPDATED_SCRIPTS).toBoolean(),
                     props.getStringPropertiesByPrefix(TAG.key)
                             .map { TagMatcher.parse(it.first.replace("${TAG.key}.", ""), it.second) }
                             .toSet(),
                     extractCheckRules(props.getNullableProperty(CHECK_RULES)),
-                    ExecutionMode.fromNullable(props.getNullableProperty(EXECUTION_MODE), defaultExecutionMode),
+                    executionMode,
+                    scriptAction,
                     driverConfig,
                     props.getProperty(VERBOSE).toBoolean()
             )
@@ -76,10 +94,12 @@ data class DatamaintainConfig @JvmOverloads constructor(val path: Path = Paths.g
     fun log() {
         logger.info { "Configuration: " }
         path.let { logger.info { "- path -> $it" } }
+        defaultScriptAction.let { logger.info { "- script action -> ${it}" } }
         identifierRegex.let { logger.info { "- identifier regex -> ${it.pattern}" } }
         blacklistedTags.let { logger.info { "- blacklisted tags -> $it" } }
         tagsToPlayAgain.let { logger.info { "- tags to play again -> $it" } }
         tagsMatchers.let { logger.info { "- tags -> $tagsMatchers" } }
+        overrideExecutedScripts.let { logger.info { "- Allow override executed script -> ${it}" } }
         checkRules.let { logger.info { "- rules -> $checkRules" } }
         executionMode.let { logger.info { "- execution mode -> $it" } }
         verbose.let { logger.info { "- verbose -> $it" } }
@@ -107,6 +127,7 @@ enum class CoreConfigKey(override val key: String,
                          override val default: String? = null) : ConfigKey {
     // GLOBAL
     VERBOSE("verbose", "false"),
+    DEFAULT_SCRIPT_ACTION("default.script.action", "RUN"),
 
     // SCAN
     SCAN_PATH("scan.path", "./scripts/"),
@@ -120,6 +141,7 @@ enum class CoreConfigKey(override val key: String,
 
     // PRUNER
     PRUNE_TAGS_TO_RUN_AGAIN("prune.tags.to.run.again"),
+    PRUNE_OVERRIDE_UPDATED_SCRIPTS("prune.scripts.override.executed", "false"),
 
     // CHECKER
     CHECK_RULES("check.rules"),

modules/core/src/main/kotlin/datamaintain/core/db/driver/DatamaintainDriver.kt

@@ -20,4 +20,9 @@ interface DatamaintainDriver {
      * Does not execute the given script, only inserts its execution in the database
      */
     fun markAsExecuted(executedScript: ExecutedScript): ExecutedScript
+
+    /**
+     * Does not execute the given script, only update its execution in the database
+     */
+    fun overrideScript(executedScript: ExecutedScript): ExecutedScript
 }

modules/core/src/main/kotlin/datamaintain/core/script/ExecutionStatus.kt

@@ -1,9 +1,8 @@
 package datamaintain.core.script
 
 enum class ExecutionStatus {
-    OK, KO, FORCE_MARKED_AS_EXECUTED, SHOULD_BE_EXECUTED;
-
-    fun correctlyExecuted() = this == OK ||
-            this == FORCE_MARKED_AS_EXECUTED ||
-            this == SHOULD_BE_EXECUTED
+    OK,
+    KO,
+    @Deprecated("Will be removed in 2.0. Useless now we store action on script")
+    FORCE_MARKED_AS_EXECUTED;
 }

modules/core/src/main/kotlin/datamaintain/core/script/FileScript.kt

@@ -1,14 +1,15 @@
 package datamaintain.core.script
 
-import java.lang.IllegalStateException
+import datamaintain.core.config.DatamaintainConfig
 import java.math.BigInteger
 import java.nio.file.Path
 import java.security.MessageDigest
 
 class FileScript @JvmOverloads constructor(
         val path: Path,
         identifierRegex: Regex,
-        override val tags: Set<Tag> = setOf()
+        override val tags: Set<Tag> = setOf(),
+        override var action: ScriptAction = DatamaintainConfig.defaultAction
 ) : ScriptWithContent {
 
     override val name: String

modules/core/src/main/kotlin/datamaintain/core/script/Script.kt

@@ -2,46 +2,44 @@ package datamaintain.core.script
 
 import datamaintain.core.step.executor.Execution
 
-import java.time.Duration
-import java.time.LocalDateTime
-
 interface Script {
     val name: String
     val checksum: String
     val identifier: String
+
+    fun fullName(): String {
+        return name + identifier
+    }
 }
 
 data class ExecutedScript @JvmOverloads constructor(
         override val name: String,
         override val checksum: String,
         override val identifier: String,
         val executionStatus: ExecutionStatus,
+        var action: ScriptAction,
         val executionDurationInMillis: Long? = null,
         val executionOutput: String? = null
 ) : Script {
     companion object {
-        fun forceMarkAsExecuted(script: ScriptWithContent) =
+        fun simulateExecuted(script: ScriptWithContent, executionStatus: ExecutionStatus) =
                 ExecutedScript(
                         script.name,
                         script.checksum,
                         script.identifier,
-                        ExecutionStatus.FORCE_MARKED_AS_EXECUTED
+                        executionStatus,
+                        script.action
                 )
 
-        fun shouldBeExecuted(script: ScriptWithContent) =
-                ExecutedScript(
-                        script.name,
-                        script.checksum,
-                        script.identifier,
-                        ExecutionStatus.SHOULD_BE_EXECUTED
-                )
+        fun build(script: ScriptWithContent, execution: Execution) = simulateExecuted(script, execution.executionStatus)
 
         fun build(script: ScriptWithContent, execution: Execution, executionDurationInMillis: Long) =
                 ExecutedScript(
                         script.name,
                         script.checksum,
                         script.identifier,
                         execution.executionStatus,
+                        script.action,
                         executionDurationInMillis,
                         execution.executionOutput
                 )
@@ -51,6 +49,7 @@ data class ExecutedScript @JvmOverloads constructor(
 interface ScriptWithContent : Script {
     val content: String
     val tags: Set<Tag>
+    var action: ScriptAction
 }
 
 

modules/core/src/main/kotlin/datamaintain/core/script/ScriptAction.kt

@@ -0,0 +1,31 @@
+package datamaintain.core.script
+
+enum class ScriptAction {
+
+    /**
+     * Indicates the script need to be executed and then marked as executed in DB.
+     */
+    RUN,
+
+    /**
+     * Indicates the script only need to be marked as executed in DB. No execution.
+     */
+    MARK_AS_EXECUTED,
+
+    /**
+     * Indicates to override an updated script that datamaintain will detect as already executed (based
+     * on the script name). No execution.
+     */
+    OVERRIDE_EXECUTED;
+
+    companion object {
+        fun fromNullable(name: String?, defaultMode: ScriptAction): ScriptAction {
+            return if (name != null) {
+                valueOf(name)
+            } else {
+                defaultMode
+            }
+        }
+    }
+
+}