Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#143 light executed script #163

Merged
merged 8 commits into from
Oct 19, 2021
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package datamaintain.core.db.driver

import datamaintain.core.script.ExecutedScript
import datamaintain.core.script.LightExecutedScript
import datamaintain.core.script.ScriptWithContent
import datamaintain.core.step.executor.Execution

Expand All @@ -9,7 +10,7 @@ abstract class DatamaintainDriver(protected val uri: String) {
/**
* Reads the executed scripts from the database and returns them
*/
abstract fun listExecutedScripts(): Sequence<ExecutedScript>
abstract fun listExecutedScripts(): Sequence<LightExecutedScript>

/**
* Executes the given script and inserts its execution in the database
Expand Down
32 changes: 31 additions & 1 deletion modules/core/src/main/kotlin/datamaintain/core/script/Script.kt
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,36 @@ interface Script {
}
}

open class LightExecutedScript(
override val name: String,
override val checksum: String,
override val identifier: String
) : Script {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false

other as LightExecutedScript

if (name != other.name) return false
if (checksum != other.checksum) return false
if (identifier != other.identifier) return false

return true
}

override fun hashCode(): Int {
var result = name.hashCode()
result = 31 * result + checksum.hashCode()
result = 31 * result + identifier.hashCode()
return result
}

override fun toString(): String {
return "LightExecutedScript(name='$name', checksum='$checksum', identifier='$identifier')"
}
}

Lysoun marked this conversation as resolved.
Show resolved Hide resolved
data class ExecutedScript @JvmOverloads constructor(
override val name: String,
override val checksum: String,
Expand All @@ -20,7 +50,7 @@ data class ExecutedScript @JvmOverloads constructor(
var action: ScriptAction? = null,
val executionDurationInMillis: Long? = null,
val executionOutput: String? = null
) : Script {
) : LightExecutedScript(name, checksum, identifier) {
companion object {
fun simulateExecuted(script: ScriptWithContent, executionStatus: ExecutionStatus) =
ExecutedScript(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package datamaintain.core.step.check.rules.contracts

import datamaintain.core.exception.DatamaintainCheckException
import datamaintain.core.script.ExecutedScript
import datamaintain.core.script.LightExecutedScript
import datamaintain.core.script.ScriptWithContent
import datamaintain.core.step.check.rules.CheckRule

abstract class FullContextCheckRule(
val executedScripts: Sequence<ExecutedScript>
val executedScripts: Sequence<LightExecutedScript>
): CheckRule {
/**
* @throws DatamaintainCheckException
Expand Down
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
package datamaintain.core.step.check.rules.implementations

import datamaintain.core.exception.DatamaintainCheckException
import datamaintain.core.script.ExecutedScript
import datamaintain.core.script.LightExecutedScript
import datamaintain.core.script.ScriptWithContent
import datamaintain.core.step.check.rules.ScriptType
import datamaintain.core.step.check.rules.contracts.FullContextCheckRule

class SameScriptsAsExecutedCheck(
executedScripts: Sequence<ExecutedScript>
executedScripts: Sequence<LightExecutedScript>
) : FullContextCheckRule(executedScripts) {
override fun check(scripts: Sequence<ScriptWithContent>) {
val executedScriptChecksumsNotFoundInScannedScripts = executedScripts
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
package datamaintain.db.driver.jdbc

import datamaintain.core.db.driver.DatamaintainDriver
import datamaintain.core.script.ExecutedScript
import datamaintain.core.script.ExecutionStatus
import datamaintain.core.script.ScriptAction
import datamaintain.core.script.ScriptWithContent
import datamaintain.core.script.*
import datamaintain.core.step.executor.Execution
import java.sql.Connection
import java.sql.DriverManager
Expand All @@ -28,14 +25,14 @@ class JdbcDriver(jdbcUri: String) : DatamaintainDriver(jdbcUri) {
}
}

override fun listExecutedScripts(): Sequence<ExecutedScript> {
override fun listExecutedScripts(): Sequence<LightExecutedScript> {
createExecutedScriptsTableIfNotExists()

val statement = connection.createStatement()
val executionOutput: ResultSet = statement.executeQuery("SELECT * from $EXECUTED_SCRIPTS_TABLE")
val executedScript = mutableListOf<ExecutedScript>()
val executionOutput: ResultSet = statement.executeQuery("SELECT name, checksum, identifier from $EXECUTED_SCRIPTS_TABLE")
val executedScript = mutableListOf<LightExecutedScript>()
while (executionOutput.next()) {
executedScript.add(executionOutput.toExecutedScript())
executedScript.add(executionOutput.toLightExecutedScript())
}
return executedScript.asSequence()
}
Expand Down Expand Up @@ -94,13 +91,9 @@ class JdbcDriver(jdbcUri: String) : DatamaintainDriver(jdbcUri) {
return executedScript
}

fun ResultSet.toExecutedScript() = ExecutedScript(
name = this.getString("name"),
checksum = this.getString("checksum"),
executionDurationInMillis = this.getLong("executionDurationInMillis"),
executionOutput = this.getString("executionOutput"),
executionStatus = ExecutionStatus.valueOf(this.getString("executionStatus")),
identifier = this.getString("identifier"),
action = ScriptAction.valueOf(this.getString("action"))
private fun ResultSet.toLightExecutedScript() = LightExecutedScript(
name = this.getString("name"),
checksum = this.getString("checksum"),
identifier = this.getString("identifier")
)
}

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
package datamaintain.db.driver.jdbc

import datamaintain.core.script.ExecutedScript
import datamaintain.core.script.ExecutionStatus
import datamaintain.core.script.FileScript
import datamaintain.core.script.ScriptAction
import datamaintain.core.script.*
import org.h2.tools.Server
import org.junit.jupiter.api.*
import strikt.api.expectThat
Expand Down Expand Up @@ -45,7 +42,7 @@ internal class JdbcDriverTest {
val executedScripts = jdbcDatamaintainDriver.listExecutedScripts()

// Then
expectThat(executedScripts.toList()).containsExactlyInAnyOrder(script1, script2)
expectThat(executedScripts.toList()).containsExactlyInAnyOrder(script1.toLightExecutedScript(), script2.toLightExecutedScript())
}

@Test
Expand All @@ -67,7 +64,7 @@ internal class JdbcDriverTest {
// Then

expectThat(jdbcDatamaintainDriver.listExecutedScripts().toList())
.containsExactlyInAnyOrder(script1, script2, script3)
.containsExactlyInAnyOrder(script1.toLightExecutedScript(), script2.toLightExecutedScript(), script3.toLightExecutedScript())
}

@Nested
Expand Down Expand Up @@ -144,8 +141,8 @@ internal class JdbcDriverTest {
// Then
expectThat(jdbcDatamaintainDriver.listExecutedScripts().toList())
.containsExactlyInAnyOrder(
script3,
script2
script3.toLightExecutedScript(),
script2.toLightExecutedScript()
)
}

Expand Down Expand Up @@ -179,3 +176,5 @@ internal class JdbcDriverTest {

}

private fun ExecutedScript.toLightExecutedScript(): LightExecutedScript = LightExecutedScript(name, checksum, identifier)

Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,6 @@ package datamaintain.db.driver.mongo
import datamaintain.core.script.ExecutedScript

interface ExecutedScriptJsonParser {
// parse a stringify json array to an Array of ExecutedScript
fun parseArrayOfExecutedScripts(executedScriptJsonArray: String): Sequence<ExecutedScript>

// Serialize an ExecutedScript to a stringify json document
fun serializeExecutedScript(executedScript:ExecutedScript): String
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
package datamaintain.db.driver.mongo

import datamaintain.core.script.LightExecutedScript

interface LightExecutedScriptJsonParser {
// parse a stringify json array to an Array of LightExecutedScript
fun parseArrayOfLightExecutedScripts(lightExecutedScriptJsonArray: String): Sequence<LightExecutedScript>
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,7 @@ package datamaintain.db.driver.mongo

import datamaintain.core.db.driver.DatamaintainDriver
import datamaintain.core.exception.DatamaintainMongoQueryException
import datamaintain.core.script.ExecutedScript
import datamaintain.core.script.ExecutionStatus
import datamaintain.core.script.FileScript
import datamaintain.core.script.ScriptWithContent
import datamaintain.core.script.*
import datamaintain.core.step.executor.Execution
import datamaintain.core.util.runProcess
import datamaintain.db.driver.mongo.serialization.KJsonParser
Expand Down Expand Up @@ -80,9 +77,9 @@ class MongoDriver(mongoUri: String,
return null
}

override fun listExecutedScripts(): Sequence<ExecutedScript> {
val executionOutput: String = executeMongoQuery("db.$EXECUTED_SCRIPTS_COLLECTION.find().toArray()")
return if (executionOutput.isNotBlank()) jsonParser.parseArrayOfExecutedScripts(executionOutput) else emptySequence()
override fun listExecutedScripts(): Sequence<LightExecutedScript> {
val executionOutput: String = executeMongoQuery("db.$EXECUTED_SCRIPTS_COLLECTION.find({}, { \"name\": 1, \"checksum\": 1, \"identifier\": 1}).toArray()")
return if (executionOutput.isNotBlank()) jsonParser.parseArrayOfLightExecutedScripts(executionOutput) else emptySequence()
}

override fun markAsExecuted(executedScript: ExecutedScript): ExecutedScript {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package datamaintain.db.driver.mongo.serialization

import datamaintain.core.script.ExecutedScript
import datamaintain.core.script.ExecutionStatus
import datamaintain.core.script.LightExecutedScript
import datamaintain.core.script.ScriptAction
import datamaintain.db.driver.mongo.ExecutedScriptJsonParser
import datamaintain.db.driver.mongo.LightExecutedScriptJsonParser
import kotlinx.serialization.ContextualSerialization
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
Expand All @@ -12,6 +14,21 @@ import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonConfiguration
import java.util.*

// Copy of LightExecutedScript, this is aim for add the Serializable annotation
// Annotation allow to serialize/deserialize this object to/from a bson document (support json only)
@Serializable
data class LightExecutedScriptDb(@SerialName("_id") @ContextualSerialization val id: String = UUID.randomUUID().toString(),
val name: String,
val checksum: String,
val identifier: String)

// Mapping function
fun LightExecutedScriptDb.toLightExecutedScript() = LightExecutedScript(
name,
checksum,
identifier
)

// Copy of ExecutedScript, this is aim for add the Serializable annotation
// Annotation allow to serialize/deserialize this object to/from a bson document (support json only)
@Serializable
Expand All @@ -26,39 +43,38 @@ data class ExecutedScriptDb(@SerialName("_id") @ContextualSerialization val id:

// Mapping function
fun ExecutedScriptDb.toExecutedScript() = ExecutedScript(
name,
checksum,
identifier,
executionStatus,
action,
executionDurationInMillis,
executionOutput
name,
checksum,
identifier,
executionStatus,
action,
executionDurationInMillis,
executionOutput
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this mapping function still necessary? It appears to me that only light executed scripts will be read from the database from now on

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indeed

)
fun ExecutedScript.toExecutedScriptDb() = ExecutedScriptDb(
name = name,
checksum = checksum,
identifier = identifier,
executionStatus = executionStatus,
action = action,
executionDurationInMillis = executionDurationInMillis,
executionOutput = executionOutput
name = name,
checksum = checksum,
identifier = identifier,
executionStatus = executionStatus,
action = action,
executionDurationInMillis = executionDurationInMillis,
executionOutput = executionOutput
)

class KJsonParser: ExecutedScriptJsonParser {
class KJsonParser: ExecutedScriptJsonParser, LightExecutedScriptJsonParser {
// Mapper between json and object
val configuration = JsonConfiguration.Stable.copy(ignoreUnknownKeys = true)
private val mapper = Json(configuration)

override fun parseArrayOfExecutedScripts(executedScriptJsonArray: String): Sequence<ExecutedScript> {

return mapper.parse(ExecutedScriptDb.serializer().list, executedScriptJsonArray)
.map { it.toExecutedScript() }
.asSequence()
}

override fun serializeExecutedScript(executedScript: ExecutedScript): String {
val executedScriptDb = executedScript.toExecutedScriptDb()

return mapper.stringify(ExecutedScriptDb.serializer(), executedScriptDb)
}

override fun parseArrayOfLightExecutedScripts(lightExecutedScriptJsonArray: String): Sequence<LightExecutedScript> {
return mapper.parse(LightExecutedScriptDb.serializer().list, lightExecutedScriptJsonArray)
.map { it.toLightExecutedScript() }
.asSequence()
}
}
Loading