Skip to content

Commit

Permalink
Fix deployments for scenarios with dict editors after model reload (#…
Browse files Browse the repository at this point in the history
  • Loading branch information
Elmacioro authored Nov 19, 2024
1 parent 5ebf989 commit 580adec
Show file tree
Hide file tree
Showing 35 changed files with 447 additions and 66 deletions.
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
package pl.touk.nussknacker.engine.api.parameter

import io.circe.generic.extras.semiauto.{deriveUnwrappedDecoder, deriveUnwrappedEncoder}
import io.circe.{Decoder, Encoder, KeyDecoder, KeyEncoder}

final case class ParameterName(value: String) {
def withBranchId(branchId: String): ParameterName = ParameterName(s"$value for branch $branchId")
}

object ParameterName {
implicit val encoder: Encoder[ParameterName] = deriveUnwrappedEncoder
implicit val decoder: Decoder[ParameterName] = deriveUnwrappedDecoder

implicit val keyEncoder: KeyEncoder[ParameterName] = KeyEncoder.encodeKeyString.contramap(_.value)
implicit val keyDecoder: KeyDecoder[ParameterName] = KeyDecoder.decodeKeyString.map(ParameterName(_))
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package pl.touk.nussknacker.engine.api.component

import io.circe.generic.JsonCodec
import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue
import pl.touk.nussknacker.engine.api.parameter.{
ParameterName,
Expand All @@ -24,6 +25,7 @@ object AdditionalUIConfigProvider {
val empty = new DefaultAdditionalUIConfigProvider(Map.empty, Map.empty)
}

@JsonCodec
case class ComponentAdditionalConfig(
parameterConfigs: Map[ParameterName, ParameterAdditionalUIConfig],
icon: Option[String] = None,
Expand All @@ -32,6 +34,7 @@ case class ComponentAdditionalConfig(
disabled: Boolean = false
)

@JsonCodec
case class ParameterAdditionalUIConfig(
required: Boolean,
initialValue: Option[FixedExpressionValue],
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package pl.touk.nussknacker.engine.api.component

import io.circe.generic.extras.semiauto.{deriveUnwrappedDecoder, deriveUnwrappedEncoder}
import io.circe.{Decoder, Encoder}
import io.circe.{Decoder, Encoder, KeyDecoder, KeyEncoder}

// TODO This class is used as a work around for the problem that the components are duplicated across processing types.
// We plan to get rid of this. After that, we could replace usages of this class by usage of ComponentId
Expand All @@ -14,6 +14,10 @@ object DesignerWideComponentId {
implicit val encoder: Encoder[DesignerWideComponentId] = deriveUnwrappedEncoder
implicit val decoder: Decoder[DesignerWideComponentId] = deriveUnwrappedDecoder

implicit val keyEncoder: KeyEncoder[DesignerWideComponentId] = KeyEncoder.encodeKeyString.contramap(_.value)
implicit val keyDecoder: KeyDecoder[DesignerWideComponentId] =
KeyDecoder.decodeKeyString.map(DesignerWideComponentId(_))

def apply(value: String): DesignerWideComponentId = new DesignerWideComponentId(value.toLowerCase)

def forBuiltInComponent(componentId: ComponentId): DesignerWideComponentId = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import pl.touk.nussknacker.engine.api.deployment.{
ProcessingTypeDeployedScenariosProvider,
ScenarioActivityManager
}
import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId}
import sttp.client3.SttpBackend

import scala.concurrent.{ExecutionContext, Future}
Expand All @@ -17,6 +18,7 @@ case class DeploymentManagerDependencies(
executionContext: ExecutionContext,
actorSystem: ActorSystem,
sttpBackend: SttpBackend[Future, Any],
configsFromProvider: Map[DesignerWideComponentId, ComponentAdditionalConfig] = Map.empty
) {
implicit def implicitExecutionContext: ExecutionContext = executionContext
implicit def implicitActorSystem: ActorSystem = actorSystem
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
package pl.touk.nussknacker.engine.util

import pl.touk.nussknacker.engine.api.component.{ComponentAdditionalConfig, DesignerWideComponentId}
import pl.touk.nussknacker.engine.api.parameter.ValueInputWithDictEditor

object AdditionalComponentConfigsForRuntimeExtractor {

// This is done to reduce data sent to Flink
def getRequiredAdditionalConfigsForRuntime(
additionalComponentConfigs: Map[DesignerWideComponentId, ComponentAdditionalConfig]
): Map[DesignerWideComponentId, ComponentAdditionalConfig] = {
getAdditionalConfigsWithDictParametersEditors(additionalComponentConfigs)
}

// This function filters additional configs provided by AdditionalUIConfigProvider
// to include only component and parameter configs with Dictionary editors.
private def getAdditionalConfigsWithDictParametersEditors(
additionalComponentConfigs: Map[DesignerWideComponentId, ComponentAdditionalConfig]
): Map[DesignerWideComponentId, ComponentAdditionalConfig] = additionalComponentConfigs
.map { case (componentId, componentAdditionalConfig) =>
val parametersWithDictEditors = componentAdditionalConfig.parameterConfigs.filter {
case (_, additionalUiConfig) =>
additionalUiConfig.valueEditor match {
case Some(_: ValueInputWithDictEditor) => true
case _ => false
}
}
componentId -> componentAdditionalConfig.copy(parameterConfigs = parametersWithDictEditors)
}
.filter(_._2.parameterConfigs.nonEmpty)

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
package pl.touk.nussknacker.engine.util

import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers
import pl.touk.nussknacker.engine.api.component.{
ComponentAdditionalConfig,
ComponentGroupName,
DesignerWideComponentId,
ParameterAdditionalUIConfig
}
import pl.touk.nussknacker.engine.api.definition.FixedExpressionValue
import pl.touk.nussknacker.engine.api.parameter.{ParameterName, ValueInputWithDictEditor}
import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractorTest.{
componentConfigWithDictionaryEditorInParameter,
componentConfigWithOnlyDictEditorParameters,
componentConfigWithoutDictionaryEditorInParameter
}

class AdditionalComponentConfigsForRuntimeExtractorTest extends AnyFunSuite with Matchers {

test("should filter only components and parameters with dictionary editors") {
val additionalConfig = Map(
DesignerWideComponentId("componentA") -> componentConfigWithDictionaryEditorInParameter,
DesignerWideComponentId("componentB") -> componentConfigWithoutDictionaryEditorInParameter,
)
val filteredResult =
AdditionalComponentConfigsForRuntimeExtractor.getRequiredAdditionalConfigsForRuntime(additionalConfig)

filteredResult shouldBe Map(
DesignerWideComponentId("componentA") -> componentConfigWithOnlyDictEditorParameters
)
}

}

object AdditionalComponentConfigsForRuntimeExtractorTest {

private val parameterAWithDictEditor = (
ParameterName("parameterA"),
ParameterAdditionalUIConfig(
required = true,
initialValue = Some(FixedExpressionValue("'someInitialValueExpression'", "someInitialValueLabel")),
hintText = None,
valueEditor = Some(ValueInputWithDictEditor("someDictA", allowOtherValue = true)),
valueCompileTimeValidation = None
)
)

private val parameterBWithDictEditor = (
ParameterName("parameterB"),
ParameterAdditionalUIConfig(
required = false,
initialValue = None,
hintText = Some("someHint"),
valueEditor = Some(ValueInputWithDictEditor("someDictB", allowOtherValue = false)),
valueCompileTimeValidation = None
)
)

private val parameterWithoutDictEditor = (
ParameterName("parameterC"),
ParameterAdditionalUIConfig(
required = true,
initialValue = None,
hintText = None,
valueEditor = None,
valueCompileTimeValidation = None
)
)

private val componentConfigWithDictionaryEditorInParameter = ComponentAdditionalConfig(
parameterConfigs = Map(
parameterAWithDictEditor,
parameterBWithDictEditor,
parameterWithoutDictEditor
),
icon = Some("someIcon"),
docsUrl = Some("someDocUrl"),
componentGroup = Some(ComponentGroupName("Service"))
)

private val componentConfigWithoutDictionaryEditorInParameter = ComponentAdditionalConfig(
parameterConfigs = Map(parameterWithoutDictEditor),
icon = Some("someOtherIcon"),
docsUrl = Some("someOtherDocUrl"),
componentGroup = Some(ComponentGroupName("Service"))
)

private val componentConfigWithOnlyDictEditorParameters = ComponentAdditionalConfig(
parameterConfigs = Map(
parameterAWithDictEditor,
parameterBWithDictEditor
),
icon = Some("someIcon"),
docsUrl = Some("someDocUrl"),
componentGroup = Some(ComponentGroupName("Service"))
)

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import pl.touk.nussknacker.engine.api.component.NodesDeploymentData
import pl.touk.nussknacker.engine.api.deployment._
import pl.touk.nussknacker.engine.api.process._
import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess
import pl.touk.nussknacker.engine.deployment.{DeploymentData, DeploymentId, User}
import pl.touk.nussknacker.engine.deployment.{AdditionalModelConfigs, DeploymentData, DeploymentId, User}
import pl.touk.nussknacker.ui.db.DbRef
import pl.touk.nussknacker.ui.process.ScenarioQuery
import pl.touk.nussknacker.ui.process.fragment.{DefaultFragmentRepository, FragmentResolver}
Expand Down Expand Up @@ -51,7 +51,8 @@ class DefaultProcessingTypeDeployedScenariosProvider(
DeploymentId.fromActionId(lastDeployAction.id),
deployingUser,
Map.empty,
NodesDeploymentData.empty
NodesDeploymentData.empty,
AdditionalModelConfigs.empty
)
val deployedScenarioDataTry =
scenarioResolver.resolveScenario(details.json).map { resolvedScenario =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,19 @@ import cats.syntax.functor._
import com.typesafe.scalalogging.LazyLogging
import db.util.DBIOActionInstances._
import pl.touk.nussknacker.engine.api.Comment
import pl.touk.nussknacker.engine.api.component.NodesDeploymentData
import pl.touk.nussknacker.engine.api.component.{
ComponentAdditionalConfig,
DesignerWideComponentId,
NodesDeploymentData
}
import pl.touk.nussknacker.engine.api.deployment.ScenarioActionName.{Cancel, Deploy}
import pl.touk.nussknacker.engine.api.deployment._
import pl.touk.nussknacker.engine.api.deployment.simple.SimpleStateStatus.ProblemStateStatus
import pl.touk.nussknacker.engine.api.deployment.simple.{SimpleProcessStateDefinitionManager, SimpleStateStatus}
import pl.touk.nussknacker.engine.api.process._
import pl.touk.nussknacker.engine.canonicalgraph.CanonicalProcess
import pl.touk.nussknacker.engine.deployment._
import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractor
import pl.touk.nussknacker.restmodel.scenariodetails.ScenarioWithDetails
import pl.touk.nussknacker.ui.api.{DeploymentCommentSettings, ListenerApiUser}
import pl.touk.nussknacker.ui.listener.ProcessChangeEvent.{OnActionExecutionFinished, OnActionFailed, OnActionSuccess}
Expand Down Expand Up @@ -55,6 +60,10 @@ class DeploymentService(
processChangeListener: ProcessChangeListener,
scenarioStateTimeout: Option[FiniteDuration],
deploymentCommentSettings: Option[DeploymentCommentSettings],
additionalComponentConfigs: ProcessingTypeDataProvider[
Map[DesignerWideComponentId, ComponentAdditionalConfig],
_
],
clock: Clock = Clock.systemUTC()
)(implicit system: ActorSystem)
extends ActionService
Expand Down Expand Up @@ -324,7 +333,8 @@ class DeploymentService(
DeploymentId.fromActionId(actionId),
user.toManagerUser,
additionalDeploymentData,
nodesDeploymentData
nodesDeploymentData,
getAdditionalModelConfigsRequiredForRuntime(processDetails.processingType)
)
} yield DeployedScenarioData(processDetails.toEngineProcessVersion, deploymentData, resolvedCanonicalProcess)
}
Expand Down Expand Up @@ -408,6 +418,14 @@ class DeploymentService(
)
}

private def getAdditionalModelConfigsRequiredForRuntime(processingType: ProcessingType)(implicit user: LoggedUser) = {
AdditionalModelConfigs(
AdditionalComponentConfigsForRuntimeExtractor.getRequiredAdditionalConfigsForRuntime(
additionalComponentConfigs.forProcessingType(processingType).getOrElse(Map.empty)
)
)
}

// TODO: check deployment id to be sure that returned status is for given deployment
override def getProcessState(
processIdWithName: ProcessIdWithName
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,21 @@ import cats.Applicative
import cats.data.{EitherT, NonEmptyList}
import com.typesafe.scalalogging.LazyLogging
import db.util.DBIOActionInstances._
import pl.touk.nussknacker.engine.api.component.NodesDeploymentData
import pl.touk.nussknacker.engine.api.component.{
ComponentAdditionalConfig,
DesignerWideComponentId,
NodesDeploymentData
}
import pl.touk.nussknacker.engine.api.deployment._
import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, VersionId}
import pl.touk.nussknacker.engine.api.process.{ProcessId, ProcessName, ProcessingType, VersionId}
import pl.touk.nussknacker.engine.api.{ProcessVersion => RuntimeVersionData}
import pl.touk.nussknacker.engine.deployment.{DeploymentData, DeploymentId => LegacyDeploymentId}
import pl.touk.nussknacker.engine.deployment.{
AdditionalModelConfigs,
DeploymentData,
DeploymentId => LegacyDeploymentId
}
import pl.touk.nussknacker.engine.newdeployment.DeploymentId
import pl.touk.nussknacker.engine.util.AdditionalComponentConfigsForRuntimeExtractor
import pl.touk.nussknacker.restmodel.validation.ValidationResults.ValidationErrors
import pl.touk.nussknacker.security.Permission
import pl.touk.nussknacker.security.Permission.Permission
Expand All @@ -19,6 +28,7 @@ import pl.touk.nussknacker.ui.process.deployment.DeploymentManagerDispatcher
import pl.touk.nussknacker.ui.process.deployment.LoggedUserConversions.LoggedUserOps
import pl.touk.nussknacker.ui.process.newdeployment.DeploymentEntityFactory.{DeploymentEntityData, WithModifiedAt}
import pl.touk.nussknacker.ui.process.newdeployment.DeploymentService._
import pl.touk.nussknacker.ui.process.processingtype.provider.ProcessingTypeDataProvider
import pl.touk.nussknacker.ui.process.repository.{DBIOActionRunner, ScenarioMetadataRepository}
import pl.touk.nussknacker.ui.process.version.ScenarioGraphVersionService
import pl.touk.nussknacker.ui.security.api.LoggedUser
Expand All @@ -42,7 +52,11 @@ class DeploymentService(
deploymentRepository: DeploymentRepository,
dmDispatcher: DeploymentManagerDispatcher,
dbioRunner: DBIOActionRunner,
clock: Clock
clock: Clock,
additionalComponentConfigs: ProcessingTypeDataProvider[
Map[DesignerWideComponentId, ComponentAdditionalConfig],
_
],
)(implicit ec: ExecutionContext)
extends LazyLogging {

Expand Down Expand Up @@ -152,11 +166,11 @@ class DeploymentService(
): EitherT[Future, RunDeploymentError, Unit] = {
val runtimeVersionData = processVersionFor(scenarioMetadata, scenarioGraphVersion)
// TODO: It shouldn't be needed
val dumbDeploymentData = DeploymentData(
val dumbDeploymentData = createDeploymentData(
LegacyDeploymentId(""),
user.toManagerUser,
Map.empty,
NodesDeploymentData.empty
user,
NodesDeploymentData.empty,
scenarioMetadata.processingType
)
for {
result <- EitherT[Future, RunDeploymentError, Unit](
Expand Down Expand Up @@ -185,11 +199,11 @@ class DeploymentService(
command: RunDeploymentCommand
): EitherT[Future, RunDeploymentError, Unit] = {
val runtimeVersionData = processVersionFor(scenarioMetadata, scenarioGraphVersion)
val deploymentData = DeploymentData(
val deploymentData = createDeploymentData(
toLegacyDeploymentId(command.id),
command.user.toManagerUser,
additionalDeploymentData = Map.empty,
command.nodesDeploymentData
command.user,
command.nodesDeploymentData,
scenarioMetadata.processingType
)
dmDispatcher
.deploymentManagerUnsafe(scenarioMetadata.processingType)(command.user)
Expand All @@ -211,6 +225,19 @@ class DeploymentService(
EitherT.pure(())
}

private def createDeploymentData(
deploymentId: LegacyDeploymentId,
loggedUser: LoggedUser,
nodesData: NodesDeploymentData,
processingType: ProcessingType
) = DeploymentData(
deploymentId = deploymentId,
user = loggedUser.toManagerUser,
additionalDeploymentData = Map.empty,
nodesData = nodesData,
additionalModelConfigs = getAdditionalModelConfigsRequiredForRuntime(processingType, loggedUser)
)

private def handleFailureDuringDeploymentRequesting(
deploymentId: DeploymentId,
ex: Throwable
Expand Down Expand Up @@ -256,6 +283,14 @@ class DeploymentService(
)
}

private def getAdditionalModelConfigsRequiredForRuntime(processingType: ProcessingType, loggedUser: LoggedUser) = {
AdditionalModelConfigs(
AdditionalComponentConfigsForRuntimeExtractor.getRequiredAdditionalConfigsForRuntime(
additionalComponentConfigs.forProcessingType(processingType)(loggedUser).getOrElse(Map.empty)
)
)
}

}

object DeploymentService {
Expand Down
Loading

0 comments on commit 580adec

Please sign in to comment.