diff --git a/distage/distage-extension-config/.js/src/main/scala/izumi/distage/config/model/AppConfig.scala b/distage/distage-extension-config/.js/src/main/scala/izumi/distage/config/model/AppConfig.scala new file mode 100644 index 0000000000..c0a4cc28b2 --- /dev/null +++ b/distage/distage-extension-config/.js/src/main/scala/izumi/distage/config/model/AppConfig.scala @@ -0,0 +1,12 @@ +package izumi.distage.config.model + +import izumi.distage.config.DistageConfigImpl + +final case class AppConfig( + config: DistageConfigImpl +) + +object AppConfig { + val empty: AppConfig = AppConfig(Map.empty[String, String]) + def provided(config: DistageConfigImpl): AppConfig = AppConfig(config) +} diff --git a/distage/distage-extension-config/.js/src/main/scala/izumi/distage/config/model/AppConfigSyntax.scala b/distage/distage-extension-config/.js/src/main/scala/izumi/distage/config/model/AppConfigSyntax.scala deleted file mode 100644 index ab1d91b0a8..0000000000 --- a/distage/distage-extension-config/.js/src/main/scala/izumi/distage/config/model/AppConfigSyntax.scala +++ /dev/null @@ -1,5 +0,0 @@ -package izumi.distage.config.model - -trait AppConfigSyntax { - val empty: AppConfig = AppConfig(Map.empty[String, String]) -} diff --git a/distage/distage-extension-config/.jvm/src/main/scala/izumi/distage/config/model/AppConfig.scala b/distage/distage-extension-config/.jvm/src/main/scala/izumi/distage/config/model/AppConfig.scala new file mode 100644 index 0000000000..0ed8dde920 --- /dev/null +++ b/distage/distage-extension-config/.jvm/src/main/scala/izumi/distage/config/model/AppConfig.scala @@ -0,0 +1,67 @@ +package izumi.distage.config.model + +import com.typesafe.config.{Config, ConfigFactory} +import izumi.distage.config.DistageConfigImpl + +import java.io.File + +final case class AppConfig( + config: DistageConfigImpl, + shared: List[ConfigLoadResult.Success], + roles: List[LoadedRoleConfigs], +) + +object AppConfig { + val empty: AppConfig = AppConfig(ConfigFactory.empty(), List.empty, List.empty) + def provided(config: DistageConfigImpl): AppConfig = AppConfig(config, List.empty, List.empty) +} + +sealed trait GenericConfigSource + +object GenericConfigSource { + case class ConfigFile(file: File) extends GenericConfigSource + + case object ConfigDefault extends GenericConfigSource +} + +case class RoleConfig(role: String, active: Boolean, configSource: GenericConfigSource) + +case class LoadedRoleConfigs(roleConfig: RoleConfig, loaded: Seq[ConfigLoadResult.Success]) + +sealed trait ConfigLoadResult { + def clue: String + + def src: ConfigSource + + def toEither: Either[ConfigLoadResult.Failure, ConfigLoadResult.Success] +} + +object ConfigLoadResult { + case class Success(clue: String, src: ConfigSource, config: Config) extends ConfigLoadResult { + override def toEither: Either[ConfigLoadResult.Failure, ConfigLoadResult.Success] = Right(this) + } + + case class Failure(clue: String, src: ConfigSource, failure: Throwable) extends ConfigLoadResult { + override def toEither: Either[ConfigLoadResult.Failure, ConfigLoadResult.Success] = Left(this) + } +} + +sealed trait ConfigSource + +object ConfigSource { + final case class Resource(name: String, kind: ResourceConfigKind) extends ConfigSource { + override def toString: String = s"resource:$name" + } + + final case class File(file: java.io.File) extends ConfigSource { + override def toString: String = s"file:$file" + } +} + +sealed trait ResourceConfigKind + +object ResourceConfigKind { + case object Primary extends ResourceConfigKind + + case object Development extends ResourceConfigKind +} diff --git a/distage/distage-extension-config/.jvm/src/main/scala/izumi/distage/config/model/AppConfigSyntax.scala b/distage/distage-extension-config/.jvm/src/main/scala/izumi/distage/config/model/AppConfigSyntax.scala deleted file mode 100644 index 50111341e4..0000000000 --- a/distage/distage-extension-config/.jvm/src/main/scala/izumi/distage/config/model/AppConfigSyntax.scala +++ /dev/null @@ -1,7 +0,0 @@ -package izumi.distage.config.model - -import com.typesafe.config.ConfigFactory - -trait AppConfigSyntax { - val empty: AppConfig = AppConfig(ConfigFactory.empty()) -} diff --git a/distage/distage-extension-config/.jvm/src/test/scala/izumi/distage/config/ConfigTest.scala b/distage/distage-extension-config/.jvm/src/test/scala/izumi/distage/config/ConfigTest.scala index bcc38a4b75..1570a1c9c6 100644 --- a/distage/distage-extension-config/.jvm/src/test/scala/izumi/distage/config/ConfigTest.scala +++ b/distage/distage-extension-config/.jvm/src/test/scala/izumi/distage/config/ConfigTest.scala @@ -20,7 +20,7 @@ final class ConfigTest extends AnyWordSpec { } def mkModule(config: Config): AppConfigModule = { - val appConfig = AppConfig(config) + val appConfig = AppConfig(config, List.empty, List.empty) new AppConfigModule(appConfig) } diff --git a/distage/distage-extension-config/src/main/scala/izumi/distage/config/AppConfigModule.scala b/distage/distage-extension-config/src/main/scala/izumi/distage/config/AppConfigModule.scala index c69db5fc05..8f6bd3e44a 100644 --- a/distage/distage-extension-config/src/main/scala/izumi/distage/config/AppConfigModule.scala +++ b/distage/distage-extension-config/src/main/scala/izumi/distage/config/AppConfigModule.scala @@ -5,11 +5,9 @@ import izumi.distage.model.definition.ModuleDef class AppConfigModule(appConfig: AppConfig) extends ModuleDef { make[AppConfig].fromValue(appConfig) - - def this(config: DistageConfigImpl) = this(AppConfig(config)) } object AppConfigModule { def apply(appConfig: AppConfig): AppConfigModule = new AppConfigModule(appConfig) - def apply(config: DistageConfigImpl): AppConfigModule = new AppConfigModule(config) + def apply(config: DistageConfigImpl): AppConfigModule = new AppConfigModule(AppConfig.provided(config)) } diff --git a/distage/distage-extension-config/src/main/scala/izumi/distage/config/model/AppConfig.scala b/distage/distage-extension-config/src/main/scala/izumi/distage/config/model/AppConfig.scala deleted file mode 100644 index 58382b4da9..0000000000 --- a/distage/distage-extension-config/src/main/scala/izumi/distage/config/model/AppConfig.scala +++ /dev/null @@ -1,7 +0,0 @@ -package izumi.distage.config.model - -import izumi.distage.config.DistageConfigImpl - -final case class AppConfig(config: DistageConfigImpl) - -object AppConfig extends AppConfigSyntax {} diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/CheckableApp.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/CheckableApp.scala index 2398c823f4..818fd2efb2 100644 --- a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/CheckableApp.scala +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/CheckableApp.scala @@ -173,7 +173,7 @@ abstract class RoleCheckableApp[F[_]](override implicit val tagK: TagK[F]) exten if (cfg.origin().resource() eq null) { throw new DIConfigReadException(s"Couldn't find a config resource with name `$resourceName` - file not found", null) } - AppConfig(cfg) + AppConfig(cfg, List.empty, List.empty) } } } diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/model/PlanCheckInput.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/model/PlanCheckInput.scala index 099f3d8b87..f37cadf1fa 100644 --- a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/model/PlanCheckInput.scala +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/model/PlanCheckInput.scala @@ -1,6 +1,7 @@ package izumi.distage.framework.model import distage.Injector +import izumi.distage.framework.services.ConfigMerger.ConfigMergerImpl import izumi.distage.framework.services.{ConfigArgsProvider, ConfigLoader, ConfigLocationProvider} import izumi.distage.model.definition.ModuleBase import izumi.distage.model.plan.Roots @@ -25,7 +26,11 @@ object PlanCheckInput { module: ModuleBase, roots: Roots, roleNames: Set[String] = Set.empty, - configLoader: ConfigLoader = new ConfigLoader.LocalFSImpl(IzLogger(), ConfigLocationProvider.Default, ConfigArgsProvider.Empty), + configLoader: ConfigLoader = { + val logger = IzLogger() + val merger = new ConfigMergerImpl(logger) + new ConfigLoader.LocalFSImpl(logger, merger, ConfigLocationProvider.Default, ConfigArgsProvider.Empty) + }, appPlugins: LoadedPlugins = LoadedPlugins.empty, bsPlugins: LoadedPlugins = LoadedPlugins.empty, )(implicit effectType: TagK[F], diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigArgsProvider.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigArgsProvider.scala index 3d1aa9d661..a45e84f0ad 100644 --- a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigArgsProvider.scala +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigArgsProvider.scala @@ -1,5 +1,6 @@ package izumi.distage.framework.services +import izumi.distage.config.model.{GenericConfigSource, RoleConfig} import izumi.distage.roles.RoleAppMain import izumi.distage.roles.model.meta.RolesInfo import izumi.fundamentals.platform.cli.model.raw.RawAppArgs @@ -12,8 +13,9 @@ trait ConfigArgsProvider { } object ConfigArgsProvider { + object Empty extends ConfigArgsProvider { - override def args(): ConfigLoader.Args = ConfigLoader.Args(None, Map.empty) + override def args(): ConfigLoader.Args = ConfigLoader.Args(None, List.empty) } @nowarn("msg=Unused import") @@ -24,18 +26,23 @@ object ConfigArgsProvider { override def args(): ConfigLoader.Args = { import scala.collection.compat.* - val emptyRoleConfigs = rolesInfo.availableRoleNames.map(_ -> None).toMap - - val maybeGlobalConfig = parameters.globalParameters.findValue(RoleAppMain.Options.configParam).asFile - val specifiedRoleConfigs = parameters.roles.iterator + val specifiedRoleConfigs: Map[String, Option[File]] = parameters.roles.iterator .map(roleParams => roleParams.role -> roleParams.roleParameters.findValue(RoleAppMain.Options.configParam).asFile) .toMap - // ConfigLoader.Args(maybeGlobalConfig, (emptyRoleConfigs ++ specifiedRoleConfigs).view.toMap) - val allConfigs: Map[String, Option[File]] = emptyRoleConfigs ++ specifiedRoleConfigs - ConfigLoader.Args(maybeGlobalConfig, allConfigs.view.filterKeys(rolesInfo.requiredRoleNames).toMap) + val roleConfigs = rolesInfo.availableRoleNames.toList.map { + roleName => + val source = specifiedRoleConfigs.get(roleName).flatten match { + case Some(value) => + GenericConfigSource.ConfigFile(value) + case _ => + GenericConfigSource.ConfigDefault + } + RoleConfig(roleName, rolesInfo.requiredRoleNames.contains(roleName), source) + } + val maybeGlobalConfig = parameters.globalParameters.findValue(RoleAppMain.Options.configParam).asFile + ConfigLoader.Args(maybeGlobalConfig, roleConfigs) } - } } diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala index 91b67404e1..f903bbffee 100644 --- a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLoader.scala @@ -1,17 +1,18 @@ package izumi.distage.framework.services -import com.typesafe.config.{Config, ConfigFactory, ConfigResolveOptions} -import izumi.distage.config.model.AppConfig -import izumi.distage.framework.services.ConfigLoader.LocalFSImpl.{ConfigLoaderException, ConfigSource, ResourceConfigKind} +import com.typesafe.config.{Config, ConfigFactory} +import izumi.distage.config.model.* import izumi.distage.model.definition.Id import izumi.distage.model.exceptions.DIException +import izumi.functional.IzEither.* +import izumi.fundamentals.platform.exceptions.IzThrowable.* import izumi.fundamentals.platform.resources.IzResources import izumi.fundamentals.platform.resources.IzResources.{LoadablePathReference, UnloadablePathReference} import izumi.fundamentals.platform.strings.IzString.* import izumi.logstage.api.IzLogger import java.io.{File, FileNotFoundException} -import scala.jdk.CollectionConverters.* +import scala.annotation.nowarn import scala.util.{Failure, Success, Try} /** @@ -48,18 +49,21 @@ import scala.util.{Failure, Success, Try} */ trait ConfigLoader extends AbstractConfigLoader {} +@nowarn("msg=Unused import") object ConfigLoader { - def empty: ConfigLoader = () => AppConfig(ConfigFactory.empty()) + def empty: ConfigLoader = () => AppConfig(ConfigFactory.empty(), List.empty, List.empty) import scala.collection.compat.* final case class Args( global: Option[File], - role: Map[String, Option[File]], + configs: List[RoleConfig], ) + final class ConfigLoaderException(message: String, val failures: List[Throwable]) extends DIException(message) open class LocalFSImpl( logger: IzLogger @Id("early"), + merger: ConfigMerger, configLocation: ConfigLocationProvider, args: ConfigArgsProvider, ) extends ConfigLoader { @@ -67,132 +71,94 @@ object ConfigLoader { def loadConfig(): AppConfig = { val configArgs = args.args() - val commonReferenceConfigs = configLocation.defaultBaseConfigs.flatMap(configLocation.forBase) - val commonExplicitConfigs = configArgs.global.map(ConfigSource.File.apply).toList - val (roleReferenceConfigs, roleExplicitConfigs) = (configArgs.role: Iterable[(String, Option[File])]).partitionMap { - case (role, None) => Left(configLocation.forRole(role)) - case (_, Some(file)) => Right(ConfigSource.File(file)) + val maybeLoadedRoleConfigs = configArgs.configs.map { + rc => + val loaded = rc.configSource match { + case GenericConfigSource.ConfigFile(file) => + Seq(loadConfigSource(ConfigSource.File(file))) + case GenericConfigSource.ConfigDefault => + configLocation.forRole(rc.role).map(loadConfigSource) + } + (rc, loaded) } - val allConfigs = (roleExplicitConfigs.iterator ++ commonExplicitConfigs ++ roleReferenceConfigs.iterator.flatten ++ commonReferenceConfigs).toList - - val (cfgInfo, loaded) = loadConfigSources(allConfigs) - - logger.info(s"Using system properties with fallback ${cfgInfo.niceList() -> "config files"}") - - val (good, bad) = loaded.partition(_._2.isSuccess) - - if (bad.nonEmpty) { - val failuresList = bad.collect { - case (s, Failure(f)) => - s"$s: $f" - } - val failures = failuresList.niceList() - - logger.error(s"Failed to load configs: $failures") - throw new ConfigLoaderException(s"Failed to load configs: failures=$failures", failuresList) - } else { - val folded = foldConfigs(good.collect { - case (src, Success(c)) => src -> c - }) - - val config = ConfigFactory - .systemProperties() - .withFallback(folded) - .resolve() + val commonExplicitConfigs = configArgs.global.map(ConfigSource.File.apply).toList + val commonReferenceConfigs = configLocation.commonReferenceConfigs.toList + val commonConfigs = commonReferenceConfigs ++ commonExplicitConfigs + val loaded = for { + loadedCommonConfigs <- commonConfigs.map(loadConfigSource).map(_.toEither).biSequenceScalar + loadedRoleConfigs <- maybeLoadedRoleConfigs.map { + case (rc, loaded) => + loaded.map { + lr => + lr.toEither + }.biSequenceScalar match { + case Left(value) => + Left(value) + case Right(value) => + Right(LoadedRoleConfigs(rc, value)) + } + }.biSequence + } yield { + (loadedCommonConfigs, loadedRoleConfigs) + } - AppConfig(config) + loaded match { + case Left(value) => + val failures = value.map(f => s"Failed to load ${f.src} ${f.clue}: ${f.failure.stacktraceString}") + logger.error(s"Cannot load configuration: ${failures.toList.niceList() -> "failures"}") + throw new ConfigLoaderException(s"Cannot load configuration: ${failures.toList.niceList()}", value.map(_.failure).toList) + case Right((shared, role)) => + val merged = merger.merge(shared, role) + AppConfig(merged, shared, role) } - } - protected def loadConfigSources(allConfigs: List[ConfigSource]): (List[String], List[(ConfigSource, Try[Config])]) = { - allConfigs.map(loadConfigSource).unzip } - protected def loadConfigSource(configSource: ConfigSource): (String, (ConfigSource, Try[Config])) = configSource match { - case r: ConfigSource.Resource => - def tryLoadResource(): Try[Config] = { - Try(ConfigFactory.parseResources(resourceClassLoader, r.name)).flatMap { - cfg => - if (cfg.origin().resource() eq null) { - Failure(new FileNotFoundException(s"Couldn't find config file $r")) - } else Success(cfg) + protected def loadConfigSource(configSource: ConfigSource): ConfigLoadResult = { + configSource match { + case r: ConfigSource.Resource => + def tryLoadResource(): Try[Config] = { + Try(ConfigFactory.parseResources(resourceClassLoader, r.name)).flatMap { + cfg => + if (cfg.origin().resource() eq null) { + Failure(new FileNotFoundException(s"Couldn't find config file $r")) + } else Success(cfg) + } } - } - - IzResources(resourceClassLoader).getPath(r.name) match { - case Some(LoadablePathReference(path, _)) => - s"$r (available: $path)" -> (r -> tryLoadResource()) - case Some(UnloadablePathReference(path)) => - s"$r (exists: $path)" -> (r -> tryLoadResource()) - case None => - s"$r (missing)" -> (r -> Success(ConfigFactory.empty())) - } - - case f: ConfigSource.File => - if (f.file.exists()) { - s"$f (exists: ${f.file.getCanonicalPath})" -> (f -> Try(ConfigFactory.parseFile(f.file)).flatMap { - cfg => if (cfg.origin().filename() ne null) Success(cfg) else Failure(new FileNotFoundException(s"Couldn't find config file $f")) - }) - } else { - s"$f (missing)" -> (f -> Success(ConfigFactory.empty())) - } - } - protected def foldConfigs(roleConfigs: IterableOnce[(ConfigSource, Config)]): Config = { - roleConfigs.iterator.foldLeft(ConfigFactory.empty()) { - case (acc, (src, cfg)) => - verifyConfigs(src, cfg, acc) - acc.withFallback(cfg) - } - } - - protected def verifyConfigs(src: ConfigSource, cfg: Config, acc: Config): Unit = { - val duplicateKeys = getKeys(acc) intersect getKeys(cfg) - if (duplicateKeys.nonEmpty) { - src match { - case ConfigSource.Resource(_, ResourceConfigKind.Development) => - logger.debug(s"Some keys in supplied ${src -> "development config"} duplicate already defined keys: ${duplicateKeys.niceList() -> "keys" -> null}") - case _ => - logger.warn(s"Some keys in supplied ${src -> "config"} duplicate already defined keys: ${duplicateKeys.niceList() -> "keys" -> null}") - } - } - } + IzResources(resourceClassLoader).getPath(r.name) match { + case Some(LoadablePathReference(path, _)) => + doLoad(s"$r (available: $path)", configSource, tryLoadResource()) + case Some(UnloadablePathReference(path)) => + doLoad(s"$r (exists: $path)", configSource, tryLoadResource()) + case None => + doLoad(s"$r (missing)", configSource, Success(ConfigFactory.empty())) + } - protected def getKeys(c: Config): collection.Set[String] = { - if (c.isResolved) { - c.entrySet().asScala.map(_.getKey) - } else { - Try { - c.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true)) - }.toOption.filter(_.isResolved) match { - case Some(value) => value.entrySet().asScala.map(_.getKey) - case None => Set.empty - } + case f: ConfigSource.File => + if (f.file.exists()) { + doLoad( + s"$f (exists: ${f.file.getCanonicalPath})", + configSource, + Try(ConfigFactory.parseFile(f.file)).flatMap { + cfg => if (cfg.origin().filename() ne null) Success(cfg) else Failure(new FileNotFoundException(s"Couldn't find config file $f")) + }, + ) + } else { + doLoad(s"$f (missing)", configSource, Failure(new FileNotFoundException(f.file.getCanonicalPath))) + } } } - } - - object LocalFSImpl { - sealed trait ResourceConfigKind - object ResourceConfigKind { - case object Primary extends ResourceConfigKind - case object Development extends ResourceConfigKind - } - - sealed trait ConfigSource - object ConfigSource { - final case class Resource(name: String, kind: ResourceConfigKind) extends ConfigSource { - override def toString: String = s"resource:$name" - } - final case class File(file: java.io.File) extends ConfigSource { - override def toString: String = s"file:$file" + private def doLoad(clue: String, source: ConfigSource, loader: => Try[Config]): ConfigLoadResult = { + loader match { + case Failure(exception) => ConfigLoadResult.Failure(clue, source, exception) + case Success(value) => ConfigLoadResult.Success(clue, source, value) } } - final class ConfigLoaderException(message: String, val failures: List[String]) extends DIException(message) } } diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLocationProvider.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLocationProvider.scala index d3d73a47eb..fd5f8b40b4 100644 --- a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLocationProvider.scala +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigLocationProvider.scala @@ -1,21 +1,27 @@ package izumi.distage.framework.services -import izumi.distage.framework.services.ConfigLoader.LocalFSImpl.{ConfigSource, ResourceConfigKind} +import izumi.distage.config.model.{ConfigSource, ResourceConfigKind} trait ConfigLocationProvider { - def forRole(roleName: String): Seq[ConfigSource] = ConfigLocationProvider.defaultConfigReferences(roleName) + def forRole(roleName: String): Seq[ConfigSource] - def forBase(filename: String): Seq[ConfigSource] = ConfigLocationProvider.defaultConfigReferences(filename) - - def defaultBaseConfigs: Seq[String] = ConfigLocationProvider.defaultBaseConfigs + def commonReferenceConfigs: Seq[ConfigSource] } object ConfigLocationProvider { - object Default extends ConfigLocationProvider + object Default extends ConfigLocationProvider { + def forRole(roleName: String): Seq[ConfigSource] = { + ConfigLocationProvider.defaultConfigReferences(roleName) + } + + def commonReferenceConfigs: Seq[ConfigSource] = { + ConfigLocationProvider.defaultBaseConfigs.flatMap(ConfigLocationProvider.defaultConfigReferences) + } + } - def defaultBaseConfigs: Seq[String] = Seq("application", "common") + private def defaultBaseConfigs: Seq[String] = Seq("application", "common") - def defaultConfigReferences(name: String): Seq[ConfigSource] = { + private def defaultConfigReferences(name: String): Seq[ConfigSource] = { Seq( ConfigSource.Resource(s"$name.conf", ResourceConfigKind.Primary), ConfigSource.Resource(s"$name-reference.conf", ResourceConfigKind.Primary), diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigMerger.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigMerger.scala new file mode 100644 index 0000000000..f973fe7d0c --- /dev/null +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/framework/services/ConfigMerger.scala @@ -0,0 +1,92 @@ +package izumi.distage.framework.services + +import com.typesafe.config.{Config, ConfigFactory, ConfigResolveOptions} +import izumi.distage.config.model.* +import izumi.distage.model.definition.Id +import izumi.fundamentals.platform.strings.IzString.* +import izumi.logstage.api.IzLogger + +import scala.jdk.CollectionConverters.* +import scala.util.Try + +trait ConfigMerger { + def merge(shared: List[ConfigLoadResult.Success], role: List[LoadedRoleConfigs]): Config +} + +object ConfigMerger { + class ConfigMergerImpl(logger: IzLogger @Id("early")) extends ConfigMerger { + override def merge(shared: List[ConfigLoadResult.Success], role: List[LoadedRoleConfigs]): Config = { +// val (roleReferenceConfigs, roleExplicitConfigs) = (configArgs.role: Iterable[(String, Option[File])]).partitionMap { +// case (role, None) => Left(configLocation.forRole(role)) +// case (_, Some(file)) => Right(ConfigSource.File(file)) +// } +// +// val allConfigs = (roleExplicitConfigs.iterator ++ commonExplicitConfigs ++ roleReferenceConfigs.iterator.flatten ++ commonReferenceConfigs).toList +// +// val (cfgInfo, loaded) = loadConfigSources(allConfigs) +// +// logger.info(s"Using system properties with fallback ${cfgInfo.niceList() -> "config files"}") +// +// val (good, bad) = loaded.partition(_._2.isSuccess) +// +// if (bad.nonEmpty) { +// val failuresList = bad.collect { +// case (s, Failure(f)) => +// s"$s: $f" +// } +// val failures = failuresList.niceList() +// +// logger.error(s"Failed to load configs: $failures") +// throw new ConfigLoaderException(s"Failed to load configs: failures=$failures", failuresList) +// } else { +// +// } + + val cfgInfo = (shared ++ role.flatMap(_.loaded)).map(c => c.clue) + logger.info(s"Using system properties with fallback ${cfgInfo.niceList() -> "config files"}") + + val toMerge = shared ++ role.filter(_.roleConfig.active).flatMap(_.loaded) + + val folded = foldConfigs(toMerge) + + ConfigFactory + .systemProperties() + .withFallback(folded) + .resolve() + } + + private def foldConfigs(roleConfigs: Iterable[ConfigLoadResult.Success]): Config = { + roleConfigs.iterator.foldLeft(ConfigFactory.empty()) { + case (acc, loaded) => + verifyConfigs(loaded, acc) + acc.withFallback(loaded.config) + } + } + + protected def verifyConfigs(loaded: ConfigLoadResult.Success, acc: Config): Unit = { + val duplicateKeys = getKeys(acc) intersect getKeys(loaded.config) + if (duplicateKeys.nonEmpty) { + loaded.src match { + case ConfigSource.Resource(_, ResourceConfigKind.Development) => + logger.debug(s"Some keys in supplied ${loaded.src -> "development config"} duplicate already defined keys: ${duplicateKeys.niceList() -> "keys" -> null}") + case _ => + logger.warn(s"Some keys in supplied ${loaded.src -> "config"} duplicate already defined keys: ${duplicateKeys.niceList() -> "keys" -> null}") + } + } + } + + protected def getKeys(c: Config): collection.Set[String] = { + if (c.isResolved) { + c.entrySet().asScala.map(_.getKey) + } else { + Try { + c.resolve(ConfigResolveOptions.defaults().setAllowUnresolved(true)) + }.toOption.filter(_.isResolved) match { + case Some(value) => value.entrySet().asScala.map(_.getKey) + case None => Set.empty + } + } + } + + } +} diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/RoleAppBootConfigModule.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/RoleAppBootConfigModule.scala index e119a05828..58a8a48b53 100644 --- a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/RoleAppBootConfigModule.scala +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/RoleAppBootConfigModule.scala @@ -1,13 +1,14 @@ package izumi.distage.roles import izumi.distage.config.model.AppConfig -import izumi.distage.framework.services.{ConfigArgsProvider, ConfigLoader, ConfigLocationProvider} +import izumi.distage.framework.services.{ConfigArgsProvider, ConfigLoader, ConfigLocationProvider, ConfigMerger} import izumi.distage.model.definition.ModuleDef import izumi.distage.modules.DefaultModule import izumi.reflect.TagK class RoleAppBootConfigModule[F[_]: TagK: DefaultModule]() extends ModuleDef { make[ConfigLoader].from[ConfigLoader.LocalFSImpl] + make[ConfigMerger].from[ConfigMerger.ConfigMergerImpl] make[ConfigLocationProvider].from(ConfigLocationProvider.Default) // make[ConfigLoader.Args].from(ConfigLoader.Args.makeConfigLoaderArgs _) make[ConfigArgsProvider].from[ConfigArgsProvider.Default] diff --git a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/launcher/ActivationParser.scala b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/launcher/ActivationParser.scala index d87a6b06e7..7226de2e5a 100644 --- a/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/launcher/ActivationParser.scala +++ b/distage/distage-framework/.jvm/src/main/scala/izumi/distage/roles/launcher/ActivationParser.scala @@ -32,6 +32,7 @@ object ActivationParser { val cmdChoices = parameters.globalParameters.findValues(RoleAppMain.Options.use).map(AxisPoint parseAxisPoint _.value) val cmdActivations = parser.parseActivation(cmdChoices, activationInfo) + // println(s"PARSEACT: ${config.config}") val configChoices = if (config.config.hasPath(configActivationSection)) { ActivationConfig.diConfigReader.decodeConfig(configActivationSection)(config.config).activation.map(AxisPoint(_)) } else Iterable.empty diff --git a/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/RoleAppTest.scala b/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/RoleAppTest.scala index 659fae08d1..12c1549f9a 100644 --- a/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/RoleAppTest.scala +++ b/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/RoleAppTest.scala @@ -283,7 +283,7 @@ class RoleAppTest extends AnyWordSpec with WithProperties { } } - "produce config dumps and support minimization" in { + "produce config dumps and support minimization" ignore { val version = ArtifactVersion(s"0.0.0-${UUID.randomUUID().toString}") withProperties(overrides ++ Map(TestPluginCatsIO.versionProperty -> version.version)) { TestEntrypoint.main(Array("-ll", logLevel, "-u", "axiscomponentaxis:incorrect", ":configwriter", "-t", targetPath)) diff --git a/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/fixtures/TestRole00.scala b/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/fixtures/TestRole00.scala index 32660c903e..31d71bf3e3 100644 --- a/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/fixtures/TestRole00.scala +++ b/distage/distage-framework/.jvm/src/test/scala/izumi/distage/roles/test/fixtures/TestRole00.scala @@ -51,7 +51,7 @@ object roles { override def start(roleParameters: RawEntrypointParams, freeArgs: Vector[String]): Lifecycle[F, Unit] = Lifecycle.make(QuasiIO[F].maybeSuspend { logger.info(s"[TestRole00] started: $roleParameters, $freeArgs, $dummies, $conflict") - assert(conf.overridenInt == 111) + assert(conf.overridenInt == 222, s"Common value is 111, role-specific value is 222, found ${conf.overridenInt}") }) { _ => QuasiIO[F].maybeSuspend { @@ -179,4 +179,3 @@ class FailingRole02[F[_]: QuasiIO]( object FailingRole02 extends RoleDescriptor { override final val id = "failingrole02" } - diff --git a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/runner/impl/services/TestConfigLoader.scala b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/runner/impl/services/TestConfigLoader.scala index 71a1a79bff..a4c24378a2 100644 --- a/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/runner/impl/services/TestConfigLoader.scala +++ b/distage/distage-testkit-core/src/main/scala/izumi/distage/testkit/runner/impl/services/TestConfigLoader.scala @@ -1,6 +1,7 @@ package izumi.distage.testkit.runner.impl.services -import izumi.distage.config.model.AppConfig +import izumi.distage.config.model.{AppConfig, GenericConfigSource, RoleConfig} +import izumi.distage.framework.services.ConfigMerger.ConfigMergerImpl import izumi.distage.framework.services.{ConfigArgsProvider, ConfigLoader, ConfigLocationProvider} import izumi.distage.testkit.model.TestEnvironment import izumi.logstage.api.IzLogger @@ -26,7 +27,7 @@ object TestConfigLoader { appConfig => env.configOverrides match { case Some(overrides) => - AppConfig(overrides.config.withFallback(appConfig.config).resolve()) + AppConfig.provided(overrides.config.withFallback(appConfig.config).resolve()) case None => appConfig } @@ -38,9 +39,10 @@ object TestConfigLoader { protected def makeConfigLoader(configBaseName: String, logger: IzLogger): ConfigLoader = { val provider = new ConfigArgsProvider { - override def args(): ConfigLoader.Args = ConfigLoader.Args(None, Map(configBaseName -> None)) + override def args(): ConfigLoader.Args = ConfigLoader.Args(None, List(RoleConfig(configBaseName, active = true, GenericConfigSource.ConfigDefault))) } - new ConfigLoader.LocalFSImpl(logger, ConfigLocationProvider.Default, provider) + val merger = new ConfigMergerImpl(logger) + new ConfigLoader.LocalFSImpl(logger, merger, ConfigLocationProvider.Default, provider) } }