From 12d4fba74d36c377e2e89747812d3a425dfdc5af Mon Sep 17 00:00:00 2001 From: Yasirmod17 Date: Wed, 18 Oct 2023 23:10:00 -0400 Subject: [PATCH 1/3] rename method --- .../relational/sqlQueryToString/sparkSQLExtension.pure | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/sparkSQLExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/sparkSQLExtension.pure index a14c2f8aad0..9710caeebc9 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/sparkSQLExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/sparkSQLExtension.pure @@ -15,7 +15,7 @@ function <> meta::relational::functions::sqlQueryToString::s function <> meta::relational::functions::sqlQueryToString::sparkSQL::createDbExtensionForSparkSQL():DbExtension[1] { - let reservedWords = sybaseReservedWords(); + let reservedWords = sparkReservedWords(); let literalProcessors = getDefaultLiteralProcessors()->putAll(getLiteralProcessorsForSparkSQL()); let literalProcessor = {type:Type[1]| $literalProcessors->get(if($type->instanceOf(Enumeration), | Enum, | $type))->toOne()}; let dynaFuncDispatch = getDynaFunctionToSqlDefault($literalProcessor)->groupBy(d| $d.funcName)->putAll( @@ -304,7 +304,8 @@ function meta::relational::functions::sqlQueryToString::sparkSQL::loadValuesToDb ,| $l->meta::relational::functions::sqlQueryToString::default::loadValuesToDbTableDefault($dbConfig)); } -function <> meta::relational::functions::sqlQueryToString::sparkSQL::sybaseReservedWords():String[*] + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::sparkReservedWords():String[*] { //Based on // http://infocenter.sybase.com/help/index.jsp?topic=/com.sybase.infocenter.dc38151.1510/html/iqrefbb/Alhakeywords.htm From ec84b2b401e0031676ca142aa78000c4f626d386 Mon Sep 17 00:00:00 2001 From: Mohammed Ibrahim Date: Thu, 19 Oct 2023 18:02:12 -0400 Subject: [PATCH 2/3] Function activation cleanup --- .../legend-engine-server/pom.xml | 10 -- .../finos/legend/engine/server/Server.java | 96 +++---------------- .../api/FunctionActivatorAPI.java | 12 ++- .../HostedServiceProtocolExtension.java | 4 + .../legend-engine-xt-snowflakeApp-api/pom.xml | 4 + .../snowflakeApp/api/SnowflakeAppService.java | 26 ++--- .../deployment/SnowflakeAppArtifact.java | 9 +- ...owflakeAppArtifactGenerationExtension.java | 33 ++++++- .../deployment/SnowflakeAppContent.java | 47 ++++++++- .../SnowflakeAppDeploymentConfiguration.java | 5 + .../deployment/SnowflakeAppGenerator.java | 71 ++++++++++++-- .../SnowflakeDeploymentManager.java | 21 ++-- .../SnowflakeAppCompilerExtension.java | 17 ++-- .../grammar/from/SnowflakeAppTreeWalker.java | 26 ++--- .../to/SnowflakeAppGrammarComposer.java | 2 + .../test/TestSnowflakeAppRoundtrip.java | 1 + ... SnowflakeAppDeploymentConfiguration.java} | 8 +- .../SnowflakeAppProtocolExtension.java | 4 + 18 files changed, 227 insertions(+), 169 deletions(-) rename legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/{SnowflakeDeploymentConfiguration.java => SnowflakeAppDeploymentConfiguration.java} (79%) diff --git a/legend-engine-config/legend-engine-server/pom.xml b/legend-engine-config/legend-engine-server/pom.xml index 5b97f0ebd47..6e94374f2f3 100644 --- a/legend-engine-config/legend-engine-server/pom.xml +++ b/legend-engine-config/legend-engine-server/pom.xml @@ -190,14 +190,6 @@ org.finos.legend.engine legend-engine-xt-functionActivator-protocol - - org.finos.legend.engine - legend-engine-xt-snowflakeApp-protocol - - - org.finos.legend.engine - legend-engine-xt-hostedService-protocol - org.finos.legend.engine legend-engine-xt-artifact-generation-api @@ -340,7 +332,6 @@ org.finos.legend.engine legend-engine-xt-snowflakeApp-api - runtime org.finos.legend.engine @@ -355,7 +346,6 @@ org.finos.legend.engine legend-engine-xt-hostedService-api - runtime org.finos.legend.engine diff --git a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java index c0933bc6f60..d6986546da6 100644 --- a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -39,30 +39,9 @@ import org.finos.legend.authentication.intermediationrule.IntermediationRule; import org.finos.legend.authentication.intermediationrule.impl.EncryptedPrivateKeyFromVaultRule; import org.finos.legend.authentication.vault.CredentialVaultProvider; -import org.finos.legend.authentication.vault.impl.EnvironmentCredentialVault; import org.finos.legend.authentication.vault.impl.PropertiesFileCredentialVault; -import org.finos.legend.authentication.vault.impl.SystemPropertiesCredentialVault; -import org.finos.legend.connection.AuthenticationMechanismConfiguration; -import org.finos.legend.connection.ConnectionFactory; -import org.finos.legend.connection.DatabaseType; -import org.finos.legend.connection.LegendEnvironment; -import org.finos.legend.connection.RelationalDatabaseStoreSupport; -import org.finos.legend.connection.StoreInstanceProvider; -import org.finos.legend.connection.impl.DefaultStoreInstanceProvider; -import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration; -import org.finos.legend.connection.impl.HACKY__SnowflakeConnectionAdapter; -import org.finos.legend.connection.impl.KerberosCredentialExtractor; -import org.finos.legend.connection.impl.KeyPairCredentialBuilder; -import org.finos.legend.connection.impl.SnowflakeConnectionBuilder; -import org.finos.legend.connection.impl.StaticJDBCConnectionBuilder; -import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; -import org.finos.legend.connection.impl.UserPasswordCredentialBuilder; -import org.finos.legend.connection.protocol.AuthenticationMechanismType; -import org.finos.legend.engine.api.analytics.BindingAnalytics; -import org.finos.legend.engine.api.analytics.ClassAnalytics; import org.finos.legend.engine.api.analytics.DataSpaceAnalytics; import org.finos.legend.engine.api.analytics.DiagramAnalytics; -import org.finos.legend.engine.api.analytics.FunctionAnalytics; import org.finos.legend.engine.api.analytics.LineageAnalytics; import org.finos.legend.engine.api.analytics.MappingAnalytics; import org.finos.legend.engine.api.analytics.StoreEntitlementAnalytics; @@ -82,19 +61,23 @@ import org.finos.legend.engine.external.shared.format.model.api.ExternalFormats; import org.finos.legend.engine.functionActivator.api.FunctionActivatorAPI; import org.finos.legend.engine.generation.artifact.api.ArtifactGenerationExtensionApi; +import org.finos.legend.engine.language.hostedService.api.HostedServiceService; +import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentConfiguration; import org.finos.legend.engine.language.pure.compiler.api.Compile; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.grammar.api.grammarToJson.GrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.grammarToJson.TransformGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.jsonToGrammar.JsonToGrammar; import org.finos.legend.engine.language.pure.grammar.api.jsonToGrammar.TransformJsonToGrammar; +import org.finos.legend.engine.language.pure.relational.api.relationalElement.RelationalElementAPI; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.RelationalOperationElementGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.RelationalOperationElementJsonToGrammar; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.TransformRelationalOperationElementGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.TransformRelationalOperationElementJsonToGrammar; import org.finos.legend.engine.language.pure.modelManager.ModelManager; import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader; -import org.finos.legend.engine.language.pure.relational.api.relationalElement.RelationalElementAPI; +import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppService; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration; import org.finos.legend.engine.plan.execution.PlanExecutor; import org.finos.legend.engine.plan.execution.api.ExecutePlanLegacy; import org.finos.legend.engine.plan.execution.api.ExecutePlanStrategic; @@ -105,8 +88,6 @@ import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool; import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration; import org.finos.legend.engine.plan.execution.service.api.ServiceModelingApi; -import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutor; -import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutorBuilder; import org.finos.legend.engine.plan.execution.stores.inMemory.plugin.InMemory; import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutor; import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutorBuilder; @@ -120,11 +101,11 @@ import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutor; import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutorBuilder; import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension; -import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration; +//import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration; import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory; import org.finos.legend.engine.protocol.pure.v1.model.PureProtocol; -import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; +//import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; import org.finos.legend.engine.query.graphQL.api.debug.GraphQLDebug; import org.finos.legend.engine.query.graphQL.api.execute.GraphQLExecute; import org.finos.legend.engine.query.graphQL.api.grammar.GraphQLGrammar; @@ -180,7 +161,7 @@ public class Server extends Application public static void main(String[] args) throws Exception { EngineUrlStreamHandlerFactory.initialize(); - new Server().run(args.length == 0 ? new String[]{"server", "legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json"} : args); + new Server().run(args.length == 0 ? new String[] {"server", "legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json"} : args); } @Override @@ -212,7 +193,7 @@ protected SwaggerBundleConfiguration getSwaggerBundleConfiguration( bootstrap.getObjectMapper().registerSubtypes(new NamedType(LegendDefaultDatabaseAuthenticationFlowProviderConfiguration.class, "legendDefault")); bootstrap.getObjectMapper().registerSubtypes(new NamedType(HostedServiceDeploymentConfiguration.class, "hostedServiceConfig")); - bootstrap.getObjectMapper().registerSubtypes(new NamedType(SnowflakeDeploymentConfiguration.class, "snowflakeAppConfig")); + bootstrap.getObjectMapper().registerSubtypes(new NamedType(SnowflakeAppDeploymentConfiguration.class, "snowflakeAppConfig")); } public CredentialProviderProvider configureCredentialProviders(List vaultConfigurations) @@ -284,10 +265,6 @@ public void run(T serverConfiguration, Environment environment) relationalExecution.setFlowProviderClass(LegendDefaultDatabaseAuthenticationFlowProvider.class); relationalExecution.setFlowProviderConfiguration(new LegendDefaultDatabaseAuthenticationFlowProviderConfiguration()); } - relationalExecution.setConnectionFactory(this.setupConnectionFactory(serverConfiguration.vaults)); - relationalExecution.setRelationalDatabaseConnectionAdapters(Lists.mutable.of( - new HACKY__SnowflakeConnectionAdapter.WithKeyPair() - )); relationalStoreExecutor = (RelationalStoreExecutor) Relational.build(serverConfiguration.relationalexecution); @@ -297,14 +274,12 @@ public void run(T serverConfiguration, Environment environment) MongoDBStoreExecutorConfiguration mongoDBExecutorConfiguration = MongoDBStoreExecutorConfiguration.newInstance().withCredentialProviderProvider(credentialProviderProvider).build(); MongoDBStoreExecutor mongoDBStoreExecutor = (MongoDBStoreExecutor) new MongoDBStoreExecutorBuilder().build(mongoDBExecutorConfiguration); - ElasticsearchV7StoreExecutor elasticsearchV7StoreExecutor = (ElasticsearchV7StoreExecutor) new ElasticsearchV7StoreExecutorBuilder().build(); - PlanExecutor planExecutor; ParallelGraphFetchExecutionExecutorPool parallelGraphFetchExecutionExecutorPool = null; if (serverConfiguration.graphFetchExecutionConfiguration != null) { GraphFetchExecutionConfiguration graphFetchExecutionConfiguration = serverConfiguration.graphFetchExecutionConfiguration; - planExecutor = PlanExecutor.newPlanExecutor(graphFetchExecutionConfiguration, relationalStoreExecutor, elasticsearchV7StoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); + planExecutor = PlanExecutor.newPlanExecutor(graphFetchExecutionConfiguration, relationalStoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); if (graphFetchExecutionConfiguration.canExecuteInParallel()) { parallelGraphFetchExecutionExecutorPool = new ParallelGraphFetchExecutionExecutorPool(graphFetchExecutionConfiguration.getParallelGraphFetchExecutionConfig(), "thread-pool for parallel graphFetch execution"); @@ -313,7 +288,7 @@ public void run(T serverConfiguration, Environment environment) } else { - planExecutor = PlanExecutor.newPlanExecutor(relationalStoreExecutor, elasticsearchV7StoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); + planExecutor = PlanExecutor.newPlanExecutor(relationalStoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); } // Session Management @@ -378,7 +353,9 @@ public void run(T serverConfiguration, Environment environment) environment.jersey().register(new ExecutePlanLegacy(planExecutor)); // Function Activator - environment.jersey().register(new FunctionActivatorAPI(modelManager, Lists.mutable.empty(), routerExtensions)); + +// environment.jersey().register(new FunctionActivatorAPI(modelManager , routerExtensions)); + environment.jersey().register(new FunctionActivatorAPI(modelManager,Lists.mutable.empty(), Lists.mutable.with(new SnowflakeAppService(planExecutor), new HostedServiceService()), routerExtensions)); // GraphQL environment.jersey().register(new GraphQLGrammar()); @@ -406,9 +383,6 @@ public void run(T serverConfiguration, Environment environment) // Analytics List entitlementServiceExtensions = EntitlementServiceExtensionLoader.extensions(); environment.jersey().register(new MappingAnalytics(modelManager)); - environment.jersey().register(new ClassAnalytics(modelManager)); - environment.jersey().register(new FunctionAnalytics(modelManager)); - environment.jersey().register(new BindingAnalytics(modelManager)); environment.jersey().register(new DiagramAnalytics(modelManager)); environment.jersey().register(new DataSpaceAnalytics(modelManager, generatorExtensions, entitlementServiceExtensions)); environment.jersey().register(new LineageAnalytics(modelManager)); @@ -423,48 +397,6 @@ public void run(T serverConfiguration, Environment environment) enableCors(environment); } - // TODO: @akphi - this is temporary, rework when we find a better way to handle the initialization of connection factory from config or some external source. - private ConnectionFactory setupConnectionFactory(List vaultConfigurations) - { - LegendEnvironment environment = new LegendEnvironment.Builder() - .withVaults( - new SystemPropertiesCredentialVault(), - new EnvironmentCredentialVault(), - new PropertiesFileCredentialVault(this.buildVaultProperties(vaultConfigurations)) - ) - .withStoreSupports( - new RelationalDatabaseStoreSupport.Builder(DatabaseType.POSTGRES) - .withIdentifier("Postgres") - .withAuthenticationMechanismConfigurations( - new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).withAuthenticationConfigurationTypes( - UserPasswordAuthenticationConfiguration.class - ).build() - ) - .build(), - new RelationalDatabaseStoreSupport.Builder(DatabaseType.SNOWFLAKE) - .withIdentifier("Snowflake") - .withAuthenticationMechanismConfigurations( - new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KEY_PAIR).withAuthenticationConfigurationTypes( - EncryptedPrivateKeyPairAuthenticationConfiguration.class - ).build() - ) - .build() - ).build(); - - StoreInstanceProvider storeInstanceProvider = new DefaultStoreInstanceProvider.Builder().build(); - return new ConnectionFactory.Builder(environment, storeInstanceProvider) - .withCredentialBuilders( - new KerberosCredentialExtractor(), - new UserPasswordCredentialBuilder(), - new KeyPairCredentialBuilder() - ) - .withConnectionBuilders( - new StaticJDBCConnectionBuilder.WithPlaintextUsernamePassword(), - new SnowflakeConnectionBuilder.WithKeyPair() - ) - .build(); - } - private void loadVaults(List vaultConfigurations) { if (vaultConfigurations != null) diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java index 7e314bd1645..000bd9952ef 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java @@ -65,6 +65,7 @@ public class FunctionActivatorAPI private final PureModel emptyModel; private final Function> routerExtensions; private List runtimeDeploymentConfig = Lists.mutable.empty(); + private MutableList> availableActivatorServices = Lists.mutable.empty(); public FunctionActivatorAPI(ModelManager modelManager, Function> routerExtensions) { @@ -73,10 +74,11 @@ public FunctionActivatorAPI(ModelManager modelManager, Function activatorConfigurations, Function> routerExtensions) + public FunctionActivatorAPI(ModelManager modelManager, List activatorConfigurations, MutableList> availableActivatorServices, Function> routerExtensions) { this(modelManager, routerExtensions); this.runtimeDeploymentConfig = activatorConfigurations; + this.availableActivatorServices = availableActivatorServices; } @GET @@ -87,7 +89,7 @@ public FunctionActivatorAPI(ModelManager modelManager, List pm) { MutableList profiles = ProfileManagerHelper.extractProfiles(pm); - MutableList values = FunctionActivatorLoader.extensions().collect(x -> x.info(emptyModel, "vX_X_X")); + MutableList values = this.availableActivatorServices.isEmpty() ? FunctionActivatorLoader.extensions().collect(x -> x.info(emptyModel, "vX_X_X")) : availableActivatorServices.collect(c -> c.info(emptyModel,"vX_X_X")); return ManageConstantResult.manageResult(profiles, values, objectMapper); } @@ -162,7 +164,11 @@ public Response renderArtifact(FunctionActivatorInput input, @ApiParam(hidden = public FunctionActivatorService getActivatorService(Root_meta_external_function_activator_FunctionActivator activator, PureModel pureModel) { - FunctionActivatorService service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst(); + FunctionActivatorService service = (FunctionActivatorService)this.availableActivatorServices.select(c -> c.supports(activator)).getFirst(); + if (service == null) + { + service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst(); + } if (service == null) { throw new RuntimeException(activator.getClass().getSimpleName() + "is not supported!"); diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/src/main/java/org/finos/legend/engine/protocol/hostedService/metamodel/HostedServiceProtocolExtension.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/src/main/java/org/finos/legend/engine/protocol/hostedService/metamodel/HostedServiceProtocolExtension.java index 81fc05ea515..b4c59da89af 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/src/main/java/org/finos/legend/engine/protocol/hostedService/metamodel/HostedServiceProtocolExtension.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/src/main/java/org/finos/legend/engine/protocol/hostedService/metamodel/HostedServiceProtocolExtension.java @@ -17,6 +17,7 @@ import org.eclipse.collections.api.block.function.Function0; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Maps; +import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; import org.finos.legend.engine.protocol.hostedService.metamodel.control.Deployment; import org.finos.legend.engine.protocol.hostedService.metamodel.control.Ownership; import org.finos.legend.engine.protocol.hostedService.metamodel.control.UserList; @@ -41,6 +42,9 @@ public List>>> getExtraProtocolSubTypeInfo ProtocolSubTypeInfo.newBuilder(Ownership.class) .withSubtype(UserList.class, "userList") .withSubtype(Deployment.class, "deployment") + .build(), + ProtocolSubTypeInfo.newBuilder(DeploymentConfiguration.class) + .withSubtype(HostedServiceDeploymentConfiguration.class, "hostedServiceDeploymentConfiguration") .build() )); } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml index afd677ee962..f9259d81c75 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml @@ -92,6 +92,10 @@ org.finos.legend.engine legend-engine-executionPlan-execution + + org.finos.legend.engine + legend-engine-pure-platform-java + org.finos.legend.engine diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java index e061f149faf..0ce172a1608 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java @@ -28,29 +28,16 @@ import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppContent; import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration; import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppGenerator; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage; -import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentManager; import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentResult; +import org.finos.legend.engine.plan.execution.PlanExecutor; import org.finos.legend.engine.plan.execution.stores.relational.config.TemporaryTestDbConfiguration; import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; -import org.finos.legend.engine.plan.generation.PlanGenerator; -import org.finos.legend.engine.plan.platform.PlanPlatform; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppProtocolExtension; import org.finos.legend.pure.generated.Root_meta_external_function_activator_FunctionActivator; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; -import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration; -import org.finos.legend.pure.generated.Root_meta_external_store_relational_runtime_RelationalDatabaseConnection; -import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy; -import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification; -import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionNode; -import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; -import org.finos.legend.pure.generated.Root_meta_relational_mapping_SQLExecutionNode; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; import org.pac4j.core.profile.CommonProfile; import java.util.List; @@ -74,6 +61,11 @@ public SnowflakeAppService(ConnectionManagerSelector connectionManager) this.snowflakeDeploymentManager = new SnowflakeDeploymentManager(new SnowflakeAppDeploymentTool(connectionManager)); } + public SnowflakeAppService(PlanExecutor executor) + { + this.snowflakeDeploymentManager = new SnowflakeDeploymentManager(executor); + } + @Override public FunctionActivatorInfo info(PureModel pureModel, String version) { @@ -94,7 +86,7 @@ public boolean supports(Root_meta_external_function_activator_FunctionActivator @Override public MutableList validate(MutableList profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function> routerExtensions) { - SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions); + SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions); int size = ((SnowflakeAppContent)artifact.content).sqlExpressions.size(); return size != 1 ? Lists.mutable.with(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", ((SnowflakeAppContent)artifact.content).sqlExpressions)) : @@ -105,14 +97,14 @@ public MutableList validate(MutableList profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, List runtimeConfigurations, Function> routerExtensions) { - SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions); + SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions); return this.snowflakeDeploymentManager.deploy(profiles, artifact, runtimeConfigurations); } @Override public SnowflakeAppArtifact renderArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, String clientVersion, Function> routerExtensions) { - return SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions); + return SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions); } @Override diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java index 9ab608335d6..b1080411d24 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java @@ -27,14 +27,15 @@ public SnowflakeAppArtifact() //empty artifact } - public SnowflakeAppArtifact(String name, MutableList sqlExpressions) + public SnowflakeAppArtifact(SnowflakeAppContent content) { - this.content = new SnowflakeAppContent(name, sqlExpressions); + this.content = content; } - public SnowflakeAppArtifact(String name, MutableList sqlExpressions, SnowflakeAppDeploymentConfiguration config) + public SnowflakeAppArtifact(SnowflakeAppContent content, SnowflakeAppDeploymentConfiguration config) { - this.content = new SnowflakeAppContent(name, sqlExpressions); + this(content); this.deploymentConfiguration = config; } + } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java index 1d916df4b67..c0d1f4aa5a2 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java @@ -15,19 +15,31 @@ package org.finos.legend.engine.language.snowflakeApp.deployment; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension; +import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.context.SDLC; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; +import org.slf4j.Logger; import java.util.List; public class SnowflakeAppArtifactGenerationExtension implements ArtifactGenerationExtension { + private static final ObjectMapper mapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(SnowflakeAppArtifactGenerationExtension.class); private static final String ROOT_PATH = "snowflakeApp"; - private static final String FILENAME = "snowflakeArtifact.json"; + private static final String FILE_NAME = "snowflakeArtifact.json"; @Override public String getKey() @@ -39,14 +51,29 @@ public String getKey() public boolean canGenerate(PackageableElement element) { return false; - // return element instanceof Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; + // return element instanceof Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; } @Override public List generate(PackageableElement element, PureModel pureModel, PureModelContextData data, String clientVersion) { - return null; + List result = Lists.mutable.empty(); + Function> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport())); + SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, (Root_meta_external_function_activator_snowflakeApp_SnowflakeApp) element, data, routerExtensions); + try + { + LOGGER.info("Generating snowflake artifact for " + element.getName()); + String content = mapper.writeValueAsString(artifact); + result.add((new Artifact(content, FILE_NAME, "json"))); + LOGGER.info("Generated snowflake artifact for " + element.getName()); + + } + catch (Exception e) + { + LOGGER.error("Error generating snowflake artifact for " + element.getName() + " reason: " + e.getMessage()); + } + return result; } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppContent.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppContent.java index 388b537959b..d8e69ee9437 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppContent.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppContent.java @@ -18,16 +18,59 @@ import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentContent; +import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.model.Database; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; public class SnowflakeAppContent extends FunctionActivatorDeploymentContent { public MutableList sqlExpressions = Lists.mutable.empty(); public String applicationName; + public String description; + public List owners; + public String groupId; + public String artifactId; + public String version; + public String creationTime; - public SnowflakeAppContent(String name, MutableList sqlExpressions) + public SnowflakeAppContent(String applicationName, MutableList sqlExpressions, AlloySDLC sdlc) { - this.applicationName = name; + this.applicationName = applicationName; this.sqlExpressions = sqlExpressions; + this.creationTime = convertToValidDate(new Date()); + if (sdlc != null) + { + this.groupId = sdlc.groupId; + this.artifactId = sdlc.artifactId; + this.version = sdlc.version; + } + } + + public SnowflakeAppContent(String applicationName, MutableList sqlExpressions, String description, List owners, AlloySDLC sdlc) + { + this(applicationName, sqlExpressions,sdlc); + this.description = description; + this.owners = owners; + + } + + public static String convertToValidDate(Date date) + { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return format.format(date); + } + + public String getVersionInfo() + { + if (this.version != null) + { + return groupId + ":" + this.artifactId + ":" + this.version; + } + return ""; } } + diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentConfiguration.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentConfiguration.java index 6308e215f8b..29e823eee5b 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentConfiguration.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentConfiguration.java @@ -20,4 +20,9 @@ public class SnowflakeAppDeploymentConfiguration extends FunctionActivatorDeploymentConfiguration { RelationalDatabaseConnection connection; + + public SnowflakeAppDeploymentConfiguration(RelationalDatabaseConnection connection) + { + this.connection = connection; + } } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppGenerator.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppGenerator.java index ee9d38e94a7..7c5ecce3fbe 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppGenerator.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppGenerator.java @@ -21,24 +21,51 @@ import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.plan.generation.PlanGenerator; import org.finos.legend.engine.plan.platform.PlanPlatform; -import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; -import org.finos.legend.pure.generated.Root_meta_external_store_relational_runtime_RelationalDatabaseConnection; -import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy; -import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification; -import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionNode; -import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; -import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; -import org.finos.legend.pure.generated.Root_meta_relational_mapping_SQLExecutionNode; +import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.context.SDLC; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.authentication.SnowflakePublicAuthenticationStrategy; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.specification.SnowflakeDatasourceSpecification; +import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp; +import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration; +import org.finos.legend.pure.generated.*; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; public class SnowflakeAppGenerator { - public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function> routerExtensions) + public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function> routerExtensions) { RichIterable sqlExpressions = extractSQLExpressions(pureModel, activator, routerExtensions); - return new SnowflakeAppArtifact(activator._applicationName(), Lists.mutable.withAll(sqlExpressions)); + + RelationalDatabaseConnection connection; + AlloySDLC sdlc = null; + if (((PureModelContextData)inputModel).getOrigin() != null) + { + SDLC sdlcInfo = ((PureModelContextData)inputModel).origin.sdlcInfo; + if (sdlcInfo instanceof AlloySDLC) + { + sdlc = (AlloySDLC) sdlcInfo; + } + } + SnowflakeAppContent content = new SnowflakeAppContent(activator._applicationName(), Lists.mutable.withAll(sqlExpressions), activator._description(), Lists.mutable.with(activator._owner()), sdlc); + if (activator._activationConfiguration() != null) + { + //identify connection + SnowflakeApp protocolActivator = Lists.mutable.withAll(((PureModelContextData) inputModel).getElementsOfType(SnowflakeApp.class)) + .select(c -> c.getPath().equals(platform_pure_basics_meta_elementToPath.Root_meta_pure_functions_meta_elementToPath_PackageableElement_1__String_1_(activator, pureModel.getExecutionSupport()))) + .getFirst(); + connection = (RelationalDatabaseConnection) Lists.mutable.withAll(((PureModelContextData) inputModel).getElementsOfType(PackageableConnection.class)) + .select(c -> c.getPath().equals(((SnowflakeAppDeploymentConfiguration)protocolActivator.activationConfiguration).activationConnection.connection)).getFirst().connectionValue; + return new SnowflakeAppArtifact(content, new org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration(connection)); + } + + return new SnowflakeAppArtifact(content); } private static RichIterable extractSQLExpressions(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function> routerExtensions) @@ -72,6 +99,30 @@ private static Object[] extractSQLExpressionsAndConnectionMetadata(PureModel pur return new Object[]{expressions, ds, as}; } + private RelationalDatabaseConnection adaptConnection(Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification datasourceSpecification, Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy authenticationStrategy) + { + RelationalDatabaseConnection connection = new RelationalDatabaseConnection(); + + SnowflakeDatasourceSpecification snowflakeDatasourceSpecification = new SnowflakeDatasourceSpecification(); + snowflakeDatasourceSpecification.accountName = datasourceSpecification._accountName(); + snowflakeDatasourceSpecification.databaseName = datasourceSpecification._databaseName(); + snowflakeDatasourceSpecification.role = datasourceSpecification._role(); + snowflakeDatasourceSpecification.warehouseName = datasourceSpecification._warehouseName(); + snowflakeDatasourceSpecification.region = datasourceSpecification._region(); + snowflakeDatasourceSpecification.cloudType = datasourceSpecification._cloudType(); + + SnowflakePublicAuthenticationStrategy snowflakeAuthenticationStrategy = new SnowflakePublicAuthenticationStrategy(); + snowflakeAuthenticationStrategy.privateKeyVaultReference = authenticationStrategy._privateKeyVaultReference(); + snowflakeAuthenticationStrategy.passPhraseVaultReference = authenticationStrategy._passPhraseVaultReference(); + snowflakeAuthenticationStrategy.publicUserName = authenticationStrategy._publicUserName(); + + connection.authenticationStrategy = snowflakeAuthenticationStrategy; + connection.datasourceSpecification = snowflakeDatasourceSpecification; + connection.type = DatabaseType.Snowflake; + + return connection; + } + private static RichIterable collectAllNodes(Root_meta_pure_executionPlan_ExecutionNode node) { return Lists.mutable.with(node).withAll(node._executionNodes().flatCollect(SnowflakeAppGenerator::collectAllNodes)); diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java index 5cd7369a2f4..02228287fa8 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java @@ -24,10 +24,11 @@ import org.finos.legend.engine.plan.execution.PlanExecutor; import org.finos.legend.engine.plan.execution.stores.relational.RelationalExecutor; import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; +import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreExecutor; import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreState; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; -import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration; import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy; import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification; @@ -50,6 +51,8 @@ public class SnowflakeDeploymentManager implements DeploymentManager c instanceof RelationalExecutor).getFirst().getStoreState()).getRelationalExecutor().getConnectionManager(); + connectionManager = ((RelationalStoreState)planExecutor.getExtraExecutors().select(c -> c instanceof RelationalStoreExecutor).getFirst().getStoreState()).getRelationalExecutor().getConnectionManager(); } @Override @@ -83,7 +86,7 @@ public SnowflakeDeploymentResult deploy(MutableList profiles, Sno { String appName = ((SnowflakeAppContent)artifact.content).applicationName; jdbcConnection.setAutoCommit(false); - this.deployImpl(jdbcConnection, appName); + this.deployImpl(jdbcConnection, (SnowflakeAppContent)artifact.content); jdbcConnection.commit(); LOGGER.info("Completed deployment successfully"); result = new SnowflakeDeploymentResult(appName, true); @@ -121,21 +124,21 @@ public java.sql.Connection getDeploymentConnection(MutableList pr return this.connectionManager.getDatabaseConnection(profiles, (DatabaseConnection) connection); } - public void deployImpl(Connection jdbcConnection, String context) throws Exception + public void deployImpl(Connection jdbcConnection, SnowflakeAppContent context) throws Exception { Statement statement = jdbcConnection.createStatement(); String deploymentTableName = this.getDeploymentTableName(jdbcConnection); - String createTableSQL = String.format("create table %s (id INTEGER, message VARCHAR(1000)) if not exists", deploymentTableName); - boolean createTableStatus = statement.execute(createTableSQL); - String insertSQL = String.format("insert into %s(id, message) values(%d, '%s')", deploymentTableName, System.currentTimeMillis(), context); + + //String createTableSQL = String.format("create table %s (id INTEGER, message VARCHAR(1000)) if not exists", deploymentTableName); + //boolean createTableStatus = statement.execute(createTableSQL); + String insertSQL = String.format("insert into %s(CREATE_DATETIME, APP_NAME, SQL_FRAGMENT, VERSION_NUMBER, OWNER) values('%s', '%s', '%s', '%s', '%s')", deploymentTableName, context.creationTime, context.applicationName, context.sqlExpressions.getFirst(), context.getVersionInfo(), Lists.mutable.withAll(context.owners).makeString(",")); boolean insertStatus = statement.execute(insertSQL); } public String getDeploymentTableName(Connection jdbcConnection) throws SQLException { String catalogName = jdbcConnection.getCatalog(); - String schema = "NATIVE_APP"; - return String.format("%s.%s.LEGEND_SNOWFLAKE_APP_DEPLOYMENT", catalogName, schema); + return String.format("%s.%s." + deploymentTable, catalogName, deploymentSchema); } public java.sql.Connection getDeploymentConnection(MutableList profiles, SnowflakeAppArtifact artifact) diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java index 2faa56790c4..2a0afc16ed1 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java @@ -19,8 +19,9 @@ import org.finos.legend.engine.language.pure.compiler.toPureGraph.CompileContext; import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension; import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.Processor; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp; -import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; +import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp_Impl; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration; @@ -29,8 +30,6 @@ import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor; -import java.util.Collections; - public class SnowflakeAppCompilerExtension implements CompilerExtension { // Here only for dependency check error ... @@ -48,12 +47,8 @@ public Iterable> getExtraProcessors() return Lists.fixedSize.of( Processor.newProcessor( SnowflakeApp.class, - org.eclipse.collections.impl.factory.Lists.fixedSize.with(SnowflakeDeploymentConfiguration.class), + org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableConnection.class), this::buildSnowflakeApp - ), - Processor.newProcessor( - SnowflakeDeploymentConfiguration.class, - this::buildDeploymentConfig ) ); } @@ -72,7 +67,7 @@ public Root_meta_external_function_activator_snowflakeApp_SnowflakeApp buildSnow ._function(func) ._description(app.description) ._owner(app.owner) - ._activationConfiguration(app.activationConfiguration != null ? buildDeploymentConfig((SnowflakeDeploymentConfiguration) app.activationConfiguration, context) : null); + ._activationConfiguration(app.activationConfiguration != null ? buildDeploymentConfig((SnowflakeAppDeploymentConfiguration) app.activationConfiguration, context) : null); } catch (Exception e) { @@ -80,10 +75,10 @@ public Root_meta_external_function_activator_snowflakeApp_SnowflakeApp buildSnow } } - public Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration buildDeploymentConfig(SnowflakeDeploymentConfiguration configuration, CompileContext context) + public Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration buildDeploymentConfig(SnowflakeAppDeploymentConfiguration configuration, CompileContext context) { return new Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration_Impl("") ._target((Root_meta_external_store_relational_runtime_RelationalDatabaseConnection) context.resolveConnection(configuration.activationConnection.connection, configuration.sourceInformation)); - // ._stage(context.pureModel.getEnumValue("meta::external::function::activator::DeploymentStage", configuration.stage.name())); + // ._stage(context.pureModel.getEnumValue("meta::external::function::activator::DeploymentStage", configuration.stage.name())); } } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java index 31444f92ce3..550ceb4c1dd 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java @@ -28,7 +28,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TaggedValue; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp; -import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; +import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import java.util.Collections; @@ -56,21 +56,6 @@ public void visit(SnowflakeAppParserGrammar.DefinitionContext ctx) { ctx.snowflakeApp().stream().map(this::visitSnowflakeApp).peek(e -> this.section.elements.add(e.getPath())).forEach(this.elementConsumer); } - if (ctx.deploymentConfig() != null && !ctx.deploymentConfig().isEmpty()) - { - ctx.deploymentConfig().stream().map(this::visitDeploymentConfig).peek(e -> this.section.elements.add(e.getPath())).forEach(this.elementConsumer); - } - } - - private SnowflakeDeploymentConfiguration visitDeploymentConfig(SnowflakeAppParserGrammar.DeploymentConfigContext ctx) - { - SnowflakeDeploymentConfiguration config = new SnowflakeDeploymentConfiguration(); - ConnectionPointer pointer = new ConnectionPointer(); - pointer.connection = PureGrammarParserUtility.fromQualifiedName(ctx.activationConnection().qualifiedName().packagePath() == null - ? Collections.emptyList() : ctx.activationConnection().qualifiedName().packagePath().identifier(), ctx.activationConnection().qualifiedName().identifier()); - pointer.sourceInformation = walkerSourceInformation.getSourceInformation(ctx.activationConnection().qualifiedName()); - config.activationConnection = pointer; - return config; } private SnowflakeApp visitSnowflakeApp(SnowflakeAppParserGrammar.SnowflakeAppContext ctx) @@ -96,6 +81,15 @@ private SnowflakeApp visitSnowflakeApp(SnowflakeAppParserGrammar.SnowflakeAppCon { snowflakeApp.description = PureGrammarParserUtility.fromGrammarString(descriptionContext.STRING().getText(), true); } + SnowflakeAppParserGrammar.ActivationContext activationContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.activation(), "activation", snowflakeApp.sourceInformation); + if (activationContext != null) + { + ConnectionPointer p = new ConnectionPointer(); + p.connection = PureGrammarParserUtility.fromQualifiedName(activationContext.qualifiedName().packagePath() == null + ? Collections.emptyList() : activationContext.qualifiedName().packagePath().identifier(), activationContext.qualifiedName().identifier()); + p.sourceInformation = walkerSourceInformation.getSourceInformation(activationContext.qualifiedName()); + snowflakeApp.activationConfiguration = new SnowflakeAppDeploymentConfiguration(p); + } return snowflakeApp; } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java index 329693d0887..4437eacdfc0 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java @@ -26,6 +26,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.externalFormat.Binding; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.externalFormat.ExternalFormatSchemaSet; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp; +import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration; import java.util.Collections; import java.util.List; @@ -53,6 +54,7 @@ private static String renderSnowflakeApp(SnowflakeApp app) " function : " + app.function + ";\n" + (app.owner == null ? "" : " owner : '" + app.owner + "';\n") + (app.description == null ? "" : " description : '" + app.description + "';\n") + + (app.activationConfiguration == null ? "" : " activationConfiguration : " + ((SnowflakeAppDeploymentConfiguration)app.activationConfiguration).activationConnection.connection + ";\n") + "}"; } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java index 54b331fcf73..16dd98acddf 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java @@ -29,6 +29,7 @@ public void testSnowflakeApp() " function : zxx(Integer[1]):String[1];\n" + " owner : 'pierre';\n" + " description : 'A super nice app!';\n" + + " activationConfiguration : a::b::connection;\n" + "}\n"); } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentConfiguration.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppDeploymentConfiguration.java similarity index 79% rename from legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentConfiguration.java rename to legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppDeploymentConfiguration.java index 0aff2c51431..7e4032b33fa 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentConfiguration.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppDeploymentConfiguration.java @@ -17,14 +17,18 @@ import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.ConnectionPointer; -public class SnowflakeDeploymentConfiguration extends DeploymentConfiguration +public class SnowflakeAppDeploymentConfiguration extends DeploymentConfiguration { public ConnectionPointer activationConnection; // public String applicationName; - public SnowflakeDeploymentConfiguration() + public SnowflakeAppDeploymentConfiguration() { + } + public SnowflakeAppDeploymentConfiguration(ConnectionPointer cp) + { + activationConnection = cp; } } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppProtocolExtension.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppProtocolExtension.java index 88d78239a52..f838fd8372b 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppProtocolExtension.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppProtocolExtension.java @@ -17,6 +17,7 @@ import org.eclipse.collections.api.block.function.Function0; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Maps; +import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; import org.finos.legend.engine.protocol.pure.v1.extension.ProtocolSubTypeInfo; import org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; @@ -34,6 +35,9 @@ public List>>> getExtraProtocolSubTypeInfo return Lists.fixedSize.with(() -> Lists.mutable.with( ProtocolSubTypeInfo.newBuilder(PackageableElement.class) .withSubtype(SnowflakeApp.class, packageJSONType) + .build(), + ProtocolSubTypeInfo.newBuilder(DeploymentConfiguration.class) + .withSubtype(SnowflakeAppDeploymentConfiguration.class, "snowflakeDeploymentConfiguration") .build() )); } From 5dc3700f75f0faddb2b424537cc2d0147c529082 Mon Sep 17 00:00:00 2001 From: Yasirmod17 Date: Thu, 19 Oct 2023 19:07:19 -0400 Subject: [PATCH 3/3] rollback file changes --- .../finos/legend/engine/server/Server.java | 94 ++++++++++++++++--- 1 file changed, 82 insertions(+), 12 deletions(-) diff --git a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java index d6986546da6..1568eefd55f 100644 --- a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -39,9 +39,30 @@ import org.finos.legend.authentication.intermediationrule.IntermediationRule; import org.finos.legend.authentication.intermediationrule.impl.EncryptedPrivateKeyFromVaultRule; import org.finos.legend.authentication.vault.CredentialVaultProvider; +import org.finos.legend.authentication.vault.impl.EnvironmentCredentialVault; import org.finos.legend.authentication.vault.impl.PropertiesFileCredentialVault; +import org.finos.legend.authentication.vault.impl.SystemPropertiesCredentialVault; +import org.finos.legend.connection.AuthenticationMechanismConfiguration; +import org.finos.legend.connection.ConnectionFactory; +import org.finos.legend.connection.DatabaseType; +import org.finos.legend.connection.LegendEnvironment; +import org.finos.legend.connection.RelationalDatabaseStoreSupport; +import org.finos.legend.connection.StoreInstanceProvider; +import org.finos.legend.connection.impl.DefaultStoreInstanceProvider; +import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration; +import org.finos.legend.connection.impl.HACKY__SnowflakeConnectionAdapter; +import org.finos.legend.connection.impl.KerberosCredentialExtractor; +import org.finos.legend.connection.impl.KeyPairCredentialBuilder; +import org.finos.legend.connection.impl.SnowflakeConnectionBuilder; +import org.finos.legend.connection.impl.StaticJDBCConnectionBuilder; +import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; +import org.finos.legend.connection.impl.UserPasswordCredentialBuilder; +import org.finos.legend.connection.protocol.AuthenticationMechanismType; +import org.finos.legend.engine.api.analytics.BindingAnalytics; +import org.finos.legend.engine.api.analytics.ClassAnalytics; import org.finos.legend.engine.api.analytics.DataSpaceAnalytics; import org.finos.legend.engine.api.analytics.DiagramAnalytics; +import org.finos.legend.engine.api.analytics.FunctionAnalytics; import org.finos.legend.engine.api.analytics.LineageAnalytics; import org.finos.legend.engine.api.analytics.MappingAnalytics; import org.finos.legend.engine.api.analytics.StoreEntitlementAnalytics; @@ -61,23 +82,19 @@ import org.finos.legend.engine.external.shared.format.model.api.ExternalFormats; import org.finos.legend.engine.functionActivator.api.FunctionActivatorAPI; import org.finos.legend.engine.generation.artifact.api.ArtifactGenerationExtensionApi; -import org.finos.legend.engine.language.hostedService.api.HostedServiceService; -import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentConfiguration; import org.finos.legend.engine.language.pure.compiler.api.Compile; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.grammar.api.grammarToJson.GrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.grammarToJson.TransformGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.jsonToGrammar.JsonToGrammar; import org.finos.legend.engine.language.pure.grammar.api.jsonToGrammar.TransformJsonToGrammar; -import org.finos.legend.engine.language.pure.relational.api.relationalElement.RelationalElementAPI; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.RelationalOperationElementGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.RelationalOperationElementJsonToGrammar; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.TransformRelationalOperationElementGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.TransformRelationalOperationElementJsonToGrammar; import org.finos.legend.engine.language.pure.modelManager.ModelManager; import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader; -import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppService; -import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration; +import org.finos.legend.engine.language.pure.relational.api.relationalElement.RelationalElementAPI; import org.finos.legend.engine.plan.execution.PlanExecutor; import org.finos.legend.engine.plan.execution.api.ExecutePlanLegacy; import org.finos.legend.engine.plan.execution.api.ExecutePlanStrategic; @@ -88,6 +105,8 @@ import org.finos.legend.engine.plan.execution.concurrent.ParallelGraphFetchExecutionExecutorPool; import org.finos.legend.engine.plan.execution.graphFetch.GraphFetchExecutionConfiguration; import org.finos.legend.engine.plan.execution.service.api.ServiceModelingApi; +import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutor; +import org.finos.legend.engine.plan.execution.stores.elasticsearch.v7.plugin.ElasticsearchV7StoreExecutorBuilder; import org.finos.legend.engine.plan.execution.stores.inMemory.plugin.InMemory; import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutor; import org.finos.legend.engine.plan.execution.stores.mongodb.plugin.MongoDBStoreExecutorBuilder; @@ -101,11 +120,13 @@ import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutor; import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutorBuilder; import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension; -//import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration; +import org.finos.legend.engine.language.hostedService.api.HostedServiceService; +import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentConfiguration; import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory; import org.finos.legend.engine.protocol.pure.v1.model.PureProtocol; +import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppService; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration; import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; -//import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; import org.finos.legend.engine.query.graphQL.api.debug.GraphQLDebug; import org.finos.legend.engine.query.graphQL.api.execute.GraphQLExecute; import org.finos.legend.engine.query.graphQL.api.grammar.GraphQLGrammar; @@ -161,7 +182,7 @@ public class Server extends Application public static void main(String[] args) throws Exception { EngineUrlStreamHandlerFactory.initialize(); - new Server().run(args.length == 0 ? new String[] {"server", "legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json"} : args); + new Server().run(args.length == 0 ? new String[]{"server", "legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json"} : args); } @Override @@ -265,6 +286,10 @@ public void run(T serverConfiguration, Environment environment) relationalExecution.setFlowProviderClass(LegendDefaultDatabaseAuthenticationFlowProvider.class); relationalExecution.setFlowProviderConfiguration(new LegendDefaultDatabaseAuthenticationFlowProviderConfiguration()); } + relationalExecution.setConnectionFactory(this.setupConnectionFactory(serverConfiguration.vaults)); + relationalExecution.setRelationalDatabaseConnectionAdapters(Lists.mutable.of( + new HACKY__SnowflakeConnectionAdapter.WithKeyPair() + )); relationalStoreExecutor = (RelationalStoreExecutor) Relational.build(serverConfiguration.relationalexecution); @@ -274,12 +299,14 @@ public void run(T serverConfiguration, Environment environment) MongoDBStoreExecutorConfiguration mongoDBExecutorConfiguration = MongoDBStoreExecutorConfiguration.newInstance().withCredentialProviderProvider(credentialProviderProvider).build(); MongoDBStoreExecutor mongoDBStoreExecutor = (MongoDBStoreExecutor) new MongoDBStoreExecutorBuilder().build(mongoDBExecutorConfiguration); + ElasticsearchV7StoreExecutor elasticsearchV7StoreExecutor = (ElasticsearchV7StoreExecutor) new ElasticsearchV7StoreExecutorBuilder().build(); + PlanExecutor planExecutor; ParallelGraphFetchExecutionExecutorPool parallelGraphFetchExecutionExecutorPool = null; if (serverConfiguration.graphFetchExecutionConfiguration != null) { GraphFetchExecutionConfiguration graphFetchExecutionConfiguration = serverConfiguration.graphFetchExecutionConfiguration; - planExecutor = PlanExecutor.newPlanExecutor(graphFetchExecutionConfiguration, relationalStoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); + planExecutor = PlanExecutor.newPlanExecutor(graphFetchExecutionConfiguration, relationalStoreExecutor, elasticsearchV7StoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); if (graphFetchExecutionConfiguration.canExecuteInParallel()) { parallelGraphFetchExecutionExecutorPool = new ParallelGraphFetchExecutionExecutorPool(graphFetchExecutionConfiguration.getParallelGraphFetchExecutionConfig(), "thread-pool for parallel graphFetch execution"); @@ -288,7 +315,7 @@ public void run(T serverConfiguration, Environment environment) } else { - planExecutor = PlanExecutor.newPlanExecutor(relationalStoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); + planExecutor = PlanExecutor.newPlanExecutor(relationalStoreExecutor, elasticsearchV7StoreExecutor, serviceStoreExecutor, mongoDBStoreExecutor, InMemory.build()); } // Session Management @@ -353,8 +380,6 @@ public void run(T serverConfiguration, Environment environment) environment.jersey().register(new ExecutePlanLegacy(planExecutor)); // Function Activator - -// environment.jersey().register(new FunctionActivatorAPI(modelManager , routerExtensions)); environment.jersey().register(new FunctionActivatorAPI(modelManager,Lists.mutable.empty(), Lists.mutable.with(new SnowflakeAppService(planExecutor), new HostedServiceService()), routerExtensions)); // GraphQL @@ -383,6 +408,9 @@ public void run(T serverConfiguration, Environment environment) // Analytics List entitlementServiceExtensions = EntitlementServiceExtensionLoader.extensions(); environment.jersey().register(new MappingAnalytics(modelManager)); + environment.jersey().register(new ClassAnalytics(modelManager)); + environment.jersey().register(new FunctionAnalytics(modelManager)); + environment.jersey().register(new BindingAnalytics(modelManager)); environment.jersey().register(new DiagramAnalytics(modelManager)); environment.jersey().register(new DataSpaceAnalytics(modelManager, generatorExtensions, entitlementServiceExtensions)); environment.jersey().register(new LineageAnalytics(modelManager)); @@ -397,6 +425,48 @@ public void run(T serverConfiguration, Environment environment) enableCors(environment); } + // TODO: @akphi - this is temporary, rework when we find a better way to handle the initialization of connection factory from config or some external source. + private ConnectionFactory setupConnectionFactory(List vaultConfigurations) + { + LegendEnvironment environment = new LegendEnvironment.Builder() + .withVaults( + new SystemPropertiesCredentialVault(), + new EnvironmentCredentialVault(), + new PropertiesFileCredentialVault(this.buildVaultProperties(vaultConfigurations)) + ) + .withStoreSupports( + new RelationalDatabaseStoreSupport.Builder(DatabaseType.POSTGRES) + .withIdentifier("Postgres") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).withAuthenticationConfigurationTypes( + UserPasswordAuthenticationConfiguration.class + ).build() + ) + .build(), + new RelationalDatabaseStoreSupport.Builder(DatabaseType.SNOWFLAKE) + .withIdentifier("Snowflake") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KEY_PAIR).withAuthenticationConfigurationTypes( + EncryptedPrivateKeyPairAuthenticationConfiguration.class + ).build() + ) + .build() + ).build(); + + StoreInstanceProvider storeInstanceProvider = new DefaultStoreInstanceProvider.Builder().build(); + return new ConnectionFactory.Builder(environment, storeInstanceProvider) + .withCredentialBuilders( + new KerberosCredentialExtractor(), + new UserPasswordCredentialBuilder(), + new KeyPairCredentialBuilder() + ) + .withConnectionBuilders( + new StaticJDBCConnectionBuilder.WithPlaintextUsernamePassword(), + new SnowflakeConnectionBuilder.WithKeyPair() + ) + .build(); + } + private void loadVaults(List vaultConfigurations) { if (vaultConfigurations != null)