Function Activation Cleanups (#2396)

* rename method * Function activation cleanup * rollback file changes
finos · Oct 20, 2023 · b2556ad · b2556ad
1 parent bb45920
commit b2556ad
Show file tree

Hide file tree

Showing 18 changed files with 219 additions and 91 deletions.
diff --git a/legend-engine-config/legend-engine-server/pom.xml b/legend-engine-config/legend-engine-server/pom.xml
@@ -190,14 +190,6 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-functionActivator-protocol</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.finos.legend.engine</groupId>
-            <artifactId>legend-engine-xt-snowflakeApp-protocol</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.finos.legend.engine</groupId>
-            <artifactId>legend-engine-xt-hostedService-protocol</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-artifact-generation-api</artifactId>
@@ -340,7 +332,6 @@
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-snowflakeApp-api</artifactId>
-            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
@@ -355,7 +346,6 @@
         <dependency>
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-xt-hostedService-api</artifactId>
-            <scope>runtime</scope>
         </dependency>
         <dependency>
             <groupId>org.finos.legend.engine</groupId>

diff --git a/...gine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java b/...gine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java
@@ -120,10 +120,12 @@
 import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutor;
 import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutorBuilder;
 import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension;
-import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration;
+import org.finos.legend.engine.language.hostedService.api.HostedServiceService;
+import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentConfiguration;
 import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory;
 import org.finos.legend.engine.protocol.pure.v1.model.PureProtocol;
-import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration;
+import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppService;
+import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration;
 import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
 import org.finos.legend.engine.query.graphQL.api.debug.GraphQLDebug;
 import org.finos.legend.engine.query.graphQL.api.execute.GraphQLExecute;
@@ -212,7 +214,7 @@ protected SwaggerBundleConfiguration getSwaggerBundleConfiguration(
 
         bootstrap.getObjectMapper().registerSubtypes(new NamedType(LegendDefaultDatabaseAuthenticationFlowProviderConfiguration.class, "legendDefault"));
         bootstrap.getObjectMapper().registerSubtypes(new NamedType(HostedServiceDeploymentConfiguration.class, "hostedServiceConfig"));
-        bootstrap.getObjectMapper().registerSubtypes(new NamedType(SnowflakeDeploymentConfiguration.class, "snowflakeAppConfig"));
+        bootstrap.getObjectMapper().registerSubtypes(new NamedType(SnowflakeAppDeploymentConfiguration.class, "snowflakeAppConfig"));
     }
 
     public CredentialProviderProvider configureCredentialProviders(List<VaultConfiguration> vaultConfigurations)
@@ -378,7 +380,7 @@ public void run(T serverConfiguration, Environment environment)
         environment.jersey().register(new ExecutePlanLegacy(planExecutor));
 
         // Function Activator
-        environment.jersey().register(new FunctionActivatorAPI(modelManager, Lists.mutable.empty(), routerExtensions));
+        environment.jersey().register(new FunctionActivatorAPI(modelManager,Lists.mutable.empty(), Lists.mutable.with(new SnowflakeAppService(planExecutor), new HostedServiceService()), routerExtensions));
 
         // GraphQL
         environment.jersey().register(new GraphQLGrammar());

diff --git a/...api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java b/...api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java
@@ -65,6 +65,7 @@ public class FunctionActivatorAPI
     private final PureModel emptyModel;
     private final Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions;
     private  List<FunctionActivatorDeploymentConfiguration> runtimeDeploymentConfig = Lists.mutable.empty();
+    private MutableList<FunctionActivatorService<? extends Root_meta_external_function_activator_FunctionActivator, ? extends FunctionActivatorDeploymentConfiguration, ? extends DeploymentResult>>  availableActivatorServices = Lists.mutable.empty();
 
     public FunctionActivatorAPI(ModelManager modelManager, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
@@ -73,10 +74,11 @@ public FunctionActivatorAPI(ModelManager modelManager, Function<PureModel, RichI
         this.emptyModel = Compiler.compile(PureModelContextData.newPureModelContextData(), DeploymentMode.PROD, null);
     }
 
-    public FunctionActivatorAPI(ModelManager modelManager,  List<FunctionActivatorDeploymentConfiguration> activatorConfigurations, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
+    public FunctionActivatorAPI(ModelManager modelManager,  List<FunctionActivatorDeploymentConfiguration> activatorConfigurations, MutableList<FunctionActivatorService<? extends Root_meta_external_function_activator_FunctionActivator, ? extends FunctionActivatorDeploymentConfiguration, ? extends DeploymentResult>> availableActivatorServices, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
         this(modelManager, routerExtensions);
         this.runtimeDeploymentConfig = activatorConfigurations;
+        this.availableActivatorServices = availableActivatorServices;
     }
 
     @GET
@@ -87,7 +89,7 @@ public FunctionActivatorAPI(ModelManager modelManager,  List<FunctionActivatorDe
     public Response list(@ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
     {
         MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(pm);
-        MutableList<FunctionActivatorInfo> values = FunctionActivatorLoader.extensions().collect(x -> x.info(emptyModel, "vX_X_X"));
+        MutableList<FunctionActivatorInfo> values = this.availableActivatorServices.isEmpty() ? FunctionActivatorLoader.extensions().collect(x -> x.info(emptyModel, "vX_X_X")) : availableActivatorServices.collect(c -> c.info(emptyModel,"vX_X_X"));
         return ManageConstantResult.manageResult(profiles, values, objectMapper);
     }
 
@@ -162,7 +164,11 @@ public Response renderArtifact(FunctionActivatorInput input, @ApiParam(hidden =
 
     public FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult> getActivatorService(Root_meta_external_function_activator_FunctionActivator activator, PureModel pureModel)
     {
-        FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult> service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst();
+        FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult> service = (FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult>)this.availableActivatorServices.select(c -> c.supports(activator)).getFirst();
+        if (service == null)
+        {
+            service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst();
+        }
         if (service == null)
         {
             throw new RuntimeException(activator.getClass().getSimpleName() + "is not supported!");

diff --git a/.../finos/legend/engine/protocol/hostedService/metamodel/HostedServiceProtocolExtension.java b/.../finos/legend/engine/protocol/hostedService/metamodel/HostedServiceProtocolExtension.java
@@ -17,6 +17,7 @@
 import org.eclipse.collections.api.block.function.Function0;
 import org.eclipse.collections.api.factory.Lists;
 import org.eclipse.collections.api.factory.Maps;
+import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration;
 import org.finos.legend.engine.protocol.hostedService.metamodel.control.Deployment;
 import org.finos.legend.engine.protocol.hostedService.metamodel.control.Ownership;
 import org.finos.legend.engine.protocol.hostedService.metamodel.control.UserList;
@@ -41,6 +42,9 @@ public List<Function0<List<ProtocolSubTypeInfo<?>>>> getExtraProtocolSubTypeInfo
                 ProtocolSubTypeInfo.newBuilder(Ownership.class)
                         .withSubtype(UserList.class, "userList")
                         .withSubtype(Deployment.class, "deployment")
+                        .build(),
+                ProtocolSubTypeInfo.newBuilder(DeploymentConfiguration.class)
+                        .withSubtype(HostedServiceDeploymentConfiguration.class, "hostedServiceDeploymentConfiguration")
                         .build()
         ));
     }

diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml
@@ -92,6 +92,10 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-executionPlan-execution</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-pure-platform-java</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.finos.legend.engine</groupId>

diff --git a/.../src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java b/.../src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java
@@ -28,29 +28,16 @@
 import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppContent;
 import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration;
 import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppGenerator;
-import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration;
-import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage;
-import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration;
 import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentManager;
 import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentResult;
+import org.finos.legend.engine.plan.execution.PlanExecutor;
 import org.finos.legend.engine.plan.execution.stores.relational.config.TemporaryTestDbConfiguration;
 import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector;
-import org.finos.legend.engine.plan.generation.PlanGenerator;
-import org.finos.legend.engine.plan.platform.PlanPlatform;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
 import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppProtocolExtension;
 import org.finos.legend.pure.generated.Root_meta_external_function_activator_FunctionActivator;
 import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
-import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration;
-import org.finos.legend.pure.generated.Root_meta_external_store_relational_runtime_RelationalDatabaseConnection;
-import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy;
-import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification;
-import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionNode;
-import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan;
 import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
-import org.finos.legend.pure.generated.Root_meta_relational_mapping_SQLExecutionNode;
-import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition;
-import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction;
 import org.pac4j.core.profile.CommonProfile;
 
 import java.util.List;
@@ -74,6 +61,11 @@ public SnowflakeAppService(ConnectionManagerSelector connectionManager)
         this.snowflakeDeploymentManager = new SnowflakeDeploymentManager(new SnowflakeAppDeploymentTool(connectionManager));
     }
 
+    public SnowflakeAppService(PlanExecutor executor)
+    {
+        this.snowflakeDeploymentManager = new SnowflakeDeploymentManager(executor);
+    }
+
     @Override
     public FunctionActivatorInfo info(PureModel pureModel, String version)
     {
@@ -94,7 +86,7 @@ public boolean supports(Root_meta_external_function_activator_FunctionActivator
     @Override
     public MutableList<? extends FunctionActivatorError> validate(MutableList<CommonProfile> profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions);
+        SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
         int size = ((SnowflakeAppContent)artifact.content).sqlExpressions.size();
         return size != 1 ?
                 Lists.mutable.with(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", ((SnowflakeAppContent)artifact.content).sqlExpressions)) :
@@ -105,14 +97,14 @@ public MutableList<? extends FunctionActivatorError> validate(MutableList<Common
     @Override
     public SnowflakeDeploymentResult publishToSandbox(MutableList<CommonProfile> profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, List<SnowflakeAppDeploymentConfiguration> runtimeConfigurations, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions);
+        SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
         return this.snowflakeDeploymentManager.deploy(profiles, artifact, runtimeConfigurations);
     }
 
     @Override
     public SnowflakeAppArtifact renderArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, String clientVersion, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        return SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions);
+        return SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
     }
 
     @Override

diff --git a/...n/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java b/...n/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java
@@ -27,14 +27,15 @@ public SnowflakeAppArtifact()
         //empty artifact
     }
 
-    public SnowflakeAppArtifact(String name, MutableList<String> sqlExpressions)
+    public SnowflakeAppArtifact(SnowflakeAppContent content)
     {
-        this.content = new SnowflakeAppContent(name, sqlExpressions);
+        this.content = content;
     }
 
-    public SnowflakeAppArtifact(String name, MutableList<String> sqlExpressions, SnowflakeAppDeploymentConfiguration config)
+    public SnowflakeAppArtifact(SnowflakeAppContent content, SnowflakeAppDeploymentConfiguration config)
     {
-        this.content = new SnowflakeAppContent(name, sqlExpressions);
+        this(content);
         this.deploymentConfiguration = config;
     }
+
 }
diff --git a/...gend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java b/...gend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java
@@ -15,19 +15,31 @@
 
 package org.finos.legend.engine.language.snowflakeApp.deployment;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.eclipse.collections.api.RichIterable;
+import org.eclipse.collections.api.block.function.Function;
+import org.eclipse.collections.impl.factory.Lists;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
 import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact;
 import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension;
+import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
+import org.finos.legend.engine.protocol.pure.v1.model.context.SDLC;
+import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
+import org.finos.legend.engine.shared.core.ObjectMapperFactory;
 import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
+import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;
+import org.slf4j.Logger;
 
 import java.util.List;
 
 public class SnowflakeAppArtifactGenerationExtension implements ArtifactGenerationExtension
 {
+    private static  final ObjectMapper mapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports();
+    private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(SnowflakeAppArtifactGenerationExtension.class);
     private static final String ROOT_PATH = "snowflakeApp";
-    private static final String FILENAME = "snowflakeArtifact.json";
+    private static final String FILE_NAME = "snowflakeArtifact.json";
 
     @Override
     public String getKey()
@@ -39,14 +51,29 @@ public String getKey()
     public boolean canGenerate(PackageableElement element)
     {
         return false;
-       // return element instanceof Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
+        // return element instanceof Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
     }
 
 
     @Override
     public List<Artifact> generate(PackageableElement element, PureModel pureModel, PureModelContextData data, String clientVersion)
     {
-        return null;
+        List<Artifact> result = Lists.mutable.empty();
+        Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport()));
+        SnowflakeAppArtifact artifact  = SnowflakeAppGenerator.generateArtifact(pureModel, (Root_meta_external_function_activator_snowflakeApp_SnowflakeApp) element, data, routerExtensions);
+        try
+        {
+            LOGGER.info("Generating snowflake artifact for " + element.getName());
+            String content = mapper.writeValueAsString(artifact);
+            result.add((new Artifact(content, FILE_NAME, "json")));
+            LOGGER.info("Generated snowflake artifact for " + element.getName());
+
+        }
+        catch (Exception e)
+        {
+            LOGGER.error("Error generating snowflake artifact for " + element.getName() + " reason: " + e.getMessage());
+        }
+        return result;
 
     }