Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Function Activation Cleanups #2396

Merged
merged 4 commits into from
Oct 20, 2023
Merged

Function Activation Cleanups #2396

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
12d4fba
rename method
Yasirmod17 Oct 19, 2023
4fd0f84
Merge branch 'finos:master' into master
Yasirmod17 Oct 19, 2023
ec84b2b
Function activation cleanup
Yasirmod17 Oct 19, 2023
5dc3700
rollback file changes
Yasirmod17 Oct 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions legend-engine-config/legend-engine-server/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,14 +190,6 @@
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-functionActivator-protocol</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-snowflakeApp-protocol</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-hostedService-protocol</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-artifact-generation-api</artifactId>
Expand Down Expand Up @@ -340,7 +332,6 @@
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-snowflakeApp-api</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
Expand All @@ -355,7 +346,6 @@
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-xt-hostedService-api</artifactId>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
Expand Down
10 changes: 6 additions & 4 deletions ...gine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,10 +120,12 @@
import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutor;
import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutorBuilder;
import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension;
import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration;
import org.finos.legend.engine.language.hostedService.api.HostedServiceService;
import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentConfiguration;
import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory;
import org.finos.legend.engine.protocol.pure.v1.model.PureProtocol;
import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration;
import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppService;
import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration;
import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
import org.finos.legend.engine.query.graphQL.api.debug.GraphQLDebug;
import org.finos.legend.engine.query.graphQL.api.execute.GraphQLExecute;
Expand Down Expand Up @@ -212,7 +214,7 @@ protected SwaggerBundleConfiguration getSwaggerBundleConfiguration(

bootstrap.getObjectMapper().registerSubtypes(new NamedType(LegendDefaultDatabaseAuthenticationFlowProviderConfiguration.class, "legendDefault"));
bootstrap.getObjectMapper().registerSubtypes(new NamedType(HostedServiceDeploymentConfiguration.class, "hostedServiceConfig"));
bootstrap.getObjectMapper().registerSubtypes(new NamedType(SnowflakeDeploymentConfiguration.class, "snowflakeAppConfig"));
bootstrap.getObjectMapper().registerSubtypes(new NamedType(SnowflakeAppDeploymentConfiguration.class, "snowflakeAppConfig"));
}

public CredentialProviderProvider configureCredentialProviders(List<VaultConfiguration> vaultConfigurations)
Expand Down Expand Up @@ -378,7 +380,7 @@ public void run(T serverConfiguration, Environment environment)
environment.jersey().register(new ExecutePlanLegacy(planExecutor));

// Function Activator
environment.jersey().register(new FunctionActivatorAPI(modelManager, Lists.mutable.empty(), routerExtensions));
environment.jersey().register(new FunctionActivatorAPI(modelManager,Lists.mutable.empty(), Lists.mutable.with(new SnowflakeAppService(planExecutor), new HostedServiceService()), routerExtensions));

// GraphQL
environment.jersey().register(new GraphQLGrammar());
Expand Down
12 changes: 9 additions & 3 deletions ...api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ public class FunctionActivatorAPI
private final PureModel emptyModel;
private final Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions;
private List<FunctionActivatorDeploymentConfiguration> runtimeDeploymentConfig = Lists.mutable.empty();
private MutableList<FunctionActivatorService<? extends Root_meta_external_function_activator_FunctionActivator, ? extends FunctionActivatorDeploymentConfiguration, ? extends DeploymentResult>> availableActivatorServices = Lists.mutable.empty();

public FunctionActivatorAPI(ModelManager modelManager, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
Expand All @@ -73,10 +74,11 @@ public FunctionActivatorAPI(ModelManager modelManager, Function<PureModel, RichI
this.emptyModel = Compiler.compile(PureModelContextData.newPureModelContextData(), DeploymentMode.PROD, null);
}

public FunctionActivatorAPI(ModelManager modelManager, List<FunctionActivatorDeploymentConfiguration> activatorConfigurations, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
public FunctionActivatorAPI(ModelManager modelManager, List<FunctionActivatorDeploymentConfiguration> activatorConfigurations, MutableList<FunctionActivatorService<? extends Root_meta_external_function_activator_FunctionActivator, ? extends FunctionActivatorDeploymentConfiguration, ? extends DeploymentResult>> availableActivatorServices, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
this(modelManager, routerExtensions);
this.runtimeDeploymentConfig = activatorConfigurations;
this.availableActivatorServices = availableActivatorServices;
}

@GET
Expand All @@ -87,7 +89,7 @@ public FunctionActivatorAPI(ModelManager modelManager, List<FunctionActivatorDe
public Response list(@ApiParam(hidden = true) @Pac4JProfileManager ProfileManager<CommonProfile> pm)
{
MutableList<CommonProfile> profiles = ProfileManagerHelper.extractProfiles(pm);
MutableList<FunctionActivatorInfo> values = FunctionActivatorLoader.extensions().collect(x -> x.info(emptyModel, "vX_X_X"));
MutableList<FunctionActivatorInfo> values = this.availableActivatorServices.isEmpty() ? FunctionActivatorLoader.extensions().collect(x -> x.info(emptyModel, "vX_X_X")) : availableActivatorServices.collect(c -> c.info(emptyModel,"vX_X_X"));
return ManageConstantResult.manageResult(profiles, values, objectMapper);
}

Expand Down Expand Up @@ -162,7 +164,11 @@ public Response renderArtifact(FunctionActivatorInput input, @ApiParam(hidden =

public FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult> getActivatorService(Root_meta_external_function_activator_FunctionActivator activator, PureModel pureModel)
{
FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult> service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst();
FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult> service = (FunctionActivatorService<Root_meta_external_function_activator_FunctionActivator, FunctionActivatorDeploymentConfiguration, DeploymentResult>)this.availableActivatorServices.select(c -> c.supports(activator)).getFirst();
if (service == null)
{
service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst();
}
if (service == null)
{
throw new RuntimeException(activator.getClass().getSimpleName() + "is not supported!");
Expand Down
4 changes: 4 additions & 0 deletions .../finos/legend/engine/protocol/hostedService/metamodel/HostedServiceProtocolExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.eclipse.collections.api.block.function.Function0;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration;
import org.finos.legend.engine.protocol.hostedService.metamodel.control.Deployment;
import org.finos.legend.engine.protocol.hostedService.metamodel.control.Ownership;
import org.finos.legend.engine.protocol.hostedService.metamodel.control.UserList;
Expand All @@ -41,6 +42,9 @@ public List<Function0<List<ProtocolSubTypeInfo<?>>>> getExtraProtocolSubTypeInfo
ProtocolSubTypeInfo.newBuilder(Ownership.class)
.withSubtype(UserList.class, "userList")
.withSubtype(Deployment.class, "deployment")
.build(),
ProtocolSubTypeInfo.newBuilder(DeploymentConfiguration.class)
.withSubtype(HostedServiceDeploymentConfiguration.class, "hostedServiceDeploymentConfiguration")
.build()
));
}
Expand Down
4 changes: 4 additions & 0 deletions legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,10 @@
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-executionPlan-execution</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.engine</groupId>
<artifactId>legend-engine-pure-platform-java</artifactId>
</dependency>

<dependency>
<groupId>org.finos.legend.engine</groupId>
Expand Down
26 changes: 9 additions & 17 deletions .../src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,29 +28,16 @@
import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppContent;
import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration;
import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppGenerator;
import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration;
import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage;
import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration;
import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentManager;
import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentResult;
import org.finos.legend.engine.plan.execution.PlanExecutor;
import org.finos.legend.engine.plan.execution.stores.relational.config.TemporaryTestDbConfiguration;
import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector;
import org.finos.legend.engine.plan.generation.PlanGenerator;
import org.finos.legend.engine.plan.platform.PlanPlatform;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppProtocolExtension;
import org.finos.legend.pure.generated.Root_meta_external_function_activator_FunctionActivator;
import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration;
import org.finos.legend.pure.generated.Root_meta_external_store_relational_runtime_RelationalDatabaseConnection;
import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy;
import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification;
import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionNode;
import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan;
import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
import org.finos.legend.pure.generated.Root_meta_relational_mapping_SQLExecutionNode;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction;
import org.pac4j.core.profile.CommonProfile;

import java.util.List;
Expand All @@ -74,6 +61,11 @@ public SnowflakeAppService(ConnectionManagerSelector connectionManager)
this.snowflakeDeploymentManager = new SnowflakeDeploymentManager(new SnowflakeAppDeploymentTool(connectionManager));
}

public SnowflakeAppService(PlanExecutor executor)
{
this.snowflakeDeploymentManager = new SnowflakeDeploymentManager(executor);
}

@Override
public FunctionActivatorInfo info(PureModel pureModel, String version)
{
Expand All @@ -94,7 +86,7 @@ public boolean supports(Root_meta_external_function_activator_FunctionActivator
@Override
public MutableList<? extends FunctionActivatorError> validate(MutableList<CommonProfile> profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions);
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
int size = ((SnowflakeAppContent)artifact.content).sqlExpressions.size();
return size != 1 ?
Lists.mutable.with(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", ((SnowflakeAppContent)artifact.content).sqlExpressions)) :
Expand All @@ -105,14 +97,14 @@ public MutableList<? extends FunctionActivatorError> validate(MutableList<Common
@Override
public SnowflakeDeploymentResult publishToSandbox(MutableList<CommonProfile> profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, List<SnowflakeAppDeploymentConfiguration> runtimeConfigurations, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions);
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
return this.snowflakeDeploymentManager.deploy(profiles, artifact, runtimeConfigurations);
}

@Override
public SnowflakeAppArtifact renderArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, String clientVersion, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
return SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions);
return SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
}

@Override
Expand Down
9 changes: 5 additions & 4 deletions ...n/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,15 @@ public SnowflakeAppArtifact()
//empty artifact
}

public SnowflakeAppArtifact(String name, MutableList<String> sqlExpressions)
public SnowflakeAppArtifact(SnowflakeAppContent content)
{
this.content = new SnowflakeAppContent(name, sqlExpressions);
this.content = content;
}

public SnowflakeAppArtifact(String name, MutableList<String> sqlExpressions, SnowflakeAppDeploymentConfiguration config)
public SnowflakeAppArtifact(SnowflakeAppContent content, SnowflakeAppDeploymentConfiguration config)
{
this.content = new SnowflakeAppContent(name, sqlExpressions);
this(content);
this.deploymentConfiguration = config;
}

}
33 changes: 30 additions & 3 deletions ...gend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,31 @@

package org.finos.legend.engine.language.snowflakeApp.deployment;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.block.function.Function;
import org.eclipse.collections.impl.factory.Lists;
import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact;
import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension;
import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.context.SDLC;
import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader;
import org.finos.legend.engine.shared.core.ObjectMapperFactory;
import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;
import org.slf4j.Logger;

import java.util.List;

public class SnowflakeAppArtifactGenerationExtension implements ArtifactGenerationExtension
{
private static final ObjectMapper mapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports();
private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(SnowflakeAppArtifactGenerationExtension.class);
private static final String ROOT_PATH = "snowflakeApp";
private static final String FILENAME = "snowflakeArtifact.json";
private static final String FILE_NAME = "snowflakeArtifact.json";

@Override
public String getKey()
Expand All @@ -39,14 +51,29 @@ public String getKey()
public boolean canGenerate(PackageableElement element)
{
return false;
// return element instanceof Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
// return element instanceof Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
}


@Override
public List<Artifact> generate(PackageableElement element, PureModel pureModel, PureModelContextData data, String clientVersion)
{
return null;
List<Artifact> result = Lists.mutable.empty();
Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport()));
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, (Root_meta_external_function_activator_snowflakeApp_SnowflakeApp) element, data, routerExtensions);
try
{
LOGGER.info("Generating snowflake artifact for " + element.getName());
String content = mapper.writeValueAsString(artifact);
result.add((new Artifact(content, FILE_NAME, "json")));
LOGGER.info("Generated snowflake artifact for " + element.getName());

}
catch (Exception e)
{
LOGGER.error("Error generating snowflake artifact for " + element.getName() + " reason: " + e.getMessage());
}
return result;

}

Expand Down
Loading
Loading