Hosted Service refactor (#2583)

* Check string length before substring * Move generation to logic to pure * fix dependencies * fix dependencies * fix dependencies * Add validation for hostedService * add validations for hosted service * Add version to Activator Artifact * refactor sandbox deploy code * fix checkstyle * more refactor * fix checkstyle * fix checkstyle * add logs * cleanup files
finos · Jan 27, 2024 · c1ba5a7 · c1ba5a7
1 parent 99645bb
commit c1ba5a7
Show file tree

Hide file tree

Showing 15 changed files with 159 additions and 94 deletions.
diff --git a/...n/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionService.java b/...n/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionService.java
@@ -64,14 +64,14 @@ public boolean supports(Root_meta_external_function_activator_FunctionActivator
     @Override
     public MutableList<? extends FunctionActivatorError> validate(Identity identity, PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, PureModelContext inputModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        BigQueryFunctionArtifact artifact = BigQueryFunctionGenerator.generateArtifact(pureModel, activator, routerExtensions);
+        BigQueryFunctionArtifact artifact = BigQueryFunctionGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
         return this.validateArtifact(artifact);
     }
 
     @Override
     public BigQueryFunctionDeploymentResult publishToSandbox(Identity identity, PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, PureModelContext inputModel, List<BigQueryFunctionDeploymentConfiguration> runtimeConfigurations, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        BigQueryFunctionArtifact artifact = BigQueryFunctionGenerator.generateArtifact(pureModel, activator, routerExtensions);
+        BigQueryFunctionArtifact artifact = BigQueryFunctionGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
         MutableList<? extends FunctionActivatorError> validationErrors = this.validateArtifact(artifact);
 
         Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration deploymentConfiguration = ((Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration) activator._activationConfiguration());
@@ -83,7 +83,7 @@ public BigQueryFunctionDeploymentResult publishToSandbox(Identity identity, Pure
     @Override
     public BigQueryFunctionArtifact renderArtifact(PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, PureModelContext inputModel, String clientVersion, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        return BigQueryFunctionGenerator.generateArtifact(pureModel, activator, routerExtensions);
+        return BigQueryFunctionGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
     }
 
     @Override

diff --git a/...g/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionGenerator.java b/...g/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionGenerator.java
@@ -23,18 +23,21 @@
 import org.finos.legend.engine.plan.generation.PlanGenerator;
 import org.finos.legend.engine.plan.platform.PlanPlatform;
 import org.finos.legend.engine.protocol.bigqueryFunction.deployment.BigQueryFunctionArtifact;
+import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
+import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
+import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.pure.generated.*;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction;
 
 public class BigQueryFunctionGenerator
 {
-    public static BigQueryFunctionArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, Function<PureModel,RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
+    public static BigQueryFunctionArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, PureModelContext inputModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
         Pair<Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification, RichIterable<String>> artifactDetails = extractArtifactDetails(pureModel, activator, routerExtensions);
         Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification bigQueryDatasourceSpecification = artifactDetails.getOne();
         RichIterable<String> sqlExpressions = artifactDetails.getTwo();
-        return new BigQueryFunctionArtifact(activator._functionName(), Lists.mutable.withAll(sqlExpressions), bigQueryDatasourceSpecification._projectId(), bigQueryDatasourceSpecification._defaultDataset());
+        return new BigQueryFunctionArtifact(activator._functionName(), Lists.mutable.withAll(sqlExpressions), bigQueryDatasourceSpecification._projectId(), bigQueryDatasourceSpecification._defaultDataset(), (AlloySDLC) ((PureModelContextData)inputModel).origin.sdlcInfo);
     }
 
     private static Pair<Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification, RichIterable<String>> extractArtifactDetails(PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)

diff --git a/...rg/finos/legend/engine/protocol/bigqueryFunction/deployment/BigQueryFunctionArtifact.java b/...rg/finos/legend/engine/protocol/bigqueryFunction/deployment/BigQueryFunctionArtifact.java
@@ -16,16 +16,18 @@
 
 import org.eclipse.collections.api.list.MutableList;
 import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorArtifact;
+import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
 
 public class BigQueryFunctionArtifact extends FunctionActivatorArtifact
 {
     public String sourceProjectId;
     public String sourceDefaultDataset;
 
-    public BigQueryFunctionArtifact(String name, MutableList<String> sqlExpressions, String sourceProjectId, String sourceDefaultDataset)
+    public BigQueryFunctionArtifact(String name, MutableList<String> sqlExpressions, String sourceProjectId, String sourceDefaultDataset, AlloySDLC sdlc)
     {
         this.content = new BigQueryFunctionContent(name, sqlExpressions);
         this.sourceProjectId = sourceProjectId;
         this.sourceDefaultDataset = sourceDefaultDataset;
+        this.version = getVersionInfo(sdlc);
     }
 }
diff --git a/.../finos/legend/engine/protocol/functionActivator/deployment/FunctionActivatorArtifact.java b/.../finos/legend/engine/protocol/functionActivator/deployment/FunctionActivatorArtifact.java
@@ -16,12 +16,25 @@
 
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
 
 @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type")
 @JsonIgnoreProperties(ignoreUnknown = true)
 public class FunctionActivatorArtifact
 {
     public FunctionActivatorDeploymentContent content;
     public FunctionActivatorDeploymentConfiguration deploymentConfiguration;
+
+    public String version;
     public String deployedLocation;
+
+
+    public String getVersionInfo(AlloySDLC sdlc)
+    {
+        if (this.version != null)
+        {
+            return sdlc.groupId + ":" + sdlc.artifactId + ":" + sdlc.version;
+        }
+        return "";
+    }
 }
diff --git a/...rc/main/java/org/finos/legend/engine/language/hostedService/api/HostedServiceService.java b/...rc/main/java/org/finos/legend/engine/language/hostedService/api/HostedServiceService.java
@@ -31,6 +31,7 @@
 import org.finos.legend.engine.language.hostedService.generation.HostedServiceArtifactGenerator;
 import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel;
 import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceProtocolExtension;
+import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 import org.finos.legend.engine.shared.core.identity.Identity;
@@ -90,7 +91,7 @@ public MutableList<? extends FunctionActivatorError> validate(Identity identity,
     @Override
     public HostedServiceArtifact renderArtifact(PureModel pureModel, Root_meta_external_function_activator_hostedService_HostedService activator, PureModelContext inputModel, String clientVersion, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        return new HostedServiceArtifact(this.hostedServiceArtifactgenerator.renderArtifact(pureModel,activator,inputModel,clientVersion,routerExtensions));
+        return new HostedServiceArtifact(activator._pattern(), this.hostedServiceArtifactgenerator.renderArtifact(pureModel,activator,inputModel,clientVersion,routerExtensions), null);
     }
 
     @Override
@@ -99,14 +100,12 @@ public List<HostedServiceDeploymentConfiguration> selectConfig(List<FunctionActi
         return Lists.mutable.withAll(configurations).select(e -> e instanceof HostedServiceDeploymentConfiguration).collect(e -> (HostedServiceDeploymentConfiguration)e);
     }
 
-
     @Override
     public HostedServiceDeploymentResult publishToSandbox(Identity identity, PureModel pureModel, Root_meta_external_function_activator_hostedService_HostedService activator, PureModelContext inputModel, List<HostedServiceDeploymentConfiguration> runtimeConfigs, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
         GenerationInfoData generation = this.hostedServiceArtifactgenerator.renderArtifact(pureModel, activator, inputModel, "vX_X_X",routerExtensions);
-        HostedServiceArtifact artifact = new HostedServiceArtifact(generation, fetchHostedService(activator, (PureModelContextData)inputModel, pureModel));
+        HostedServiceArtifact artifact = new HostedServiceArtifact(activator._pattern(), generation, fetchHostedService(activator, (PureModelContextData)inputModel, pureModel), (AlloySDLC) ((PureModelContextData)inputModel).origin.sdlcInfo);
         return this.hostedServiceDeploymentManager.deploy(identity, artifact, runtimeConfigs);
-//        return new HostedServiceDeploymentResult();
     }
 
     public static PureModelContextData fetchHostedService(Root_meta_external_function_activator_hostedService_HostedService activator, PureModelContextData data, PureModel pureModel)
@@ -126,5 +125,4 @@ private static String fullName(org.finos.legend.engine.protocol.pure.v1.model.pa
         return e._package + "::" + e.name;
     }
 
-
 }
diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-generation/pom.xml b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-generation/pom.xml
@@ -94,6 +94,14 @@
 
         <!-- ENGINE -->
 
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpclient</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.httpcomponents</groupId>
+            <artifactId>httpcore</artifactId>
+        </dependency>
         <!-- ECLIPSE COLLECTIONS -->
         <dependency>
             <groupId>org.eclipse.collections</groupId>

diff --git a/...d/engine/language/hostedService/generation/deployment/HostedServiceDeploymentManager.java b/...d/engine/language/hostedService/generation/deployment/HostedServiceDeploymentManager.java
@@ -15,18 +15,33 @@
 package org.finos.legend.engine.language.hostedService.generation.deployment;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.BasicCookieStore;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.util.EntityUtils;
+import org.eclipse.collections.api.factory.Lists;
+import org.eclipse.collections.api.list.MutableList;
 import org.finos.legend.engine.functionActivator.deployment.DeploymentManager;
 import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorArtifact;
-import org.finos.legend.engine.protocol.hostedService.deployment.HostedServiceArtifact;
-import org.finos.legend.engine.protocol.hostedService.deployment.HostedServiceDeploymentConfiguration;
-import org.finos.legend.engine.protocol.hostedService.deployment.HostedServiceDeploymentResult;
+import org.finos.legend.engine.protocol.hostedService.deployment.*;
 import org.finos.legend.engine.shared.core.ObjectMapperFactory;
 import org.finos.legend.engine.shared.core.identity.Identity;
+import org.finos.legend.engine.shared.core.identity.credential.LegendKerberosCredential;
+import org.finos.legend.engine.shared.core.kerberos.HttpClientBuilder;
+import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import javax.security.auth.Subject;
+import java.security.PrivilegedExceptionAction;
 import java.util.List;
 
 public class HostedServiceDeploymentManager implements  DeploymentManager<HostedServiceArtifact, HostedServiceDeploymentResult, HostedServiceDeploymentConfiguration>
 {
+    private static final Logger LOGGER = LoggerFactory.getLogger(HostedServiceDeploymentManager.class);
 
     public static ObjectMapper mapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports();
 
@@ -37,61 +52,70 @@ public boolean canDeploy(FunctionActivatorArtifact element)
 
     public HostedServiceDeploymentResult deploy(Identity identity, HostedServiceArtifact artifact)
     {
-        return new HostedServiceDeploymentResult();
+        return doDeploy(identity, (HostedServiceDeploymentConfiguration) artifact.deploymentConfiguration, artifact);
     }
 
-
     public HostedServiceDeploymentResult deploy(Identity identity, HostedServiceArtifact artifact, List<HostedServiceDeploymentConfiguration> availableRuntimeConfigurations)
     {
-        String host;
-        String path;
-        int port;
+        HostedServiceDeploymentConfiguration deployConf;
+        MutableList<HostedServiceDeploymentConfiguration> c = Lists.mutable.withAll(availableRuntimeConfigurations);
+        if (artifact.deploymentConfiguration == null
+                && c.select(conf -> conf.destination.equals(((HostedServiceDeploymentConfiguration) (artifact.deploymentConfiguration)).destination)).size() > 0)
+        {
+            deployConf = c.getFirst();
+        }
+        else
+        {
+            deployConf = (HostedServiceDeploymentConfiguration) artifact.deploymentConfiguration;
+        }
 
-//        if (artifact.deploymentConfiguration == null || activator._activationConfiguration()._stage()._name().equals(DeploymentStage.SANDBOX.name()))
-//        {
-//            if (availableRuntimeConfigurations.size() > 0)
-//            {
-//                host = availableRuntimeConfigurations.get(0).host;
-//                path = availableRuntimeConfigurations.get(0).path;
-//                port = availableRuntimeConfigurations.get(0).port;
-//            }
-//            else
-//            {
-//                throw new EngineException("No available configuration for sandbox deployment");
-//            }
-//            try
-//            {
-//                HttpPost request = new HttpPost(new URIBuilder()
-//                        .setScheme("http")
-//                        .setHost(host)
-//                        .setPort(port)
-//                        .setPath(path)
-//                        .build());
-//                StringEntity stringEntity = new StringEntity(mapper.writeValueAsString(artifact));
-//                stringEntity.setContentType("application/json");
-//                request.setEntity(stringEntity);
-//                CloseableHttpClient httpclient = (CloseableHttpClient) HttpClientBuilder.getHttpClient(new BasicCookieStore());
-//                Subject s = ProfileManagerHelper.extractSubject(profiles);
-//                Subject.doAs(s, (PrivilegedExceptionAction<HostedServiceDeploymentResult>) () ->
-//                            {
-//                                HttpResponse response = httpclient.execute(request);
-//                                return new HostedServiceDeploymentResult();
-//                            });
-//            }
-//            catch (Exception e)
-//            {
-//                throw new EngineException("No available configuration for sandbox deployment");
-//
-//            }
-//        }
-//        else if (activator._activationConfiguration() != null)
-//        {
-//            host = ((Root_meta_external_function_activator_hostedService_HostedServiceDeploymentConfiguration)activator._activationConfiguration())._;
-//            path = availableRuntimeConfigurations.get(0).path;
-//            port = availableRuntimeConfigurations.get(0).port;
-//        }
-        return new HostedServiceDeploymentResult();
+        return doDeploy(identity, deployConf, artifact);
     }
 
+    public HostedServiceDeploymentResult doDeploy(Identity identity, HostedServiceDeploymentConfiguration deployConf, HostedServiceArtifact artifact)
+    {
+        HostedServiceDeploymentResult result = new HostedServiceDeploymentResult();
+        try
+        {
+            HttpPost request = new HttpPost(new URIBuilder()
+                    .setScheme("https")
+                    .setHost(deployConf.domain)
+                    .setPort(deployConf.port)
+                    .setPath(deployConf.path)
+                    .build());
+            StringEntity stringEntity = new StringEntity(mapper.writeValueAsString(artifact));
+            stringEntity.setContentType("application/json");
+            request.setEntity(stringEntity);
+            CloseableHttpClient httpclient = (CloseableHttpClient) HttpClientBuilder.getHttpClient(new BasicCookieStore());
+            Subject subject = identity.getCredential(LegendKerberosCredential.class).get().getSubject();
+            Subject.doAs(subject, (PrivilegedExceptionAction<String>) () ->
+            {
+                HttpResponse response = httpclient.execute(request);
+                if (response.getStatusLine().getStatusCode() != 200)
+                {
+                    result.error = EntityUtils.toString(response.getEntity());
+                }
+                else
+                {
+                    result.successful = true;
+                    result.deploymentLocation = buildDeployStub(deployConf, artifact);
+                }
+                return "done";
+            });
+            //LOGGER.info("Done deploying hosted service");
+        }
+        catch (Exception e)
+        {
+            LOGGER.error("Error deploying hosted service", e);
+            throw new EngineException(e.getMessage());
 
+        }
+        return result;
+    }
+
+    public String buildDeployStub(HostedServiceDeploymentConfiguration config, HostedServiceArtifact artifact)
+    {
+        //change to UI
+        return "http://" + config.domain + ":" + config.port + config.path + ((HostedServiceContent)artifact.content).pattern;
+    }
 }
diff --git a/...java/org/finos/legend/engine/protocol/hostedService/deployment/HostedServiceArtifact.java b/...java/org/finos/legend/engine/protocol/hostedService/deployment/HostedServiceArtifact.java
@@ -14,9 +14,9 @@
 
 package org.finos.legend.engine.protocol.hostedService.deployment;
 
-//import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorArtifact;
 import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorArtifact;
 import org.finos.legend.engine.protocol.hostedService.deployment.model.GenerationInfo;
+import org.finos.legend.engine.protocol.pure.v1.model.context.AlloySDLC;
 import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
 
 public class HostedServiceArtifact extends FunctionActivatorArtifact
@@ -27,14 +27,16 @@ public HostedServiceArtifact()
 
     }
 
-    public HostedServiceArtifact(GenerationInfo info)
+    public HostedServiceArtifact(String pattern, GenerationInfo info, AlloySDLC sdlc)
     {
-        this.content = new HostedServiceContent(info);
+        this.content = new HostedServiceContent(pattern, info);
+        this.version = getVersionInfo(sdlc);
     }
 
-    public HostedServiceArtifact(GenerationInfo info, PureModelContextData serviceData)
+    public HostedServiceArtifact(String pattern, GenerationInfo info, PureModelContextData serviceData, AlloySDLC sdlc)
     {
-        this.content = new HostedServiceContent(info, serviceData);
+        this.content = new HostedServiceContent(pattern, info, serviceData);
+        this.version = getVersionInfo(sdlc);
     }
 
 

diff --git a/.../java/org/finos/legend/engine/protocol/hostedService/deployment/HostedServiceContent.java b/.../java/org/finos/legend/engine/protocol/hostedService/deployment/HostedServiceContent.java
@@ -20,18 +20,20 @@
 
 public class HostedServiceContent extends FunctionActivatorDeploymentContent
 {
+    public String pattern;
 
     public GenerationInfo info;
     public PureModelContextData serviceData;
 
-    public HostedServiceContent(GenerationInfo info)
+    public HostedServiceContent(String pattern, GenerationInfo info)
     {
+        this.pattern = pattern;
         this.info = info;
     }
 
-    public HostedServiceContent(GenerationInfo info, PureModelContextData serviceData)
+    public HostedServiceContent(String pattern, GenerationInfo info, PureModelContextData serviceData)
     {
-        this.info = info;
+        this(pattern, info);
         this.serviceData = serviceData;
     }
 }