Uplift Activation

finos · Nov 12, 2023 · 6a8bfe3 · 6a8bfe3
1 parent fc4c7f5
commit 6a8bfe3
Show file tree

Hide file tree

Showing 16 changed files with 113 additions and 25 deletions.
diff --git a/.../finos/legend/engine/protocol/functionActivator/deployment/FunctionActivatorArtifact.java b/.../finos/legend/engine/protocol/functionActivator/deployment/FunctionActivatorArtifact.java
@@ -14,6 +14,9 @@
 
 package org.finos.legend.engine.protocol.functionActivator.deployment;
 
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type")
 public class FunctionActivatorArtifact
 {
     public FunctionActivatorDeploymentContent content;

diff --git a/...ngine/protocol/functionActivator/deployment/FunctionActivatorDeploymentConfiguration.java b/...ngine/protocol/functionActivator/deployment/FunctionActivatorDeploymentConfiguration.java
@@ -14,6 +14,9 @@
 
 package org.finos.legend.engine.protocol.functionActivator.deployment;
 
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type")
 public class FunctionActivatorDeploymentConfiguration
 {
 }
diff --git a/...gend/engine/protocol/functionActivator/deployment/FunctionActivatorDeploymentContent.java b/...gend/engine/protocol/functionActivator/deployment/FunctionActivatorDeploymentContent.java
@@ -14,6 +14,9 @@
 
 package org.finos.legend.engine.protocol.functionActivator.deployment;
 
+import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
+@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type")
 public class FunctionActivatorDeploymentContent
 {
 }
diff --git a/...g/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java b/...g/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java
@@ -120,21 +120,38 @@ public SnowflakeDeploymentResult fakeDeploy(Root_meta_pure_alloy_connections_all
         }
     }
 
-    public java.sql.Connection getDeploymentConnection(Identity identity, RelationalDatabaseConnection connection)
+    public Connection getDeploymentConnection(Identity identity, RelationalDatabaseConnection connection)
     {
         return this.connectionManager.getDatabaseConnection(identity, (DatabaseConnection) connection);
     }
 
-    public void deployImpl(Connection jdbcConnection, SnowflakeAppContent context) throws Exception
+    public void deployImpl(Connection jdbcConnection, SnowflakeAppContent context) throws SQLException
     {
-        Statement statement = jdbcConnection.createStatement();
-        String deploymentTableName = this.getDeploymentTableName(jdbcConnection);
+        String catalogName = jdbcConnection.getCatalog();
+        MutableList<String> statements = generateStatements(catalogName, context);
+        for (String s: statements)
+        {
+            Statement statement = jdbcConnection.createStatement();
+            statement.execute(s);
+        }
+    }
 
-        //String createTableSQL = String.format("create table %s (id INTEGER, message VARCHAR(1000)) if not exists", deploymentTableName);
-        //boolean createTableStatus = statement.execute(createTableSQL);
-        String insertSQL = String.format("insert into %s(CREATE_DATETIME, APP_NAME, SQL_FRAGMENT, VERSION_NUMBER, OWNER, DESCRIPTION) values('%s', '%s', '%s', '%s', '%s', '%s')",
-                deploymentTableName, context.creationTime, context.applicationName, context.sqlExpressions.getFirst(), context.getVersionInfo(), Lists.mutable.withAll(context.owners).makeString(","), context.description);
-        statement.execute(insertSQL);
+    public MutableList<String> generateStatements(String catalogName, SnowflakeAppContent content)
+    {
+        MutableList<String> statements = Lists.mutable.empty();
+        if (content.type.equals("STAGE"))
+        {
+            String deploymentTableName = String.format("%s.%s." + deploymentTable, catalogName, deploymentSchema);
+            statements.add(String.format("insert into %s(CREATE_DATETIME, APP_NAME, SQL_FRAGMENT, VERSION_NUMBER, OWNER, DESCRIPTION) values('%s', '%s', '%s', '%s', '%s', '%s');",
+                    deploymentTableName, content.creationTime, content.applicationName, content.sqlExpressions.getFirst(), content.getVersionInfo(), Lists.mutable.withAll(content.owners).makeString(","), content.description));
+
+        }
+        else
+        {
+            statements.add(String.format("CREATE OR REPLACE FUNCTION %S.%S.%s() RETURNS TABLE (%s) as $$ %s $$;", catalogName, deploymentSchema, content.applicationName, content.functionArguments, content.sqlExpressions.getFirst(), content.description));
+            statements.add(String.format("CREATE OR REPLACE SECURE FUNCTION %S.%S.%s() RETURNS TABLE (%s) as $$ %s $$;", catalogName, deploymentSchema, content.applicationName, content.functionArguments, content.sqlExpressions.getFirst(), content.description));
+        }
+        return statements;
     }
 
     public String getDeploymentTableName(Connection jdbcConnection) throws SQLException
@@ -143,7 +160,7 @@ public String getDeploymentTableName(Connection jdbcConnection) throws SQLExcept
         return String.format("%s.%s." + deploymentTable, catalogName, deploymentSchema);
     }
 
-    public java.sql.Connection getDeploymentConnection(Identity identity, SnowflakeAppArtifact artifact)
+    public Connection getDeploymentConnection(Identity identity, SnowflakeAppArtifact artifact)
     {
         RelationalDatabaseConnection connection = extractConnectionFromArtifact(artifact);
         return this.connectionManager.getDatabaseConnection(identity, connection);

diff --git a/...gend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java b/...gend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java
@@ -22,11 +22,7 @@
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection;
 import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp;
 import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration;
-import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp;
-import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp_Impl;
-import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration;
-import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration_Impl;
-import org.finos.legend.pure.generated.Root_meta_external_store_relational_runtime_RelationalDatabaseConnection;
+import org.finos.legend.pure.generated.*;
 import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction;
 import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor;
 
@@ -67,6 +63,7 @@ public Root_meta_external_function_activator_snowflakeApp_SnowflakeApp buildSnow
                         ._function(func)
                         ._description(app.description)
                         ._owner(app.owner)
+                        ._type(app.type != null ? context.pureModel.getEnumValue("meta::external::function::activator::snowflakeApp::SnowflakeDeploymentType", app.type.toString()) : context.pureModel.getEnumValue("meta::external::function::activator::snowflakeApp::SnowflakeDeploymentType", "FULL"))
                         ._activationConfiguration(app.activationConfiguration != null ? buildDeploymentConfig((SnowflakeAppDeploymentConfiguration) app.activationConfiguration, context) : null);
         }
         catch (Exception e)

diff --git a/...n/java/org/finos/legend/engine/language/snowflakeApp/generator/SnowflakeAppGenerator.java b/...n/java/org/finos/legend/engine/language/snowflakeApp/generator/SnowflakeAppGenerator.java
@@ -43,7 +43,10 @@ public class SnowflakeAppGenerator
 
     public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
-        RichIterable<String> sqlExpressions = extractSQLExpressions(pureModel, activator, routerExtensions);
+        PackageableFunction<?> function = activator._function();
+        Root_meta_pure_executionPlan_ExecutionPlan executionPlan = PlanGenerator.generateExecutionPlanAsPure((FunctionDefinition<?>) function, null, null, null, pureModel, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel));
+        RichIterable<String> sqlExpressions = extractSQLExpressions(executionPlan);
+        String functionColumns = executionPlan._rootExecutionNode()._resultType() instanceof Root_meta_pure_executionPlan_TDSResultType ? generateFunctionReturnColumns((Root_meta_pure_executionPlan_TDSResultType)executionPlan._rootExecutionNode()._resultType()) : "";
 
         RelationalDatabaseConnection connection;
         AlloySDLC sdlc = null;
@@ -55,7 +58,7 @@ public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_me
                 sdlc = (AlloySDLC) sdlcInfo;
             }
         }
-        SnowflakeAppContent content = new SnowflakeAppContent(activator._applicationName(), Lists.mutable.withAll(sqlExpressions), activator._description(), Lists.mutable.with(activator._owner()), sdlc);
+        SnowflakeAppContent content = new SnowflakeAppContent(activator._applicationName(), Lists.mutable.withAll(sqlExpressions), activator._description(), functionColumns, activator._type()._name(), Lists.mutable.with(activator._owner()), sdlc);
         if (activator._activationConfiguration() != null)
         {
             //identify connection
@@ -70,17 +73,22 @@ public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_me
         return new SnowflakeAppArtifact(content);
     }
 
-    private static RichIterable<String> extractSQLExpressions(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
+    private static RichIterable<String> extractSQLExpressions(Root_meta_pure_executionPlan_ExecutionPlan executionPlan)
     {
-        PackageableFunction<?> function = activator._function();
-        Root_meta_pure_executionPlan_ExecutionPlan executionPlan = PlanGenerator.generateExecutionPlanAsPure((FunctionDefinition<?>) function, null, null, null, pureModel, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel));
+
         Root_meta_pure_executionPlan_ExecutionNode node = executionPlan._rootExecutionNode();
         return collectAllNodes(node)
                 .selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class)
                 .collect(Root_meta_relational_mapping_SQLExecutionNode::_sqlQuery)
                 .select(x -> !x.toLowerCase().startsWith("alter"));
     }
 
+    private static String generateFunctionReturnColumns(Root_meta_pure_executionPlan_TDSResultType planResult)
+    {
+        return Lists.mutable.withAll(planResult._tdsColumns()).collect(c ->
+                c._name().replace(" ","_").replace("/","_")  + " " +  "VARCHAR(16777216)").makeString(" , ");
+    }
+
     private static Object[] extractSQLExpressionsAndConnectionMetadata(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
     {
         PackageableFunction<?> function = activator._function();

diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml
@@ -120,6 +120,10 @@
             <groupId>org.finos.legend.engine</groupId>
             <artifactId>legend-engine-protocol-pure</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.finos.legend.engine</groupId>
+            <artifactId>legend-engine-shared-core</artifactId>
+        </dependency>
         <!-- ENGINE -->
 
         <!-- ECLIPSE COLLECTIONS -->

diff --git a/...lr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppLexerGrammar.g4 b/...lr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppLexerGrammar.g4
@@ -7,6 +7,7 @@ SNOWFLAKE_APP__APPLICATION_NAME:                'applicationName';
 SNOWFLAKE_APP__DESCRIPTION:                     'description';
 SNOWFLAKE_APP__FUNCTION:                        'function';
 SNOWFLAKE_APP__OWNER:                           'owner';
+SNOWFLAKE_APP__TYPE:                           'type';
 SNOWFLAKE_APP__ACTIVATION:                      'activationConfiguration';
 
 // ------------------------------------- CONFIGURATION -------------------------------

diff --git a/...r4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 b/...r4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4
@@ -13,6 +13,7 @@ identifier:     VALID_STRING | STRING |
                 SNOWFLAKE_APP__DESCRIPTION |
                 SNOWFLAKE_APP__FUNCTION |
                 SNOWFLAKE_APP__OWNER |
+                SNOWFLAKE_APP__TYPE |
                 SNOWFLAKE_APP__ACTIVATION|
                 CONFIGURATION| DEPLOYMENT_STAGE
                 | ACTIVATION_CONNECTION |
@@ -34,6 +35,7 @@ snowflakeApp:                   SNOWFLAKE_APP stereotypes? taggedValues? qualifi
                                                 | description
                                                 | function
                                                 | owner
+                                                | type
                                                 | activation
                                             )*
                                         BRACE_CLOSE;
@@ -51,6 +53,8 @@ function:                       SNOWFLAKE_APP__FUNCTION COLON functionIdentifier
 
 owner :                         SNOWFLAKE_APP__OWNER COLON STRING SEMI_COLON;
 
+type :                         SNOWFLAKE_APP__TYPE COLON identifier SEMI_COLON;
+
 activation:                     SNOWFLAKE_APP__ACTIVATION COLON qualifiedName SEMI_COLON ;
 
 // ----------------------------------- Deployment ------------------------------------------------------

diff --git a/...va/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java b/...va/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java
@@ -21,6 +21,7 @@
 import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserUtility;
 import org.finos.legend.engine.language.pure.grammar.from.antlr4.SnowflakeAppParserGrammar;
 import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage;
+import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.ConnectionPointer;
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.StereotypePtr;
@@ -29,6 +30,7 @@
 import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection;
 import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp;
 import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration;
+import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppType;
 import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;
 
 import java.util.Collections;
@@ -76,6 +78,19 @@ private SnowflakeApp visitSnowflakeApp(SnowflakeAppParserGrammar.SnowflakeAppCon
         {
             snowflakeApp.owner = PureGrammarParserUtility.fromGrammarString(ownerContext.STRING().getText(), true);
         }
+        SnowflakeAppParserGrammar.TypeContext typeContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.type(), "type", snowflakeApp.sourceInformation);
+        if (typeContext != null)
+        {
+            try
+            {
+                snowflakeApp.type = SnowflakeAppType.valueOf(PureGrammarParserUtility.fromIdentifier(typeContext.identifier()));
+            }
+            catch (Exception e)
+            {
+                throw new EngineException("Unknown type '" + PureGrammarParserUtility.fromIdentifier(typeContext.identifier()) + "'", this.walkerSourceInformation.getSourceInformation(typeContext), EngineErrorType.PARSER);
+            }
+        }
+
         SnowflakeAppParserGrammar.DescriptionContext descriptionContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.description(), "description", snowflakeApp.sourceInformation);
         if (descriptionContext != null)
         {

diff --git a/...org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java b/...org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java
@@ -54,6 +54,7 @@ private static String renderSnowflakeApp(SnowflakeApp app)
                 "   function : " + app.function + ";\n" +
                 (app.owner == null ? "" : "   owner : '" + app.owner + "';\n") +
                 (app.description == null ? "" : "   description : '" + app.description + "';\n") +
+                (app.type == null ? "" : "   type : " + app.type.name() + ";\n") +
                 (app.activationConfiguration == null ? "" : "   activationConfiguration : " + ((SnowflakeAppDeploymentConfiguration)app.activationConfiguration).activationConnection.connection + ";\n") +
                 "}";
     }
@@ -79,7 +80,7 @@ public List<Function3<List<PackageableElement>, PureGrammarComposerContext, Stri
     }
 
     @Override
-    public List<Function3<List<PackageableElement>, PureGrammarComposerContext, List<String>, PureGrammarComposerExtension.PureFreeSectionGrammarComposerResult>> getExtraFreeSectionComposers()
+    public List<Function3<List<PackageableElement>, PureGrammarComposerContext, List<String>, PureFreeSectionGrammarComposerResult>> getExtraFreeSectionComposers()
     {
         return Collections.singletonList((elements, context, composedSections) ->
         {

diff --git a/...org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java b/...org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java
@@ -29,6 +29,7 @@ public void testSnowflakeApp()
                 "   function : zxx(Integer[1]):String[1];\n" +
                 "   owner : 'pierre';\n" +
                 "   description : 'A super nice app!';\n" +
+                "   type : STAGE;\n" +
                 "   activationConfiguration : a::b::connection;\n" +
                 "}\n");
     }

diff --git a/...in/java/org/finos/legend/engine/protocol/snowflakeApp/deployment/SnowflakeAppContent.java b/...in/java/org/finos/legend/engine/protocol/snowflakeApp/deployment/SnowflakeAppContent.java
@@ -26,7 +26,8 @@
 public class SnowflakeAppContent extends FunctionActivatorDeploymentContent
 {
     public MutableList<String> sqlExpressions = Lists.mutable.empty();
-
+    public String functionArguments;
+    public String type;
     public String applicationName;
     public String description;
     public List<String> owners;
@@ -40,11 +41,13 @@ public SnowflakeAppContent()
         //Empty constructor for Jackson
     }
 
-    public SnowflakeAppContent(String applicationName, MutableList<String> sqlExpressions, AlloySDLC sdlc)
+    public SnowflakeAppContent(String applicationName, MutableList<String> sqlExpressions, String functionArguments, String type, AlloySDLC sdlc)
     {
         this.applicationName = applicationName;
         this.sqlExpressions = sqlExpressions;
         this.creationTime = convertToValidDate(new Date());
+        this.functionArguments = functionArguments;
+        this.type = type;
         if (sdlc != null)
         {
             this.groupId = sdlc.groupId;
@@ -53,9 +56,9 @@ public SnowflakeAppContent(String applicationName, MutableList<String> sqlExpres
         }
     }
 
-    public SnowflakeAppContent(String applicationName, MutableList<String> sqlExpressions, String description, List<String> owners, AlloySDLC sdlc)
+    public SnowflakeAppContent(String applicationName, MutableList<String> sqlExpressions, String description, String functionArguments, String type,List<String> owners, AlloySDLC sdlc)
     {
-        this(applicationName, sqlExpressions,sdlc);
+        this(applicationName, sqlExpressions, functionArguments, type, sdlc);
         this.description = description;
         this.owners = owners;
 

diff --git a/...l/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeApp.java b/...l/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeApp.java
@@ -30,4 +30,5 @@ public class SnowflakeApp extends FunctionActivator
     public String applicationName;
     public String description;
     public String owner;
+    public SnowflakeAppType type;
 }
diff --git a/...c/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppType.java b/...c/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeAppType.java
@@ -0,0 +1,20 @@
+// Copyright 2023 Goldman Sachs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.finos.legend.engine.protocol.snowflakeApp.metamodel;
+
+public enum SnowflakeAppType
+{
+    STAGE, FULL
+}
diff --git a/...engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/metamodel/metamodel.pure b/...engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/metamodel/metamodel.pure
@@ -1,10 +1,12 @@
 import meta::external::function::activator::*;
+import meta::external::function::activator::snowflakeApp::*;
 
 Class meta::external::function::activator::snowflakeApp::SnowflakeApp extends FunctionActivator
 {
   applicationName : String[1];
   description : String[0..1];
   owner : String[0..1];
+  type: SnowflakeDeploymentType[0..1] ;//default to full
 }
 
  Class meta::external::function::activator::snowflakeApp::SnowflakeDeploymentConfiguration extends DeploymentConfiguration
@@ -17,6 +19,11 @@ Class meta::external::function::activator::snowflakeApp::SnowflakeApp extends Fu
 
  }
 
+Enum meta::external::function::activator::snowflakeApp::SnowflakeDeploymentType
+{
+  STAGE, FULL
+}
+
 // This section needs to be code generated from the section above
 Class meta::protocols::pure::vX_X_X::metamodel::function::activator::snowflakeApp::SnowflakeApp extends meta::protocols::pure::vX_X_X::metamodel::function::activator::FunctionActivator
 {