From b2f4534c75f83c84aaaa3c6b779a931d4bbcddbb Mon Sep 17 00:00:00 2001 From: Mohammed Ibrahim Date: Wed, 17 Jul 2024 09:49:16 -0400 Subject: [PATCH] update UDTF model and Validation (#2965) --- .../executionPlan/executionPlan.pure | 17 + .../replaceFreeMarkerOps.pure | 4 +- .../relationalMappingExecution.pure | 3 +- .../snowflakeApp/api/SnowflakeAppError.java | 7 +- .../snowflakeApp/api/SnowflakeAppService.java | 34 +- .../SnowflakeAppDeploymentManager.java | 49 ++- .../deployment/SnowflakeGrantInfo.java | 33 ++ .../SnowflakeAppCompilerExtension.java | 3 + .../generator/SnowflakeAppGenerator.java | 15 +- .../pom.xml | 10 +- .../from/antlr4/SnowflakeAppLexerGrammar.g4 | 2 + .../from/antlr4/SnowflakeAppParserGrammar.g4 | 8 + .../grammar/from/SnowflakeAppTreeWalker.java | 20 ++ .../to/SnowflakeAppGrammarComposer.java | 2 + .../test/TestSnowflakeAppRoundtrip.java | 17 + .../grammar/test/TestSnowflakeParsing.java | 17 +- .../deployment/SnowflakeAppContent.java | 18 ++ .../snowflakeApp/metamodel/SnowflakeApp.java | 3 +- .../metamodel/SnowflakePermissionScheme.java | 21 ++ .../generation/generation.pure | 66 ++-- .../metamodel/metamodel.pure | 17 +- .../showcase/showcaseModel.pure | 12 +- .../showcase/showcaseNativeApps.pure | 2 +- .../tests/testSnowflakeAppGeneration.pure | 292 +++++++++--------- 24 files changed, 461 insertions(+), 211 deletions(-) create mode 100644 legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeGrantInfo.java create mode 100644 legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakePermissionScheme.java diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure index cb9818b19f7..1f6004ff0e2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure @@ -45,6 +45,22 @@ Class meta::relational::mapping::TempTableColumnMetaData parametersForGetter : Map[0..1]; } +Class meta::relational::mapping::QueryMetadata +{ + +} + +Class meta::relational::mapping::TableInfo extends QueryMetadata +{ + info: TableIdentifier[*]; +} + +Class meta::relational::mapping::TableIdentifier +{ + schema: String[1]; + table: String[1]; +} + Class meta::relational::mapping::SQLExecutionNode extends ExecutionNode { sqlComment : String[0..1]; @@ -53,6 +69,7 @@ Class meta::relational::mapping::SQLExecutionNode extends ExecutionNode onConnectionCloseRollbackQuery : String[0..1]; resultColumns : SQLResultColumn[*]; connection: DatabaseConnection[1]; + metadata: QueryMetadata[*]; } Class meta::relational::mapping::RelationalSaveNode extends ExecutionNode diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/functionActivationPostProcessor/replaceFreeMarkerOps.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/functionActivationPostProcessor/replaceFreeMarkerOps.pure index 22178c3ae8a..f51d6cc7d22 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/functionActivationPostProcessor/replaceFreeMarkerOps.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/postprocessor/functionActivationPostProcessor/replaceFreeMarkerOps.pure @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::relational::metamodel::relation::*; import meta::relational::functions::pureToSqlQuery::*; import meta::relational::postProcessor::*; import meta::relational::functions::pureToSqlQuery::metamodel::*; @@ -21,6 +22,7 @@ import meta::core::runtime::*; import meta::relational::metamodel::*; import meta::relational::mapping::*; + function meta::relational::postProcessor::ReplaceFreeMarkerWithTableFunctionParamHolder(query:SQLQuery[1]): PostProcessorResult[1] { let transformFunc = {r: RelationalOperationElement[1], uniqueId: Integer[1] | @@ -36,7 +38,7 @@ function meta::relational::postProcessor::ReplaceFreeMarkerWithTableFunctionPara let transformedQuery = $query->transform($transformFunc, ^Map()).first; - ^PostProcessorResult(query = $transformedQuery->cast(@SQLQuery)); + ^PostProcessorResult(query = $transformedQuery->cast(@SQLQuery)); } function <> meta::relational::postProcessor::replaceVarPlaceHolder(v: VarPlaceHolder[1]): RelationalOperationElement[1] diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure index 37776a4b419..07e8eeadacd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure @@ -172,7 +172,8 @@ function meta::relational::mapping::generateSQLExecutionNode(query:SQLQuery[1], resultColumns = $query->match([sel:SelectSQLQuery[1] | $sel.columns->map(c| ^SQLResultColumn(label = $c->columnLabel($connection, $extensions), dataType = $c->getRelationalTypeFromRelationalOperationElement()->translateCoreTypeToDbSpecificType(^TranslationContext(dbType = $connection.type)))), a:Any[1] | []]), resultType = ^ResultType(type = Any), connection = $connection->meta::relational::mapping::updateConnection($extensions)->cast(@DatabaseConnection), - supportFunctions = relationalPlanSupportFunctions($connection)->concatenate($enumMapSupportFunctions) + supportFunctions = relationalPlanSupportFunctions($connection)->concatenate($enumMapSupportFunctions), + metadata = ^meta::relational::mapping::TableInfo(info = $query->meta::relational::milestoning::getQueryTableAliases()->map(a|$a.relationalElement)->cast(@Table)->removeDuplicatesBy(t|$t.schema.name+'.'+$t.name)->map(tab| ^meta::relational::mapping::TableIdentifier(schema=$tab.schema.name, table=$tab.name))) ); } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppError.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppError.java index 97e23a6cdd1..8b6229ddcb5 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppError.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppError.java @@ -21,9 +21,14 @@ public class SnowflakeAppError extends FunctionActivatorError { public MutableList foundSQLs; - public SnowflakeAppError(String message, MutableList foundSQLs) + public SnowflakeAppError(String message, MutableList foundSQLs) { super(message); this.foundSQLs = foundSQLs; } + + public SnowflakeAppError(String message) + { + super(message); + } } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java index fa66d387892..05e544d6593 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java @@ -20,6 +20,7 @@ import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.list.mutable.FastList; import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeGrantInfo; import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; import org.finos.legend.engine.functionActivator.service.FunctionActivatorError; import org.finos.legend.engine.functionActivator.service.FunctionActivatorService; @@ -92,23 +93,40 @@ public boolean supports(Root_meta_external_function_activator_FunctionActivator @Override public MutableList validate(Identity identity, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function> routerExtensions) { - SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, true, routerExtensions); - return validate(artifact); + SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions); + return validate(identity, artifact); } - public MutableList validate(SnowflakeAppArtifact artifact) + public MutableList validate(Identity identity, SnowflakeAppArtifact artifact) { - int size = ((SnowflakeAppContent)artifact.content).sqlExpressions.select(e -> !e.toLowerCase().endsWith("to role public;")).size(); - return size != 1 ? - Lists.mutable.with(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", ((SnowflakeAppContent)artifact.content).sqlExpressions)) : - Lists.mutable.empty(); + MutableList errors = Lists.mutable.empty(); + SnowflakeAppContent content = (SnowflakeAppContent)artifact.content; + if (!content.sqlExpressions.isEmpty()) + { + int size = content.sqlExpressions.select(e -> !e.toLowerCase().endsWith("to role public;")).size(); + if (size != 1) + { + errors.add(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", content.sqlExpressions)); + } + } + if (content.permissionScope.equals("SEQUESTERED")) + { + MutableList grantsToRole = this.snowflakeDeploymentManager.getGrants(identity, artifact); + MutableList contentTables = ((SnowflakeAppContent) artifact.content).usedTables.collect(String::toUpperCase); + MutableList violations = grantsToRole.select(g -> g.privilege.equals("SELECT") && contentTables.contains(g.objectName.toUpperCase())); + if (!violations.isEmpty()) + { + errors.add(new SnowflakeAppError("Privilege Violation for SEQUESTERED scheme. Deployment Role contains SELECT permissions on tables: " + violations.collect(v -> v.objectName).makeString("[", ",", "]"))); + } + } + return errors; } @Override public SnowflakeDeploymentResult publishToSandbox(Identity identity, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, List runtimeConfigurations, Function> routerExtensions) { SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions); - MutableList validationError = validate(artifact); + MutableList validationError = validate(identity, artifact); if (validationError.isEmpty()) { return this.snowflakeDeploymentManager.deploy(identity, artifact, runtimeConfigurations); diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java index 906b92c773a..ab7d2c552e8 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java @@ -22,6 +22,7 @@ import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorArtifact; import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppDeploymentTool; import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.specification.SnowflakeDatasourceSpecification; import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppArtifact; import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration; import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppContent; @@ -54,6 +55,8 @@ public class SnowflakeAppDeploymentManager implements DeploymentManager statements = generateStatements(catalogName, context); + MutableList statements = generateStatements(catalogName, content); for (String s: statements) { Statement statement = jdbcConnection.createStatement(); @@ -152,10 +155,18 @@ public void deployImpl(Connection jdbcConnection, SnowflakeAppContent context) t public MutableList generateStatements(String catalogName, SnowflakeAppContent content) { MutableList statements = Lists.mutable.empty(); - statements.add(String.format("CREATE OR REPLACE SECURE FUNCTION %S.%S.%s", catalogName, deploymentSchema, content.sqlExpressions.getFirst())); - if (content.sqlExpressions.size() > 1) + if (!content.sqlExpressions.isEmpty()) + { + statements.add(String.format("CREATE OR REPLACE SECURE FUNCTION %S.%S.%s", catalogName, deploymentSchema, content.sqlExpressions.getFirst())); + if (content.sqlExpressions.size() > 1) + { + statements.add(String.format("GRANT USAGE ON FUNCTION %S.%S.%S", catalogName, deploymentSchema, content.sqlExpressions.get(1))); + } + } + else { - statements.add(String.format("GRANT USAGE ON FUNCTION %S.%S.%S", catalogName, deploymentSchema, content.sqlExpressions.get(1))); + statements.add(String.format(content.createStatement, catalogName)); + statements.add(String.format(content.grantStatement, catalogName)); } return statements; } @@ -177,6 +188,32 @@ public RelationalDatabaseConnection extractConnectionFromArtifact(SnowflakeAppAr return ((SnowflakeAppDeploymentConfiguration)artifact.deploymentConfiguration).connection; } + public MutableList getGrants(Identity identity, SnowflakeAppArtifact artifact) + { + RelationalDatabaseConnection rel = ((SnowflakeAppDeploymentConfiguration)artifact.deploymentConfiguration).connection; + MutableList grants = Lists.mutable.empty(); + try (Connection jdbcConnection = this.getDeploymentConnection(identity, artifact)) + { + Statement role = jdbcConnection.createStatement(); + ResultSet roleResult = role.executeQuery("SELECT CURRENT_ROLE()"); + roleResult.next(); + String currentRole = roleResult.getString(1); + roleResult.close(); + Statement s = jdbcConnection.createStatement(); + ResultSet res = s.executeQuery(String.format("SHOW GRANTS TO ROLE %S LIMIT %S;", currentRole, limit)); + while (res.next()) + { + grants.add(new SnowflakeGrantInfo(res.getString("privilege"), res.getString("granted_on"), res.getString("name"), res.getString("grantee_name"), res.getString("granted_by"))); + } + s.close(); + } + catch (Exception e) + { + LOGGER.info("Unable to query for grants for role. Error: " + e.getMessage()); + } + return grants; + } + public ImmutableList getDeployed(Identity identity, RelationalDatabaseConnection connection) throws Exception { ImmutableList deployments = null; diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeGrantInfo.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeGrantInfo.java new file mode 100644 index 00000000000..a5cbb944626 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeGrantInfo.java @@ -0,0 +1,33 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.snowflakeApp.deployment; + +public class SnowflakeGrantInfo +{ + public String privilege; + public String objectType; + public String objectName; + public String granteeRole; + public String grantee; + + public SnowflakeGrantInfo(String privilege, String objectType, String objectName, String granteeRole, String grantee) + { + this.privilege = privilege; + this.objectType = objectType; + this.objectName = objectName; + this.granteeRole = granteeRole; + this.grantee = grantee; + } +} diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java index d630283c8a6..0f9c20c6f5c 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java @@ -26,6 +26,7 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration; +import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakePermissionScheme; import org.finos.legend.pure.generated.*; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor; @@ -72,6 +73,8 @@ public Root_meta_external_function_activator_snowflakeApp_SnowflakeApp buildSnow ._applicationName(app.applicationName) ._function(func) ._description(app.description) + ._usageRole(app.usageRole) + ._permissionScheme(app.permissionScheme != null ? context.pureModel.getEnumValue("meta::external::function::activator::snowflakeApp::SnowflakePermissionScheme", app.permissionScheme.toString()) : context.pureModel.getEnumValue("meta::external::function::activator::snowflakeApp::SnowflakePermissionScheme", SnowflakePermissionScheme.DEFAULT.toString())) ._ownership(new Root_meta_external_function_activator_DeploymentOwnership_Impl("")._id(((DeploymentOwner)app.ownership).id)) ._activationConfiguration(app.activationConfiguration != null ? buildDeploymentConfig((SnowflakeAppDeploymentConfiguration) app.activationConfiguration, context) : null); } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/main/java/org/finos/legend/engine/language/snowflakeApp/generator/SnowflakeAppGenerator.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/main/java/org/finos/legend/engine/language/snowflakeApp/generator/SnowflakeAppGenerator.java index b119d475a7d..e1fac536c1c 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/main/java/org/finos/legend/engine/language/snowflakeApp/generator/SnowflakeAppGenerator.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-generator/src/main/java/org/finos/legend/engine/language/snowflakeApp/generator/SnowflakeAppGenerator.java @@ -43,13 +43,7 @@ public class SnowflakeAppGenerator public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function> routerExtensions) { - return generateArtifact(pureModel, activator, inputModel, false, routerExtensions); - } - - public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, boolean createStatementOnly, Function> routerExtensions) - { - String sqlFunctionExpression = core_snowflakeapp_generation_generation.Root_meta_external_function_activator_snowflakeApp_generation_generateArtifact_SnowflakeApp_1__Extension_MANY__String_1_(activator, routerExtensions.apply(pureModel), pureModel.getExecutionSupport()); - String inputParamStub = core_snowflakeapp_generation_generation.Root_meta_external_function_activator_snowflakeApp_generation_generateInputParamsStub_Function_1__Boolean_1__String_1_(activator._function(), false, pureModel.getExecutionSupport()); + Root_meta_external_function_activator_snowflakeApp_generation_Artifact fullArtifact = core_snowflakeapp_generation_generation.Root_meta_external_function_activator_snowflakeApp_generation_generateFullArtifact_SnowflakeApp_1__Extension_MANY__Artifact_1_(activator, routerExtensions.apply(pureModel), pureModel.getExecutionSupport()); RelationalDatabaseConnection connection; AlloySDLC sdlc = null; if (((PureModelContextData)inputModel).getOrigin() != null) @@ -60,7 +54,7 @@ public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_me sdlc = (AlloySDLC) sdlcInfo; } } - SnowflakeAppContent content = new SnowflakeAppContent(activator._applicationName(), Lists.mutable.of(sqlFunctionExpression), activator._description(), ((Root_meta_external_function_activator_DeploymentOwnership)activator._ownership())._id(), sdlc); + SnowflakeAppContent content = new SnowflakeAppContent(activator._applicationName(), fullArtifact._createQuery(), fullArtifact._grantStatement(), activator._permissionScheme().toString(), activator._description(), ((Root_meta_external_function_activator_DeploymentOwnership)activator._ownership())._id(), Lists.mutable.withAll(fullArtifact._tables())); if (activator._activationConfiguration() != null) { //identify connection @@ -71,11 +65,6 @@ public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_me .select(c -> c.getPath().equals(((org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration)protocolActivator.activationConfiguration).activationConnection.connection)).getFirst().connectionValue; SnowflakeDatasourceSpecification ds = (SnowflakeDatasourceSpecification)connection.datasourceSpecification; String deployedLocation = String.format("https://app.%s.privatelink.snowflakecomputing.com/%s/%s/data/databases/%S", ds.region, ds.region, ds.accountName, ds.databaseName); - if (!createStatementOnly) - { - String generatedGrantStatement = generateGrantStatement(activator._applicationName(), inputParamStub); - content.addGrantStatement(generatedGrantStatement); - } return new SnowflakeAppArtifact(content, new SnowflakeAppDeploymentConfiguration(connection), deployedLocation, sdlc); } return new SnowflakeAppArtifact(content, sdlc); diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml index c004a36132e..b14f835728f 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml @@ -124,6 +124,10 @@ org.finos.legend.engine legend-engine-protocol-pure + + org.finos.legend.engine + legend-engine-shared-core + @@ -151,12 +155,6 @@ junit test - - org.finos.legend.engine - legend-engine-shared-core - test-jar - test - org.finos.legend.engine legend-engine-language-pure-grammar diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppLexerGrammar.g4 b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppLexerGrammar.g4 index a3893ae2093..0b46ab04a38 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppLexerGrammar.g4 +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppLexerGrammar.g4 @@ -10,6 +10,8 @@ SNOWFLAKE_APP__OWNER: 'ownership'; SNOWFLAKE_APP__OWNER_DEPLOYMENT: 'Deployment'; SNOWFLAKE_APP__OWNER_DEPLOYMENT_ID: 'identifier'; SNOWFLAKE_APP__ACTIVATION: 'activationConfiguration'; +SNOWFLAKE_APP__PERMISSION: 'permissionScheme'; +SNOWFLAKE_APP__USAGE_ROLE: 'usageRole'; // ------------------------------------- CONFIGURATION ------------------------------- CONFIGURATION: 'SnowflakeAppDeploymentConfiguration'; diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 index 889c87ad222..7e2d02b669f 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 @@ -16,6 +16,8 @@ identifier: VALID_STRING | STRING | SNOWFLAKE_APP__OWNER_DEPLOYMENT | SNOWFLAKE_APP__OWNER_DEPLOYMENT_ID | SNOWFLAKE_APP__ACTIVATION| + SNOWFLAKE_APP__USAGE_ROLE | + SNOWFLAKE_APP__PERMISSION | CONFIGURATION| DEPLOYMENT_STAGE | ACTIVATION_CONNECTION | ALL | @@ -37,6 +39,8 @@ snowflakeApp: SNOWFLAKE_APP stereotypes? taggedValues? qualifi | function | ownership | activation + | role + | scheme )* BRACE_CLOSE; @@ -49,6 +53,10 @@ applicationName: SNOWFLAKE_APP__APPLICATION_NAME COLON STRING SEM description: SNOWFLAKE_APP__DESCRIPTION COLON STRING SEMI_COLON; +scheme: SNOWFLAKE_APP__PERMISSION COLON identifier SEMI_COLON; + +role: SNOWFLAKE_APP__USAGE_ROLE COLON STRING SEMI_COLON; + function: SNOWFLAKE_APP__FUNCTION COLON functionIdentifier SEMI_COLON; ownership : SNOWFLAKE_APP__OWNER COLON diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java index 47c22a1a73f..8920bbd4448 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java @@ -21,6 +21,7 @@ import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserUtility; import org.finos.legend.engine.language.pure.grammar.from.antlr4.SnowflakeAppParserGrammar; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentOwner; +import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType; import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementType; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; @@ -31,6 +32,8 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration; +import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakePermissionScheme; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import java.util.Collections; import java.util.List; @@ -83,6 +86,23 @@ private SnowflakeApp visitSnowflakeApp(SnowflakeAppParserGrammar.SnowflakeAppCon { snowflakeApp.description = PureGrammarParserUtility.fromGrammarString(descriptionContext.STRING().getText(), true); } + SnowflakeAppParserGrammar.RoleContext roleContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.role(), "role", snowflakeApp.sourceInformation); + if (roleContext != null) + { + snowflakeApp.usageRole = PureGrammarParserUtility.fromGrammarString(roleContext.STRING().getText(), true); + } + SnowflakeAppParserGrammar.SchemeContext schemeContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.scheme(), "scheme", snowflakeApp.sourceInformation); + if (schemeContext != null) + { + try + { + snowflakeApp.permissionScheme = SnowflakePermissionScheme.valueOf(PureGrammarParserUtility.fromIdentifier(schemeContext.identifier())); + } + catch (Exception e) + { + throw new EngineException("Unknown permission scheme '" + PureGrammarParserUtility.fromIdentifier(schemeContext.identifier()) + "'", this.walkerSourceInformation.getSourceInformation(schemeContext), EngineErrorType.PARSER); + } + } SnowflakeAppParserGrammar.ActivationContext activationContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.activation(), "activation", snowflakeApp.sourceInformation); if (activationContext != null) { diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java index 54cd8d5ad65..82ef0c10e99 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/to/SnowflakeAppGrammarComposer.java @@ -59,6 +59,8 @@ private static String renderSnowflakeApp(SnowflakeApp app) " function : " + app.function.path + ";\n" + " ownership : Deployment { identifier: '" + ((DeploymentOwner)app.ownership).id + "'};\n" + (app.description == null ? "" : " description : '" + app.description + "';\n") + + (app.usageRole == null ? "" : " usageRole : '" + app.usageRole + "';\n") + + (app.permissionScheme == null ? "" : " permissionScheme : " + app.permissionScheme + ";\n") + (app.activationConfiguration == null ? "" : " activationConfiguration : " + ((SnowflakeAppDeploymentConfiguration)app.activationConfiguration).activationConnection.connection + ";\n") + "}"; } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java index 3f7d80ac8e9..18b9f0e1234 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeAppRoundtrip.java @@ -33,6 +33,23 @@ public void testSnowflakeApp() "}\n"); } + @Test + public void testSnowflakeAppWithSchemeAndRole() + { + test("###Snowflake\n" + + "SnowflakeApp <> {a::A.val = 'ok'} xx::MyApp\n" + + "{\n" + + " applicationName : 'MyApp';\n" + + " function : zxx(Integer[1]):String[1];\n" + + " ownership : Deployment { identifier: 'pierre'};\n" + + " description : 'A super nice app!';\n" + + " usageRole : 'PRIVATE';\n" + + " permissionScheme : SEQUESTERED;\n" + + " activationConfiguration : a::b::connection;\n" + + "}\n"); + } + + @Test public void testSnowflakeAppMinimal() { diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeParsing.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeParsing.java index 7519081e01b..244db97b126 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeParsing.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/test/java/org/finos/legend/engine/language/snowflakeApp/grammar/test/TestSnowflakeParsing.java @@ -49,7 +49,7 @@ public void testGetParserErrorWrongProperty() "SnowflakeApp x::A\n" + "{\n" + " applicatioName : 'sass';\n" + - "}\n", "PARSER error at [4:4-17]: Unexpected token 'applicatioName'. Valid alternatives: ['applicationName', 'description', 'function', 'ownership', 'activationConfiguration']"); + "}\n", "PARSER error at [4:4-17]: Unexpected token 'applicatioName'. Valid alternatives: ['applicationName', 'description', 'function', 'ownership', 'activationConfiguration', 'permissionScheme', 'usageRole']"); } @Test @@ -84,4 +84,17 @@ public void testGetParserErrorMissingFunction() "}\n", "PARSER error at [2:1-6:1]: Field 'function' is required"); } -} \ No newline at end of file + @Test + public void testGetParserErrorWrongScheme() + { + test("###Snowflake\n" + + "SnowflakeApp x::A\n" + + "{\n" + + " function : a::f():String[1];" + + " applicationName : 'MyApp';\n" + + " permissionScheme : WHATSCHEME;\n" + + " ownership : Deployment { identifier: 'pierre'};\n" + + "}\n", "PARSER error at [5:4-33]: Unknown permission scheme 'WHATSCHEME'"); + } + +} diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/deployment/SnowflakeAppContent.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/deployment/SnowflakeAppContent.java index 1d10e19b911..125e4cf6f5b 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/deployment/SnowflakeAppContent.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/deployment/SnowflakeAppContent.java @@ -24,11 +24,16 @@ public class SnowflakeAppContent extends FunctionActivatorDeploymentContent { + @Deprecated public MutableList sqlExpressions = Lists.mutable.empty(); + public String createStatement; + public String grantStatement; public String applicationName; public String description; public String ownership; public String creationTime; + public String permissionScope; + public MutableList usedTables = Lists.mutable.empty(); public SnowflakeAppContent() { @@ -51,6 +56,19 @@ public SnowflakeAppContent(String applicationName, MutableList sqlExpres } + public SnowflakeAppContent(String applicationName, String createStatement, String grantStatement, String permissionScope, String description, String ownership, MutableList usedTables) + { + this.applicationName = applicationName; + this.createStatement = createStatement; + this.grantStatement = grantStatement; + this.permissionScope = permissionScope; + this.description = description; + this.ownership = ownership; + this.creationTime = convertToValidDate(new Date());; + this.usedTables = usedTables; + } + + public static String convertToValidDate(Date date) { SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeApp.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeApp.java index d0a4422ee0f..61f03090895 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeApp.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeApp.java @@ -26,5 +26,6 @@ public class SnowflakeApp extends FunctionActivator { public String applicationName; public String description; - // public String owner; + public String usageRole; + public SnowflakePermissionScheme permissionScheme; } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakePermissionScheme.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakePermissionScheme.java new file mode 100644 index 00000000000..fc471906bf6 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakePermissionScheme.java @@ -0,0 +1,21 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.snowflakeApp.metamodel; + +public enum SnowflakePermissionScheme +{ + DEFAULT, + SEQUESTERED +} diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure index 8952143b83d..8f1b0ddcbaa 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/generation/generation.pure @@ -9,15 +9,20 @@ Class meta::external::function::activator::snowflakeApp::SnowFlakeAppExecutionCo { } -function meta::external::function::activator::snowflakeApp::generation::generateArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1]):String[1] +Class meta::external::function::activator::snowflakeApp::generation::Artifact { - let extensions = meta::external::format::shared::externalFormatExtension()->concatenate(meta::relational::extension::relationalExtensions()); - meta::external::function::activator::snowflakeApp::generation::generateArtifact($s, $extensions); + createQuery:String[1]; + grantStatement:String[1]; + tables: String[*]; } -function meta::external::function::activator::snowflakeApp::generation::generateArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1], extensions:meta::pure::extension::Extension[*]):String[1] +function meta::external::function::activator::snowflakeApp::generation::defaultDeploymentSchema():String[1] { + 'LEGEND_NATIVE_APPS' +} +function meta::external::function::activator::snowflakeApp::generation::generateFullArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1], extensions:meta::pure::extension::Extension[*]):Artifact[1] +{ let inputParamsStub = generateInputParamsStub($s.function, true); let context = ^meta::external::function::activator::snowflakeApp::SnowFlakeAppExecutionContext(postProcessor = meta::relational::postProcessor::ReplaceFreeMarkerWithTableFunctionParamHolder_SQLQuery_1__PostProcessorResult_1_); @@ -25,9 +30,29 @@ function meta::external::function::activator::snowflakeApp::generation::generat let plan = meta::pure::executionPlan::executionPlan($s.function->cast(@ConcreteFunctionDefinition), $contextWithEnumPushDown, $extensions, noDebug()); let resultStub = generateResultTypeStub($plan.rootExecutionNode.resultType->cast(@TDSResultType), $extensions); - let generatedQuery = $plan.rootExecutionNode->allNodes($extensions)->filter(n|$n->instanceOf(SQLExecutionNode))->cast(@SQLExecutionNode)->filter(node|$node.sqlQuery->toLower()->startsWith('select'))->last().sqlQuery->toOne('candidate query not found'); + let generatedQueryNode = $plan.rootExecutionNode->allNodes($extensions)->filter(n|$n->instanceOf(SQLExecutionNode))->cast(@SQLExecutionNode)->filter(node|$node.sqlQuery->toLower()->startsWith('select'))->last()->toOne('candidate query not found'); + let query = 'CREATE OR REPLACE SECURE FUNCTION %S.' + defaultDeploymentSchema() + '.' +$s.applicationName->toUpper() + $inputParamsStub + ' RETURNS TABLE ('+ $resultStub+ ') LANGUAGE SQL AS $$ '+ $generatedQueryNode.sqlQuery +' $$;'; + let grant = 'GRANT USAGE ON FUNCTION %S.' + defaultDeploymentSchema() + '.'+ $s.applicationName->toUpper()+ generateInputParamsStub($s.function, false) + ' to role '+ if($s.usageRole->isNotEmpty(), |$s.usageRole->toOne()->toUpper(),|'PUBLIC')+';' ; + let artifact = ^Artifact(createQuery = $query, grantStatement = $grant, tables = $generatedQueryNode.metadata->filter(m| $m->instanceOf(TableInfo))->cast(@TableInfo)->toOne('No table info found').info->map(i|$i.schema+'.'+$i.table)); + +} + +function meta::external::function::activator::snowflakeApp::generation::generateArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1]):String[1] +{ + let extensions = meta::external::format::shared::externalFormatExtension()->concatenate(meta::relational::extension::relationalExtensions()); + meta::external::function::activator::snowflakeApp::generation::generateArtifact($s, $extensions); +} + +function meta::external::function::activator::snowflakeApp::generation::generateFullArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1]):Artifact[1] +{ + let extensions = meta::external::format::shared::externalFormatExtension()->concatenate(meta::relational::extension::relationalExtensions()); + meta::external::function::activator::snowflakeApp::generation::generateFullArtifact($s, $extensions); +} + +function meta::external::function::activator::snowflakeApp::generation::generateArtifact(s: meta::external::function::activator::snowflakeApp::SnowflakeApp[1], extensions:meta::pure::extension::Extension[*]):String[1] +{ + meta::external::function::activator::snowflakeApp::generation::generateFullArtifact($s, $extensions).createQuery; - $s.applicationName->toUpper() + $inputParamsStub + ' RETURNS TABLE ('+ $resultStub+ ') LANGUAGE SQL AS $$ '+ $generatedQuery +' $$;'; } function meta::external::function::activator::snowflakeApp::generation::generateResultTypeStub(r: TDSResultType[1], extensions:meta::pure::extension::Extension[*]):String[1] @@ -40,38 +65,41 @@ function meta::external::function::activator::snowflakeApp::generation::generat { let params = $f->functionType().parameters->evaluateAndDeactivate(); assertEmpty($params->filter(p | $p.multiplicity != PureOne), 'not implemented, only multiplicity PureOne for snowflake app params is supported: ' + $params->filter(p | $p.multiplicity != PureOne)->map(p|$p.name)->joinStrings(', ')); - let typeMap = pureTypeToSnowflakeTypeMap(); + let typeMap = typeMap(); '(' + $params->map(p|if($includeParamName,|'"' + $p.name + '" ',|'') + if($p.genericType.rawType->toOne()->instanceOf(Enumeration), | 'VARCHAR', | $typeMap->get($p.genericType.rawType->toOne())->toOne('Relational type missing for type: ' + $p.genericType.rawType->toOne()->toString())) )->joinStrings(',') + ')'; } -function <> meta::external::function::activator::snowflakeApp::generation::pureTypeToSnowflakeTypeMap():Map[1] +function <> meta::external::function::activator::snowflakeApp::generation::typeMap():Map[1] { [ - pair(Integer, 'number'), + pair(Integer, 'INTEGER'), pair(Float, 'FLOAT'), - pair(Number, 'number'), + pair(Number, 'NUMBER'), pair(String, 'VARCHAR'), - pair(Date, 'date'), - pair(DateTime, 'timestamp'), - pair(StrictDate, 'date'), - pair(Boolean, 'boolean') + pair(Date, 'DATE'), + pair(DateTime, 'TIMESTAMP'), + pair(StrictDate, 'DATE'), + pair(Boolean, 'BOOLEAN'), + pair(Decimal, 'DECIMAL') ]->newMap() } function <> meta::external::function::activator::snowflakeApp::generation::pureTypeToSnowflakeTypeMatcher(c:meta::pure::tds::TDSColumn[1]):String[1] { - let tdsTypeToRelationalTypeMap = pureTypeToSnowflakeTypeMap(); - if($c.sourceDataType->isNotEmpty(), - |$c.sourceDataType->toOne()->match( + println($c,2); + let tdsTypeToRelationalTypeMap = typeMap(); + if($c.type->isNotEmpty(), + | $tdsTypeToRelationalTypeMap->get($c.type->toOne())->toOne('Relational type missing for type: '+ $c.type->toOne()->toString()), + | $c.sourceDataType->toOne('Source Data Type Missing for column: '+ +$c.name) + ->match( [ v:meta::relational::metamodel::datatype::Varchar[1] | 'VARCHAR', b:meta::relational::metamodel::datatype::Bit[1]|'BOOLEAN', all:meta::relational::metamodel::datatype::DataType[1]| $all->meta::relational::metamodel::datatype::dataTypeToSqlText() - ]), - | $tdsTypeToRelationalTypeMap->get($c.type->toOne('Column type missing for column: '+$c.name))->toOne('Relational type missing for type: '+ $c.type->toOne()->toString()) + ]) ); } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/metamodel/metamodel.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/metamodel/metamodel.pure index d6c821818ca..35e10147b97 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/metamodel/metamodel.pure +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/metamodel/metamodel.pure @@ -5,12 +5,21 @@ Class meta::external::function::activator::snowflakeApp::SnowflakeApp extends Fu { applicationName : String[1]; description : String[0..1]; + usageRole: String[0..1]; + deploymentSchema: String[0..1]; + permissionScheme: SnowflakePermissionScheme[0..1]; } - Class meta::external::function::activator::snowflakeApp::SnowflakeDeploymentConfiguration extends DeploymentConfiguration - { - target: meta::external::store::relational::runtime::RelationalDatabaseConnection[1]; - } +Enum meta::external::function::activator::snowflakeApp::SnowflakePermissionScheme +{ + DEFAULT, + SEQUESTERED +} + +Class meta::external::function::activator::snowflakeApp::SnowflakeDeploymentConfiguration extends DeploymentConfiguration +{ + target: meta::external::store::relational::runtime::RelationalDatabaseConnection[1]; +} Class meta::external::function::activator::snowflakeApp::SnowflakeDeploymentResult extends DeploymentResult { diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure index 795152bd04b..18a4ea4501e 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseModel.pure @@ -15,6 +15,7 @@ Class meta::external::function::activator::snowflakeApp::tests::model::simple::P extraInformation : String[0..1]; manager : PersonX[0..1]; gender: Gender[0..1]; + genderFromInt: Gender[0..1]; age : Integer[0..1]; nickName : String[0..1]; activeEmployment: Boolean[0..1]; @@ -80,7 +81,8 @@ Mapping meta::external::function::activator::snowflakeApp::tests::simpleRelation ( firstName : personTable.FIRSTNAME, age : personTable.AGE, - gender: EnumerationMapping genderEnum: personTable.GENDER + gender: EnumerationMapping genderEnum: personTable.GENDER, + genderFromInt: EnumerationMapping genderEnum2: personTable.GENDER2 ), scope([dbInc]default.personTable) ( @@ -95,6 +97,12 @@ Mapping meta::external::function::activator::snowflakeApp::tests::simpleRelation M: 'male', F: ['female', 'Female'] } + + Gender: EnumerationMapping genderEnum2 + { + M: 1, + F: 2 + } ) @@ -123,7 +131,7 @@ Mapping meta::external::function::activator::snowflakeApp::tests::simpleRelation ###Relational Database meta::external::function::activator::snowflakeApp::tests::dbInc ( - Table personTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT, GENDER VARCHAR(200), LAST_UPDATED TIMESTAMP) + Table personTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT, GENDER VARCHAR(200), GENDER2 INT,LAST_UPDATED TIMESTAMP) Table validPersonTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT) Table PersonTableExtension (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT, birthDate DATE) Table differentPersonTable (ID INT PRIMARY KEY, FIRSTNAME VARCHAR(200), LASTNAME VARCHAR(200), AGE INT, ADDRESSID INT, FIRMID INT, MANAGERID INT) diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure index e1aa3a5cfb0..d9cdbfd0722 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/showcase/showcaseNativeApps.pure @@ -36,7 +36,7 @@ function meta::external::function::activator::snowflakeApp::tests::simpleApp():A activationConfiguration = defaultConfig() , function = meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunction__TabularDataSet_1_ ); - let generatedQuery = $app->generateArtifact(); + let generatedQuery = $app->generateFullArtifact(); //isMulti } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/tests/testSnowflakeAppGeneration.pure b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/tests/testSnowflakeAppGeneration.pure index 0e0a511a572..3a10e04ce53 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/tests/testSnowflakeAppGeneration.pure +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/src/main/resources/core_snowflakeapp/tests/testSnowflakeAppGeneration.pure @@ -1,146 +1,146 @@ -// Copyright 2024 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::external::function::activator::snowflakeApp::generation::*; -import meta::external::function::activator::*; -import meta::external::function::activator::snowflakeApp::*; -import meta::pure::mapping::*; -import meta::external::function::activator::snowflakeApp::tests::*; -import meta::external::function::activator::snowflakeApp::tests::model::simple::*; - - -function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunction():Boolean[1] -{ - let expected = 'APP1() RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".FIRSTNAME = \'haha\' $$;'; - assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunction__TabularDataSet_1_, $expected); -} - -function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithParams():Boolean[1] -{ - let expected = 'APP1("age" number) RETURNS TABLE ("FIRSTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName" from personTable as "root" where ("root".AGE is not null and "root".AGE > age) $$;'; - assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithParams_Integer_1__TabularDataSet_1_, $expected); -} - -function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithStringParams():Boolean[1] -{ - let expected = 'APP1("firstName" VARCHAR) RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".FIRSTNAME = firstName $$;'; - assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithStringParams_String_1__TabularDataSet_1_, $expected); -} - -function <> meta::external::function::activator::snowflakeApp::tests::testEnumPushDownWithFilter():Boolean[1] -{ - let expected = 'APP1() RETURNS TABLE ("FIRSTNAME" VARCHAR,"GENDER" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", case when "root".GENDER = \'male\' then \'M\' when "root".GENDER in (\'female\', \'Female\') then \'F\' else null end as "Gender" from personTable as "root" where "root".GENDER = \'male\' $$;'; - assertSnowflakeArtifactForFunction( meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumPushDown__TabularDataSet_1_, $expected); -} - -function <> meta::external::function::activator::snowflakeApp::tests::testRelationalfunctionWithDateTimeHardCoded():Boolean[1] -{ - // the time zone info in the connection should be ignored in the generated sql - let expected = 'APP1() RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".LAST_UPDATED = \'2024-02-27 16:21:53\'::timestamp $$;'; - assertSnowflakeArtifactForFunction( meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeHardCoded__TabularDataSet_1_, $expected); -} - -function <> meta::external::function::activator::snowflakeApp::tests::testRelationalfunctionWithDateTimeParam():Boolean[1] -{ - // the time zone info in the connection should be ignored in the generated sql - let expected = 'APP1("asOf" timestamp) RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".LAST_UPDATED = asOf $$;'; - assertSnowflakeArtifactForFunction( meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeParam_DateTime_1__TabularDataSet_1_, $expected); -} - -// not supported yet -function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithCollectionParams():Boolean[1] -{ - let expected = ''; - assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithCollectionParams_Integer_MANY__TabularDataSet_1_, $expected); -} - -function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithEnumParams():Boolean[1] -{ - let expected = 'APP1("gender" VARCHAR) RETURNS TABLE ("FIRSTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName" from personTable as "root" where gender = case when "root".GENDER = \'male\' then \'M\' when "root".GENDER in (\'female\', \'Female\') then \'F\' else null end $$;'; - assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumParam_Gender_1__TabularDataSet_1_, $expected); -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunction():TabularDataSet[1] -{ - PersonX.all()->filter(p|$p.firstName == 'haha')->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) - ->from(simpleRelationalMapping, testRuntime(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithParams(age: Integer[1]):TabularDataSet[1] -{ - PersonX.all()->filter(p|$p.age > $age)->project([col(p|$p.firstName, 'firstName')]) - ->from(simpleRelationalMapping, testRuntime(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithStringParams(firstName: String[1]):TabularDataSet[1] -{ - PersonX.all()->filter(p|$p.firstName == $firstName)->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) - ->from(simpleRelationalMapping, testRuntime(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithCollectionParams(ages: Integer[*]):TabularDataSet[1] -{ - PersonX.all()->filter(p| $p.age->in($ages) )->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) - ->from(simpleRelationalMapping, testRuntime(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithCollectionParamsUsingContains(ages: Integer[*]):TabularDataSet[1] -{ - PersonX.all()->filter(p| $ages->contains($p.age->toOne()) )->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) - ->from(simpleRelationalMapping, testRuntime(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumPushDown():TabularDataSet[1] -{ - PersonX.all()->filter(p|$p.gender == Gender.M)->project([col(p|$p.firstName, 'firstName'), col(p|$p.gender, 'Gender')]) - ->from(simpleRelationalMapping, testRuntime(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumParam(gender: Gender[1]):TabularDataSet[1] -{ - PersonX.all()->filter(p|$p.gender == $gender)->project([col(p|$p.firstName, 'firstName')]) - ->from(simpleRelationalMapping, testRuntime(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeHardCoded():TabularDataSet[1] -{ - PersonX.all()->filter(p|$p.lastUpdated == %2024-02-27T16:21:53)->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) - ->from(simpleRelationalMapping, testRuntimeWithTimeZone(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeParam(asOf: DateTime[1]):TabularDataSet[1] -{ - PersonX.all()->filter(p|$p.lastUpdated ==$asOf)->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) - ->from(simpleRelationalMapping, testRuntimeWithTimeZone(dbInc)) -} - -function meta::external::function::activator::snowflakeApp::tests::assertSnowflakeArtifactForFunction(function: PackageableFunction[1], expected: String[1]): Boolean[1] -{ - meta::external::function::activator::snowflakeApp::tests::assertSnowflakeArtifactForFunction($function, $expected, []); -} - -function meta::external::function::activator::snowflakeApp::tests::assertSnowflakeArtifactForFunction(function: PackageableFunction[1], expected: String[1], extensions:meta::pure::extension::Extension[*]): Boolean[1] -{ - let app = ^SnowflakeApp - ( - applicationName = 'App1', - ownership = ^DeploymentOwnership(id = 'owner1'), - description = 'bla bla', - activationConfiguration = defaultConfig(), - function = $function - ); - - let generatedQuery = if($extensions->isNotEmpty(),| $app->generateArtifact($extensions),|$app->generateArtifact()); - assertEquals($expected, $generatedQuery); -} +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::external::function::activator::snowflakeApp::generation::*; +import meta::external::function::activator::*; +import meta::external::function::activator::snowflakeApp::*; +import meta::pure::mapping::*; +import meta::external::function::activator::snowflakeApp::tests::*; +import meta::external::function::activator::snowflakeApp::tests::model::simple::*; + + +function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunction():Boolean[1] +{ + let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1() RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".FIRSTNAME = \'haha\' $$;'; + assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunction__TabularDataSet_1_, $expected); +} + +function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithParams():Boolean[1] +{ + let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1("age" INTEGER) RETURNS TABLE ("FIRSTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName" from personTable as "root" where ("root".AGE is not null and "root".AGE > age) $$;'; + assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithParams_Integer_1__TabularDataSet_1_, $expected); +} + +function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithStringParams():Boolean[1] +{ + let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1("firstName" VARCHAR) RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".FIRSTNAME = firstName $$;'; + assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithStringParams_String_1__TabularDataSet_1_, $expected); +} + +function <> meta::external::function::activator::snowflakeApp::tests::testEnumPushDownWithFilter():Boolean[1] +{ + let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1() RETURNS TABLE ("FIRSTNAME" VARCHAR,"GENDER" VARCHAR,"GENDERFROMINT" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", case when "root".GENDER = \'male\' then \'M\' when "root".GENDER in (\'female\', \'Female\') then \'F\' else null end as "Gender", case when "root".GENDER2 = 1 then \'M\' when "root".GENDER2 = 2 then \'F\' else null end as "GenderFromInt" from personTable as "root" where "root".GENDER = \'male\' $$;'; + assertSnowflakeArtifactForFunction( meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumPushDown__TabularDataSet_1_, $expected); +} + +function <> meta::external::function::activator::snowflakeApp::tests::testRelationalfunctionWithDateTimeHardCoded():Boolean[1] +{ + // the time zone info in the connection should be ignored in the generated sql + let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1() RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".LAST_UPDATED = \'2024-02-27 16:21:53\'::timestamp $$;'; + assertSnowflakeArtifactForFunction( meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeHardCoded__TabularDataSet_1_, $expected); +} + +function <> meta::external::function::activator::snowflakeApp::tests::testRelationalfunctionWithDateTimeParam():Boolean[1] +{ + // the time zone info in the connection should be ignored in the generated sql + let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1("asOf" TIMESTAMP) RETURNS TABLE ("FIRSTNAME" VARCHAR,"LASTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName" from personTable as "root" where "root".LAST_UPDATED = asOf $$;'; + assertSnowflakeArtifactForFunction( meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeParam_DateTime_1__TabularDataSet_1_, $expected); +} + +// not supported yet +function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithCollectionParams():Boolean[1] +{ + let expected = ''; + assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithCollectionParams_Integer_MANY__TabularDataSet_1_, $expected); +} + +function <> meta::external::function::activator::snowflakeApp::tests::testSimpleRelationalfunctionWithEnumParams():Boolean[1] +{ + let expected = 'CREATE OR REPLACE SECURE FUNCTION %S.LEGEND_NATIVE_APPS.APP1("gender" VARCHAR) RETURNS TABLE ("FIRSTNAME" VARCHAR) LANGUAGE SQL AS $$ select "root".FIRSTNAME as "firstName" from personTable as "root" where gender = case when "root".GENDER = \'male\' then \'M\' when "root".GENDER in (\'female\', \'Female\') then \'F\' else null end $$;'; + assertSnowflakeArtifactForFunction(meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumParam_Gender_1__TabularDataSet_1_, $expected); +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunction():TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.firstName == 'haha')->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithParams(age: Integer[1]):TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.age > $age)->project([col(p|$p.firstName, 'firstName')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithStringParams(firstName: String[1]):TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.firstName == $firstName)->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithCollectionParams(ages: Integer[*]):TabularDataSet[1] +{ + PersonX.all()->filter(p| $p.age->in($ages) )->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithCollectionParamsUsingContains(ages: Integer[*]):TabularDataSet[1] +{ + PersonX.all()->filter(p| $ages->contains($p.age->toOne()) )->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumPushDown():TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.gender == Gender.M)->project([col(p|$p.firstName, 'firstName'), col(p|$p.gender, 'Gender'), col(p|$p.genderFromInt, 'GenderFromInt')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionEnumParam(gender: Gender[1]):TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.gender == $gender)->project([col(p|$p.firstName, 'firstName')]) + ->from(simpleRelationalMapping, testRuntime(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeHardCoded():TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.lastUpdated == %2024-02-27T16:21:53)->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) + ->from(simpleRelationalMapping, testRuntimeWithTimeZone(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::simpleRelationalfunctionWithDateTimeParam(asOf: DateTime[1]):TabularDataSet[1] +{ + PersonX.all()->filter(p|$p.lastUpdated ==$asOf)->project([col(p|$p.firstName, 'firstName'), col(p|$p.lastName, 'lastName')]) + ->from(simpleRelationalMapping, testRuntimeWithTimeZone(dbInc)) +} + +function meta::external::function::activator::snowflakeApp::tests::assertSnowflakeArtifactForFunction(function: PackageableFunction[1], expected: String[1]): Boolean[1] +{ + meta::external::function::activator::snowflakeApp::tests::assertSnowflakeArtifactForFunction($function, $expected, []); +} + +function meta::external::function::activator::snowflakeApp::tests::assertSnowflakeArtifactForFunction(function: PackageableFunction[1], expected: String[1], extensions:meta::pure::extension::Extension[*]): Boolean[1] +{ + let app = ^SnowflakeApp + ( + applicationName = 'App1', + ownership = ^DeploymentOwnership(id = 'owner1'), + description = 'bla bla', + activationConfiguration = defaultConfig(), + function = $function + ); + + let generatedQuery = if($extensions->isNotEmpty(),| $app->generateArtifact($extensions),|$app->generateArtifact()); + assertEquals($expected, $generatedQuery); +}