Skip to content

Commit

Permalink
update UDTF model and Validation (#2965)
Browse files Browse the repository at this point in the history
  • Loading branch information
@Yasirmod17
Yasirmod17 authored Jul 17, 2024
1 parent d044a3d commit b2f4534
Show file tree
Hide file tree
Showing 24 changed files with 461 additions and 211 deletions.
17 changes: 17 additions & 0 deletions ...Store-pure/src/main/resources/core_relational/relational/executionPlan/executionPlan.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,22 @@ Class meta::relational::mapping::TempTableColumnMetaData
parametersForGetter : Map<String, Any>[0..1];
}

Class meta::relational::mapping::QueryMetadata
{

}

Class meta::relational::mapping::TableInfo extends QueryMetadata
{
info: TableIdentifier[*];
}

Class meta::relational::mapping::TableIdentifier
{
schema: String[1];
table: String[1];
}

Class meta::relational::mapping::SQLExecutionNode extends ExecutionNode
{
sqlComment : String[0..1];
Expand All @@ -53,6 +69,7 @@ Class meta::relational::mapping::SQLExecutionNode extends ExecutionNode
onConnectionCloseRollbackQuery : String[0..1];
resultColumns : SQLResultColumn[*];
connection: DatabaseConnection[1];
metadata: QueryMetadata[*];
}

Class meta::relational::mapping::RelationalSaveNode extends ExecutionNode
Expand Down
4 changes: 3 additions & 1 deletion ...tional/relational/postprocessor/functionActivationPostProcessor/replaceFreeMarkerOps.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import meta::relational::metamodel::relation::*;
import meta::relational::functions::pureToSqlQuery::*;
import meta::relational::postProcessor::*;
import meta::relational::functions::pureToSqlQuery::metamodel::*;
Expand All @@ -21,6 +22,7 @@ import meta::core::runtime::*;
import meta::relational::metamodel::*;
import meta::relational::mapping::*;


function meta::relational::postProcessor::ReplaceFreeMarkerWithTableFunctionParamHolder(query:SQLQuery[1]): PostProcessorResult[1]
{
let transformFunc = {r: RelationalOperationElement[1], uniqueId: Integer[1] |
Expand All @@ -36,7 +38,7 @@ function meta::relational::postProcessor::ReplaceFreeMarkerWithTableFunctionPara

let transformedQuery = $query->transform($transformFunc, ^Map<RelationalOperationElement, RelationalOperationElement>()).first;

^PostProcessorResult(query = $transformedQuery->cast(@SQLQuery));
^PostProcessorResult(query = $transformedQuery->cast(@SQLQuery));
}

function <<access.private>> meta::relational::postProcessor::replaceVarPlaceHolder(v: VarPlaceHolder[1]): RelationalOperationElement[1]
Expand Down
3 changes: 2 additions & 1 deletion ...lStore-pure/src/main/resources/core_relational/relational/relationalMappingExecution.pure
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,8 @@ function meta::relational::mapping::generateSQLExecutionNode(query:SQLQuery[1],
resultColumns = $query->match([sel:SelectSQLQuery[1] | $sel.columns->map(c| ^SQLResultColumn(label = $c->columnLabel($connection, $extensions), dataType = $c->getRelationalTypeFromRelationalOperationElement()->translateCoreTypeToDbSpecificType(^TranslationContext(dbType = $connection.type)))), a:Any[1] | []]),
resultType = ^ResultType(type = Any),
connection = $connection->meta::relational::mapping::updateConnection($extensions)->cast(@DatabaseConnection),
supportFunctions = relationalPlanSupportFunctions($connection)->concatenate($enumMapSupportFunctions)
supportFunctions = relationalPlanSupportFunctions($connection)->concatenate($enumMapSupportFunctions),
metadata = ^meta::relational::mapping::TableInfo(info = $query->meta::relational::milestoning::getQueryTableAliases()->map(a|$a.relationalElement)->cast(@Table)->removeDuplicatesBy(t|$t.schema.name+'.'+$t.name)->map(tab| ^meta::relational::mapping::TableIdentifier(schema=$tab.schema.name, table=$tab.name)))
);

}
Expand Down
7 changes: 6 additions & 1 deletion ...pi/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppError.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,14 @@ public class SnowflakeAppError extends FunctionActivatorError
{
public MutableList<String> foundSQLs;

public <V> SnowflakeAppError(String message, MutableList<String> foundSQLs)
public SnowflakeAppError(String message, MutableList<String> foundSQLs)
{
super(message);
this.foundSQLs = foundSQLs;
}

public SnowflakeAppError(String message)
{
super(message);
}
}
34 changes: 26 additions & 8 deletions .../src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.eclipse.collections.impl.factory.Lists;
import org.eclipse.collections.impl.list.mutable.FastList;
import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo;
import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeGrantInfo;
import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorDeploymentConfiguration;
import org.finos.legend.engine.functionActivator.service.FunctionActivatorError;
import org.finos.legend.engine.functionActivator.service.FunctionActivatorService;
Expand Down Expand Up @@ -92,23 +93,40 @@ public boolean supports(Root_meta_external_function_activator_FunctionActivator
@Override
public MutableList<? extends FunctionActivatorError> validate(Identity identity, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, true, routerExtensions);
return validate(artifact);
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
return validate(identity, artifact);
}

public MutableList<? extends FunctionActivatorError> validate(SnowflakeAppArtifact artifact)
public MutableList<? extends FunctionActivatorError> validate(Identity identity, SnowflakeAppArtifact artifact)
{
int size = ((SnowflakeAppContent)artifact.content).sqlExpressions.select(e -> !e.toLowerCase().endsWith("to role public;")).size();
return size != 1 ?
Lists.mutable.with(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", ((SnowflakeAppContent)artifact.content).sqlExpressions)) :
Lists.mutable.empty();
MutableList<FunctionActivatorError> errors = Lists.mutable.empty();
SnowflakeAppContent content = (SnowflakeAppContent)artifact.content;
if (!content.sqlExpressions.isEmpty())
{
int size = content.sqlExpressions.select(e -> !e.toLowerCase().endsWith("to role public;")).size();
if (size != 1)
{
errors.add(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", content.sqlExpressions));
}
}
if (content.permissionScope.equals("SEQUESTERED"))
{
MutableList<SnowflakeGrantInfo> grantsToRole = this.snowflakeDeploymentManager.getGrants(identity, artifact);
MutableList<String> contentTables = ((SnowflakeAppContent) artifact.content).usedTables.collect(String::toUpperCase);
MutableList<SnowflakeGrantInfo> violations = grantsToRole.select(g -> g.privilege.equals("SELECT") && contentTables.contains(g.objectName.toUpperCase()));
if (!violations.isEmpty())
{
errors.add(new SnowflakeAppError("Privilege Violation for SEQUESTERED scheme. Deployment Role contains SELECT permissions on tables: " + violations.collect(v -> v.objectName).makeString("[", ",", "]")));
}
}
return errors;
}

@Override
public SnowflakeDeploymentResult publishToSandbox(Identity identity, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, List<SnowflakeAppDeploymentConfiguration> runtimeConfigurations, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, inputModel, routerExtensions);
MutableList<? extends FunctionActivatorError> validationError = validate(artifact);
MutableList<? extends FunctionActivatorError> validationError = validate(identity, artifact);
if (validationError.isEmpty())
{
return this.snowflakeDeploymentManager.deploy(identity, artifact, runtimeConfigurations);
Expand Down
49 changes: 43 additions & 6 deletions ...g/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorArtifact;
import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppDeploymentTool;
import org.finos.legend.engine.protocol.functionActivator.deployment.FunctionActivatorDeploymentConfiguration;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.specification.SnowflakeDatasourceSpecification;
import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppArtifact;
import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration;
import org.finos.legend.engine.protocol.snowflakeApp.deployment.SnowflakeAppContent;
Expand Down Expand Up @@ -54,6 +55,8 @@ public class SnowflakeAppDeploymentManager implements DeploymentManager<Snowflak
private ConnectionManagerSelector connectionManager;
private static final String deploymentSchema = "LEGEND_NATIVE_APPS";
private static final String deploymentTable = "APP_METADATA";
private static final String limit = "10000";


private static String deployStub = "/schemas/" + deploymentSchema + "/user-function/%S()";

Expand Down Expand Up @@ -135,13 +138,13 @@ public SnowflakeDeploymentResult fakeDeploy(Root_meta_pure_alloy_connections_all

public Connection getDeploymentConnection(Identity identity, RelationalDatabaseConnection connection)
{
return this.connectionManager.getDatabaseConnection(identity, (DatabaseConnection) connection);
return this.connectionManager.getDatabaseConnection(identity, connection);
}

public void deployImpl(Connection jdbcConnection, SnowflakeAppContent context) throws SQLException
public void deployImpl(Connection jdbcConnection, SnowflakeAppContent content) throws SQLException
{
String catalogName = jdbcConnection.getCatalog();
MutableList<String> statements = generateStatements(catalogName, context);
MutableList<String> statements = generateStatements(catalogName, content);
for (String s: statements)
{
Statement statement = jdbcConnection.createStatement();
Expand All @@ -152,10 +155,18 @@ public void deployImpl(Connection jdbcConnection, SnowflakeAppContent context) t
public MutableList<String> generateStatements(String catalogName, SnowflakeAppContent content)
{
MutableList<String> statements = Lists.mutable.empty();
statements.add(String.format("CREATE OR REPLACE SECURE FUNCTION %S.%S.%s", catalogName, deploymentSchema, content.sqlExpressions.getFirst()));
if (content.sqlExpressions.size() > 1)
if (!content.sqlExpressions.isEmpty())
{
statements.add(String.format("CREATE OR REPLACE SECURE FUNCTION %S.%S.%s", catalogName, deploymentSchema, content.sqlExpressions.getFirst()));
if (content.sqlExpressions.size() > 1)
{
statements.add(String.format("GRANT USAGE ON FUNCTION %S.%S.%S", catalogName, deploymentSchema, content.sqlExpressions.get(1)));
}
}
else
{
statements.add(String.format("GRANT USAGE ON FUNCTION %S.%S.%S", catalogName, deploymentSchema, content.sqlExpressions.get(1)));
statements.add(String.format(content.createStatement, catalogName));
statements.add(String.format(content.grantStatement, catalogName));
}
return statements;
}
Expand All @@ -177,6 +188,32 @@ public RelationalDatabaseConnection extractConnectionFromArtifact(SnowflakeAppAr
return ((SnowflakeAppDeploymentConfiguration)artifact.deploymentConfiguration).connection;
}

public MutableList<SnowflakeGrantInfo> getGrants(Identity identity, SnowflakeAppArtifact artifact)
{
RelationalDatabaseConnection rel = ((SnowflakeAppDeploymentConfiguration)artifact.deploymentConfiguration).connection;
MutableList<SnowflakeGrantInfo> grants = Lists.mutable.empty();
try (Connection jdbcConnection = this.getDeploymentConnection(identity, artifact))
{
Statement role = jdbcConnection.createStatement();
ResultSet roleResult = role.executeQuery("SELECT CURRENT_ROLE()");
roleResult.next();
String currentRole = roleResult.getString(1);
roleResult.close();
Statement s = jdbcConnection.createStatement();
ResultSet res = s.executeQuery(String.format("SHOW GRANTS TO ROLE %S LIMIT %S;", currentRole, limit));
while (res.next())
{
grants.add(new SnowflakeGrantInfo(res.getString("privilege"), res.getString("granted_on"), res.getString("name"), res.getString("grantee_name"), res.getString("granted_by")));
}
s.close();
}
catch (Exception e)
{
LOGGER.info("Unable to query for grants for role. Error: " + e.getMessage());
}
return grants;
}

public ImmutableList<DeploymentInfo> getDeployed(Identity identity, RelationalDatabaseConnection connection) throws Exception
{
ImmutableList<DeploymentInfo> deployments = null;
Expand Down
33 changes: 33 additions & 0 deletions ...ain/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeGrantInfo.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright 2023 Goldman Sachs
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org.finos.legend.engine.language.snowflakeApp.deployment;

public class SnowflakeGrantInfo
{
public String privilege;
public String objectType;
public String objectName;
public String granteeRole;
public String grantee;

public SnowflakeGrantInfo(String privilege, String objectType, String objectName, String granteeRole, String grantee)
{
this.privilege = privilege;
this.objectType = objectType;
this.objectName = objectName;
this.granteeRole = granteeRole;
this.grantee = grantee;
}
}
3 changes: 3 additions & 0 deletions ...gend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Function;
import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeApp;
import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration;
import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakePermissionScheme;
import org.finos.legend.pure.generated.*;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction;
import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor;
Expand Down Expand Up @@ -72,6 +73,8 @@ public Root_meta_external_function_activator_snowflakeApp_SnowflakeApp buildSnow
._applicationName(app.applicationName)
._function(func)
._description(app.description)
._usageRole(app.usageRole)
._permissionScheme(app.permissionScheme != null ? context.pureModel.getEnumValue("meta::external::function::activator::snowflakeApp::SnowflakePermissionScheme", app.permissionScheme.toString()) : context.pureModel.getEnumValue("meta::external::function::activator::snowflakeApp::SnowflakePermissionScheme", SnowflakePermissionScheme.DEFAULT.toString()))
._ownership(new Root_meta_external_function_activator_DeploymentOwnership_Impl("")._id(((DeploymentOwner)app.ownership).id))
._activationConfiguration(app.activationConfiguration != null ? buildDeploymentConfig((SnowflakeAppDeploymentConfiguration) app.activationConfiguration, context) : null);
}
Expand Down
15 changes: 2 additions & 13 deletions ...n/java/org/finos/legend/engine/language/snowflakeApp/generator/SnowflakeAppGenerator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,7 @@ public class SnowflakeAppGenerator

public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
return generateArtifact(pureModel, activator, inputModel, false, routerExtensions);
}

public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, boolean createStatementOnly, Function<PureModel, RichIterable<? extends Root_meta_pure_extension_Extension>> routerExtensions)
{
String sqlFunctionExpression = core_snowflakeapp_generation_generation.Root_meta_external_function_activator_snowflakeApp_generation_generateArtifact_SnowflakeApp_1__Extension_MANY__String_1_(activator, routerExtensions.apply(pureModel), pureModel.getExecutionSupport());
String inputParamStub = core_snowflakeapp_generation_generation.Root_meta_external_function_activator_snowflakeApp_generation_generateInputParamsStub_Function_1__Boolean_1__String_1_(activator._function(), false, pureModel.getExecutionSupport());
Root_meta_external_function_activator_snowflakeApp_generation_Artifact fullArtifact = core_snowflakeapp_generation_generation.Root_meta_external_function_activator_snowflakeApp_generation_generateFullArtifact_SnowflakeApp_1__Extension_MANY__Artifact_1_(activator, routerExtensions.apply(pureModel), pureModel.getExecutionSupport());
RelationalDatabaseConnection connection;
AlloySDLC sdlc = null;
if (((PureModelContextData)inputModel).getOrigin() != null)
Expand All @@ -60,7 +54,7 @@ public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_me
sdlc = (AlloySDLC) sdlcInfo;
}
}
SnowflakeAppContent content = new SnowflakeAppContent(activator._applicationName(), Lists.mutable.of(sqlFunctionExpression), activator._description(), ((Root_meta_external_function_activator_DeploymentOwnership)activator._ownership())._id(), sdlc);
SnowflakeAppContent content = new SnowflakeAppContent(activator._applicationName(), fullArtifact._createQuery(), fullArtifact._grantStatement(), activator._permissionScheme().toString(), activator._description(), ((Root_meta_external_function_activator_DeploymentOwnership)activator._ownership())._id(), Lists.mutable.withAll(fullArtifact._tables()));
if (activator._activationConfiguration() != null)
{
//identify connection
Expand All @@ -71,11 +65,6 @@ public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_me
.select(c -> c.getPath().equals(((org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppDeploymentConfiguration)protocolActivator.activationConfiguration).activationConnection.connection)).getFirst().connectionValue;
SnowflakeDatasourceSpecification ds = (SnowflakeDatasourceSpecification)connection.datasourceSpecification;
String deployedLocation = String.format("https://app.%s.privatelink.snowflakecomputing.com/%s/%s/data/databases/%S", ds.region, ds.region, ds.accountName, ds.databaseName);
if (!createStatementOnly)
{
String generatedGrantStatement = generateGrantStatement(activator._applicationName(), inputParamStub);
content.addGrantStatement(generatedGrantStatement);
}
return new SnowflakeAppArtifact(content, new SnowflakeAppDeploymentConfiguration(connection), deployedLocation, sdlc);
}
return new SnowflakeAppArtifact(content, sdlc);
Expand Down
Loading

0 comments on commit b2f4534

Please sign in to comment.