From 050cb60ffa013dc1a14cf125fb3b44e1cc301a56 Mon Sep 17 00:00:00 2001 From: Mohammed Ibrahim Date: Mon, 6 Nov 2023 12:41:00 -0500 Subject: [PATCH] BigQuery Activator: Sandbox (#2438) --- CODEOWNERS | 1 + .../pom.xml | 15 ++ .../collection/generation/TestExtensions.java | 10 +- .../legend-engine-server/pom.xml | 19 ++ .../finos/legend/engine/server/Server.java | 2 + .../finos/legend/engine/ide/PureIDELight.java | 1 + .../pom.xml | 203 ++++++++++++++++++ .../api/BigQueryFunctionError.java | 30 +++ .../api/BigQueryFunctionService.java | 98 +++++++++ .../deployment/BigQueryFunctionArtifact.java | 31 +++ ...ryFunctionArtifactGenerationExtension.java | 46 ++++ .../deployment/BigQueryFunctionContent.java | 31 +++ ...gQueryFunctionDeploymentConfiguration.java | 23 ++ .../BigQueryFunctionDeploymentManager.java | 98 +++++++++ .../BigQueryFunctionDeploymentResult.java | 34 +++ .../deployment/BigQueryFunctionGenerator.java | 60 ++++++ ...Activator.service.FunctionActivatorService | 1 + ...tion.extension.ArtifactGenerationExtension | 1 + .../bigqueryFunction/api/TestValidation.java | 47 ++++ .../pom.xml | 102 +++++++++ .../BigQueryFunctionCompilerExtension.java | 84 ++++++++ ...er.toPureGraph.extension.CompilerExtension | 1 + ...igQueryFunctionCompilationFromGrammar.java | 67 ++++++ .../pom.xml | 163 ++++++++++++++ .../antlr4/BigQueryFunctionLexerGrammar.g4 | 15 ++ .../antlr4/BigQueryFunctionParserGrammar.g4 | 67 ++++++ ...igQueryFunctionGrammarParserExtension.java | 70 ++++++ .../from/BigQueryFunctionTreeWalker.java | 140 ++++++++++++ .../to/BigQueryFunctionGrammarComposer.java | 90 ++++++++ ....from.extension.PureGrammarParserExtension | 1 + ....to.extension.PureGrammarComposerExtension | 1 + .../test/TestBigQueryFunctionRoundtrip.java | 46 ++++ .../grammar/test/TestBigQueryParsing.java | 74 +++++++ .../pom.xml | 46 ++++ .../metamodel/BigQueryFunction.java | 27 +++ ...gQueryFunctionDeploymentConfiguration.java | 28 +++ .../BigQueryFunctionProtocolExtension.java | 49 +++++ ...ol.pure.v1.extension.PureProtocolExtension | 1 + .../pom.xml | 156 ++++++++++++++ ...igQueryFunctionCodeRepositoryProvider.java | 28 +++ ...lesystem.repository.CodeRepositoryProvider | 1 + .../core_bigqueryfunction.definition.json | 9 + .../metamodel/metamodel.pure | 26 +++ legend-engine-xts-bigqueryFunction/pom.xml | 36 ++++ pom.xml | 31 +++ 45 files changed, 2109 insertions(+), 1 deletion(-) create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/pom.xml create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionError.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionService.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifact.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifactGenerationExtension.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionContent.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentConfiguration.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentManager.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentResult.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionGenerator.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.functionActivator.service.FunctionActivatorService create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/test/java/org/finos/legend/engine/language/bigqueryFunction/api/TestValidation.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/pom.xml create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/BigQueryFunctionCompilerExtension.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/test/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/test/TestBigQueryFunctionCompilationFromGrammar.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/pom.xml create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionLexerGrammar.g4 create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionParserGrammar.g4 create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionGrammarParserExtension.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionTreeWalker.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/to/BigQueryFunctionGrammarComposer.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryFunctionRoundtrip.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryParsing.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/pom.xml create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunction.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionDeploymentConfiguration.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionProtocolExtension.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/resources/META-INF/services/org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/pom.xml create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/java/org/finos/legend/pure/code/core/CoreBigQueryFunctionCodeRepositoryProvider.java create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction.definition.json create mode 100644 legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction/metamodel/metamodel.pure create mode 100644 legend-engine-xts-bigqueryFunction/pom.xml diff --git a/CODEOWNERS b/CODEOWNERS index 13e84bd3873..a246f0d3e3f 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -8,6 +8,7 @@ /legend-engine-xts-analytics/** @finos/legend-engine-maintainers /legend-engine-xts-authentication/** @finos/legend-engine-maintainers /legend-engine-xts-avro/** @finos/legend-engine-maintainers +/legend-engine-xts-bigqueryFunction/** @finos/legend-engine-maintainers /legend-engine-xts-changetoken/** @finos/legend-engine-maintainers /legend-engine-xts-daml/** @finos/legend-engine-maintainers /legend-engine-xts-data-push/** @finos/legend-engine-maintainers diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml index c55dba62e78..8dbc3a47afc 100644 --- a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml +++ b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml @@ -484,6 +484,21 @@ + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-compiler + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-grammar + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-protocol + + + org.finos.legend.engine diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java b/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java index 38f93c3cddd..c1865923cf6 100644 --- a/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java +++ b/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java @@ -28,6 +28,9 @@ import org.finos.legend.engine.generation.DataSpaceAnalyticsArtifactGenerationExtension; import org.finos.legend.engine.generation.OpenApiArtifactGenerationExtension; import org.finos.legend.engine.generation.SearchDocumentArtifactGenerationExtension; +import org.finos.legend.engine.language.bigqueryFunction.compiler.toPureGraph.BigQueryFunctionCompilerExtension; +import org.finos.legend.engine.language.bigqueryFunction.grammar.from.BigQueryFunctionGrammarParserExtension; +import org.finos.legend.engine.language.bigqueryFunction.grammar.to.BigQueryFunctionGrammarComposer; import org.finos.legend.engine.language.graphQL.grammar.integration.GraphQLGrammarParserExtension; import org.finos.legend.engine.language.graphQL.grammar.integration.GraphQLPureGrammarComposerExtension; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; @@ -77,7 +80,7 @@ import org.finos.legend.engine.language.sql.grammar.integration.SQLGrammarParserExtension; import org.finos.legend.engine.language.sql.grammar.integration.SQLPureGrammarComposerExtension; import org.finos.legend.engine.language.stores.elasticsearch.v7.from.ElasticsearchGrammarParserExtension; -import org.finos.legend.engine.protocol.mongodb.schema.metamodel.MongoDBPureProtocolExtension; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunctionProtocolExtension; import org.finos.legend.pure.code.core.ElasticsearchPureCoreExtension; import org.finos.legend.engine.language.stores.elasticsearch.v7.to.ElasticsearchGrammarComposerExtension; import org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension; @@ -267,6 +270,7 @@ protected Iterable> getExpected .with(org.finos.legend.engine.protocol.pure.v1.CorePureProtocolExtension.class) .with(org.finos.legend.engine.protocol.pure.v1.DataSpaceProtocolExtension.class) .with(SnowflakeAppProtocolExtension.class) + .with(BigQueryFunctionProtocolExtension.class) .with(org.finos.legend.engine.protocol.pure.v1.DiagramProtocolExtension.class) .with(org.finos.legend.engine.protocol.pure.v1.GenerationProtocolExtension.class) .with(org.finos.legend.engine.protocol.pure.v1.PersistenceProtocolExtension.class) @@ -311,6 +315,7 @@ protected Iterable> getExp .with(CorePureGrammarParser.class) .with(DataSpaceParserExtension.class) .with(SnowflakeAppGrammarParserExtension.class) + .with(BigQueryFunctionGrammarParserExtension.class) .with(DiagramParserExtension.class) .with(ExternalFormatGrammarParserExtension.class) .with(GenerationParserExtension.class) @@ -337,6 +342,7 @@ protected Iterable> getE .with(CorePureGrammarComposer.class) .with(DataSpaceGrammarComposerExtension.class) .with(SnowflakeAppGrammarComposer.class) + .with(BigQueryFunctionGrammarComposer.class) .with(DiagramGrammarComposerExtension.class) .with(ExternalFormatGrammarComposerExtension.class) .with(GenerationGrammarComposerExtension.class) @@ -368,6 +374,7 @@ protected Iterable> getExpectedComp return Lists.mutable.>empty() .with(org.finos.legend.engine.language.pure.compiler.toPureGraph.DiagramCompilerExtension.class) .with(SnowflakeAppCompilerExtension.class) + .with(BigQueryFunctionCompilerExtension.class) .with(org.finos.legend.engine.language.pure.compiler.toPureGraph.DataSpaceCompilerExtension.class) .with(org.finos.legend.engine.language.pure.compiler.toPureGraph.TextCompilerExtension.class) .with(org.finos.legend.engine.language.pure.compiler.toPureGraph.CoreCompilerExtension.class) @@ -519,6 +526,7 @@ protected Iterable getExpectedCodeRepositories() .with("core_servicestore") .with("core_authentication") .with("core_snowflakeapp") + .with("core_bigqueryfunction") .with("core_text_metamodel") .with("core_external_language_java") .with("core_java_platform_binding") diff --git a/legend-engine-config/legend-engine-server/pom.xml b/legend-engine-config/legend-engine-server/pom.xml index db6965216d9..05dd0cc7f1d 100644 --- a/legend-engine-config/legend-engine-server/pom.xml +++ b/legend-engine-config/legend-engine-server/pom.xml @@ -194,6 +194,10 @@ org.finos.legend.engine legend-engine-xt-functionActivator-protocol + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-protocol + org.finos.legend.engine legend-engine-xt-artifact-generation-api @@ -341,6 +345,21 @@ org.finos.legend.engine legend-engine-xt-snowflakeApp-generator + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-compiler + runtime + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-grammar + runtime + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-api + runtime + org.finos.legend.engine legend-engine-xt-hostedService-compiler diff --git a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java index d13bb16292c..95db1c44e09 100644 --- a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -123,6 +123,7 @@ import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutor; import org.finos.legend.engine.plan.execution.stores.service.plugin.ServiceStoreExecutorBuilder; import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunctionDeploymentConfiguration; import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory; import org.finos.legend.engine.protocol.pure.v1.model.PureProtocol; import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; @@ -219,6 +220,7 @@ protected SwaggerBundleConfiguration getSwaggerBundleConfiguration( bootstrap.getObjectMapper().registerSubtypes(new NamedType(LegendDefaultDatabaseAuthenticationFlowProviderConfiguration.class, "legendDefault")); bootstrap.getObjectMapper().registerSubtypes(new NamedType(HostedServiceDeploymentConfiguration.class, "hostedServiceConfig")); bootstrap.getObjectMapper().registerSubtypes(new NamedType(SnowflakeAppDeploymentConfiguration.class, "snowflakeAppConfig")); + bootstrap.getObjectMapper().registerSubtypes(new NamedType(BigQueryFunctionDeploymentConfiguration.class, "snowflakeAppConfig")); } public CredentialProviderProvider configureCredentialProviders(List vaultConfigurations) diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java index 0008f70baee..5a20a181b37 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java @@ -45,6 +45,7 @@ protected MutableList buildRepositories(SourceLocationCon .with(this.buildCore("legend-engine-xts-mastery/legend-engine-xt-mastery-pure", "mastery")) .with(this.buildCore("legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure", "function_activator")) .with(this.buildCore("legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure", "snowflakeapp")) + .with(this.buildCore("legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure", "bigqueryfunction")) .with(this.buildCore("legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure", "hostedservice")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure", "relational")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-javaPlatformBinding-pure", "relational-java-platform-binding")) diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/pom.xml b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/pom.xml new file mode 100644 index 00000000000..55bf430025a --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/pom.xml @@ -0,0 +1,203 @@ + + + + + org.finos.legend.engine + legend-engine-xts-bigqueryFunction + 4.34.1-SNAPSHOT + + 4.0.0 + + legend-engine-xt-bigqueryFunction-api + jar + Legend Engine - XT - BigQuery Function - API + + + + + org.finos.legend.pure + legend-pure-m3-core + + + + + + org.finos.legend.engine + legend-engine-language-pure-compiler + + + org.finos.legend.engine + legend-engine-language-pure-modelManager + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + + + org.finos.legend.engine + legend-engine-pure-code-core-extension + + + org.finos.legend.engine + legend-engine-executionPlan-generation + + + org.finos.legend.engine + legend-engine-shared-core + + + org.finos.legend.engine + legend-engine-xt-functionActivator-pure + + + org.finos.legend.engine + legend-engine-protocol-pure + + + org.finos.legend.engine + legend-engine-xt-functionActivator-api + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + + + org.finos.legend.engine + legend-engine-xt-relationalStore-bigquery-pure + + + org.finos.legend.engine + legend-engine-xt-relationalStore-protocol + + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-pure + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-protocol + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-compiler + runtime + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-grammar + runtime + + + org.finos.legend.engine + legend-engine-language-pure-dsl-generation + + + + + javax.ws.rs + javax.ws.rs-api + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + + + + + com.google.cloud + google-cloud-bigquery + 2.29.0 + + + org.checkerframework + checker-qual + + + com.google.errorprone + error_prone_annotations + + + javax.annotation + javax.annotation-api + + + com.google.j2objc + j2objc-annotations + + + commons-logging + commons-logging + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + + + org.apache.arrow + arrow-format + + + + + + + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + + + + + + org.slf4j + slf4j-api + + + + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-xt-relationalStore-grammar + test + + + org.finos.legend.engine + legend-engine-configuration + test + + + org.glassfish.jersey.core + jersey-common + test + + + + \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionError.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionError.java new file mode 100644 index 00000000000..d167c28bd59 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionError.java @@ -0,0 +1,30 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.api; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ImmutableList; +import org.finos.legend.engine.functionActivator.service.FunctionActivatorError; + +public class BigQueryFunctionError extends FunctionActivatorError +{ + public ImmutableList foundSQLs; + + public BigQueryFunctionError(String message, Iterable foundSQLs) + { + super(message); + this.foundSQLs = Lists.immutable.withAll(foundSQLs); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionService.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionService.java new file mode 100644 index 00000000000..56f786ad33d --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/api/BigQueryFunctionService.java @@ -0,0 +1,98 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.api; + +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; +import org.finos.legend.engine.functionActivator.service.FunctionActivatorError; +import org.finos.legend.engine.functionActivator.service.FunctionActivatorService; +import org.finos.legend.engine.language.bigqueryFunction.deployment.*; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunctionProtocolExtension; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext; +import org.finos.legend.pure.generated.*; +import org.finos.legend.engine.shared.core.identity.Identity; + +import java.util.List; + +public class BigQueryFunctionService implements FunctionActivatorService +{ + private final BigQueryFunctionDeploymentManager bigQueryFunctionDeploymentManager; + + public BigQueryFunctionService() + { + this.bigQueryFunctionDeploymentManager = new BigQueryFunctionDeploymentManager(); + } + + @Override + public FunctionActivatorInfo info(PureModel pureModel, String version) + { + return new FunctionActivatorInfo( + "BigQuery Function", + "Create a BigQuery Function that can activate in BigQuery.", + "meta::protocols::pure::" + version + "::metamodel::function::activator::bigQueryFunction::BigQueryFunction", + BigQueryFunctionProtocolExtension.packageJSONType, + pureModel); + } + + @Override + public boolean supports(Root_meta_external_function_activator_FunctionActivator functionActivator) + { + return functionActivator instanceof Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction; + } + + @Override + public MutableList validate(Identity identity, PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, PureModelContext inputModel, Function> routerExtensions) + { + BigQueryFunctionArtifact artifact = BigQueryFunctionGenerator.generateArtifact(pureModel, activator, routerExtensions); + return this.validateArtifact(artifact); + } + + @Override + public BigQueryFunctionDeploymentResult publishToSandbox(Identity identity, PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, PureModelContext inputModel, List runtimeConfigurations, Function> routerExtensions) + { + BigQueryFunctionArtifact artifact = BigQueryFunctionGenerator.generateArtifact(pureModel, activator, routerExtensions); + MutableList validationErrors = this.validateArtifact(artifact); + + Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration deploymentConfiguration = ((Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration) activator._activationConfiguration()); + return validationErrors.notEmpty() ? + new BigQueryFunctionDeploymentResult(validationErrors.collect(e -> e.message)) : + this.bigQueryFunctionDeploymentManager.deployImpl(artifact, deploymentConfiguration); + } + + @Override + public BigQueryFunctionArtifact renderArtifact(PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, PureModelContext inputModel, String clientVersion, Function> routerExtensions) + { + return BigQueryFunctionGenerator.generateArtifact(pureModel, activator, routerExtensions); + } + + @Override + public List selectConfig(List configurations) + { + return Lists.mutable.withAll(configurations).select(e -> e instanceof BigQueryFunctionDeploymentConfiguration).collect(e -> (BigQueryFunctionDeploymentConfiguration) e); + } + + private MutableList validateArtifact(BigQueryFunctionArtifact artifact) + { + int size = ((BigQueryFunctionContent)artifact.content).sqlExpressions.size(); + return size == 1 ? + Lists.fixedSize.empty() : + Lists.fixedSize.with(new BigQueryFunctionError("BigQuery Function can't be used with a plan containing '" + size + "' SQL expressions", ((BigQueryFunctionContent)artifact.content).sqlExpressions)); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifact.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifact.java new file mode 100644 index 00000000000..ef867daac1e --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifact.java @@ -0,0 +1,31 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.deployment; + +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorArtifact; + +public class BigQueryFunctionArtifact extends FunctionActivatorArtifact +{ + public String sourceProjectId; + public String sourceDefaultDataset; + + public BigQueryFunctionArtifact(String name, MutableList sqlExpressions, String sourceProjectId, String sourceDefaultDataset) + { + this.content = new BigQueryFunctionContent(name, sqlExpressions); + this.sourceProjectId = sourceProjectId; + this.sourceDefaultDataset = sourceDefaultDataset; + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifactGenerationExtension.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifactGenerationExtension.java new file mode 100644 index 00000000000..0dd3ca8877d --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionArtifactGenerationExtension.java @@ -0,0 +1,46 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.deployment; + +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; +import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; + +import java.util.List; + +public class BigQueryFunctionArtifactGenerationExtension implements ArtifactGenerationExtension +{ + private static final String ROOT_PATH = "bigQueryFunction"; + + @Override + public String getKey() + { + return ROOT_PATH; + } + + @Override + public boolean canGenerate(PackageableElement element) + { + return false; + } + + @Override + public List generate(PackageableElement element, PureModel pureModel, PureModelContextData data, String clientVersion) + { + return null; + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionContent.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionContent.java new file mode 100644 index 00000000000..13c8aa1aa4c --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionContent.java @@ -0,0 +1,31 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.deployment; + +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentContent; + +public class BigQueryFunctionContent extends FunctionActivatorDeploymentContent +{ + public final MutableList sqlExpressions; + + public final String functionName; + + public BigQueryFunctionContent(String name, MutableList sqlExpressions) + { + this.functionName = name; + this.sqlExpressions = sqlExpressions; + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentConfiguration.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentConfiguration.java new file mode 100644 index 00000000000..8ca43c04e1c --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentConfiguration.java @@ -0,0 +1,23 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.deployment; + +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; + +public class BigQueryFunctionDeploymentConfiguration extends FunctionActivatorDeploymentConfiguration +{ + RelationalDatabaseConnection connection; +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentManager.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentManager.java new file mode 100644 index 00000000000..bcd5834edb7 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentManager.java @@ -0,0 +1,98 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.deployment; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.RoutineId; +import com.google.cloud.bigquery.RoutineInfo; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.utility.Iterate; +import org.finos.legend.engine.functionActivator.deployment.DeploymentManager; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorArtifact; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.pure.generated.Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration; +import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification; +import org.pac4j.core.profile.CommonProfile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; + +/** + * These deployment functions assume that the artifact has already been validated. + */ +public class BigQueryFunctionDeploymentManager implements DeploymentManager +{ + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryFunctionDeploymentManager.class); + + @Override + public boolean canDeploy(FunctionActivatorArtifact activatorArtifact) + { + return activatorArtifact instanceof BigQueryFunctionArtifact; + } + + @Override + public BigQueryFunctionDeploymentResult deploy(Identity identity, BigQueryFunctionArtifact artifact) + { + return new BigQueryFunctionDeploymentResult("", false); + } + + @Override + public BigQueryFunctionDeploymentResult deploy(Identity identity, BigQueryFunctionArtifact artifact, List availableRuntimeConfigurations) + { + return new BigQueryFunctionDeploymentResult("", false); + } + + public BigQueryFunctionDeploymentResult deployImpl(BigQueryFunctionArtifact artifact, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration deploymentConfiguration) + { + LOGGER.info("Starting deployment"); + BigQueryFunctionDeploymentResult result; + try + { + Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification datasourceSpecification = (Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification) deploymentConfiguration._target()._datasourceSpecification(); + String dataset = datasourceSpecification._defaultDataset(); + String projectId = datasourceSpecification._projectId(); + + BigQueryFunctionContent functionContent = (BigQueryFunctionContent) artifact.content; + BigQuery bigQuery = BigQueryOptions.newBuilder().setProjectId(projectId).build().getService(); + RoutineId routineId = RoutineId.of(projectId, dataset, functionContent.functionName); + + String sqlExpression = Iterate.getOnly(functionContent.sqlExpressions); + String sourceProjectId = artifact.sourceProjectId; + String sourceDefaultDataset = artifact.sourceDefaultDataset; + // TODO: Include projectId in core relational BigQuery SQL statement construction + String fullyQualifiedSqlExpression = sqlExpression.replace(sourceDefaultDataset, String.format("`%s.%s`", sourceProjectId, sourceDefaultDataset)); + RoutineInfo routineInfo = + RoutineInfo + .newBuilder(routineId) + .setRoutineType("TABLE_VALUED_FUNCTION") + .setLanguage("SQL") + .setBody(fullyQualifiedSqlExpression) + .build(); + bigQuery.create(routineInfo); + + LOGGER.info("Completed deployment successfully"); + result = new BigQueryFunctionDeploymentResult(functionContent.functionName, true); + } + catch (Exception e) + { + LOGGER.info("Completed deployment with error"); + result = new BigQueryFunctionDeploymentResult(Lists.mutable.with(e.getMessage())); + } + return result; + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentResult.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentResult.java new file mode 100644 index 00000000000..fcc83a9693d --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionDeploymentResult.java @@ -0,0 +1,34 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.deployment; + +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.functionActivator.deployment.DeploymentResult; + +public class BigQueryFunctionDeploymentResult extends DeploymentResult +{ + public MutableList errors; + + public BigQueryFunctionDeploymentResult(String activatorIdentifier, boolean result) + { + this.successful = result; + this.activatorIdentifier = activatorIdentifier; + } + + public BigQueryFunctionDeploymentResult(MutableList errors) + { + this.errors = errors; + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionGenerator.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionGenerator.java new file mode 100644 index 00000000000..3e917dcd558 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/java/org/finos/legend/engine/language/bigqueryFunction/deployment/BigQueryFunctionGenerator.java @@ -0,0 +1,60 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.deployment; + +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.tuple.Pair; +import org.eclipse.collections.impl.tuple.Tuples; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.plan.platform.PlanPlatform; +import org.finos.legend.pure.generated.*; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; + +public class BigQueryFunctionGenerator +{ + public static BigQueryFunctionArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, Function> routerExtensions) + { + Pair> artifactDetails = extractArtifactDetails(pureModel, activator, routerExtensions); + Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification bigQueryDatasourceSpecification = artifactDetails.getOne(); + RichIterable sqlExpressions = artifactDetails.getTwo(); + return new BigQueryFunctionArtifact(activator._functionName(), Lists.mutable.withAll(sqlExpressions), bigQueryDatasourceSpecification._projectId(), bigQueryDatasourceSpecification._defaultDataset()); + } + + private static Pair> extractArtifactDetails(PureModel pureModel, Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction activator, Function> routerExtensions) + { + PackageableFunction function = activator._function(); + Root_meta_pure_executionPlan_ExecutionPlan executionPlan = PlanGenerator.generateExecutionPlanAsPure((FunctionDefinition) function, null, null, null, pureModel, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel)); + RichIterable sqlExecutionNodes = + collectAllNodes(executionPlan._rootExecutionNode()).selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class); + + Root_meta_external_store_relational_runtime_RelationalDatabaseConnection relationalDatabaseConnection = (Root_meta_external_store_relational_runtime_RelationalDatabaseConnection) sqlExecutionNodes.getAny()._connection(); + Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification bigQueryDatasourceSpecification = ((Root_meta_pure_alloy_connections_alloy_specification_BigQueryDatasourceSpecification) relationalDatabaseConnection._datasourceSpecification()); + + return Tuples.pair( + bigQueryDatasourceSpecification, + sqlExecutionNodes + .collect(Root_meta_relational_mapping_SQLExecutionNode::_sqlQuery) + .select(x -> !x.toLowerCase().startsWith("alter"))); + } + + private static RichIterable collectAllNodes(Root_meta_pure_executionPlan_ExecutionNode node) + { + return Lists.mutable.with(node).withAll(node._executionNodes().flatCollect(BigQueryFunctionGenerator::collectAllNodes)); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.functionActivator.service.FunctionActivatorService b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.functionActivator.service.FunctionActivatorService new file mode 100644 index 00000000000..6fafe23bd24 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.functionActivator.service.FunctionActivatorService @@ -0,0 +1 @@ +org.finos.legend.engine.language.bigqueryFunction.api.BigQueryFunctionService \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension new file mode 100644 index 00000000000..5d10f14211b --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.bigqueryFunction.deployment.BigQueryFunctionArtifactGenerationExtension diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/test/java/org/finos/legend/engine/language/bigqueryFunction/api/TestValidation.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/test/java/org/finos/legend/engine/language/bigqueryFunction/api/TestValidation.java new file mode 100644 index 00000000000..9bcf6f77dd9 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-api/src/test/java/org/finos/legend/engine/language/bigqueryFunction/api/TestValidation.java @@ -0,0 +1,47 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.api; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.finos.legend.engine.functionActivator.api.FunctionActivatorAPI; +import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.modelManager.ModelManager; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.deployment.DeploymentMode; +import org.junit.Assert; +import org.junit.Test; + +import javax.ws.rs.core.Response; +import java.util.List; + +public class TestValidation +{ + private final FunctionActivatorAPI api = new FunctionActivatorAPI(new ModelManager(DeploymentMode.TEST), (PureModel pureModel) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(pureModel.getExecutionSupport()))); + + @Test + public void testList() throws Exception + { + Response response = api.list(null); + System.out.println(response.getEntity().toString()); + List info = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().readValue(response.getEntity().toString(), new TypeReference>(){}); + Assert.assertEquals(1, info.size()); + Assert.assertEquals("BigQuery Function", info.get(0).name); + Assert.assertEquals("Create a BigQuery Function that can activate in BigQuery.", info.get(0).description); + Assert.assertEquals("meta::protocols::pure::vX_X_X::metamodel::function::activator::bigQueryFunction::BigQueryFunction", info.get(0).configuration.topElement); + Assert.assertEquals(8, info.get(0).configuration.model.size()); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/pom.xml b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/pom.xml new file mode 100644 index 00000000000..7d72a13dd4a --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/pom.xml @@ -0,0 +1,102 @@ + + + + + org.finos.legend.engine + legend-engine-xts-bigqueryFunction + 4.34.1-SNAPSHOT + + 4.0.0 + + legend-engine-xt-bigqueryFunction-compiler + jar + Legend Engine - XT - BigQuery Function - Compiler + + + + + + org.finos.legend.pure + legend-pure-m3-core + + + + + + org.finos.legend.engine + legend-engine-language-pure-compiler + + + org.finos.legend.engine + legend-engine-xt-functionActivator-pure + + + org.finos.legend.engine + legend-engine-protocol-pure + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-pure + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-protocol + + + + + + org.eclipse.collections + eclipse-collections-api + + + + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-grammar + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test-jar + test + + + org.finos.legend.engine + legend-engine-language-pure-compiler + test-jar + test + + + + \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/BigQueryFunctionCompilerExtension.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/BigQueryFunctionCompilerExtension.java new file mode 100644 index 00000000000..50d127e75b1 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/BigQueryFunctionCompilerExtension.java @@ -0,0 +1,84 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.compiler.toPureGraph; + +import org.eclipse.collections.api.factory.Lists; +import org.finos.legend.engine.code.core.CoreFunctionActivatorCodeRepositoryProvider; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.CompileContext; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.Processor; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunction; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunctionDeploymentConfiguration; +import org.finos.legend.pure.generated.*; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; +import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor; + +import java.util.Collections; + +public class BigQueryFunctionCompilerExtension implements CompilerExtension +{ + // Here only for dependency check error ... + CoreFunctionActivatorCodeRepositoryProvider forDependencies; + + @Override + public CompilerExtension build() + { + return new BigQueryFunctionCompilerExtension(); + } + + @Override + public Iterable> getExtraProcessors() + { + return Lists.fixedSize.of( + Processor.newProcessor( + BigQueryFunction.class, + Lists.fixedSize.with(BigQueryFunctionDeploymentConfiguration.class), + this::buildBigQueryFunction + ), + Processor.newProcessor( + BigQueryFunctionDeploymentConfiguration.class, + this::buildDeploymentConfig + ) + ); + } + + public Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction buildBigQueryFunction(BigQueryFunction bigQueryFunction, CompileContext context) + { + try + { + PackageableFunction func = (PackageableFunction) context.resolvePackageableElement(FunctionDescriptor.functionDescriptorToId(bigQueryFunction.function), bigQueryFunction.sourceInformation); + return new Root_meta_external_function_activator_bigQueryFunction_BigQueryFunction_Impl( + bigQueryFunction.name, + null, + context.pureModel.getClass("meta::external::function::activator::bigQueryFunction::BigQueryFunction")) + ._functionName(bigQueryFunction.functionName) + ._function(func) + ._description(bigQueryFunction.description) + ._owner(bigQueryFunction.owner) + ._activationConfiguration(bigQueryFunction.activationConfiguration != null ? buildDeploymentConfig((BigQueryFunctionDeploymentConfiguration) bigQueryFunction.activationConfiguration, context) : null); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + public Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration buildDeploymentConfig(BigQueryFunctionDeploymentConfiguration configuration, CompileContext context) + { + return new Root_meta_external_function_activator_bigQueryFunction_BigQueryFunctionDeploymentConfiguration_Impl("") + ._target((Root_meta_external_store_relational_runtime_RelationalDatabaseConnection) context.resolveConnection(configuration.activationConnection.connection, configuration.sourceInformation)); + // ._stage(context.pureModel.getEnumValue("meta::external::function::activator::DeploymentStage", configuration.stage.name())); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension new file mode 100644 index 00000000000..8441e6e90c5 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.bigqueryFunction.compiler.toPureGraph.BigQueryFunctionCompilerExtension \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/test/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/test/TestBigQueryFunctionCompilationFromGrammar.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/test/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/test/TestBigQueryFunctionCompilationFromGrammar.java new file mode 100644 index 00000000000..413445fd0f6 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-compiler/src/test/java/org/finos/legend/engine/language/bigqueryFunction/compiler/toPureGraph/test/TestBigQueryFunctionCompilationFromGrammar.java @@ -0,0 +1,67 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.compiler.toPureGraph.test; + +import org.finos.legend.engine.language.pure.compiler.test.TestCompilationFromGrammar; +import org.junit.Test; + +public class TestBigQueryFunctionCompilationFromGrammar extends TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite +{ + @Override + public String getDuplicatedElementTestCode() + { + return "Class anything::Name {}\n" + + "###Mapping\n" + + "Mapping anything::somethingelse ()\n" + + "###BigQuery\n" + + "BigQueryFunction anything::Name\n" + + "{" + + " functionName : 'name';\n" + + " function : a::f():String[1];" + + "}\n"; + } + + @Override + public String getDuplicatedElementTestExpectedErrorMessage() + { + return "COMPILATION error at [5:1-7:32]: Duplicated element 'anything::Name'"; + } + + @Test + public void testHappyPath() + { + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test( + "function a::f():String[1]{'ok';}\n" + + "###BigQuery\n" + + "BigQueryFunction app::pack::MyApp\n" + + "{" + + " functionName : 'name';\n" + + " function : a::f():String[1];" + + "}\n", null); + } + + @Test + public void testFunctionError() + { + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test( + "function a::f():String[1]{'ok';}\n" + + "###BigQuery\n" + + "BigQueryFunction app::pack::MyApp\n" + + "{" + + " functionName : 'name';\n" + + " function : a::fz():String[1];" + + "}\n", " at [3:1-5:33]: Error in 'app::pack::MyApp': org.finos.legend.engine.shared.core.operational.errorManagement.EngineException: Can't find the packageable element 'a::fz__String_1_'"); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/pom.xml b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/pom.xml new file mode 100644 index 00000000000..e49ad97cdac --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/pom.xml @@ -0,0 +1,163 @@ + + + + + org.finos.legend.engine + legend-engine-xts-bigqueryFunction + 4.34.1-SNAPSHOT + + 4.0.0 + + legend-engine-xt-bigqueryFunction-grammar + jar + Legend Engine - XT - BigQuery Function - Grammar + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + copy-antlr-core-grammar + initialize + + unpack + + + + + org.finos.legend.engine + legend-engine-language-pure-grammar + jar + false + ${project.build.directory} + antlr/*.g4 + + + + + + + + org.antlr + antlr4-maven-plugin + + + + antlr4 + + + true + true + true + target/antlr + ${project.build.directory}/generated-sources + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + + test-jar + + + + + + + + + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-protocol + + + org.finos.legend.engine + legend-engine-language-pure-grammar + + + org.finos.legend.engine + legend-engine-protocol-pure + + + + + + org.eclipse.collections + eclipse-collections-api + + + org.eclipse.collections + eclipse-collections + + + + + + org.antlr + antlr4-runtime + compile + + + + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-shared-core + test-jar + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test-jar + test + + + + \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionLexerGrammar.g4 b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionLexerGrammar.g4 new file mode 100644 index 00000000000..469bf5791b5 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionLexerGrammar.g4 @@ -0,0 +1,15 @@ +lexer grammar BigQueryFunctionLexerGrammar; + +import M3LexerGrammar; + +BIGQUERY_FUNCTION: 'BigQueryFunction'; +BIGQUERY_FUNCTION__FUNCTION_NAME: 'functionName'; +BIGQUERY_FUNCTION__DESCRIPTION: 'description'; +BIGQUERY_FUNCTION__FUNCTION: 'function'; +BIGQUERY_FUNCTION__OWNER: 'owner'; +BIGQUERY_FUNCTION__ACTIVATION: 'activationConfiguration'; + +// ------------------------------------- CONFIGURATION ------------------------------- +CONFIGURATION: 'BigQueryFunctionDeploymentConfiguration'; +ACTIVATION_CONNECTION: 'activationConnection'; +DEPLOYMENT_STAGE: 'stage'; \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionParserGrammar.g4 b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionParserGrammar.g4 new file mode 100644 index 00000000000..7ece7329da8 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/BigQueryFunctionParserGrammar.g4 @@ -0,0 +1,67 @@ +parser grammar BigQueryFunctionParserGrammar; + +import M3ParserGrammar; + +options +{ + tokenVocab = BigQueryFunctionLexerGrammar; +} + +identifier: VALID_STRING | STRING | + BIGQUERY_FUNCTION | + BIGQUERY_FUNCTION__FUNCTION_NAME | + BIGQUERY_FUNCTION__DESCRIPTION | + BIGQUERY_FUNCTION__FUNCTION | + BIGQUERY_FUNCTION__OWNER | + BIGQUERY_FUNCTION__ACTIVATION| + CONFIGURATION| DEPLOYMENT_STAGE + | ACTIVATION_CONNECTION | + ALL | + LET | + ALL_VERSIONS | + ALL_VERSIONS_IN_RANGE | + TO_BYTES_FUNCTION + ; +// -------------------------------------- DEFINITION -------------------------------------- + +definition: (bigQueryFunction | deploymentConfig)* + EOF +; +bigQueryFunction: BIGQUERY_FUNCTION stereotypes? taggedValues? qualifiedName + BRACE_OPEN + ( + functionName + | description + | function + | owner + | activation + )* + BRACE_CLOSE; + +stereotypes: LESS_THAN LESS_THAN stereotype (COMMA stereotype)* GREATER_THAN GREATER_THAN; +stereotype: qualifiedName DOT identifier; +taggedValues: BRACE_OPEN taggedValue (COMMA taggedValue)* BRACE_CLOSE; +taggedValue: qualifiedName DOT identifier EQUAL STRING; + +functionName: BIGQUERY_FUNCTION__FUNCTION_NAME COLON STRING SEMI_COLON; + +description: BIGQUERY_FUNCTION__DESCRIPTION COLON STRING SEMI_COLON; + +function: BIGQUERY_FUNCTION__FUNCTION COLON functionIdentifier SEMI_COLON; + +owner : BIGQUERY_FUNCTION__OWNER COLON STRING SEMI_COLON; + +activation: BIGQUERY_FUNCTION__ACTIVATION COLON qualifiedName SEMI_COLON ; + +// ----------------------------------- Deployment ------------------------------------------------------ +deploymentConfig: CONFIGURATION qualifiedName + BRACE_OPEN + activationConnection + BRACE_CLOSE +; + +activationConnection: ACTIVATION_CONNECTION COLON qualifiedName SEMI_COLON +; + +stage: DEPLOYMENT_STAGE COLON STRING SEMI_COLON +; \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionGrammarParserExtension.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionGrammarParserExtension.java new file mode 100644 index 00000000000..a2c960bbd38 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionGrammarParserExtension.java @@ -0,0 +1,70 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.grammar.from; + +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.grammar.from.ParserErrorListener; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserContext; +import org.finos.legend.engine.language.pure.grammar.from.SectionSourceCode; +import org.finos.legend.engine.language.pure.grammar.from.SourceCodeParserInfo; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.BigQueryFunctionLexerGrammar; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.BigQueryFunctionParserGrammar; +import org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension; +import org.finos.legend.engine.language.pure.grammar.from.extension.SectionParser; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.Section; + +import java.util.function.Consumer; + +public class BigQueryFunctionGrammarParserExtension implements PureGrammarParserExtension +{ + public static final String NAME = "BigQuery"; + + @Override + public Iterable getExtraSectionParsers() + { + return Lists.fixedSize.of(SectionParser.newParser(NAME, BigQueryFunctionGrammarParserExtension::parseSection)); + } + + private static Section parseSection(SectionSourceCode sectionSourceCode, Consumer elementConsumer, PureGrammarParserContext context) + { + SourceCodeParserInfo parserInfo = getBigQueryFunctionInfo(sectionSourceCode); + DefaultCodeSection section = new DefaultCodeSection(); + section.parserName = sectionSourceCode.sectionType; + section.sourceInformation = parserInfo.sourceInformation; + + BigQueryFunctionTreeWalker walker = new BigQueryFunctionTreeWalker(parserInfo.input, parserInfo.walkerSourceInformation, elementConsumer, section); + walker.visit((BigQueryFunctionParserGrammar.DefinitionContext) parserInfo.rootContext); + + return section; + } + + private static SourceCodeParserInfo getBigQueryFunctionInfo(SectionSourceCode sectionSourceCode) + { + CharStream input = CharStreams.fromString(sectionSourceCode.code); + ParserErrorListener errorListener = new ParserErrorListener(sectionSourceCode.walkerSourceInformation, BigQueryFunctionParserGrammar.VOCABULARY); + BigQueryFunctionLexerGrammar lexer = new BigQueryFunctionLexerGrammar(input); + lexer.removeErrorListeners(); + lexer.addErrorListener(errorListener); + BigQueryFunctionParserGrammar parser = new BigQueryFunctionParserGrammar(new CommonTokenStream(lexer)); + parser.removeErrorListeners(); + parser.addErrorListener(errorListener); + return new SourceCodeParserInfo(sectionSourceCode.code, input, sectionSourceCode.sourceInformation, sectionSourceCode.walkerSourceInformation, lexer, parser, parser.definition()); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionTreeWalker.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionTreeWalker.java new file mode 100644 index 00000000000..306d4a91388 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/from/BigQueryFunctionTreeWalker.java @@ -0,0 +1,140 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.grammar.from; + +import org.antlr.v4.runtime.CharStream; +import org.eclipse.collections.impl.factory.Lists; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.grammar.from.ParseTreeWalkerSourceInformation; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserUtility; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.BigQueryFunctionParserGrammar; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunctionDeploymentConfiguration; +import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.ConnectionPointer; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.StereotypePtr; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TagPtr; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TaggedValue; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.DefaultCodeSection; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunction; + +import java.util.Collections; +import java.util.List; +import java.util.function.Consumer; + +public class BigQueryFunctionTreeWalker +{ + private final CharStream input; + private final ParseTreeWalkerSourceInformation walkerSourceInformation; + private final Consumer elementConsumer; + private final DefaultCodeSection section; + + public BigQueryFunctionTreeWalker(CharStream input, ParseTreeWalkerSourceInformation walkerSourceInformation, Consumer elementConsumer, DefaultCodeSection section) + { + this.input = input; + this.walkerSourceInformation = walkerSourceInformation; + this.elementConsumer = elementConsumer; + this.section = section; + } + + public void visit(BigQueryFunctionParserGrammar.DefinitionContext ctx) + { + if (ctx.bigQueryFunction() != null && !ctx.bigQueryFunction().isEmpty()) + { + ctx.bigQueryFunction().stream().map(this::visitBigQueryFunction).peek(e -> this.section.elements.add(e.getPath())).forEach(this.elementConsumer); + } + if (ctx.deploymentConfig() != null && !ctx.deploymentConfig().isEmpty()) + { + ctx.deploymentConfig().stream().map(this::visitDeploymentConfig).peek(e -> this.section.elements.add(e.getPath())).forEach(this.elementConsumer); + } + } + + private BigQueryFunctionDeploymentConfiguration visitDeploymentConfig(BigQueryFunctionParserGrammar.DeploymentConfigContext ctx) + { + BigQueryFunctionDeploymentConfiguration config = new BigQueryFunctionDeploymentConfiguration(); + ConnectionPointer pointer = new ConnectionPointer(); + pointer.connection = PureGrammarParserUtility.fromQualifiedName(ctx.activationConnection().qualifiedName().packagePath() == null + ? Collections.emptyList() : ctx.activationConnection().qualifiedName().packagePath().identifier(), ctx.activationConnection().qualifiedName().identifier()); + pointer.sourceInformation = walkerSourceInformation.getSourceInformation(ctx.activationConnection().qualifiedName()); + config.activationConnection = pointer; + return config; + } + + private BigQueryFunction visitBigQueryFunction(BigQueryFunctionParserGrammar.BigQueryFunctionContext ctx) + { + BigQueryFunction bigQueryFunction = new BigQueryFunction(); + bigQueryFunction.name = PureGrammarParserUtility.fromIdentifier(ctx.qualifiedName().identifier()); + bigQueryFunction._package = ctx.qualifiedName().packagePath() == null ? "" : PureGrammarParserUtility.fromPath(ctx.qualifiedName().packagePath().identifier()); + bigQueryFunction.sourceInformation = walkerSourceInformation.getSourceInformation(ctx); + bigQueryFunction.stereotypes = ctx.stereotypes() == null ? Lists.mutable.empty() : this.visitStereotypes(ctx.stereotypes()); + bigQueryFunction.taggedValues = ctx.taggedValues() == null ? Lists.mutable.empty() : this.visitTaggedValues(ctx.taggedValues()); + + BigQueryFunctionParserGrammar.FunctionNameContext functionNameContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.functionName(), "functionName", bigQueryFunction.sourceInformation); + bigQueryFunction.functionName = PureGrammarParserUtility.fromGrammarString(functionNameContext.STRING().getText(), true); + BigQueryFunctionParserGrammar.FunctionContext functionContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.function(), "function", bigQueryFunction.sourceInformation); + bigQueryFunction.function = functionContext.functionIdentifier().getText(); + BigQueryFunctionParserGrammar.OwnerContext ownerContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.owner(), "owner", bigQueryFunction.sourceInformation); + if (ownerContext != null) + { + bigQueryFunction.owner = PureGrammarParserUtility.fromGrammarString(ownerContext.STRING().getText(), true); + } + BigQueryFunctionParserGrammar.DescriptionContext descriptionContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.description(), "description", bigQueryFunction.sourceInformation); + if (descriptionContext != null) + { + bigQueryFunction.description = PureGrammarParserUtility.fromGrammarString(descriptionContext.STRING().getText(), true); + } + BigQueryFunctionParserGrammar.ActivationContext activationContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.activation(), "activationConfiguration", bigQueryFunction.sourceInformation); + if (activationContext != null) + { + BigQueryFunctionDeploymentConfiguration config = new BigQueryFunctionDeploymentConfiguration(); + ConnectionPointer connectionPointer = new ConnectionPointer(); + connectionPointer.connection = activationContext.qualifiedName().getText(); + config.activationConnection = connectionPointer; + + bigQueryFunction.activationConfiguration = config; + } + return bigQueryFunction; + } + + private List visitTaggedValues(BigQueryFunctionParserGrammar.TaggedValuesContext ctx) + { + return ListIterate.collect(ctx.taggedValue(), taggedValueContext -> + { + TaggedValue taggedValue = new TaggedValue(); + TagPtr tagPtr = new TagPtr(); + taggedValue.tag = tagPtr; + tagPtr.profile = PureGrammarParserUtility.fromQualifiedName(taggedValueContext.qualifiedName().packagePath() == null ? Collections.emptyList() : taggedValueContext.qualifiedName().packagePath().identifier(), taggedValueContext.qualifiedName().identifier()); + tagPtr.value = PureGrammarParserUtility.fromIdentifier(taggedValueContext.identifier()); + taggedValue.value = PureGrammarParserUtility.fromGrammarString(taggedValueContext.STRING().getText(), true); + taggedValue.tag.profileSourceInformation = this.walkerSourceInformation.getSourceInformation(taggedValueContext.qualifiedName()); + taggedValue.tag.sourceInformation = this.walkerSourceInformation.getSourceInformation(taggedValueContext.identifier()); + taggedValue.sourceInformation = this.walkerSourceInformation.getSourceInformation(taggedValueContext); + return taggedValue; + }); + } + + private List visitStereotypes(BigQueryFunctionParserGrammar.StereotypesContext ctx) + { + return ListIterate.collect(ctx.stereotype(), stereotypeContext -> + { + StereotypePtr stereotypePtr = new StereotypePtr(); + stereotypePtr.profile = PureGrammarParserUtility.fromQualifiedName(stereotypeContext.qualifiedName().packagePath() == null ? Collections.emptyList() : stereotypeContext.qualifiedName().packagePath().identifier(), stereotypeContext.qualifiedName().identifier()); + stereotypePtr.value = PureGrammarParserUtility.fromIdentifier(stereotypeContext.identifier()); + stereotypePtr.profileSourceInformation = this.walkerSourceInformation.getSourceInformation(stereotypeContext.qualifiedName()); + stereotypePtr.sourceInformation = this.walkerSourceInformation.getSourceInformation(stereotypeContext); + return stereotypePtr; + }); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/to/BigQueryFunctionGrammarComposer.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/to/BigQueryFunctionGrammarComposer.java new file mode 100644 index 00000000000..7c2f5ef1d8a --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/java/org/finos/legend/engine/language/bigqueryFunction/grammar/to/BigQueryFunctionGrammarComposer.java @@ -0,0 +1,90 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.grammar.to; + +import org.eclipse.collections.api.block.function.Function3; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.utility.Iterate; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.bigqueryFunction.grammar.from.BigQueryFunctionGrammarParserExtension; +import org.finos.legend.engine.language.pure.grammar.to.PureGrammarComposerContext; +import org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunction; +import org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunctionDeploymentConfiguration; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; + +import java.util.Collections; +import java.util.List; + +import static org.finos.legend.engine.language.pure.grammar.to.HelperDomainGrammarComposer.renderAnnotations; + +public class BigQueryFunctionGrammarComposer implements PureGrammarComposerExtension +{ + private static String renderElement(PackageableElement element) + { + if (element instanceof BigQueryFunction) + { + return renderBigQueryFunction((BigQueryFunction) element); + } + return "/* Can't transform element '" + element.getPath() + "' in this section */"; + } + + private static String renderBigQueryFunction(BigQueryFunction app) + { + String packageName = app._package == null || app._package.isEmpty() ? app.name : app._package + "::" + app.name; + + return "BigQueryFunction " + renderAnnotations(app.stereotypes, app.taggedValues) + packageName + "\n" + + "{\n" + + " functionName : '" + app.functionName + "';\n" + + " function : " + app.function + ";\n" + + (app.owner == null ? "" : " owner : '" + app.owner + "';\n") + + (app.description == null ? "" : " description : '" + app.description + "';\n") + + (app.activationConfiguration == null ? "" : " activationConfiguration : " + ((BigQueryFunctionDeploymentConfiguration) app.activationConfiguration).activationConnection.connection + ";\n") + + "}"; + } + + @Override + public List, PureGrammarComposerContext, String, String>> getExtraSectionComposers() + { + return Lists.fixedSize.with((elements, context, sectionName) -> + { + if (!BigQueryFunctionGrammarParserExtension.NAME.equals(sectionName)) + { + return null; + } + return ListIterate.collect(elements, element -> + { + if (element instanceof BigQueryFunction) + { + return renderBigQueryFunction((BigQueryFunction) element); + } + return "/* Can't transform element '" + element.getPath() + "' in this section */"; + }).makeString("\n\n"); + }); + } + + @Override + public List, PureGrammarComposerContext, List, PureGrammarComposerExtension.PureFreeSectionGrammarComposerResult>> getExtraFreeSectionComposers() + { + return Collections.singletonList((elements, context, composedSections) -> + { + MutableList composableElements = Iterate.select(elements, e -> (e instanceof BigQueryFunction), Lists.mutable.empty()); + return composableElements.isEmpty() + ? null + : new PureFreeSectionGrammarComposerResult(composableElements.asLazy().collect(BigQueryFunctionGrammarComposer::renderElement).makeString("###" + BigQueryFunctionGrammarParserExtension.NAME + "\n", "\n\n", ""), composableElements); + }); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension new file mode 100644 index 00000000000..5db153ec50a --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.bigqueryFunction.grammar.from.BigQueryFunctionGrammarParserExtension \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension new file mode 100644 index 00000000000..c96ed6ec60c --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.grammar.to.extension.PureGrammarComposerExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.bigqueryFunction.grammar.to.BigQueryFunctionGrammarComposer \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryFunctionRoundtrip.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryFunctionRoundtrip.java new file mode 100644 index 00000000000..be4a0658090 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryFunctionRoundtrip.java @@ -0,0 +1,46 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.grammar.test; + +import org.finos.legend.engine.language.pure.grammar.test.TestGrammarRoundtrip; +import org.junit.Test; + +public class TestBigQueryFunctionRoundtrip extends TestGrammarRoundtrip.TestGrammarRoundtripTestSuite +{ + @Test + public void testBigQueryFunction() + { + test("###BigQuery\n" + + "BigQueryFunction <> {a::A.val = 'ok'} xx::MyApp\n" + + "{\n" + + " functionName : 'MyApp';\n" + + " function : zxx(Integer[1]):String[1];\n" + + " owner : 'pierre';\n" + + " description : 'A super nice app!';\n" + + " activationConfiguration : com::gs::test::TestConnection;\n" + + "}\n"); + } + + @Test + public void testBigQueryFunctionMinimal() + { + test("###BigQuery\n" + + "BigQueryFunction xx::MyApp\n" + + "{\n" + + " functionName : 'MyApp';\n" + + " function : zxx(Integer[1]):String[1];\n" + + "}\n"); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryParsing.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryParsing.java new file mode 100644 index 00000000000..e700a9fd43e --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-grammar/src/test/java/org/finos/legend/engine/language/bigqueryFunction/grammar/test/TestBigQueryParsing.java @@ -0,0 +1,74 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.bigqueryFunction.grammar.test; + +import org.antlr.v4.runtime.Vocabulary; +import org.eclipse.collections.impl.list.mutable.ListAdapter; +import org.finos.legend.engine.language.pure.grammar.from.antlr4.BigQueryFunctionParserGrammar; +import org.finos.legend.engine.language.pure.grammar.test.TestGrammarParser; +import org.junit.Test; + +import java.util.List; + +public class TestBigQueryParsing extends TestGrammarParser.TestGrammarParserTestSuite +{ + @Override + public Vocabulary getParserGrammarVocabulary() + { + return BigQueryFunctionParserGrammar.VOCABULARY; + } + + @Override + public String getParserGrammarIdentifierInclusionTestCode(List keywords) + { + return "###BigQuery\n" + + "BigQueryFunction " + ListAdapter.adapt(keywords).makeString("::") + "\n" + + "{\n" + + " function : a::f():String[1];" + + " functionName : 'sass';\n" + + "}\n"; + } + + @Test + public void testGetParserErrorWrongProperty() + { + test("###BigQuery\n" + + "BigQueryFunction x::A\n" + + "{\n" + + " functioName : 'sass';\n" + + "}\n", "PARSER error at [4:4-14]: Unexpected token 'functioName'. Valid alternatives: ['functionName', 'description', 'function', 'owner', 'activationConfiguration']"); + } + + @Test + public void testGetParserErrorMissingApplicationName() + { + test("###BigQuery\n" + + "BigQueryFunction x::A\n" + + "{\n" + + " owner : 'pierre';\n" + + "}\n", "PARSER error at [2:1-5:1]: Field 'functionName' is required"); + } + + @Test + public void testGetParserErrorMissingFunction() + { + test("###BigQuery\n" + + "BigQueryFunction x::A\n" + + "{\n" + + " functionName : 'MyApp';\n" + + " owner : 'pierre';\n" + + "}\n", "PARSER error at [2:1-6:1]: Field 'function' is required"); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/pom.xml b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/pom.xml new file mode 100644 index 00000000000..73f2779c33f --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/pom.xml @@ -0,0 +1,46 @@ + + + + + org.finos.legend.engine + legend-engine-xts-bigqueryFunction + 4.34.1-SNAPSHOT + + 4.0.0 + + legend-engine-xt-bigqueryFunction-protocol + jar + Legend Engine - XT - BigQuery Function - Protocol + + + + org.finos.legend.engine + legend-engine-protocol-pure + + + + org.finos.legend.engine + legend-engine-xt-functionActivator-protocol + ${project.version} + + + + org.eclipse.collections + eclipse-collections-api + + + \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunction.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunction.java new file mode 100644 index 00000000000..89cc7e74bf2 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunction.java @@ -0,0 +1,27 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.bigqueryFunction.metamodel; + +import org.finos.legend.engine.protocol.functionActivator.metamodel.FunctionActivator; + +//------------------------------------------------------------ +// Should be generated out of the Pure protocol specification +//------------------------------------------------------------ +public class BigQueryFunction extends FunctionActivator +{ + public String functionName; + public String description; + public String owner; +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionDeploymentConfiguration.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionDeploymentConfiguration.java new file mode 100644 index 00000000000..790a0ae97a0 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionDeploymentConfiguration.java @@ -0,0 +1,28 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.bigqueryFunction.metamodel; + +import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.ConnectionPointer; + +public class BigQueryFunctionDeploymentConfiguration extends DeploymentConfiguration +{ + public ConnectionPointer activationConnection; + + public BigQueryFunctionDeploymentConfiguration() + { + + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionProtocolExtension.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionProtocolExtension.java new file mode 100644 index 00000000000..b1ea493736d --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/java/org/finos/legend/engine/protocol/bigqueryFunction/metamodel/BigQueryFunctionProtocolExtension.java @@ -0,0 +1,49 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.bigqueryFunction.metamodel; + +import org.eclipse.collections.api.block.function.Function0; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.finos.legend.engine.protocol.pure.v1.extension.ProtocolSubTypeInfo; +import org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; + +import java.util.List; +import java.util.Map; + +public class BigQueryFunctionProtocolExtension implements PureProtocolExtension +{ + public static String packageJSONType = "bigQueryFunction"; + + @Override + public List>>> getExtraProtocolSubTypeInfoCollectors() + { + return Lists.fixedSize.with(() -> Lists.mutable.with( + ProtocolSubTypeInfo.newBuilder(PackageableElement.class) + .withSubtype(BigQueryFunction.class, packageJSONType) + .build(), + ProtocolSubTypeInfo.newBuilder(PackageableElement.class) + .withSubtype(BigQueryFunctionDeploymentConfiguration.class, packageJSONType + "Config") + .build() + )); + } + + @Override + public Map, String> getExtraProtocolToClassifierPathMap() + { + return Maps.mutable.with(BigQueryFunction.class, "meta::external::function::activator::bigQueryFunction::BigQueryFunction"); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/resources/META-INF/services/org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/resources/META-INF/services/org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension new file mode 100644 index 00000000000..7ab4de68c73 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-protocol/src/main/resources/META-INF/services/org.finos.legend.engine.protocol.pure.v1.extension.PureProtocolExtension @@ -0,0 +1 @@ +org.finos.legend.engine.protocol.bigqueryFunction.metamodel.BigQueryFunctionProtocolExtension \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/pom.xml b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/pom.xml new file mode 100644 index 00000000000..0ab4ea41ccc --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/pom.xml @@ -0,0 +1,156 @@ + + + + + + org.finos.legend.engine + legend-engine-xts-bigqueryFunction + 4.34.1-SNAPSHOT + + 4.0.0 + + legend-engine-xt-bigqueryFunction-pure + jar + Legend Engine - XT - BigQuery Function - PAR/JAVA + + + + + org.finos.legend.pure + legend-pure-maven-generation-par + + src/main/resources + ${legend.pure.version} + + core_bigqueryfunction + + + ${project.basedir}/src/main/resources/core_bigqueryfunction.definition.json + + + + + generate-sources + + build-pure-jar + + + + + + org.finos.legend.engine + legend-engine-xt-functionActivator-pure + ${project.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + org.finos.legend.pure + legend-pure-maven-generation-java + + + compile + + build-pure-compiled-jar + + + true + true + modular + true + + core_bigqueryfunction + + + + + + + org.finos.legend.engine + legend-engine-xt-functionActivator-pure + ${project.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + + + + + org.finos.legend.pure + legend-pure-m4 + + + org.finos.legend.pure + legend-pure-m3-core + + + org.finos.legend.pure + legend-pure-runtime-java-engine-compiled + + + + org.finos.legend.engine + legend-engine-pure-platform-java + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + + + org.finos.legend.engine + legend-engine-pure-platform-functions-java + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + + + org.finos.legend.engine + legend-engine-xt-functionActivator-pure + + + + org.eclipse.collections + eclipse-collections + + + org.eclipse.collections + eclipse-collections-api + + + diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/java/org/finos/legend/pure/code/core/CoreBigQueryFunctionCodeRepositoryProvider.java b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/java/org/finos/legend/pure/code/core/CoreBigQueryFunctionCodeRepositoryProvider.java new file mode 100644 index 00000000000..2817e421404 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/java/org/finos/legend/pure/code/core/CoreBigQueryFunctionCodeRepositoryProvider.java @@ -0,0 +1,28 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.code.core; + +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository; +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider; +import org.finos.legend.pure.m3.serialization.filesystem.repository.GenericCodeRepository; + +public class CoreBigQueryFunctionCodeRepositoryProvider implements CodeRepositoryProvider +{ + @Override + public CodeRepository repository() + { + return GenericCodeRepository.build("core_bigqueryfunction.definition.json"); + } +} diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider new file mode 100644 index 00000000000..a6852036018 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider @@ -0,0 +1 @@ +org.finos.legend.pure.code.core.CoreBigQueryFunctionCodeRepositoryProvider \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction.definition.json b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction.definition.json new file mode 100644 index 00000000000..0987cf1d134 --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction.definition.json @@ -0,0 +1,9 @@ +{ + "name": "core_bigqueryfunction", + "pattern": "(meta::external::function::activator::bigQueryFunction|meta::protocols)(::.*)?", + "dependencies": [ + "platform", + "core_function_activator", + "core_relational" + ] +} \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction/metamodel/metamodel.pure b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction/metamodel/metamodel.pure new file mode 100644 index 00000000000..0bfb77ad25d --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/legend-engine-xt-bigqueryFunction-pure/src/main/resources/core_bigqueryfunction/metamodel/metamodel.pure @@ -0,0 +1,26 @@ +import meta::external::function::activator::*; + +Class meta::external::function::activator::bigQueryFunction::BigQueryFunction extends FunctionActivator +{ + functionName : String[1]; + description : String[0..1]; + owner : String[0..1]; +} + +Class meta::external::function::activator::bigQueryFunction::BigQueryFunctionDeploymentConfiguration extends DeploymentConfiguration +{ + target: meta::external::store::relational::runtime::RelationalDatabaseConnection[1]; +} + +Class meta::external::function::activator::bigQueryFunction::BigQueryFunctionDeploymentResult extends DeploymentResult +{ + +} + +// This section needs to be code generated from the section above +Class meta::protocols::pure::vX_X_X::metamodel::function::activator::bigQueryFunction::BigQueryFunction extends meta::protocols::pure::vX_X_X::metamodel::function::activator::FunctionActivator +{ + functionName : String[1]; + description : String[0..1]; + owner : String[0..1]; +} \ No newline at end of file diff --git a/legend-engine-xts-bigqueryFunction/pom.xml b/legend-engine-xts-bigqueryFunction/pom.xml new file mode 100644 index 00000000000..6afc3e4ae9d --- /dev/null +++ b/legend-engine-xts-bigqueryFunction/pom.xml @@ -0,0 +1,36 @@ + + + + + org.finos.legend.engine + legend-engine + 4.34.1-SNAPSHOT + + 4.0.0 + + legend-engine-xts-bigqueryFunction + pom + Legend Engine - XTS - BigQuery Function + + + legend-engine-xt-bigqueryFunction-api + legend-engine-xt-bigqueryFunction-compiler + legend-engine-xt-bigqueryFunction-grammar + legend-engine-xt-bigqueryFunction-protocol + legend-engine-xt-bigqueryFunction-pure + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index e4c19e20450..c429648312a 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,7 @@ legend-engine-xts-functionActivator legend-engine-xts-snowflakeApp + legend-engine-xts-bigqueryFunction legend-engine-xts-service legend-engine-xts-persistence legend-engine-xts-hostedService @@ -642,6 +643,36 @@ legend-engine-xt-snowflakeApp-generator ${project.version} + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-protocol + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-pure + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-compiler + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-grammar + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-bigqueryFunction-api + ${project.version} + + + + + + org.finos.legend.engine legend-engine-xt-authentication-pure