diff --git a/CODEOWNERS b/CODEOWNERS index 4ec886084ef..9ae1e6d5b14 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,10 +1,10 @@ -* @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari +* @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs /.github/** @finos/legend-engine-maintainers /docs/** @finos/legend-engine-maintainers /legend-engine-application-query/** @finos/legend-engine-maintainers /legend-engine-config/** @finos/legend-engine-maintainers -/legend-engine-core/** @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari -/legend-engine-pure/** @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari +/legend-engine-core/** @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs +/legend-engine-pure/** @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs /legend-engine-xts-analytics/** @finos/legend-engine-maintainers /legend-engine-xts-authentication/** @finos/legend-engine-maintainers /legend-engine-xts-avro/** @finos/legend-engine-maintainers @@ -17,7 +17,7 @@ /legend-engine-xts-flatdata/** @finos/legend-engine-maintainers /legend-engine-xts-functionActivator/** @finos/legend-engine-maintainers /legend-engine-xts-generation/** @finos/legend-engine-maintainers -/legend-engine-xts-graphQL/** @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari +/legend-engine-xts-graphQL/** @pierredebelen @epsstan @davidharte-gs @gs-ssh16 @hardikmaheshwari @aormerod-gs /legend-engine-xts-haskell/** @finos/legend-engine-maintainers /legend-engine-xts-iceberg/** @finos/legend-engine-maintainers /legend-engine-xts-java/** @finos/legend-engine-maintainers @@ -37,4 +37,4 @@ /legend-engine-xts-text/** @finos/legend-engine-maintainers /legend-engine-xts-xml/** @finos/legend-engine-maintainers .gitignore @finos/legend-engine-maintainers -pom.xml @finos/legend-engine-maintainers \ No newline at end of file +pom.xml @finos/legend-engine-maintainers diff --git a/docs/externalFormat/extensions/FlatData/integration-with-external-format-contract.md b/docs/externalFormat/extensions/FlatData/integration-with-external-format-contract.md index 82a985bfffb..2c1f372ae7b 100644 --- a/docs/externalFormat/extensions/FlatData/integration-with-external-format-contract.md +++ b/docs/externalFormat/extensions/FlatData/integration-with-external-format-contract.md @@ -6,45 +6,45 @@ This page explains the integration of Flatdata with External Format Contract Flatdata integrates with external format ecosystem via following contracts : -1. [Flatdata ExternalFormat contract instance](../../../../legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/externalFormatContract.pure) -2. [Flatdata ExternalFormat LegendJavaPlatformBindingDescriptor](../../../../legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/descriptor.pure) -3. [Flatdata External Format Extension](../../../../legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataExternalFormatExtension.java) -4. [Flatadata Runtime extension](../../../../legend-engine-xt-flatdata-runtime/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataRuntimeExtension.java) +1. [Flatdata ExternalFormat contract instance](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/externalFormatContract.pure) +2. [Flatdata ExternalFormat LegendJavaPlatformBindingDescriptor](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/descriptor.pure) +3. [Flatdata External Format Extension](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataExternalFormatExtension.java) +4. [Flatadata Runtime extension](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-runtime/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataRuntimeExtension.java) ### Flatdata module structure Flatdata source code is distributed across 6 core modules as explained below : 1. `legend-engine-xt-flatdata-pure`: This module contains pure code for flatdata format. It hosts - - [Flatdata ExternalFormat contract instance](../../../../legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/externalFormatContract.pure) - - [Flatdata metamodel](../../../../legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/metamodel.pure) - - [Flatdata to Pure models transformation logic](../../../../legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/transformation/toPure) - - [Flatdata from Pure models transformation logic](../../../../legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/transformation/fromPure) - - [Flatdata binding validation logic](../../../../legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/binding/validation) + - [Flatdata ExternalFormat contract instance](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/externalFormatContract.pure) + - [Flatdata metamodel](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/metamodel/metamodel.pure) + - [Flatdata to Pure models transformation logic](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/transformation/toPure) + - [Flatdata from Pure models transformation logic](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/transformation/fromPure) + - [Flatdata binding validation logic](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/src/main/resources/core_external_format_flatdata/binding/validation) 2. `legend-engine-xt-flatdata-shared`: This is a lightweight module containing contracts/code shared by generation/execution logic. It hosts - - [Flatdata protocol classes](../../../../legend-engine-xt-flatdata-shared/pom.xml) : These are generated using a plugin. - - [Flatdata driver contracts](../../../../legend-engine-xt-flatdata-shared/src/main/java/org/finos/legend/engine/external/format/flatdata/driver/spi) - - [Flatdata core drivers](../../../../legend-engine-xt-flatdata-shared/src/main/java/org/finos/legend/engine/external/format/flatdata/driver/core) + - [Flatdata protocol classes](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/pom.xml) : These are generated using a plugin. + - [Flatdata driver contracts](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/src/main/java/org/finos/legend/engine/external/format/flatdata/driver/spi) + - [Flatdata core drivers](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/src/main/java/org/finos/legend/engine/external/format/flatdata/driver/core) 3. `legend-engine-xt-flatdata-model`: This module links flatdata to external format ecosystem. It hosts - - [Flatdata External Format Extension](../../../../legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataExternalFormatExtension.java) - - [Flatdata Parser](../../../../legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/grammar/fromPure) - - [Flatdata Composer](../../../../legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/grammar/toPure) - - [Flatdata Compiler](../../../../legend-engine-xt-flatdata-model/pom.xml) : This transformation is generated using a plugin - - [Flatdata Transformer](../../../../legend-engine-xt-flatdata-model/pom.xml) : This transformation is generated using a plugin - - [Flatdata Extension Service File](../../../../legend-engine-xt-flatdata-model/src/main/resources/META-INF/services/org.finos.legend.engine.external.shared.format.model.ExternalFormatExtension) + - [Flatdata External Format Extension](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataExternalFormatExtension.java) + - [Flatdata Parser](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/grammar/fromPure) + - [Flatdata Composer](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/src/main/java/org/finos/legend/engine/external/format/flatdata/grammar/toPure) + - [Flatdata Compiler](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/pom.xml) : This transformation is generated using a plugin + - [Flatdata Transformer](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/pom.xml) : This transformation is generated using a plugin + - [Flatdata Extension Service File](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/src/main/resources/META-INF/services/org.finos.legend.engine.external.shared.format.model.ExternalFormatExtension) 4. `legend-engine-xt-flatdata-javaPlatformBinding-pure`: This module contains pure code to generate java classes for serialization/deserialization to/from flatdata. It hosts - - [Flatdata ExternalFormat LegendJavaPlatformBindingDescriptor](../../../../legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/descriptor.pure) - - [Flatdata internalize descriptor](../../../../legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/internalize.pure) - - [Flatdata externalize descriptor](../../../../legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/externalize.pure) + - [Flatdata ExternalFormat LegendJavaPlatformBindingDescriptor](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/descriptor.pure) + - [Flatdata internalize descriptor](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/internalize.pure) + - [Flatdata externalize descriptor](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/src/main/resources/core_external_format_flatdata_java_platform_binding/legendJavaPlatformBinding/externalize.pure) 5. `legend-engine-xt-flatdata-runtime`: This module contains logic to execute externalize/internalize on Flatdata. It hosts - - [Flatadata runtime extension](../../../../legend-engine-xt-flatdata-runtime/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataRuntimeExtension.java) - - [Flatdata runtime extension Service File](../../../../legend-engine-xt-flatdata-runtime/src/main/resources/META-INF/services) + - [Flatadata runtime extension](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-runtime/src/main/java/org/finos/legend/engine/external/format/flatdata/FlatDataRuntimeExtension.java) + - [Flatdata runtime extension Service File](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-runtime/src/main/resources/META-INF/services) -6. `legend-engine-xt-flatdata-test`: This module contains logic to execute flatdata tests in maven pipeline. +6. `legend-engine-xt-flatdata-javaPlatformBinding-test`: This module contains logic to execute flatdata tests in maven pipeline. -Apart from these core modules we believe flatdata is a generic format which should cater to variety of file formats. Hence we have structured flatdata to be extensible and support variety of other file formats (like bloomberg) via [FlatDataDriverDescription](../../../../legend-engine-xt-flatdata-shared/src/main/java/org/finos/legend/engine/external/format/flatdata/driver/spi/FlatDataDriverDescription.java) implementations
+Apart from these core modules we believe flatdata is a generic format which should cater to variety of file formats. Hence we have structured flatdata to be extensible and support variety of other file formats (like bloomberg) via [FlatDataDriverDescription](../../../../legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/src/main/java/org/finos/legend/engine/external/format/flatdata/driver/spi/FlatDataDriverDescription.java) implementations
Module: `legend-engine-xt-flatdata-driver-bloomberg` serves as an example how flatdata can be extended for more formats. diff --git a/docs/externalFormat/steps-to-add-new-external-format.md b/docs/externalFormat/steps-to-add-new-external-format.md index a0e036e48d0..ce59e90b2f3 100644 --- a/docs/externalFormat/steps-to-add-new-external-format.md +++ b/docs/externalFormat/steps-to-add-new-external-format.md @@ -6,7 +6,7 @@ This page describes steps required to add support for a new external format. ### Pure -- Implement [ExternalFormatContract](../../legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure) : External Format Contract defines your format in legend ecosystem and links all its offered features and configurations. It is defined by following fields +- Implement [ExternalFormatContract](../../legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure) : External Format Contract defines your format in legend ecosystem and links all its offered features and configurations. It is defined by following fields - `id` : Provide id for your External Format - `contentTypes` : Provide content types for your External Format - `externalFormatMetamodel` : Define Metamodel for External Format @@ -15,13 +15,13 @@ This page describes steps required to add support for a new external format. - Everything except id and schema metamodel is optional for defining an external format. You may choose to add features you need with your external format in parts and as you need them.
Each external format can choose to implement following features - - `externalFormatToPureDescriptor` : Pure Model generation from External Format Schema - This is needed when you want to allow users of your external format to generate Pure models from external format schema. - - To add support for this you need to implement [ExternalFormatToPureDescriptor](../../legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure). This requires you to - + - To add support for this you need to implement [ExternalFormatToPureDescriptor](../../legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure). This requires you to - - Model SchemaToModelConfiguration for your external format - Implement a function which takes in an instance of SchemaSet (holding schema instances of your external format) and modeled configuration and generates packageable elements - Provide a default config for your SchemaToModelConfiguration - `externalFormatFromPureDescriptor` : External Format Schema generation from Pure Model - This is needed when you want to allow users of your external format to generate external format schema from authored pure models - - To add support for this you need to implement [ExternalFormatFromPureDescriptor](../../legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure). This requires you to - + - To add support for this you need to implement [ExternalFormatFromPureDescriptor](../../legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure). This requires you to - - Model ModelToSchemaConfiguration for your external format - Implement a function which takes in an instance of ModelUnit and modeled configuration and generates SchemaSet - Provide a default config for your ModelToSchemaConfiguration @@ -33,15 +33,15 @@ This page describes steps required to add support for a new external format. - This is needed when you want to allow users to serialize instances into your external format with custom configurations - This is used at plan generation/execution time. - To add support for this you need to - - - Model subtype of [ExternalFormatExternalizeConfig](../../legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure) + - Model subtype of [ExternalFormatExternalizeConfig](../../legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure) - Author algorithm to dynamically generate platform binding requirements (for JAVA see below) - `internalizeConfig` : Defines configurations needed to deserialize data aligning with external format into instances. - This is needed when you want to allow users to deserialize data aligning with external format into instances to be used with legend ecosystem with custom configurations - This is used at plan generation/execution time. - To add support for this you need to - - - Model [ExternalFormatInternalizeConfig](../../legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure) + - Model [ExternalFormatInternalizeConfig](../../legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/binding/externalFormat/externalFormatContract.pure) - Author algorithm to dynamically generate platform binding requirements (for JAVA see below) - - `internalizeReturnsChecked` : Defines whether your format returns deserialized instances wrapped in [Checked](../../legend-engine-pure-code-compiled-core/src/main/resources/core/pure/dataQuality/dataQuality.pure) structure or not. + - `internalizeReturnsChecked` : Defines whether your format returns deserialized instances wrapped in [Checked](../../legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/dataQuality/dataQuality.pure) structure or not. - If your external format support internalize (deserialization) feature then you can choose to return deserialized instances wrapped in checked structure. - Returning checked instances allows you to provide record level lineage and defect information to your users. - `Default Value`: false @@ -50,18 +50,18 @@ This page describes steps required to add support for a new external format. - This field links serialize tree representation for this source record metamodel and defines structure in which record level lineage information will be presented to the user -- Implement [ExternalFormatLegendJavaPlatformBindingDescriptor](../../legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/descriptor.pure) : ExternalFormatLegendJavaPlatformBindingDescriptor is a description of algorithms required to integrate your format with JAVA implementation of Legend query engine +- Implement [ExternalFormatLegendJavaPlatformBindingDescriptor](../../legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/descriptor.pure) : ExternalFormatLegendJavaPlatformBindingDescriptor is a description of algorithms required to integrate your format with JAVA implementation of Legend query engine - `externalFormatContract` : Links your ExternalFormatContract defined above - `enginePlatformDependencies` : Inject Engine Dependencies for your format into JAVA Platform Binding conventions. - These are the java interfaces which your algorithms can use when generating JAVA platform binding code. - These dependencies serve as the contract between plan generation and execution. - Now you can choose to implement either or both of serialization/deserialization support for your format - `externalizeDescriptor` : Description of algorithms required to generate code to support serialization. To add this feature you need to - - - Provide instance of [ExternalFormatExternalizeBindingDescriptor](../../legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/descriptor.pure) + - Provide instance of [ExternalFormatExternalizeBindingDescriptor](../../legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/descriptor.pure) - `externalizeGenerator` : Author a function which takes in ExternalFormatExternalizeExecutionNode, Path, GenerationContext, DebugContext and return GeneratedCode. GenerateCode contain entry point and JAVA classes which would be added to the execution plan. - `externalizePreparator` : You may optionally need to provide an implementation to inject some types into TypeInfoSet to support your generations. Based on basic analysis of query certain types are injected into TypeInfoSet for every ExternalFormatContract - `internalizeDescriptor` : Description of algorithms required to generate code to support deserialization. To add this feature you need to - - - Provide instance of [ExternalFormatInternalizeBindingDescriptor](../../legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/descriptor.pure) + - Provide instance of [ExternalFormatInternalizeBindingDescriptor](../../legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/descriptor.pure) - `internalizeGenerator` : Author a function which takes in ExternalFormatInternalizeExecutionNode, Path, GenerationContext, DebugContext and return GeneratedCode. GenerateCode contain entry point and JAVA classes which would be added to the execution plan - `internalizePreparator` : You may optionally need to provide an implementation to inject some types into TypeInfoSet to support your generations. Based on basic analysis of query certain types are injected into TypeInfoSet for every ExternalFormatContract @@ -163,7 +163,7 @@ This page describes steps required to add support for a new external format. ``` -- Implement [ExternalFormatExtension](../../legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/ExternalFormatExtension.java) : Basic extension required to integrate external format in legend ecosystem +- Implement [ExternalFormatExtension](../../legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/ExternalFormatExtension.java) : Basic extension required to integrate external format in legend ecosystem - Each and every external format need to provide implementation for this interface to integrate the format with legend ecosystem. - To implement this interface you need to provide implementation for following 3 functions - - `getExternalFormatContract` : Link ExternalFormatContract modeled in PURE @@ -171,25 +171,25 @@ This page describes steps required to add support for a new external format. - `metamodelToText` : Implement algorithm to generate PURE grammar from instances of metamodel classes. You should use transformer/composer defined in previous step. - Define service file with your implementation class, so that your implementation is picked via ServiceLoaders -- Implement [ExternalFormatSchemaGenerationExtension](../../legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/transformation/fromModel/ExternalFormatSchemaGenerationExtension.java): Extension required to integrate your format with Legend schema generation capabilities +- Implement [ExternalFormatSchemaGenerationExtension](../../legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/transformation/fromModel/ExternalFormatSchemaGenerationExtension.java): Extension required to integrate your format with Legend schema generation capabilities - Implement this to integrate with Legend's schema generation. - - This extends [ExternalFormatExtension](../../legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/ExternalFormatExtension.java) + - This extends [ExternalFormatExtension](../../legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/ExternalFormatExtension.java) - To implement this interface you need to implement following functionality in addition to those listed above - `compileModelToSchemaConfiguration` : Compiles instances of JAVA classes for ModelToSchemaConfiguration into its metamodel counterpart - Author ModelToSchemaConfiguration in JAVA aligning with what you would have implemented in PURE - Implement algorithm to translate instances of JAVA classes for ModelToSchemaConfiguration into its metamodel counterpart - `Prerequiste` : You need to implement `externalFormatFromPureDescriptor` in PURE contract for your format -- Implement [ExternalFormatModelGenerationExtension](../../legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/transformation/toModel/ExternalFormatModelGenerationExtension.java): Extension required to integrate your format with Legend model generation capabilities +- Implement [ExternalFormatModelGenerationExtension](../../legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/transformation/toModel/ExternalFormatModelGenerationExtension.java): Extension required to integrate your format with Legend model generation capabilities - Implement this to integrate with Legend's model generation. - - This extends [ExternalFormatExtension](../../legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/ExternalFormatExtension.java) + - This extends [ExternalFormatExtension](../../legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/src/main/java/org/finos/legend/engine/external/shared/format/model/ExternalFormatExtension.java) - To implement this interface you need to implement following functionality in addition to those listed above - `compileSchemaToModelConfiguration` : Compiles instances of JAVA classes for SchemaToModelConfiguration into its metamodel counterpart - Author SchemaToModelConfiguration in JAVA aligning with what you would have implemented in PURE - Implement algorithm to translate instances of JAVA classes for SchemaToModelConfiguration into its metamodel counterpart - `Prerequiste` : You need to implement `externalFormatToPureDescriptor` in PURE contract for your format -- Implement [ExternalFormatRuntimeExtension](../../legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatRuntimeExtension.java) : Extension to integrate your format with Legend's query execution engine +- Implement [ExternalFormatRuntimeExtension](../../legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/ExternalFormatRuntimeExtension.java) : Extension to integrate your format with Legend's query execution engine - Implement this to allow your users to use serialization/deserialization feature with your external format. - To implement this interface you need to provide implementation for following functions - - `getContentTypes` : Provide contentTypes for your format @@ -232,7 +232,7 @@ We recommend following module structure as a starting point - - legend-engine-xt-{format}-javaPlatformBinding-pure : Module to host pure code for generation of java code to support execution on java engine. Implementation of ExternalFormatLegendJavaPlatformBindingDescriptor is defined here. - legend-engine-xt-{format}-model : Module to host implementation of ExternalFormatExtension, ExternalFormatSchemaGenerationExtension, ExternalFormatModelGenerationExtension - legend-engine-xt-{format}-runtime : Module to host implementation of ExternalFormatRuntimeExtension -- legend-engine-xt-{format}-test : Module to run defined tests in maven pipeline +- legend-engine-xt-{format}-javaPlatformBinding-test : Module to run defined tests in maven pipeline using java platform binding #### Dependency Graph @@ -243,11 +243,14 @@ graph LR Runtime(legend-engine-xt-format-runtime) Pure(legend-engine-xt-format-pure) PureJava(legend-engine-xt-format-javaPlatformBinding-pure) + JavaTest(legend-engine-xt-format-javaPlatformBinding-test) PureJava --> Pure Model --> Pure Model --> Shared Runtime --> Shared + JavaTest --> PureJava + JavaTest --> Runtime ``` As mentioned these are recommendations at best. Please feel free to diverge from this. Please feel free to explore existing formats to understand the structure more diff --git a/legend-engine-application-query/pom.xml b/legend-engine-application-query/pom.xml index 08e8b49dd68..a769efc77e2 100644 --- a/legend-engine-application-query/pom.xml +++ b/legend-engine-application-query/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-application-query diff --git a/legend-engine-config/legend-engine-configuration/pom.xml b/legend-engine-config/legend-engine-configuration/pom.xml index acf6b79a5ee..28ffc2ff09f 100644 --- a/legend-engine-config/legend-engine-configuration/pom.xml +++ b/legend-engine-config/legend-engine-configuration/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-config - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-configuration diff --git a/legend-engine-config/legend-engine-connection-integration-tests/pom.xml b/legend-engine-config/legend-engine-connection-integration-tests/pom.xml index 6953e8a064b..3228943aa0c 100644 --- a/legend-engine-config/legend-engine-connection-integration-tests/pom.xml +++ b/legend-engine-config/legend-engine-connection-integration-tests/pom.xml @@ -15,11 +15,12 @@ limitations under the License. --> - + org.finos.legend.engine legend-engine-config - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-connection-integration-tests @@ -74,26 +75,7 @@ - - - - - - - - - - - - - - - - junit - junit - test - org.junit.jupiter junit-jupiter-api @@ -104,6 +86,11 @@ junit-jupiter-engine test + + net.bytebuddy + byte-buddy + test + diff --git a/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/AbstractConnectionFactoryTest.java b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/AbstractConnectionFactoryTest.java index 880898bb024..e5b6435ece9 100644 --- a/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/AbstractConnectionFactoryTest.java +++ b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/AbstractConnectionFactoryTest.java @@ -17,19 +17,23 @@ import org.finos.legend.authentication.vault.CredentialVault; import org.finos.legend.authentication.vault.impl.EnvironmentCredentialVault; import org.finos.legend.authentication.vault.impl.SystemPropertiesCredentialVault; +import org.finos.legend.connection.AuthenticationMechanismConfiguration; import org.finos.legend.connection.Authenticator; import org.finos.legend.connection.ConnectionFactory; import org.finos.legend.connection.DatabaseType; -import org.finos.legend.connection.EnvironmentConfiguration; import org.finos.legend.connection.IdentityFactory; import org.finos.legend.connection.IdentitySpecification; +import org.finos.legend.connection.impl.InstrumentedStoreInstanceProvider; +import org.finos.legend.connection.LegendEnvironment; import org.finos.legend.connection.RelationalDatabaseStoreSupport; import org.finos.legend.connection.StoreInstance; +import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration; import org.finos.legend.connection.impl.KerberosCredentialExtractor; import org.finos.legend.connection.impl.KeyPairCredentialBuilder; import org.finos.legend.connection.impl.SnowflakeConnectionBuilder; +import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; import org.finos.legend.connection.impl.UserPasswordCredentialBuilder; -import org.finos.legend.connection.jdbc.StaticJDBCConnectionBuilder; +import org.finos.legend.connection.impl.StaticJDBCConnectionBuilder; import org.finos.legend.connection.protocol.AuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationMechanismType; import org.finos.legend.engine.shared.core.identity.Identity; @@ -41,8 +45,9 @@ public abstract class AbstractConnectionFactoryTest { protected static final String TEST_STORE_INSTANCE_NAME = "test-store"; - protected EnvironmentConfiguration environmentConfiguration; + protected LegendEnvironment environment; protected IdentityFactory identityFactory; + protected InstrumentedStoreInstanceProvider storeInstanceProvider; protected ConnectionFactory connectionFactory; @BeforeEach @@ -50,47 +55,43 @@ public void initialize() { this.setup(); - EnvironmentConfiguration.Builder environmentConfigurationBuilder = new EnvironmentConfiguration.Builder() + LegendEnvironment.Builder environmentBuilder = new LegendEnvironment.Builder() .withVaults( new SystemPropertiesCredentialVault(), new EnvironmentCredentialVault() ) .withStoreSupports( - new RelationalDatabaseStoreSupport.Builder() + new RelationalDatabaseStoreSupport.Builder(DatabaseType.POSTGRES) .withIdentifier("Postgres") - .withDatabase(DatabaseType.POSTGRES) - .withAuthenticationMechanisms( - AuthenticationMechanismType.USER_PASSWORD + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).withAuthenticationConfigurationTypes( + UserPasswordAuthenticationConfiguration.class + ).build() ) .build(), - new RelationalDatabaseStoreSupport.Builder() + new RelationalDatabaseStoreSupport.Builder(DatabaseType.SNOWFLAKE) .withIdentifier("Snowflake") - .withDatabase(DatabaseType.SNOWFLAKE) - .withAuthenticationMechanisms( - AuthenticationMechanismType.KEY_PAIR -// AuthenticationMechanismType.OAUTH + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KEY_PAIR).withAuthenticationConfigurationTypes( + EncryptedPrivateKeyPairAuthenticationConfiguration.class + ).build() ) .build() - ) - .withAuthenticationMechanisms( - AuthenticationMechanismType.USER_PASSWORD, - AuthenticationMechanismType.API_KEY, - AuthenticationMechanismType.KEY_PAIR, - AuthenticationMechanismType.KERBEROS ); CredentialVault credentialVault = this.getCredentialVault(); if (credentialVault != null) { - environmentConfigurationBuilder.withVault(credentialVault); + environmentBuilder.withVault(credentialVault); } - this.environmentConfiguration = environmentConfigurationBuilder.build(); + this.environment = environmentBuilder.build(); - this.identityFactory = new IdentityFactory.Builder(environmentConfiguration) + this.identityFactory = new IdentityFactory.Builder(this.environment) .build(); - this.connectionFactory = new ConnectionFactory.Builder(environmentConfiguration) + this.storeInstanceProvider = new InstrumentedStoreInstanceProvider(); + this.connectionFactory = new ConnectionFactory.Builder(this.environment, this.storeInstanceProvider) .withCredentialBuilders( new KerberosCredentialExtractor(), new UserPasswordCredentialBuilder(), @@ -129,12 +130,12 @@ public CredentialVault getCredentialVault() @Test public void runTest() throws Exception { - this.connectionFactory.injectStoreInstance(this.getStoreInstance()); + this.storeInstanceProvider.injectStoreInstance(this.getStoreInstance()); Identity identity = this.getIdentity(); AuthenticationConfiguration authenticationConfiguration = this.getAuthenticationConfiguration(); Authenticator authenticator = this.connectionFactory.getAuthenticator(identity, TEST_STORE_INSTANCE_NAME, authenticationConfiguration); - T connection = this.connectionFactory.getConnection(authenticator); + T connection = this.connectionFactory.getConnection(identity, authenticator); this.runTestWithConnection(connection); System.out.println("Successfully established and checked connection!"); diff --git a/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestJDBCConnectionManager.java b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestJDBCConnectionManager.java new file mode 100644 index 00000000000..7abf243d608 --- /dev/null +++ b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestJDBCConnectionManager.java @@ -0,0 +1,295 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.connection.test; + +import net.bytebuddy.asm.Advice; +import org.finos.legend.authentication.vault.impl.PropertiesFileCredentialVault; +import org.finos.legend.connection.AuthenticationMechanismConfiguration; +import org.finos.legend.connection.Authenticator; +import org.finos.legend.connection.ConnectionFactory; +import org.finos.legend.connection.DatabaseType; +import org.finos.legend.connection.IdentityFactory; +import org.finos.legend.connection.IdentitySpecification; +import org.finos.legend.connection.JDBCConnectionBuilder; +import org.finos.legend.connection.LegendEnvironment; +import org.finos.legend.connection.PostgresTestContainerWrapper; +import org.finos.legend.connection.RelationalDatabaseStoreSupport; +import org.finos.legend.connection.StoreInstance; +import org.finos.legend.connection.impl.InstrumentedStoreInstanceProvider; +import org.finos.legend.connection.impl.JDBCConnectionManager; +import org.finos.legend.connection.impl.StaticJDBCConnectionBuilder; +import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; +import org.finos.legend.connection.impl.UserPasswordCredentialBuilder; +import org.finos.legend.connection.protocol.AuthenticationConfiguration; +import org.finos.legend.connection.protocol.AuthenticationMechanismType; +import org.finos.legend.connection.protocol.ConnectionSpecification; +import org.finos.legend.connection.protocol.StaticJDBCConnectionSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.authentication.vault.PropertiesFileSecret; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.SQLTransientConnectionException; +import java.util.Properties; + +public class TestJDBCConnectionManager +{ + PostgresTestContainerWrapper postgresContainer; + private static final String TEST_STORE_INSTANCE_NAME = "test-store"; + + private LegendEnvironment environment; + private IdentityFactory identityFactory; + private InstrumentedStoreInstanceProvider storeInstanceProvider; + private ConnectionFactory connectionFactory; + private StoreInstance storeInstance; + + @BeforeEach + public void setup() + { + postgresContainer = PostgresTestContainerWrapper.build(); + postgresContainer.start(); + + Properties properties = new Properties(); + properties.put("passwordRef", this.postgresContainer.getPassword()); + + LegendEnvironment.Builder environmentBuilder = new LegendEnvironment.Builder() + .withVaults(new PropertiesFileCredentialVault(properties)) + .withStoreSupports( + new RelationalDatabaseStoreSupport.Builder(DatabaseType.POSTGRES) + .withIdentifier("Postgres") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).withAuthenticationConfigurationTypes( + UserPasswordAuthenticationConfiguration.class + ).build() + ) + .build() + ); + + this.environment = environmentBuilder.build(); + this.identityFactory = new IdentityFactory.Builder(this.environment) + .build(); + this.storeInstanceProvider = new InstrumentedStoreInstanceProvider(); + ConnectionSpecification connectionSpecification = new StaticJDBCConnectionSpecification( + this.postgresContainer.getHost(), + this.postgresContainer.getPort(), + this.postgresContainer.getDatabaseName() + ); + this.storeInstance = new StoreInstance.Builder(this.environment) + .withIdentifier(TEST_STORE_INSTANCE_NAME) + .withStoreSupportIdentifier("Postgres") + .withConnectionSpecification(connectionSpecification) + .build(); + } + + @AfterEach + public void cleanUp() + { + postgresContainer.stop(); + + JDBCConnectionManager.getInstance().flushPool(); + } + + @Test + public void testBasicConnectionPooling() throws Exception + { + JDBCConnectionBuilder customizedJDBCConnectionBuilder = new StaticJDBCConnectionBuilder.WithPlaintextUsernamePassword(); + customizedJDBCConnectionBuilder.setConnectionPoolConfig( + new JDBCConnectionManager.ConnectionPoolConfig.Builder() + .withMaxPoolSize(2) + .withConnectionTimeout(1000L) + .build() + ); + this.connectionFactory = new ConnectionFactory.Builder(this.environment, this.storeInstanceProvider) + .withCredentialBuilders( + new UserPasswordCredentialBuilder() + ) + .withConnectionBuilders( + customizedJDBCConnectionBuilder + ) + .build(); + this.storeInstanceProvider.injectStoreInstance(this.storeInstance); + Identity identity = identityFactory.createIdentity( + new IdentitySpecification.Builder() + .withName("test-user") + .build() + ); + ConnectionSpecification connectionSpecification = this.storeInstance.getConnectionSpecification(); + AuthenticationConfiguration authenticationConfiguration = new UserPasswordAuthenticationConfiguration( + postgresContainer.getUser(), + new PropertiesFileSecret("passwordRef") + ); + Authenticator authenticator = this.connectionFactory.getAuthenticator(identity, TEST_STORE_INSTANCE_NAME, authenticationConfiguration); + + JDBCConnectionManager connectionManager = JDBCConnectionManager.getInstance(); + Assertions.assertEquals(0, connectionManager.getPoolSize()); + + // 1. Get a connection, this should initialize the pool as well as create a new connection in the empty pool + // this connection should be active + Connection connection0 = this.connectionFactory.getConnection(identity, authenticator); + + String poolName = JDBCConnectionManager.getPoolName(identity, connectionSpecification, authenticationConfiguration); + JDBCConnectionManager.ConnectionPool connectionPool = connectionManager.getPool(poolName); + + // 2. Close the connection, verify that the pool keeps this connection around in idle state + Connection underlyingConnection0 = connection0.unwrap(Connection.class); + connection0.close(); + + Assertions.assertEquals(1, connectionPool.getTotalConnections()); + Assertions.assertEquals(0, connectionPool.getActiveConnections()); + Assertions.assertEquals(1, connectionPool.getIdleConnections()); + + // 3. Get a new connection, the pool should return the idle connection and create no new connection + Connection connection1 = this.connectionFactory.getConnection(identity, authenticator); + + Assertions.assertEquals(underlyingConnection0, connection1.unwrap(Connection.class)); + Assertions.assertEquals(1, connectionPool.getTotalConnections()); + Assertions.assertEquals(1, connectionPool.getActiveConnections()); + Assertions.assertEquals(0, connectionPool.getIdleConnections()); + + // 4. Get another connection while the first one is still alive and used, a new connection + // will be created in the pool + this.connectionFactory.getConnection(identity, authenticator); + + Assertions.assertEquals(2, connectionPool.getTotalConnections()); + Assertions.assertEquals(2, connectionPool.getActiveConnections()); + Assertions.assertEquals(0, connectionPool.getIdleConnections()); + + // 5. Get yet another connection while the first and second one are still alive and used, this will + // exceed the pool size, throwing an error + Assertions.assertThrows(SQLTransientConnectionException.class, () -> + { + this.connectionFactory.getConnection(identity, authenticator); + }); + } + + @Test + public void testConnectionPoolingForDifferentIdentities() throws Exception + { + this.connectionFactory = new ConnectionFactory.Builder(this.environment, this.storeInstanceProvider) + .withCredentialBuilders( + new UserPasswordCredentialBuilder() + ) + .withConnectionBuilders( + new StaticJDBCConnectionBuilder.WithPlaintextUsernamePassword() + ) + .build(); + this.storeInstanceProvider.injectStoreInstance(this.storeInstance); + Identity identity1 = identityFactory.createIdentity( + new IdentitySpecification.Builder() + .withName("testUser1") + .build() + ); + Identity identity2 = identityFactory.createIdentity( + new IdentitySpecification.Builder() + .withName("testUser2") + .build() + ); + ConnectionSpecification connectionSpecification = this.storeInstance.getConnectionSpecification(); + AuthenticationConfiguration authenticationConfiguration = new UserPasswordAuthenticationConfiguration( + postgresContainer.getUser(), + new PropertiesFileSecret("passwordRef") + ); + + JDBCConnectionManager connectionManager = JDBCConnectionManager.getInstance(); + Assertions.assertEquals(0, connectionManager.getPoolSize()); + + // 1. Get a new connection for identity1, which should initialize a pool + this.connectionFactory.getConnection(identity1, this.connectionFactory.getAuthenticator(identity1, TEST_STORE_INSTANCE_NAME, authenticationConfiguration)); + + String poolName1 = JDBCConnectionManager.getPoolName(identity1, connectionSpecification, authenticationConfiguration); + JDBCConnectionManager.ConnectionPool connectionPool1 = connectionManager.getPool(poolName1); + + Assertions.assertEquals(1, connectionManager.getPoolSize()); + Assertions.assertEquals(1, connectionPool1.getTotalConnections()); + Assertions.assertEquals(1, connectionPool1.getActiveConnections()); + Assertions.assertEquals(0, connectionPool1.getIdleConnections()); + + // 2. Get a new connection for identity2, which should initialize another pool + this.connectionFactory.getConnection(identity2, this.connectionFactory.getAuthenticator(identity2, TEST_STORE_INSTANCE_NAME, authenticationConfiguration)); + + String poolName2 = JDBCConnectionManager.getPoolName(identity2, connectionSpecification, authenticationConfiguration); + JDBCConnectionManager.ConnectionPool connectionPool2 = connectionManager.getPool(poolName2); + + Assertions.assertEquals(2, connectionManager.getPoolSize()); + Assertions.assertEquals(1, connectionPool2.getTotalConnections()); + Assertions.assertEquals(1, connectionPool2.getActiveConnections()); + Assertions.assertEquals(0, connectionPool2.getIdleConnections()); + } + + @Test + public void testRetryOnBrokenConnection() + { + // + } + + public static class CustomAdvice + { + @Advice.OnMethodExit + public static void intercept(@Advice.Return(readOnly = false) String value) + { + System.out.println("intercepted: " + value); + value = "hi: " + value; + } + } + +// public static class MyWay +// { +// } +// +// private static class InstrumentedStaticJDBCConnectionBuilder +// { +// static class WithPlaintextUsernamePassword extends StaticJDBCConnectionBuilder.WithPlaintextUsernamePassword +// { +// WithPlaintextUsernamePassword(Function hikariConfigHandler) +// { +// this.connectionManager = new InstrumentedJDBCConnectionManager(hikariConfigHandler); +// } +// +// @Override +// public JDBCConnectionManager getConnectionManager() +// { +// return this.connectionManager; +// } +// +// @Override +// protected Type[] actualTypeArguments() +// { +// Type genericSuperClass = this.getClass().getSuperclass().getGenericSuperclass(); +// ParameterizedType parameterizedType = (ParameterizedType) genericSuperClass; +// return parameterizedType.getActualTypeArguments(); +// } +// } +// } +// +// private static class InstrumentedJDBCConnectionManager extends JDBCConnectionManager +// { +// private final Function hikariConfigHandler; +// +// InstrumentedJDBCConnectionManager(Function hikariConfigHandler) +// { +// this.hikariConfigHandler = hikariConfigHandler; +// } +// +//// @Override +//// protected void handleHikariConfig(HikariConfig config) +//// { +//// config.setRegisterMbeans(true); +//// this.hikariConfigHandler.apply(config); +//// } +// } +} diff --git a/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestPostgresConnection.java b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestPostgresConnection.java index 1f92986d3e6..9e311f05fa8 100644 --- a/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestPostgresConnection.java +++ b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestPostgresConnection.java @@ -16,10 +16,11 @@ import org.finos.legend.authentication.vault.CredentialVault; import org.finos.legend.authentication.vault.impl.PropertiesFileCredentialVault; +import org.finos.legend.connection.AuthenticationMechanismConfiguration; import org.finos.legend.connection.PostgresTestContainerWrapper; import org.finos.legend.connection.StoreInstance; import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; -import org.finos.legend.connection.jdbc.StaticJDBCConnectionSpecification; +import org.finos.legend.connection.protocol.StaticJDBCConnectionSpecification; import org.finos.legend.connection.protocol.AuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationMechanismType; import org.finos.legend.connection.protocol.ConnectionSpecification; @@ -30,7 +31,7 @@ import java.sql.Statement; import java.util.Properties; -import static org.junit.Assume.assumeTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; public class TestPostgresConnection { @@ -48,7 +49,7 @@ public void setup() } catch (Exception e) { - assumeTrue("Can't start PostgreSQLContainer", false); + assumeTrue(false, "Can't start PostgreSQLContainer"); } } @@ -77,11 +78,11 @@ public StoreInstance getStoreInstance() this.postgresContainer.getPort(), this.postgresContainer.getDatabaseName() ); - return new StoreInstance.Builder(this.environmentConfiguration) + return new StoreInstance.Builder(this.environment) .withIdentifier(TEST_STORE_INSTANCE_NAME) .withStoreSupportIdentifier("Postgres") - .withAuthenticationMechanisms( - AuthenticationMechanismType.USER_PASSWORD + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).build() ) .withConnectionSpecification(connectionSpecification) .build(); diff --git a/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestSnowflakeConnection.java b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestSnowflakeConnection.java index 2d522ff96d8..825725a5eea 100644 --- a/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestSnowflakeConnection.java +++ b/legend-engine-config/legend-engine-connection-integration-tests/src/test/java/org/finos/legend/engine/connection/test/TestSnowflakeConnection.java @@ -16,6 +16,7 @@ import org.finos.legend.authentication.vault.CredentialVault; import org.finos.legend.authentication.vault.impl.PropertiesFileCredentialVault; +import org.finos.legend.connection.AuthenticationMechanismConfiguration; import org.finos.legend.connection.StoreInstance; import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationConfiguration; @@ -29,14 +30,14 @@ import java.sql.Statement; import java.util.Properties; -import static org.junit.Assume.assumeTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; public class TestSnowflakeConnection { public static class WithKeyPair extends AbstractConnectionFactoryTest { - private static final String TEST_SNOWFLAKE_PK = "TEST_SNOWFLAKE_PK"; - private static final String TEST_SNOWFLAKE_PK_PASSPHRASE = "TEST_SNOWFLAKE_PK_PASSPHRASE"; + private static final String CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK = "CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK"; + private static final String CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK_PASSPHRASE = "CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK_PASSPHRASE"; private String snowflakePrivateKey; private String snowflakePassPhrase; @@ -45,12 +46,12 @@ public void setup() { try { - this.snowflakePrivateKey = this.environmentConfiguration.lookupVaultSecret(new EnvironmentCredentialVaultSecret(TEST_SNOWFLAKE_PK), null); - this.snowflakePassPhrase = this.environmentConfiguration.lookupVaultSecret(new EnvironmentCredentialVaultSecret(TEST_SNOWFLAKE_PK_PASSPHRASE), null); + this.snowflakePrivateKey = this.environment.lookupVaultSecret(new EnvironmentCredentialVaultSecret(CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK), null); + this.snowflakePassPhrase = this.environment.lookupVaultSecret(new EnvironmentCredentialVaultSecret(CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK_PASSPHRASE), null); } catch (Exception e) { - assumeTrue("Can't retrieve Snowflake test instance key-pair info (TEST_SNOWFLAKE_PK, TEST_SNOWFLAKE_PK_PASSPHRASE)", false); + assumeTrue(false, String.format("Can't retrieve Snowflake connection key-pair info (%s, %s environment variables are expected)", CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK, CONNECTION_INTEGRATION_TEST__SNOWFLAKE_PK_PASSPHRASE)); } } @@ -79,11 +80,11 @@ public StoreInstance getStoreInstance() connectionSpecification.region = "us-east-2"; connectionSpecification.cloudType = "aws"; connectionSpecification.role = "SUMMIT_DEV"; - return new StoreInstance.Builder(this.environmentConfiguration) + return new StoreInstance.Builder(this.environment) .withIdentifier(TEST_STORE_INSTANCE_NAME) .withStoreSupportIdentifier("Snowflake") - .withAuthenticationMechanisms( - AuthenticationMechanismType.KEY_PAIR + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KEY_PAIR).build() ) .withConnectionSpecification(connectionSpecification) .build(); diff --git a/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml b/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml index c5718331ea0..7ae5876854c 100644 --- a/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml +++ b/legend-engine-config/legend-engine-extensions-collection-execution/pom.xml @@ -19,7 +19,7 @@ legend-engine-config org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml index 7d0531e7440..62e94574827 100644 --- a/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml +++ b/legend-engine-config/legend-engine-extensions-collection-generation/pom.xml @@ -19,7 +19,7 @@ legend-engine-config org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -295,6 +295,11 @@ legend-engine-xt-relationalStore-sybaseiq-pure runtime + + org.finos.legend.engine + legend-engine-xt-relationalStore-sparksql-pure + runtime + org.finos.legend.engine legend-engine-xt-relationalStore-javaPlatformBinding-pure diff --git a/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java b/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java index 5ae46944c3d..da2039c880d 100644 --- a/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java +++ b/legend-engine-config/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java @@ -509,6 +509,7 @@ protected Iterable getExpectedCodeRepositories() .with("core_relational_presto") .with("core_relational_sybase") .with("core_relational_sybaseiq") + .with("core_relational_sparksql") .with("core_relational_store_entitlement") .with("core_servicestore") .with("core_authentication") diff --git a/legend-engine-config/legend-engine-pure-code-compiled-core-configuration/pom.xml b/legend-engine-config/legend-engine-pure-code-compiled-core-configuration/pom.xml index 3bb342ccfdf..558bd130fdc 100644 --- a/legend-engine-config/legend-engine-pure-code-compiled-core-configuration/pom.xml +++ b/legend-engine-config/legend-engine-pure-code-compiled-core-configuration/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-config - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-config/legend-engine-server-integration-tests/pom.xml b/legend-engine-config/legend-engine-server-integration-tests/pom.xml index 1f9f12ce1e4..6de9ae3c274 100644 --- a/legend-engine-config/legend-engine-server-integration-tests/pom.xml +++ b/legend-engine-config/legend-engine-server-integration-tests/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-config - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-server-integration-tests diff --git a/legend-engine-config/legend-engine-server-support-core/pom.xml b/legend-engine-config/legend-engine-server-support-core/pom.xml index f159071b85f..c1e3b6084a8 100644 --- a/legend-engine-config/legend-engine-server-support-core/pom.xml +++ b/legend-engine-config/legend-engine-server-support-core/pom.xml @@ -3,7 +3,7 @@ legend-engine-config org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-config/legend-engine-server/pom.xml b/legend-engine-config/legend-engine-server/pom.xml index f1a088aa20b..02789e87e86 100644 --- a/legend-engine-config/legend-engine-server/pom.xml +++ b/legend-engine-config/legend-engine-server/pom.xml @@ -19,14 +19,12 @@ org.finos.legend.engine legend-engine-config - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-server Legend Engine - Server - - jar @@ -134,6 +132,11 @@ legend-engine-xt-relationalStore-sybaseiq-pure runtime + + org.finos.legend.engine + legend-engine-xt-relationalStore-sparksql-pure + runtime + org.finos.legend.engine legend-engine-language-pure-grammar-api @@ -669,6 +672,23 @@ + + org.finos.legend.engine + legend-engine-xt-authentication-connection-factory + + + org.finos.legend.engine + legend-engine-xt-relationalStore-connection + + + org.finos.legend.engine + legend-engine-xt-relationalStore-postgres-connection + runtime + + + org.finos.legend.engine + legend-engine-xt-relationalStore-snowflake-connection + diff --git a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java index 60fa1a7cfdc..ad248e0062b 100644 --- a/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java +++ b/legend-engine-config/legend-engine-server/src/main/java/org/finos/legend/engine/server/Server.java @@ -39,7 +39,25 @@ import org.finos.legend.authentication.intermediationrule.IntermediationRule; import org.finos.legend.authentication.intermediationrule.impl.EncryptedPrivateKeyFromVaultRule; import org.finos.legend.authentication.vault.CredentialVaultProvider; +import org.finos.legend.authentication.vault.impl.EnvironmentCredentialVault; import org.finos.legend.authentication.vault.impl.PropertiesFileCredentialVault; +import org.finos.legend.authentication.vault.impl.SystemPropertiesCredentialVault; +import org.finos.legend.connection.AuthenticationMechanismConfiguration; +import org.finos.legend.connection.ConnectionFactory; +import org.finos.legend.connection.DatabaseType; +import org.finos.legend.connection.LegendEnvironment; +import org.finos.legend.connection.RelationalDatabaseStoreSupport; +import org.finos.legend.connection.StoreInstanceProvider; +import org.finos.legend.connection.impl.DefaultStoreInstanceProvider; +import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration; +import org.finos.legend.connection.impl.HACKY__SnowflakeConnectionAdapter; +import org.finos.legend.connection.impl.KerberosCredentialExtractor; +import org.finos.legend.connection.impl.KeyPairCredentialBuilder; +import org.finos.legend.connection.impl.SnowflakeConnectionBuilder; +import org.finos.legend.connection.impl.StaticJDBCConnectionBuilder; +import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; +import org.finos.legend.connection.impl.UserPasswordCredentialBuilder; +import org.finos.legend.connection.protocol.AuthenticationMechanismType; import org.finos.legend.engine.api.analytics.DataSpaceAnalytics; import org.finos.legend.engine.api.analytics.DiagramAnalytics; import org.finos.legend.engine.api.analytics.LineageAnalytics; @@ -67,13 +85,13 @@ import org.finos.legend.engine.language.pure.grammar.api.grammarToJson.TransformGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.jsonToGrammar.JsonToGrammar; import org.finos.legend.engine.language.pure.grammar.api.jsonToGrammar.TransformJsonToGrammar; -import org.finos.legend.engine.language.pure.relational.api.relationalElement.RelationalElementAPI; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.RelationalOperationElementGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.RelationalOperationElementJsonToGrammar; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.TransformRelationalOperationElementGrammarToJson; import org.finos.legend.engine.language.pure.grammar.api.relationalOperationElement.TransformRelationalOperationElementJsonToGrammar; import org.finos.legend.engine.language.pure.modelManager.ModelManager; import org.finos.legend.engine.language.pure.modelManager.sdlc.SDLCLoader; +import org.finos.legend.engine.language.pure.relational.api.relationalElement.RelationalElementAPI; import org.finos.legend.engine.plan.execution.PlanExecutor; import org.finos.legend.engine.plan.execution.api.ExecutePlanLegacy; import org.finos.legend.engine.plan.execution.api.ExecutePlanStrategic; @@ -100,8 +118,8 @@ import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration; import org.finos.legend.engine.protocol.pure.v1.PureProtocolObjectMapperFactory; import org.finos.legend.engine.protocol.pure.v1.model.PureProtocol; -import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; import org.finos.legend.engine.query.graphQL.api.debug.GraphQLDebug; import org.finos.legend.engine.query.graphQL.api.execute.GraphQLExecute; import org.finos.legend.engine.query.graphQL.api.grammar.GraphQLGrammar; @@ -157,7 +175,7 @@ public class Server extends Application public static void main(String[] args) throws Exception { EngineUrlStreamHandlerFactory.initialize(); - new Server().run(args.length == 0 ? new String[] {"server", "legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json"} : args); + new Server().run(args.length == 0 ? new String[]{"server", "legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json"} : args); } @Override @@ -261,6 +279,10 @@ public void run(T serverConfiguration, Environment environment) relationalExecution.setFlowProviderClass(LegendDefaultDatabaseAuthenticationFlowProvider.class); relationalExecution.setFlowProviderConfiguration(new LegendDefaultDatabaseAuthenticationFlowProviderConfiguration()); } + relationalExecution.setConnectionFactory(this.setupConnectionFactory(serverConfiguration.vaults)); + relationalExecution.setRelationalDatabaseConnectionAdapters(Lists.mutable.of( + new HACKY__SnowflakeConnectionAdapter.WithKeyPair() + )); relationalStoreExecutor = (RelationalStoreExecutor) Relational.build(serverConfiguration.relationalexecution); @@ -349,7 +371,7 @@ public void run(T serverConfiguration, Environment environment) environment.jersey().register(new ExecutePlanLegacy(planExecutor)); // Function Activator - environment.jersey().register(new FunctionActivatorAPI(modelManager, serverConfiguration.activatorConfiguration, routerExtensions)); + environment.jersey().register(new FunctionActivatorAPI(modelManager, Lists.mutable.empty(), routerExtensions)); // GraphQL environment.jersey().register(new GraphQLGrammar()); @@ -391,6 +413,48 @@ public void run(T serverConfiguration, Environment environment) enableCors(environment); } + // TODO: @akphi - this is temporary, rework when we find a better way to handle the initialization of connection factory from config or some external source. + private ConnectionFactory setupConnectionFactory(List vaultConfigurations) + { + LegendEnvironment environment = new LegendEnvironment.Builder() + .withVaults( + new SystemPropertiesCredentialVault(), + new EnvironmentCredentialVault(), + new PropertiesFileCredentialVault(this.buildVaultProperties(vaultConfigurations)) + ) + .withStoreSupports( + new RelationalDatabaseStoreSupport.Builder(DatabaseType.POSTGRES) + .withIdentifier("Postgres") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).withAuthenticationConfigurationTypes( + UserPasswordAuthenticationConfiguration.class + ).build() + ) + .build(), + new RelationalDatabaseStoreSupport.Builder(DatabaseType.SNOWFLAKE) + .withIdentifier("Snowflake") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KEY_PAIR).withAuthenticationConfigurationTypes( + EncryptedPrivateKeyPairAuthenticationConfiguration.class + ).build() + ) + .build() + ).build(); + + StoreInstanceProvider storeInstanceProvider = new DefaultStoreInstanceProvider.Builder().build(); + return new ConnectionFactory.Builder(environment, storeInstanceProvider) + .withCredentialBuilders( + new KerberosCredentialExtractor(), + new UserPasswordCredentialBuilder(), + new KeyPairCredentialBuilder() + ) + .withConnectionBuilders( + new StaticJDBCConnectionBuilder.WithPlaintextUsernamePassword(), + new SnowflakeConnectionBuilder.WithKeyPair() + ) + .build(); + } + private void loadVaults(List vaultConfigurations) { if (vaultConfigurations != null) diff --git a/legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json b/legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json index acd17396d0d..8106f02d379 100644 --- a/legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json +++ b/legend-engine-config/legend-engine-server/src/test/resources/org/finos/legend/engine/server/test/userTestConfig.json @@ -80,12 +80,5 @@ }, "errorhandlingconfiguration": { "enabled": true - }, - "activatorConfiguration":[ - {"_type":"hostedServiceConfig", "host": "127.0.0.1", - "port":9090, - "path": "/api/service/v1/registerHostedService", - "stage": "SANDBOX" - } - ] -} \ No newline at end of file + } +} diff --git a/legend-engine-config/pom.xml b/legend-engine-config/pom.xml index 14faa9b8896..8bc10e8ab9a 100644 --- a/legend-engine-config/pom.xml +++ b/legend-engine-config/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-dependencies/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-dependencies/pom.xml index eace5c1cbd8..e3c2d9be4e5 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-dependencies/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-dependencies/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-executionPlan-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-executionPlan-dependencies diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-api/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-api/pom.xml index 289088d20c7..97a581c99cd 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-api/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-executionPlan-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-executionPlan-execution-api diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-authorizer/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-authorizer/pom.xml index 31f83d41dc9..67188fdcbcb 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-authorizer/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-authorizer/pom.xml @@ -3,7 +3,7 @@ legend-engine-core-executionPlan-execution org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-store-inMemory/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-store-inMemory/pom.xml index 4696a11dc8c..65c758f45a1 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-store-inMemory/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution-store-inMemory/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-executionPlan-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-executionPlan-execution-store-inMemory diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/pom.xml index d089f5fb53d..49018781741 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-executionPlan-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-executionPlan-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-executionPlan-execution diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/pom.xml index 62b63333d3d..c34283b87bf 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-executionPlan-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatSerializeResult.java b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatSerializeResult.java index 1c41eec625f..ff0c13475c8 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatSerializeResult.java +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatSerializeResult.java @@ -14,6 +14,9 @@ package org.finos.legend.engine.external.shared.runtime.write; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.engine.plan.execution.result.Result; import org.finos.legend.engine.plan.execution.result.ResultVisitor; @@ -52,6 +55,21 @@ public Serializer getSerializer(SerializationFormat format) return new ExternalFormatDefaultSerializer(externalFormatWriter, this); } + @Override + public String flush(Serializer serializer) + { + try + { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + externalFormatWriter.writeDataAsString(bos); + return bos.toString(StandardCharsets.UTF_8.name()); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + @Override public void close() { diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatWriter.java b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatWriter.java index aa2742dceb1..77a0d4d9556 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatWriter.java +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/main/java/org/finos/legend/engine/external/shared/runtime/write/ExternalFormatWriter.java @@ -20,4 +20,10 @@ public abstract class ExternalFormatWriter { public abstract void writeData(OutputStream stream) throws IOException; + + public void writeDataAsString(OutputStream outputStream) throws IOException + { + writeData(outputStream); + } + } diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/test/java/org/finos/legend/engine/external/shared/runtime/fixtures/firmModel/Firm.java b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/test/java/org/finos/legend/engine/external/shared/runtime/fixtures/firmModel/Firm.java index de291c706f2..87ee15e38b1 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/test/java/org/finos/legend/engine/external/shared/runtime/fixtures/firmModel/Firm.java +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/legend-engine-external-shared-format-runtime/src/test/java/org/finos/legend/engine/external/shared/runtime/fixtures/firmModel/Firm.java @@ -53,8 +53,12 @@ public String getPureClassName() private List addresses; + private int addressesSize; + private List employees; + private int employeesSize; + public String getName() { return this.nameSize == 0 ? null : this.name; @@ -95,6 +99,7 @@ public void _addressesAdd(AddressUse value) this.addresses = new ArrayList(); } this.addresses.add(value); + this.addressesSize++; } public List getEmployees() @@ -109,6 +114,7 @@ void _employeesAddImpl(Person value) this.employees = new ArrayList(); } this.employees.add(value); + this.employeesSize++; } public void _employeesAdd(Person value) @@ -128,10 +134,6 @@ public List checkMultiplicities() { defects.add(BasicDefect.newClassStructureDefect("Invalid multiplicity for ranking: expected [0..1] found [" + this.rankingSize + "]", "meta::external::format::shared::testpack::simple::Firm")); } - if (this.addresses.size() < 1L) - { - defects.add(BasicDefect.newClassStructureDefect("Invalid multiplicity for addresses: expected [1..*] found [" + this.addresses.size() + "]", "meta::external::format::shared::testpack::simple::Firm")); - } return defects; } diff --git a/legend-engine-core/legend-engine-core-executionPlan-execution/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-execution/pom.xml index fcc3ae58002..5fdf54971e3 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-execution/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-execution/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-core - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-executionPlan-generation/legend-engine-executionPlan-generation/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-generation/legend-engine-executionPlan-generation/pom.xml index 0a4dfb27845..0c73f22f72f 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-generation/legend-engine-executionPlan-generation/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-generation/legend-engine-executionPlan-generation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-executionPlan-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-executionPlan-generation/pom.xml b/legend-engine-core/legend-engine-core-executionPlan-generation/pom.xml index 28c44758958..9ce432c6c7e 100644 --- a/legend-engine-core/legend-engine-core-executionPlan-generation/pom.xml +++ b/legend-engine-core/legend-engine-core-executionPlan-generation/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-core - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/pom.xml index e2a0b970c41..e10ac455a12 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-external-shared-format-model/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler-api/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler-api/pom.xml index 0280a519bdd..56bfa71742d 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler-api/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-compiler-api diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml index d481a687937..89f2bbf1130 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-compiler diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/Milestoning.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/Milestoning.java index 7a03d75c02e..a397fb44af4 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/Milestoning.java +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/Milestoning.java @@ -289,8 +289,19 @@ private static QualifiedProperty newSingleDateMilestoningQualifiedPropertyNoA ._rawType(context.pureModel.getType("meta::pure::metamodel::function::property::QualifiedProperty")) ._typeArguments(Lists.fixedSize.of(PureModel.buildFunctionType(Lists.mutable.of(thisVar), qualifiedProperty._genericType(), originalProperty._multiplicity(), context.pureModel))); + SimpleFunctionExpression finalExpression = filterExpression; + if (originalProperty._multiplicity() == context.pureModel.getMultiplicity("one")) + { + finalExpression = new Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::valuespecification::SimpleFunctionExpression")) + ._func(context.pureModel.getFunction("meta::pure::functions::multiplicity::toOne_T_MANY__T_1_", true)) + ._functionName("toOne") + ._genericType(qualifiedProperty._genericType()) + ._multiplicity(context.pureModel.getMultiplicity("one")) + ._parametersValues(Lists.fixedSize.of(filterExpression)); + } + qualifiedProperty._classifierGenericType(classifierGenericType); - qualifiedProperty._expressionSequence(Lists.fixedSize.of(filterExpression)); + qualifiedProperty._expressionSequence(Lists.fixedSize.of(finalExpression)); return qualifiedProperty; } @@ -357,12 +368,23 @@ private static MutableList> newSingleDateMilestoningQualifi ._multiplicity(context.pureModel.getMultiplicity("zeromany")) ._parametersValues(Lists.fixedSize.of(filterLhs, filterInstanceValue)); + SimpleFunctionExpression finalExpression = filterExpression; + if (originalProperty._multiplicity() == context.pureModel.getMultiplicity("one")) + { + finalExpression = new Root_meta_pure_metamodel_valuespecification_SimpleFunctionExpression_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::valuespecification::SimpleFunctionExpression")) + ._func(context.pureModel.getFunction("meta::pure::functions::multiplicity::toOne_T_MANY__T_1_", true)) + ._functionName("toOne") + ._genericType(qualifiedProperty._genericType()) + ._multiplicity(context.pureModel.getMultiplicity("one")) + ._parametersValues(Lists.fixedSize.of(filterExpression)); + } + GenericType classifierGenericType = new Root_meta_pure_metamodel_type_generics_GenericType_Impl("", null, context.pureModel.getClass("meta::pure::metamodel::type::generics::GenericType")) ._rawType(context.pureModel.getType("meta::pure::metamodel::function::property::QualifiedProperty")) ._typeArguments(Lists.fixedSize.of(PureModel.buildFunctionType(Lists.mutable.of(thisVar).withAll(datesToCompare.collect(Functions.firstOfPair())), qualifiedProperty._genericType(), originalProperty._multiplicity(), context.pureModel))); qualifiedProperty._classifierGenericType(classifierGenericType); - qualifiedProperty._expressionSequence(Lists.fixedSize.of(filterExpression)); + qualifiedProperty._expressionSequence(Lists.fixedSize.of(finalExpression)); return Lists.mutable.of(qualifiedProperty); } diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java index ca961cb9c47..4b65ee3aa26 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/handlers/Handlers.java @@ -913,6 +913,7 @@ private void registerStrings() register("meta::pure::functions::string::replace_String_1__String_1__String_1__String_1_", true, ps -> res("String", "one")); register("meta::pure::functions::string::reverseString_String_1__String_1_", true, ps -> res("String", "one")); register("meta::pure::functions::string::split_String_1__String_1__String_MANY_", true, ps -> res("String", "zeroMany")); + register("meta::pure::functions::string::splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_", false, ps -> res("String", "zeroOne")); register(m(m(h("meta::pure::functions::string::substring_String_1__Integer_1__Integer_1__String_1_", true, ps -> res("String", "one"), ps -> ps.size() == 3)), m(h("meta::pure::functions::string::substring_String_1__Integer_1__String_1_", true, ps -> res("String", "one"), ps -> true)))); register("meta::pure::functions::string::toLower_String_1__String_1_", true, ps -> res("String", "one")); @@ -2040,6 +2041,7 @@ private Map buildDispatch() map.put("meta::pure::functions::string::reverseString_String_1__String_1_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::splitOnCamelCase_String_1__String_MANY_", (List ps) -> ps.size() == 1 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::split_String_1__String_1__String_MANY_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name()))); + map.put("meta::pure::functions::string::splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_", (List ps) -> ps.size() == 3 && matchZeroOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name())) && isOne(ps.get(2)._multiplicity()) && ("Nil".equals(ps.get(2)._genericType()._rawType()._name()) || "Integer".equals(ps.get(2)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::startsWith_String_$0_1$__String_1__Boolean_1_", (List ps) -> ps.size() == 2 && matchZeroOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::startsWith_String_1__String_1__Boolean_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name()))); map.put("meta::pure::functions::string::substringAfter_String_1__String_1__String_1_", (List ps) -> ps.size() == 2 && isOne(ps.get(0)._multiplicity()) && ("Nil".equals(ps.get(0)._genericType()._rawType()._name()) || "String".equals(ps.get(0)._genericType()._rawType()._name())) && isOne(ps.get(1)._multiplicity()) && ("Nil".equals(ps.get(1)._genericType()._rawType()._name()) || "String".equals(ps.get(1)._genericType()._rawType()._name()))); diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/fromGrammar/TestDomainCompilationFromGrammar.java b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/fromGrammar/TestDomainCompilationFromGrammar.java index dc855abb8e5..86a18fe9604 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/fromGrammar/TestDomainCompilationFromGrammar.java +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/test/fromGrammar/TestDomainCompilationFromGrammar.java @@ -2295,6 +2295,66 @@ public void testMilestoningSimplePropertiesInNonMilestonedClassesAreNotRestricte Assert.assertTrue(biTemporalAddressProperties.allSatisfy(p -> Lists.immutable.with("processingDate", "processingDateComplex", "businessDate", "businessDateComplex").contains(p.getName()))); } + @Test + public void testMilestoningPropertiesQualfiedExpressionForToOne() + { + String grammar = "###Pure\n" + + "Class <> test::ProcessingTemporalAddress\n" + + "{\n" + + " DunmmyProperty : String[1];\n" + + "}\n" + + "\n" + + "Class <> test::BusinessTemporalAddress\n" + + "{\n" + + " DunmmyProperty : String[1];\n" + + "}\n" + + "\n" + + "Class <> test::BiTemporalAddress\n" + + "{\n" + + " DunmmyProperty : String[1];\n" + + "}\n" + + "\n" + + "Class test::Person\n" + + "{\n" + + " processingTemporalAddress1 : test::ProcessingTemporalAddress[1];\n" + + " businessTemporalAddress1 : test::BusinessTemporalAddress[1];\n" + + " biTemporalAddress1 : test::BiTemporalAddress[1];\n" + + " processingTemporalAddress2 : test::ProcessingTemporalAddress[0..1];\n" + + " businessTemporalAddress2 : test::BusinessTemporalAddress[0..1];\n" + + " biTemporalAddress2 : test::BiTemporalAddress[0..1];\n" + + " processingTemporalAddress3 : test::ProcessingTemporalAddress[*];\n" + + " businessTemporalAddress3 : test::BusinessTemporalAddress[*];\n" + + " biTemporalAddress3 : test::BiTemporalAddress[*];\n" + + "}"; + PureModel pm = test(grammar).getTwo(); + + RichIterable> personQualifiedProperties = pm.getClass("test::Person")._qualifiedProperties(); + boolean checkQualifiedExpressionForprocessingTemporalAddress1 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("processingTemporalAddress1"))); + boolean checkQualifiedExpressionForbusinessTemporalAddress1 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("businessTemporalAddress1"))); + boolean checkQualifiedExpressionForbiTemporalAddress1 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("biTemporalAddress1"))); + boolean checkQualifiedExpressionForprocessingTemporalAddress2 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("processingTemporalAddress2"))); + boolean checkQualifiedExpressionForbusinessTemporalAddress2 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("businessTemporalAddress2"))); + boolean checkQualifiedExpressionForbiTemporalAddress2 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("biTemporalAddress2"))); + boolean checkQualifiedExpressionForprocessingTemporalAddress3 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("processingTemporalAddress3"))); + boolean checkQualifiedExpressionForbusinessTemporalAddress3 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("businessTemporalAddress3"))); + boolean checkQualifiedExpressionForbiTemporalAddress3 = checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(personQualifiedProperties.detect(p -> p.getName().equals("biTemporalAddress3"))); + Assert.assertTrue("Qualfied expression generated for processingTemporalAddress should have toOne() as topLevel functionExpression", checkQualifiedExpressionForprocessingTemporalAddress1); + Assert.assertTrue("Qualfied expression generated for BusinessTemporalAddress should have toOne() as topLevel functionExpression", checkQualifiedExpressionForbusinessTemporalAddress1); + Assert.assertTrue("Qualfied expression generated for BiTemporalAddress should have toOne() as topLevel functionExpression", checkQualifiedExpressionForbiTemporalAddress1); + Assert.assertTrue("Qualfied expression generated for processingTemporalAddress should not have toOne() as topLevel functionExpression", !checkQualifiedExpressionForprocessingTemporalAddress2); + Assert.assertTrue("Qualfied expression generated for BusinessTemporalAddress should not have toOne() as topLevel functionExpression", !checkQualifiedExpressionForbusinessTemporalAddress2); + Assert.assertTrue("Qualfied expression generated for BiTemporalAddress should not have toOne() as topLevel functionExpression", !checkQualifiedExpressionForbiTemporalAddress2); + Assert.assertTrue("Qualfied expression generated for processingTemporalAddress should not have toOne() as topLevel functionExpression", !checkQualifiedExpressionForprocessingTemporalAddress3); + Assert.assertTrue("Qualfied expression generated for BusinessTemporalAddress should not have toOne() as topLevel functionExpression", !checkQualifiedExpressionForbusinessTemporalAddress3); + Assert.assertTrue("Qualfied expression generated for BiTemporalAddress should not have toOne() as topLevel functionExpression", !checkQualifiedExpressionForbiTemporalAddress3); + } + + private boolean checkQualifiedExpressionForToOneAsTopLevelFunctionExpression(QualifiedProperty generatedMilestoningClassQualifiedProperty) + { + FunctionExpression topLevelExpression = ((FunctionExpression) generatedMilestoningClassQualifiedProperty._expressionSequence().getFirst()); + return topLevelExpression._func()._functionName().equals("toOne"); + } + @Test public void testMilestoningSimplePropertiesAreNotOverridenByUserProperties() { @@ -2353,13 +2413,22 @@ public void testMilestoningSimplePropertiesAreNotOverridenByUserProperties() private boolean generatedMilestoningQualifgiedPropertyUsesGeneratedMilestoningProperty(Property generatedMilestoningClassSimpleProperty, QualifiedProperty generatedMilestoningClassQualifiedProperty) { - FunctionExpression topLevelExpression = (FunctionExpression) ((LambdaFunction) ((InstanceValue) ((FunctionExpression) generatedMilestoningClassQualifiedProperty._expressionSequence().getFirst())._parametersValues().toList().get(1))._values().getOnly())._expressionSequence().getOnly(); - if (topLevelExpression._func()._functionName().equals("and")) + FunctionExpression topLevelExpression = ((FunctionExpression) generatedMilestoningClassQualifiedProperty._expressionSequence().getFirst()); + FunctionExpression simplifiedExpression; + if (topLevelExpression._func()._functionName().equals("toOne")) + { + simplifiedExpression = (FunctionExpression) ((LambdaFunction) ((InstanceValue) ((FunctionExpression) (topLevelExpression._parametersValues().toList().get(0)))._parametersValues().toList().get(1))._values().getOnly())._expressionSequence().getOnly(); + } + else + { + simplifiedExpression = (FunctionExpression) ((LambdaFunction) ((InstanceValue) topLevelExpression._parametersValues().toList().get(1))._values().getOnly())._expressionSequence().getOnly(); + } + if (simplifiedExpression._func()._functionName().equals("and")) { int idx = generatedMilestoningClassSimpleProperty.getName().equals("processingDate") ? 0 : 1; - topLevelExpression = (FunctionExpression) topLevelExpression._parametersValues().toList().get(idx); + simplifiedExpression = (FunctionExpression) simplifiedExpression._parametersValues().toList().get(idx); } - Property filterMilestoningDateProperty = (Property) ((FunctionExpression) topLevelExpression._parametersValues().toList().get(0))._func(); + Property filterMilestoningDateProperty = (Property) ((FunctionExpression) simplifiedExpression._parametersValues().toList().get(0))._func(); return generatedMilestoningClassSimpleProperty.equals(filterMilestoningDateProperty); } diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar-api/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar-api/pom.xml index 66917c3dd0f..1a7b491da29 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar-api/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-grammar-api diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar/pom.xml index e086f318371..973cef500c9 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-grammar diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager-sdlc/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager-sdlc/pom.xml index a8331ce3dca..71bb9445572 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager-sdlc/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager-sdlc/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-modelManager-sdlc diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager/pom.xml index cb100c63dbb..be2fc6469e2 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-language-pure-modelManager/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-modelManager diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-api/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-api/pom.xml index cd1308df5be..41ee6422e0c 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-api/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-protocol-api diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation-pure/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation-pure/pom.xml index 8f3724d88f5..366fed6b963 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation-pure/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-protocol-generation-pure diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation/pom.xml index 41e0e0a9a99..823790e4a7a 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-generation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-protocol-generation diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/pom.xml index c2de39358a9..934ff2bfd97 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-protocol-pure diff --git a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol/pom.xml b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol/pom.xml index 665296998e4..f63fd2764a6 100644 --- a/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/legend-engine-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-language-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-protocol diff --git a/legend-engine-core/legend-engine-core-language-pure/pom.xml b/legend-engine-core/legend-engine-core-language-pure/pom.xml index 4a20dce6c5c..970dbd37fe8 100644 --- a/legend-engine-core/legend-engine-core-language-pure/pom.xml +++ b/legend-engine-core/legend-engine-core-language-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-core - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-query-pure/legend-engine-query-pure/pom.xml b/legend-engine-core/legend-engine-core-query-pure/legend-engine-query-pure/pom.xml index 361b5f3ed17..e55e7b0532b 100644 --- a/legend-engine-core/legend-engine-core-query-pure/legend-engine-query-pure/pom.xml +++ b/legend-engine-core/legend-engine-core-query-pure/legend-engine-query-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-query-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-query-pure diff --git a/legend-engine-core/legend-engine-core-query-pure/pom.xml b/legend-engine-core/legend-engine-core-query-pure/pom.xml index d871a339550..10be27ceb23 100644 --- a/legend-engine-core/legend-engine-core-query-pure/pom.xml +++ b/legend-engine-core/legend-engine-core-query-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-core - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-shared/legend-engine-shared-core/pom.xml b/legend-engine-core/legend-engine-core-shared/legend-engine-shared-core/pom.xml index 1f59eba5072..15d8c371a6e 100644 --- a/legend-engine-core/legend-engine-core-shared/legend-engine-shared-core/pom.xml +++ b/legend-engine-core/legend-engine-core-shared/legend-engine-shared-core/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-shared - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-shared-core diff --git a/legend-engine-core/legend-engine-core-shared/legend-engine-shared-javaCompiler/pom.xml b/legend-engine-core/legend-engine-core-shared/legend-engine-shared-javaCompiler/pom.xml index d3322c19d8c..c6a67446138 100644 --- a/legend-engine-core/legend-engine-core-shared/legend-engine-shared-javaCompiler/pom.xml +++ b/legend-engine-core/legend-engine-core-shared/legend-engine-shared-javaCompiler/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-shared - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-shared-javaCompiler diff --git a/legend-engine-core/legend-engine-core-shared/pom.xml b/legend-engine-core/legend-engine-core-shared/pom.xml index c027315e943..235b224f155 100644 --- a/legend-engine-core/legend-engine-core-shared/pom.xml +++ b/legend-engine-core/legend-engine-core-shared/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-core - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-test/legend-engine-test-data-generation/pom.xml b/legend-engine-core/legend-engine-core-test/legend-engine-test-data-generation/pom.xml index 9217dfcf3df..99a284088ed 100644 --- a/legend-engine-core/legend-engine-core-test/legend-engine-test-data-generation/pom.xml +++ b/legend-engine-core/legend-engine-core-test/legend-engine-test-data-generation/pom.xml @@ -3,7 +3,7 @@ org.finos.legend.engine legend-engine-core-test - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-mapping/pom.xml b/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-mapping/pom.xml index 4b46db9c1b0..ef911453207 100644 --- a/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-mapping/pom.xml +++ b/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-mapping/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-test - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-shared/pom.xml b/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-shared/pom.xml index eda15403a64..05c819f64a9 100644 --- a/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-shared/pom.xml +++ b/legend-engine-core/legend-engine-core-test/legend-engine-test-runner-shared/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-test - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-test-runner-shared diff --git a/legend-engine-core/legend-engine-core-test/legend-engine-test-server-shared/pom.xml b/legend-engine-core/legend-engine-core-test/legend-engine-test-server-shared/pom.xml index 8eb6a0982af..512449b9849 100644 --- a/legend-engine-core/legend-engine-core-test/legend-engine-test-server-shared/pom.xml +++ b/legend-engine-core/legend-engine-core-test/legend-engine-test-server-shared/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-test - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-test-server-shared diff --git a/legend-engine-core/legend-engine-core-test/legend-engine-testable/pom.xml b/legend-engine-core/legend-engine-core-test/legend-engine-testable/pom.xml index 7fb35544c79..9a034b49bcc 100644 --- a/legend-engine-core/legend-engine-core-test/legend-engine-testable/pom.xml +++ b/legend-engine-core/legend-engine-core-test/legend-engine-testable/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-core-test - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-testable diff --git a/legend-engine-core/legend-engine-core-test/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/assertion/TestAssertionEvaluator.java b/legend-engine-core/legend-engine-core-test/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/assertion/TestAssertionEvaluator.java index 8738cf297a2..83b7daa2e1d 100644 --- a/legend-engine-core/legend-engine-core-test/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/assertion/TestAssertionEvaluator.java +++ b/legend-engine-core/legend-engine-core-test/legend-engine-testable/src/main/java/org/finos/legend/engine/testable/assertion/TestAssertionEvaluator.java @@ -48,13 +48,23 @@ public AssertionStatus visit(TestAssertion testAssertion) { if (testAssertion instanceof EqualTo) { - if (!(result instanceof ConstantResult)) + + Object actual; + Object expected = ((EqualTo) testAssertion).expected.accept(new PrimitiveValueSpecificationToObjectVisitor());; + if (result instanceof ConstantResult) + { + actual = ((ConstantResult) result).getValue(); + } + else if (result instanceof StreamingResult) + { + actual = ((StreamingResult) result).flush(((StreamingResult) result).getSerializer(this.serializationFormat)); + } + + else { throw new UnsupportedOperationException("Result type - " + result.getClass().getSimpleName() + " not supported with EqualTo Assert !!"); } - Object expected = ((EqualTo) testAssertion).expected.accept(new PrimitiveValueSpecificationToObjectVisitor()); - Object actual = ((ConstantResult) result).getValue(); AssertionStatus assertionStatus; if (expected.equals(actual)) diff --git a/legend-engine-core/legend-engine-core-test/pom.xml b/legend-engine-core/legend-engine-core-test/pom.xml index 6ff6e63d86e..9d9af090890 100644 --- a/legend-engine-core/legend-engine-core-test/pom.xml +++ b/legend-engine-core/legend-engine-core-test/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-core - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-core/pom.xml b/legend-engine-core/pom.xml index ed0f8c97ee0..5d32e53982a 100644 --- a/legend-engine-core/pom.xml +++ b/legend-engine-core/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/pom.xml b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/pom.xml index dfcf4e7eb36..c1878c5dbc9 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/pom.xml +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-code - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-code-compiled-core diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure index 2c6f06c0352..48d8a454c6e 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/legend/test/handlersTest.pure @@ -382,6 +382,7 @@ Class meta::legend::test::handlers::model::TestString replace(){$this.string->replace($this.string, $this.string)}:String[1]; reverseString(){$this.string->reverseString()}:String[1]; split(){$this.string->split($this.string)}:String[*]; + splitPart(){$this.string->splitPart($this.string, 0)}:String[*]; rtrim(){$this.string->rtrim()}:String[1]; substringTwo(){$this.string->substring(1,2)}:String[1]; substring(){$this.string->substring(1)}:String[1]; diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure index c1e9bc5da32..2ec81292d02 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/stringExtension.pure @@ -167,6 +167,14 @@ function {doc.doc = 'Un-camel case / humanize the provided string using provided ->joinStrings(' '); } +function {doc.doc = 'Split the string and select the part'} + meta::pure::functions::string::splitPart(str:String[0..1], token:String[1], part:Integer[1]):String[0..1] +{ + if ($str->isEmpty(), + | [], + | $str->toOne()->split($token)->at($part)); +} + function {doc.doc = 'Split the proivded string on camel case'} meta::pure::functions::string::splitOnCamelCase(str:String[1]):String[*] { diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/stringExtension.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/stringExtension.pure index ac993d65cf9..92134809154 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/stringExtension.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/corefunctions/tests/stringExtension.pure @@ -110,6 +110,14 @@ function <> {test.excludePlatform = 'Java compiled'} meta::pure::func assertEquals('abab', repeatString('ab', 2)); } +function <> meta::pure::functions::string::tests::testSplitPart():Boolean[1] +{ + assertEquals([], splitPart([], 'a', 1)); + assertEquals('abc', splitPart('abc', 'd', 0)); + assertEquals('a', splitPart('abc', 'b', 0)); + assertEquals('c', splitPart('abc', 'b', 1)); +} + function <> meta::pure::functions::string::tests::testSplitOnCamelCase():Boolean[1] { let pairs = [ diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/protocol/vX_X_X/transfers/valueSpecification.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/protocol/vX_X_X/transfers/valueSpecification.pure index be571b393fa..b62703c5df5 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/protocol/vX_X_X/transfers/valueSpecification.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/protocol/vX_X_X/transfers/valueSpecification.pure @@ -693,7 +693,8 @@ function meta::protocols::pure::vX_X_X::transformation::fromPureGraph::toPureGra ->concatenate([ pair(meta::pure::mapping::from_TabularDataSet_1__Mapping_1__Runtime_1__TabularDataSet_1_.name->toOne(), meta::pure::mapping::from_TabularDataSet_1__Mapping_1__Runtime_1__TabularDataSet_1_), pair(meta::pure::mapping::from_TabularDataSet_1__Mapping_1__Runtime_1__ExecutionContext_1__TabularDataSet_1_.name->toOne(), meta::pure::mapping::from_TabularDataSet_1__Mapping_1__Runtime_1__ExecutionContext_1__TabularDataSet_1_), - pair(meta::pure::runtime::currentUserId__String_1_.name->toOne(), meta::pure::runtime::currentUserId__String_1_) + pair(meta::pure::runtime::currentUserId__String_1_.name->toOne(), meta::pure::runtime::currentUserId__String_1_), + pair(meta::pure::functions::hash::hash_String_1__HashType_1__String_1_.name->toOne(), meta::pure::functions::hash::hash_String_1__HashType_1__String_1_) ]) ->removeDuplicatesBy(p | $p.second) ->newMultiValueMap(); diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/preeval.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/preeval.pure index 0b3a399c4f4..c132f44d392 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/preeval.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/preeval.pure @@ -1,1090 +1,1092 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::pure::router::preeval::*; -import meta::pure::extension::*; -import meta::pure::metamodel::path::*; -import meta::pure::router::extension::*; -import meta::pure::router::utils::*; - -Class meta::pure::router::preeval::State -{ - inScopeVars : Map>[1]; - rollingInScopeVars : Map>[1]; - inScopeTypeParams : Map[1]; - - debug : DebugContext[1]; - depth : Integer[0..1]; - path : FunctionDefinition[*]; - - shouldInlineFxn : FunctionDefinition<{Function[1] -> Boolean[1]}>[1]; - stopPreeval : Function<{Any[1] -> Boolean[1]}>[1]; - - toString() { $this->simpleToString() }:String[1]; -} - -Class meta::pure::router::preeval::PrevalWrapper -{ - canPreval : Boolean[1]; - value : T[1]; - openVars : String[*]; - modified : Boolean[1]; - - toString() { $this->simpleToString() }:String[1]; -} - - -// ==================================================================================================================================================================== -// Public Apis -// ==================================================================================================================================================================== - -function <> meta::pure::router::preeval::preval(f:FunctionDefinition[1], extensions:Extension[*]):FunctionDefinition[1] -{ - preval($f, $extensions, noDebug()); -} - -function <> meta::pure::router::preeval::preval(f:FunctionDefinition[1], extensions:Extension[*], debug:DebugContext[1]):FunctionDefinition[1] -{ - preval($f, newMap([]->cast(@Pair)), $f->openVariableValues(), $extensions, $debug).value->toOne()->cast($f); -} - -function <> meta::pure::router::preeval::preval(f : FunctionDefinition[1], vars:Map[1], inScopeVars:Map>[1], extensions:Extension[*]):PrevalWrapper>[1] -{ - preval($f, $vars, $inScopeVars, $extensions, noDebug()); -} - -function <> meta::pure::router::preeval::preval(f : FunctionDefinition[1], vars:Map[1], inScopeVars:Map>[1], extensions:Extension[*], debug:DebugContext[1]):PrevalWrapper>[1] -{ - let state = ^meta::pure::router::preeval::State(inScopeVars = $inScopeVars, - shouldInlineFxn = defaultFunctionInlineStrategy($extensions), - stopPreeval = defaultPreevalStopStrategy($extensions), - rollingInScopeVars = $inScopeVars, - debug = $debug, - inScopeTypeParams = ^Map()); - preval($f, $state, $extensions); -} - -function <> meta::pure::router::preeval::preval(f : FunctionDefinition[1], state:State[1], extensions:Extension[*]):PrevalWrapper>[1] -{ - let r = prevalInternal($f->evaluateAndDeactivate(), $state, $extensions)->toOne()->cast(@PrevalWrapper>); - - if(!$r->anyModified(), - | - $state->printDebug(|'No changes made for: '); - $state->printDebug(|$f->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::toPureGrammar::toPure($extensions));, - | - let res = $r.value->toOne(); - - // $state->printDebug('Transformed From:'); - // $state->printDebug(|$f->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::json::toJSON(50000)); - // $state->printDebug(|$f->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::protocols::pure::vX_X_X::transformation::toPureGrammar::toPure($extensions)); - - $state->printDebug('Transformed To:'); - $state->printDebug(|$res->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::json::toJSON(50000)); - $state->printDebug(|$res->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::toPureGrammar::toPure($extensions)); - - $state->printDebug($res); - ); - - $r; -} - -function <> meta::pure::router::preeval::preval(f : FunctionExpression[1], inScopeVars:Map>[1], extensions:Extension[*], debug:DebugContext[1]):ValueSpecification[1] -{ - let state = ^meta::pure::router::preeval::State(inScopeVars = $inScopeVars, - shouldInlineFxn = defaultFunctionInlineStrategy($extensions), - stopPreeval = defaultPreevalStopStrategy($extensions), - rollingInScopeVars = $inScopeVars, - debug = $debug, - inScopeTypeParams = ^Map()); - prevalInternal($f, $state, $extensions).value->cast(@ValueSpecification); -} - -// ==================================================================================================================================================================== -// Processor Functions -// ==================================================================================================================================================================== - -function <> meta::pure::router::preeval::prevalInternal(item : Any[1], origState : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] -{ - let state = ^$origState(depth = if($origState.depth->isEmpty(), | 0, | $origState.depth->toOne() + 1), - path = if(!$item->instanceOf(FunctionDefinition), | $origState.path, | $origState.path->concatenate($item->cast(@FunctionDefinition)))); - - - $state->printDebugWithDepth(|'Processing ' + $item->type()->makeString() + ' ' - + $item->evaluateAndDeactivate() - ->match([ - iv:InstanceValue[1] | $iv.values->map(x|$x->type())->makeString(), - sfe:SimpleFunctionExpression[1] | '(' + $sfe.func->elementToPath() + ')', - ve:VariableExpression[1] | '(' + $ve.name + ')', - a:Any[*]|''] - ) - + ', inScopeVars: ' + $state.inScopeVars->keys()->joinStrings('[', ',', ']')); - - let newItem = $item->evaluateAndDeactivate() - ->match([ - {lambda : LambdaFunction[1]| - let fnArgNames = $lambda->functionType().parameters.name; - let fnOpenVariables = $lambda->openVariableValues()->putAll($state.inScopeVars->keyValues()->filter(kv | !$kv.first->in($fnArgNames))->newMap()); - - $lambda->prevalFunctionDefinition(^$state(inScopeVars = $fnOpenVariables), $extensions); - }, - {fd : FunctionDefinition[1]| - $fd->prevalFunctionDefinition(^$state(inScopeVars = newMap([])), $extensions); // Assigning empty map to inScopeVars as function definition can't have openVars - }, - {sfe : SimpleFunctionExpression[1]| - let handlers = [ - // Avoid handling known 'false' case (which may not be evalulatable / have runtime issues, e.g. a toOne() after an isNotEmpty condition) - ^Pair, LambdaFunction<{->PrevalWrapper[1]}>>(first = if_Boolean_1__Function_1__Function_1__T_m_, second = {| - let conditionPreVal = $sfe.parametersValues->at(0)->prevalInternal($state, $extensions); - if($conditionPreVal.value->toOne()->cast(@ValueSpecification)->isInstanceValue($state.inScopeVars), - | let valueValueSpecification = if($conditionPreVal.value->cast(@InstanceValue).values == true, - | $sfe.parametersValues->at(1), - | $sfe.parametersValues->at(2) - ); - let newSfe = ^$sfe(func = eval_Function_1__V_m_, parametersValues = $valueValueSpecification); - - $newSfe->prevalInternal($state, $extensions)->markModified();, - | prevalGenericSimpleFunctionExpression($sfe, $state, $extensions); - ); - }), - - // Avoid handling conditions after the first false (which may not be evalulatable / have runtime issues, e.g. a toOne() after an isNotEmpty condition) - ^Pair, LambdaFunction<{->PrevalWrapper[1]}>>(first = meta::pure::functions::boolean::and_Boolean_1__Boolean_1__Boolean_1_, second = {| - prevalBooleanSimpleFunctionExpression($sfe, $state, $extensions); - }), - - // Avoid handling conditions after the first true (which may not be evalulatable / have runtime issues, e.g. a toOne() after an isEmpty condition) - ^Pair, LambdaFunction<{->PrevalWrapper[1]}>>(first = meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_, second = {| - prevalBooleanSimpleFunctionExpression($sfe, $state, $extensions); - }) - - // TODO: - // 1. Optimise and_Boolean_MANY__Boolean_1_ and or_Boolean_$1_MANY$__Boolean_1_ - // 1. Optimise plus(many), minus(many) and times(many) (plus_String_MANY__String_1_, plus_Float_MANY__Float_1_, times_Number_MANY__Number_1_, minus_Float_MANY__Float_1_) - ]; - - let handler = $handlers->filter(p|$p.first == $sfe.func).second; - - if($handler->isNotEmpty(), - | $handler->toOne()->eval(), - | prevalGenericSimpleFunctionExpression($sfe, $state, $extensions)); - }, - {ve : VariableExpression[1]| - let varValue = $state.inScopeVars->get($ve.name).values; - let iv = if($state.inScopeVars->get($ve.name)->isEmpty(), - | ^PrevalWrapper(value = $ve, canPreval = true, modified = false, openVars = [$ve.name]), - | $varValue->match([ - n : Nil[0]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), - iv : InstanceValue[1]| $iv, - ve : VariableExpression[1]| $ve->evaluateAndDeactivate(), - sfe : SimpleFunctionExpression[1]| $sfe, - a : meta::pure::tds::AggregateValue[*]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), - a : meta::pure::functions::collection::AggregateValue[*]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), - a : meta::pure::tds::BasicColumnSpecification[*]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), - r : meta::pure::graphFetch::RootGraphFetchTree[1]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate();, - b : meta::external::format::shared::binding::Binding[1]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate();, - s : meta::pure::store::Store[1]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate();, - a : Any[*]| assert($state.stopPreeval->eval($a), | 'Unsupported type: ' + $a->type()->match([pe:PackageableElement[1]|$pe->elementToPath(), t:Type[1]|$t->makeString()])); - ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(); - ]) - ->map(x| - let newState = ^$state(inScopeVars = $state.inScopeVars->keyValues()->filter(p|$p.first != $ve.name)->newMap()); - $x->prevalInternal($newState, $extensions); - ) - ->markModified() - ); - }, - {iv : InstanceValue[1]| - let subVals = $iv.values->map(x|$x->prevalInternal($state, $extensions)); - let r = if(!$subVals->anyModified(), - | pair(false, $iv), - | let cleanIv = ^$iv(values = $subVals.value->map(x|$x->match([subIv:InstanceValue[1]|if($subIv.values->size() == 1, |$subIv.values, |$subIv), a:Any[*]|$a]))); - let cleanIv2 = ^$cleanIv(genericType = $cleanIv.genericType->resolveGenericType($state).value, - multiplicity = if($cleanIv.multiplicity == PureOne && $cleanIv.values->isEmpty(), - | $cleanIv.multiplicity, //This seems to be a quirk of Cast(@ABC) - | $cleanIv.values->size()->toMultiplicity()) - ); - pair(true, $cleanIv2); - ); - ^PrevalWrapper(value = $r.second, canPreval = $subVals->canPreval(), openVars = $subVals->openVars($state.inScopeVars), modified = $r.first); - }, - {k : KeyExpression[1]| - let e = $k.expression->prevalInternal($state, $extensions)->toOne(); - let ke = if(!$e->anyModified(), - | pair(false, $k), - | pair(true, ^$k(expression = $e.value->cast(@ValueSpecification)->toOne()))); - - ^PrevalWrapper(value = $ke.second, canPreval = $e.canPreval, openVars = $e->openVars($state.inScopeVars), modified = $ke.first); - }, - {p : ColumnSpecification[1]| - $p->match([ - bcs : BasicColumnSpecification[1]| - let f = $bcs.func->evaluateAndDeactivate(); - let newFuncWrapper = prevalInternal($f, ^$state(inScopeTypeParams = ^Map(), inScopeVars = $state.inScopeVars->putAll($f->openVariableValues())), $extensions); - - if(!$newFuncWrapper->anyModified(), - | ^PrevalWrapper(value = $bcs, canPreval = true, modified = false), - | ^PrevalWrapper(value = ^$bcs(func = $newFuncWrapper.value->cast($f)), canPreval = true, modified = true); - );, - o:ColumnSpecification[1] | ^PrevalWrapper(value = $o, canPreval = true, modified = false); - ]); - - }, - {av : meta::pure::tds::AggregateValue[1]| - let mapFn = $av.mapFn->evaluateAndDeactivate(); - let aggFn = $av.aggregateFn->evaluateAndDeactivate(); - - let newFuncWrapper_map = prevalInternal($mapFn, $state, $extensions); - let newFuncWrapper_agg = prevalInternal($aggFn, $state, $extensions); - - if(![$newFuncWrapper_map, $newFuncWrapper_agg]->anyModified(), - | ^PrevalWrapper(value = $av, canPreval = true, modified = false), - | ^PrevalWrapper(value = ^$av(mapFn = $newFuncWrapper_map.value->cast($mapFn), aggregateFn = $newFuncWrapper_agg.value->cast($aggFn)), canPreval = true, modified = true); - ); - }, - {av : meta::pure::functions::collection::AggregateValue[1]| - let mapFn = $av.mapFn->evaluateAndDeactivate(); - let aggFn = $av.aggregateFn->evaluateAndDeactivate(); - - let newFuncWrapper_map = prevalInternal($mapFn, $state, $extensions); - let newFuncWrapper_agg = prevalInternal($aggFn, $state, $extensions); - - if(![$newFuncWrapper_map, $newFuncWrapper_agg]->anyModified(), - | ^PrevalWrapper(value = $av, canPreval = true, modified = false), - | ^PrevalWrapper(value = ^$av(mapFn = $newFuncWrapper_map.value->cast($mapFn), aggregateFn = $newFuncWrapper_agg.value->cast($aggFn)), canPreval = true, modified = true); - ); - }, - {r: meta::pure::graphFetch::RootGraphFetchTree[1] | ^PrevalWrapper(value = $r, canPreval = true, modified = false);}, - {b: meta::external::format::shared::binding::Binding[1] | ^PrevalWrapper(value = $b, canPreval = true, modified = false);}, - {s: meta::pure::store::Store[1] | ^PrevalWrapper(value = $s, canPreval = true, modified = false);}, - {a : Any[1]| - assert($state.stopPreeval->eval($a), | 'Unsupported type: ' + $a->type()->match([pe:PackageableElement[1]|$pe->elementToPath(), t:Type[1]|$t->makeString()])); - ^PrevalWrapper(value = $a, canPreval = true, modified = false); - } - ]); - - $state->printDebugWithDepth(|'Returning ' + $newItem.value->type()->makeString() + ' ' + $newItem.value->match([iv:InstanceValue[1]|$iv.values->map(x|$x->type())->makeString(), sfe:SimpleFunctionExpression[1]|'(' + $sfe.func->elementToPath() + ')', ve : VariableExpression[1]|'(' + $ve.name + ')', a:Any[*]|'']) + ' ' + $newItem.toString()); - $newItem; -} - -function <> meta::pure::router::preeval::prevalFunctionDefinition(fd : FunctionDefinition[1], origState : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] -{ - let items = $fd.expressionSequence; - - let r = range($items->size()) - ->fold({index, p| - let item = $items->at($index); - let state = $p.first; - - let isLastExpression = ($index == ($items->size()-1)); - - if(!$isLastExpression && !$item->isLetFunction(), - | - $state->printDebugWithDepth(|'Dropping non-variable assignement non-return statement: ' + $item->type()->makeString() + ' ' - + $item->match([ - iv:InstanceValue[1]|$iv.values->map(x|$x->type())->makeString(), - sfe:SimpleFunctionExpression[1]|'(' + $sfe.func->elementToPath() + ')', - ve : VariableExpression[1]|'(' + $ve.name + ')', - a:Any[*]|'' - ]) - + ', inScopeVars: ' + $state.inScopeVars->keys()->joinStrings('[', ',', ']')); - - pair($state, $p.second);, - | - let r = $item->prevalInternal($state, $extensions); - - let isLetFunctionValueSpecification = $r.value->instanceOf(ValueSpecification) && $r.value->cast(@ValueSpecification)->isLetFunction(); - - if(!$isLetFunctionValueSpecification, - | pair($state, list(if($p.second.values->isEmpty(), | $r, | $p.second.values->concatenate($r)))), - | - let fe = $r.value->cast(@FunctionExpression); - - let varName = $fe.parametersValues->at(0)->reactivate($state.inScopeVars)->cast(@String)->toOne(); - let varValue = $fe.parametersValues->at(1) - ->match([ - iv:InstanceValue[1]|$iv.values, - fe:ValueSpecification[1]|$fe - ]); - - $state->printDebugWithDepth(|'adding variable ' + $varName + ' to rolling scope, with value ' + $varValue->makeString()); - - let shouldInlineVariableExpression = ($varValue->match([v:VariableExpression[1]|true, a: Any[*]|false])) - || ($r->canPreval() && $r->areAllInScope($state.inScopeVars) && !$varValue->match([v:LambdaFunction[1]|true, a: Any[*]|false])); - - let newRollingInScopeVars = $state.rollingInScopeVars->put($varName, list($varValue)); - let newInScopeVars = if(!$shouldInlineVariableExpression, - | $state.inScopeVars, - | $state.inScopeVars->put($varName, list($varValue)) - ); - let newState = ^$state(inScopeVars = $newInScopeVars, rollingInScopeVars = $newRollingInScopeVars); - - let expValue = if($isLastExpression, - | ^PrevalWrapper(canPreval = $r.canPreval, modified = true, openVars =$r.openVars, value = $varValue->match([vs:ValueSpecification[1]|$vs, a:Any[*]|^InstanceValue(genericType = $fe.genericType, multiplicity = $a->size()->toMultiplicity(), values = $a)])), - | if($shouldInlineVariableExpression, - | [], - | $r - ) - ); - if($expValue->isEmpty(), - | pair($newState, $p.second), - | pair($newState, list(if($p.second.values->isEmpty(), | $expValue, | $p.second.values->concatenate($expValue)))) - ); - ); - ); - }, [pair($origState, list([]->cast(@PrevalWrapper)))] - ).second.values; - - - /* - This should not be necessary, but seems to be required otherwise fold can end up wrapping InstanceValues in InstanceValues, - which map doesn't seem to - */ - let newExpressionSequncePrevalWrappers = $r->evaluateAndDeactivate() - ->match([ - iv:InstanceValue[1]|$iv.values->cast(@PrevalWrapper), - a:PrevalWrapper[*]|$a - ]); - - let value = if(($newExpressionSequncePrevalWrappers->size() == $items->size()) && !$newExpressionSequncePrevalWrappers->anyModified(), - | pair(false, $fd), - | let newFd = ^$fd(expressionSequence = $newExpressionSequncePrevalWrappers.value->cast(@ValueSpecification)->toOneMany()) - ->match([ - lf : LambdaFunction[1]| ^$lf(openVariables = $lf.openVariables->map(v|$v->resolveVariable($origState)) - ->distinct() - ->intersection($newExpressionSequncePrevalWrappers.openVars->map(v|$v->resolveVariable($origState)))), - fd2:FunctionDefinition[1]| $fd2 - ]); - pair(true, $newFd); - ); - - ^PrevalWrapper>( - value = $value.second, - canPreval = $newExpressionSequncePrevalWrappers->canPreval(), - openVars = $fd->match([lf : LambdaFunction[1]|$value.second->cast(@LambdaFunction).openVariables, fd:FunctionDefinition[1]|[]]), - modified = $value.first - ); -} - -function <> meta::pure::router::preeval::prevalBooleanSimpleFunctionExpression(sfe : SimpleFunctionExpression[1], state : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] -{ - assert($sfe.func->in([and_Boolean_1__Boolean_1__Boolean_1_, or_Boolean_1__Boolean_1__Boolean_1_])); - assertEquals(2, $sfe.parametersValues->size()); - - let shortcutVal = !($sfe.func == and_Boolean_1__Boolean_1__Boolean_1_); - - let newParam1Wrapper = $sfe.parametersValues->at(0)->prevalInternal($state, $extensions); - - if($newParam1Wrapper.value->toOne()->isInstanceValue($state.inScopeVars), - | if(($newParam1Wrapper.value->cast(@InstanceValue).values == $shortcutVal), - | $newParam1Wrapper->markModified(), - | let newParam2Wrapper = $sfe.parametersValues->at(1)->prevalInternal($state, $extensions); - $newParam2Wrapper->markModified();), - | let newParam2Wrapper = $sfe.parametersValues->at(1)->prevalInternal($state, $extensions); - - if($newParam2Wrapper.value->toOne()->isInstanceValue($state.inScopeVars), - | if($newParam2Wrapper.value->cast(@InstanceValue).values != $shortcutVal, - | $newParam1Wrapper->markModified(), - | $newParam2Wrapper->markModified()), - | let newParamWrappers = [$newParam1Wrapper, $newParam2Wrapper]; - ^PrevalWrapper( - value = ^$sfe(parametersValues = $newParamWrappers.value->cast(@ValueSpecification)), - canPreval = $newParamWrappers->canPreval(), - openVars = $newParamWrappers->openVars($state.inScopeVars), - modified = $newParamWrappers->size() != $sfe.parametersValues->size() || $newParamWrappers->anyModified() - ); - ); - ); -} - -function <> meta::pure::router::preeval::prevalGenericSimpleFunctionExpression(sfe : SimpleFunctionExpression[1], state : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] -{ - let newParamWrappers = if($sfe.parametersValues->isEmpty(), - | [], - | $sfe.func->match([p:Property[1]|'this', f:Function[1]|$f->functionType().parameters.name]) - ->zip($sfe.parametersValues) - ->evaluateAndDeactivate() - ->map(p| $state->printDebugWithDepth(|'Processing parameter: ' + $p.first); - let r = $p.second->prevalInternal($state, $extensions); - $state->printDebugWithDepth(|'Completed processing parameter: ' + $p.first); - $r;) - ); - let newSfeModified = $newParamWrappers->anyModified(); - - let newSfe = if($newSfeModified, - | ^$sfe(parametersValues = $newParamWrappers.value->cast(@ValueSpecification) - ->map(pv|^$pv(genericType = $pv.genericType->resolveGenericType($state).value)), - genericType = $sfe.genericType->resolveGenericType($state).value), - | $sfe); - - let canPrevalFunc = !($newSfe.func->hasStereotype('SideEffectFunction', functionType) || $newSfe.func->hasStereotype('NotImplementedFunction', functionType) || $state.stopPreeval->eval($newSfe)); - - if(!$canPrevalFunc, - | $state->printDebugWithDepth(|'Unable to perform preval: ' + $newSfe.func->elementToPath() + ' (' + $newSfe.func->instanceOf(NativeFunction)->makeString() + ')'); - - ^PrevalWrapper( - value = $newSfe, - canPreval = ($canPrevalFunc || ($newSfe.func == letFunction_String_1__T_m__T_m_)) && $newParamWrappers->canPreval(), - openVars = $newParamWrappers->openVars($state.inScopeVars), - modified = $newSfeModified - );, - | if(shouldInline($newSfe.func, $state), - | - $state->printDebugWithDepth(|'Inlining: ' + $newSfe.func->elementToPath()); - assert($newSfe.func->instanceOf(FunctionDefinition) && $newSfe.func->cast(@FunctionDefinition).expressionSequence->size() == 1); - - let newState = $newSfe->addToScope($state)->map(s|^$s(path = $state.path->concatenate($newSfe.func->cast(@FunctionDefinition)))); - - let subSfe1 = $newSfe.func->cast(@FunctionDefinition).expressionSequence->toOne()->evaluateAndDeactivate(); - let subSfe = ^$subSfe1(genericType = $subSfe1.genericType->resolveGenericType($newState).value); - - let sPreValWrapper = $subSfe->prevalInternal($newState, $extensions); - - $sPreValWrapper->markModified();, - | if($newSfe->canInlineEvalFunctionExpression(), - | - $state->printDebugWithDepth(|'Expanding eval: ' + $newSfe.func->elementToPath()); - - let func = $newSfe.parametersValues->at(0)->match([ - iv:InstanceValue[1]|$iv.values->toOne(), - vs:ValueSpecification[*]|$vs - ])->cast(@FunctionDefinition)->toOne(); - - let newState = $newSfe->addToScope($state)->addToScope($func, $sfe.resolvedTypeParameters, $sfe.parametersValues->drop(1)); - - let subSfe1 = $func.expressionSequence->toOne()->evaluateAndDeactivate(); - let subSfe = ^$subSfe1(genericType = $subSfe1.genericType->resolveGenericType($newState).value); - - let sPreValWrapper = $subSfe->prevalInternal($newState, $extensions); - $sPreValWrapper->markModified();, - | - let notPrevalReason = if($newSfe.parametersValues->exists(pv|!$pv->isInstanceValue($state.inScopeVars)), - | 'params are not instance values';, - | if (!$newParamWrappers->canPreval(), - | 'params can not preval';, - | if(!$newParamWrappers->areAllInScope($state.inScopeVars), - | 'open variables not in scope', - | if($newSfe.func == meta::pure::functions::lang::cast_Any_m__T_1__T_m_ && $newParamWrappers.value->at(0)->map(x|$x->instanceOf(InstanceValue) && $x->cast(@InstanceValue).values->isEmpty()), - | 'cast of empty collection', - | [] - ) - ); - ) - ); - if($notPrevalReason->isNotEmpty(), - | - let handlers = [] - ->cast(@PairBoolean[1]}>, Function<{SimpleFunctionExpression[1]->PrevalWrapper[1]}>>) - ->concatenate([ - pair( - {theSfe:SimpleFunctionExpression[1]| - $newSfe.func.name == 'columns' - && $newSfe.func->instanceOf(AbstractProperty) - && $newSfe.func->cast(@AbstractProperty).owner == TabularDataSet - && $newParamWrappers->areAllInScope($state.rollingInScopeVars) - }, - {theSfe:SimpleFunctionExpression[1]| - /* - Simplistically we could just re-activate the expression here as well, however there are some - TDS functions that are not implemented in the "Pure" implementation and only work in relational - queries. Therefore we have to specifically redirect to a helper function to resolve the schema - instead - */ - let val = meta::pure::tds::schema::resolveSchema($newSfe.parametersValues->toOne(), $state.rollingInScopeVars, $extensions); - - let iv = ^InstanceValue(genericType = $newSfe.genericType,multiplicity = $val->size()->toMultiplicity(), values = $val)->evaluateAndDeactivate(); - ^PrevalWrapper(value = $iv, canPreval = true, modified = true); - }), - pair( - {theSfe:SimpleFunctionExpression[1]|$newSfe.func->in([map_T_m__Function_1__V_m_, map_T_$0_1$__Function_1__V_$0_1$_, map_T_MANY__Function_1__V_MANY_]) && $newSfe.parametersValues->forAll(pv|$pv->isInstanceValue($state.inScopeVars))}, - {theSfe:SimpleFunctionExpression[1]| - // This special handling is required to cover cases where 'fixed'/ InstanceValue inputs are being mapped - // using a function that can't be pre-evalled (because it has external input), so wouldn't normally be modified. - // However we can actually just expand the lambda applied to each of the inputs, to elimniate the Map. - - $state->printDebugWithDepth(|'Expanding: ' + $newSfe.func->elementToPath()); - - let inputVals = $newSfe.parametersValues->at(0)->reactivate($state.inScopeVars) - ->map(v|$v->match([ - vs:ValueSpecification[1]|$vs, - a:Any[*]| ^InstanceValue( - genericType = $newSfe.parametersValues->at(0).genericType, - multiplicity = PureOne, - values = $v->toOne() - )->evaluateAndDeactivate() - ])); - - let lambda = $newSfe.parametersValues->at(1)->cast(@InstanceValue).values->toOne()->cast(@FunctionDefinition); - //TODO: This should really be pre-checked as part of the If condition to get here (so that the assertion never fails) - assert($lambda->instanceOf(FunctionDefinition) && $lambda->cast(@FunctionDefinition).expressionSequence->size() == 1); - - let paramName = $lambda->functionType().parameters->toOne().name; - - let resultValWrappers = $inputVals->map(v| - let newState = $state->addToScope($lambda, [], $v); - - let subSfe1 = $lambda.expressionSequence->toOne()->evaluateAndDeactivate(); - let subSfe = ^$subSfe1(genericType = $sfe.genericType->resolveGenericType($newState).value); - - $subSfe->prevalInternal($newState, $extensions); - ); - - let resultVals = $resultValWrappers.value->map(v|$v->match([iv:InstanceValue[0..1]|$iv->evaluateAndDeactivate().values, a:Any[*]|$a])); - - let r = ^InstanceValue( - genericType = $newSfe.genericType, - multiplicity = $resultVals->size()->toMultiplicity(), - values = $resultVals - )->evaluateAndDeactivate(); - - ^PrevalWrapper( - value = $r, - canPreval = $resultValWrappers->canPreval(), - openVars = $resultValWrappers->openVars($state.inScopeVars), - modified = true - ); - }), - pair( - {theSfe:SimpleFunctionExpression[1]|$newSfe.func == fold_T_MANY__Function_1__V_m__V_m_ && $newSfe.parametersValues->forAll(pv|$pv->isInstanceValue($state.inScopeVars))}, - {theSfe:SimpleFunctionExpression[1]| - // This special handling is required to cover cases where 'fixed'/ InstanceValue inputs are being folded - // using a function that can't be pre-evalled (because it has external input), so wouldn't normally be modified. - // However we can actually just repeat apply the lambda applied to each of the inputs, to elimniate the Fold. - $state->printDebugWithDepth(|'Expanding: ' + $newSfe.func->elementToPath()); - - let inputVals = $newSfe.parametersValues->at(0)->reactivate($state.inScopeVars) - ->map(v| $v->match([ - vs:ValueSpecification[1]|$vs, - a:Any[1]| - ^InstanceValue( - genericType = $newSfe.parametersValues->at(0).genericType, - multiplicity = PureOne, - values = $v->toOne() - ) - ])); - - let lambda = $newSfe.parametersValues->at(1)->cast(@InstanceValue).values->toOne()->cast(@FunctionDefinition);//->evaluateAndDeactivate(); - //TODO: This should really be pre-checked as part of the If condition to get here (so that the assertion never fails) - assert($lambda->instanceOf(FunctionDefinition) && $lambda->cast(@FunctionDefinition).expressionSequence->size() == 1); - - let intialAccumlatorVal = $newSfe.parametersValues->at(2); - - let resultVal = $inputVals->fold({v,a| - let newState = $state->addToScope($lambda, [], $v->concatenate($a.value->cast(@ValueSpecification))->evaluateAndDeactivate()); - - let subSfe1 = $lambda.expressionSequence->toOne(); - let subSfe = ^$subSfe1(genericType = $sfe.genericType->resolveGenericType($newState).value); - - $subSfe->prevalInternal($newState, $extensions); - }, $intialAccumlatorVal->prevalInternal($state, $extensions)); - - $resultVal->markModified(); - }), - pair( - {theSfe:SimpleFunctionExpression[1]| - ($newSfe.func == concatenate_T_MANY__T_MANY__T_MANY_) - && ($newSfe.parametersValues.multiplicity->forAll(m|$m->in([PureOne, PureZero]) || $m->isMultiplicityConcrete() && $m->hasUpperBound() && $m->hasLowerBound() && ($m.lowerBound.value == $m.upperBound.value))) - /* - This last condition should not be needed, but there are circumstances where the inferred type of some lambdas is incorrect. - So this is guard to avoid trying to pre-optimise in those situations (where in the example below, we'd optimised to the lambda - simply return $index because accumulator is detected as a PureZero) - - let input = {| - [1,2]->fold({index, accumulator| - $accumulator->concatenate($index); - }, []) - }; - - let foldLambda = $input->evaluateAndDeactivate().expressionSequence->cast(@SimpleFunctionExpression).parametersValues->at(1)->cast(@InstanceValue).values->cast(@FunctionDefinition)->toOne(); - let foldLambdaAccumulatorParam = $foldLambda->functionType().parameters->at(1); - - assertEquals(ZeroMany, $input->functionReturnMultiplicity(), 'Original lambda return multiplicty expected (zero many)'); - assertEquals($foldLambda->functionReturnMultiplicity(), $input->functionReturnMultiplicity(), | 'Fold lambda return multiplicity matches the original function multiplicity'); - //fails - assertEquals($foldLambda->functionReturnMultiplicity(), $foldLambdaAccumulatorParam.multiplicity, 'Accumulator parameter multiplicity matches the return multiplicty of the fold lambda: %r vs %r', [$foldLambda->functionReturnMultiplicity(), $foldLambdaAccumulatorParam.multiplicity]); - - assertEquals(Integer, $input->functionReturnType().rawType, | 'Original lambda return type expected (integer)'); - assertEquals($foldLambda->functionReturnType().rawType, $input->functionReturnType().rawType, | 'Fold lambda matchs original lambda return type'); - //fails - assertEquals($foldLambda->functionReturnType().rawType, $foldLambdaAccumulatorParam.genericType.rawType, 'Accumulator parameter matches lambda return type: %r vs %r', [$foldLambda->functionReturnType().rawType->toOne(), $foldLambdaAccumulatorParam.genericType.rawType->toOne()]); - - */ - && $newParamWrappers->filter(pw|$pw.value->match([v:VariableExpression[1]|$v.multiplicity == PureZero, a:Any[*]|false]))->areAllInScope($state.inScopeVars) - }, - {theSfe:SimpleFunctionExpression[1]| - $state->printDebugWithDepth(|'Handling expand: ' + $newSfe.func->elementToPath()); - $state->printDebugWithDepth(|'Parameter multiplicities: ' + $newSfe.parametersValues.multiplicity->map(x|$x->makeString())->joinStrings('[',',',']')); - - let r = if($newSfe.parametersValues->at(0).multiplicity == PureZero, - | $newSfe.parametersValues->at(1);, - | if($newSfe.parametersValues->at(1).multiplicity == PureZero, - | $newSfe.parametersValues->at(0);, - | let parametersValues = $newSfe.parametersValues - ->map(x|$x->match([iv:InstanceValue[*]| $iv.values;, a:Any[*]| $a])); - - ^InstanceValue( - genericType = $newSfe.genericType->resolveGenericType($state).value, - multiplicity = $parametersValues->size()->toMultiplicity(), - values = $parametersValues - )->evaluateAndDeactivate() - ->map(x|^$x(values = $parametersValues)); - ) - ); - - ^PrevalWrapper( - value = $r, - canPreval = $newParamWrappers->canPreval(), - openVars = $newParamWrappers->openVars($state.inScopeVars), - modified = true - ); - }), - pair( - {theSfe:SimpleFunctionExpression[1]|$newSfe.func == meta::pure::functions::lang::cast_Any_m__T_1__T_m_ && $newParamWrappers.value->at(0)->map(x|$x->instanceOf(InstanceValue) && $x->cast(@InstanceValue).values->isEmpty())}, - {theSfe:SimpleFunctionExpression[1]| - //This special handling should not be required but []->cast(@MyType) seems to fail in compiled mode - - $state->printDebugWithDepth(|'Handling cast of empty collection: ' + $newSfe.func->elementToPath()); - - - let r = ^InstanceValue( - genericType = $newSfe.genericType->resolveGenericType($state).value, - multiplicity = $newSfe.multiplicity, - values = [] - )->evaluateAndDeactivate(); - - ^PrevalWrapper( - value = $r, - canPreval = true, - openVars = [], - modified = true - ); - }), - pair( - {theSfe:SimpleFunctionExpression[1] | - //TODO getAll check is only needed as router does not route FunctionDefinition wth constant result. - isFilterFunctionReturningConstant($theSfe, false) && !isGetAll($theSfe.parametersValues->at(0))}, - {theSfe:SimpleFunctionExpression[1] | - $state->printDebugWithDepth(|'Handling filter which returns false: ' + $newSfe.func->elementToPath()); - - let r = ^InstanceValue( - genericType = ^GenericType(rawType = Nil), - multiplicity = PureZero - )->evaluateAndDeactivate(); - - ^PrevalWrapper( - value = $r, - canPreval = true, - openVars = [], - modified = true - ); - } - ), - pair( - {theSfe:SimpleFunctionExpression[1] | isFilterFunctionReturningConstant($theSfe, true)}, - {theSfe:SimpleFunctionExpression[1] | - $state->printDebugWithDepth(|'Handling filter which returns true: ' + $newSfe.func->elementToPath()); - - ^PrevalWrapper( - value = $theSfe.parametersValues->at(0), - canPreval = true, - openVars = [], - modified = true - ); - } - ), - pair( - {theSfe:SimpleFunctionExpression[1] | ($theSfe.func == toOneMany_T_MANY__T_$1_MANY$_) && - $theSfe.parametersValues->match([ - vs:ValueSpecification[1]|$vs.multiplicity->isMultiplicityConcrete() && $vs.multiplicity->hasLowerBound() - && ($vs.multiplicity->getLowerBound() > 0), - a:Any[*]|false - ])}, - {theSfe:SimpleFunctionExpression[1] | - $state->printDebugWithDepth(|'Handling toOneMany'); - - $newParamWrappers->toOne()->map(pw|^$pw(modified=true)); - } - ), - pair( - {theSfe:SimpleFunctionExpression[1] | ($theSfe.func == toOne_T_MANY__T_1_) && - $theSfe.parametersValues->match([ - vs:VariableExpression[1]|$vs.multiplicity->isMultiplicityConcrete() - && $vs.multiplicity == PureOne, - a:Any[*]|false - ])}, - {theSfe:SimpleFunctionExpression[1] | - $state->printDebugWithDepth(|'Handling toOne'); - - $newParamWrappers->toOne()->map(pw|^$pw(modified=true)); - } - ) - ]); - - let handler = $handlers->filter(p|$p.first->eval($newSfe))->first(); - - if($handler->isNotEmpty(), - | $handler->toOne().second->eval($newSfe), - | $state->printDebugWithDepth(|'Not prevalling: ' + $newSfe.func->elementToPath() + ' (' + $notPrevalReason->toOne() + ')'); - - ^PrevalWrapper( - value = $newSfe, - canPreval = $newParamWrappers->canPreval(), - openVars = $newParamWrappers->openVars($state.inScopeVars), - modified = $newSfeModified - ); - );, - | - $state->printDebugWithDepth(|'Performing preval: ' + $newSfe.func->elementToPath() + ' (can recativate dynamically: '+ $newSfe->canReactivateDynamically()->makeString() + ')'); - - let valX = $newSfe->reactivate($state.inScopeVars)->cast(@Any); - - let val = $valX->map(x|$x->match([ - {bcs : BasicColumnSpecification[1]| prevalInternal($bcs, ^$state(inScopeTypeParams = ^Map(), inScopeVars = ^Map>()), $extensions).value}, - {av : meta::pure::functions::collection::AggregateValue[1]| prevalInternal($av, ^$state(inScopeTypeParams = ^Map(), inScopeVars = ^Map>()), $extensions).value}, - {av : meta::pure::tds::AggregateValue[1]| prevalInternal($av, ^$state(inScopeTypeParams = ^Map(), inScopeVars = ^Map>()), $extensions).value}, - {a:Any[1]|$a} - ])); - - let iv = $val->match([ - x : InstanceValue[1]| $x, - a : Any[*]| ^InstanceValue(genericType = $newSfe.genericType,multiplicity = $val->size()->toMultiplicity(), values = $val); - ])->evaluateAndDeactivate(); - ^PrevalWrapper(value = $iv, canPreval = true, modified = true); - ); - ); - ); - ); -} - -function meta::pure::router::preeval::isGetAll(v:ValueSpecification[0..1]):Boolean[1] -{ - $v->match([ - s:SimpleFunctionExpression[1] | meta::pure::router::routing::isGetAllFunction($s.func) || $s.parametersValues->first()->isGetAll(), - a:Any[*] | false; - ]); -} - -function <> meta::pure::router::preeval::isFilterFunctionReturningConstant(sfe:SimpleFunctionExpression[1], value:Boolean[1]) : Boolean[1] -{ - let isFilter = $sfe.func == filter_T_MANY__Function_1__T_MANY_; - - if ($isFilter, - | - let es = $sfe.parametersValues->last()->cast(@InstanceValue).values->cast(@LambdaFunction).expressionSequence->evaluateAndDeactivate(); - $es->last()->toOne()->instanceOf(InstanceValue) && $es->last()->toOne()->cast(@InstanceValue).values == $value;, - | false); -} - -// ==================================================================================================================================================================== -// State Building Functions -// ==================================================================================================================================================================== - -function <> meta::pure::router::preeval::defaultFunctionInlineStrategy(extensions:Extension[*]) : FunctionDefinition<{Function[1]->Boolean[1]}>[1] -{ - {f:Function[1] | $f->in([col_Function_1__String_1__BasicColumnSpecification_1_, - col_Function_1__String_1__String_1__BasicColumnSpecification_1_, - // TODO: We should confirm presence of below 2 functions - meta::pure::functions::date::mostRecentDayOfWeek_DayOfWeek_1__Date_1_, - meta::pure::functions::date::previousDayOfWeek_DayOfWeek_1__Date_1_]) - || (!meta::pure::router::routing::shouldStop($f, $extensions) && !($f->instanceOf(NativeFunction)))} -} - -function <> meta::pure::router::preeval::defaultPreevalStopStrategy(extensions:Extension[*]) : FunctionDefinition<{Any[1]->Boolean[1]}>[1] -{ - {a:Any[1] | stopPreeval($a, $extensions)}; -} - -function meta::pure::router::preeval::getPreevalStateWithAdditionalStopInlineFunc(inScopeVars:Map>[1], extensions:meta::pure::extension::Extension[*], stopInlineFunctions:Function[*]):meta::pure::router::preeval::State[1] -{ - ^meta::pure::router::preeval::State(inScopeVars = $inScopeVars, - shouldInlineFxn = {f:Function[1] | meta::pure::router::preeval::defaultFunctionInlineStrategy($extensions)->eval($f) - && !$f->in($stopInlineFunctions)}, - stopPreeval = meta::pure::router::preeval::defaultPreevalStopStrategy($extensions), - rollingInScopeVars = $inScopeVars, - debug = noDebug(), - inScopeTypeParams = ^Map()); -} - -function <> meta::pure::router::preeval::stopPreeval(value : Any[*], extensions:Extension[*]) : Boolean[1] -{ - $extensions.routerExtensions().shouldStopPreeval->exists(p | $p->eval($value)) || - $value->match([ - p : String[*]| true, - p : Float[*]| true, - p : Decimal[*] | true, - p : Enum[*]| true, - p : StrictDate[*]| true, - p : LatestDate[*]| true, - p : Integer[*]| true, - p : Boolean[*]| true, - p : DateTime[*]| true, - p : SortInformation[*]| true, - p : Enumeration[*]| true, - p : Class[*]| true, - p : Pair[*]| true, - p : Property[*]| true, - p : Type[*]| true, - p : Path[*]|true, - p : meta::pure::mapping::Mapping[1]| true, - p : meta::pure::runtime::PackageableRuntime[1]| true, - p : meta::pure::runtime::Runtime[1]| true, - p : meta::pure::runtime::Connection[1] | true, - p : meta::pure::runtime::ExecutionContext[1]| true, - p : meta::pure::tds::TDSColumn[*]| true, - p : TdsOlapRank[*]| true, - p : TdsOlapAggregation[*]| true, - p : meta::pure::graphFetch::execution::AlloySerializationConfig[1]| true, - f : SimpleFunctionExpression[1]| $f.func->in([letFunction_String_1__T_m__T_m_]), - a : Any[*]| $a->type() - ->match([ - c:Class[1]|$c->isSimpleType(), - o:Any[1]|false - ]) - ]); -} - -function <> meta::pure::router::preeval::isSimpleType(c : Class[1]) : Boolean[1] -{ - $c->isSimpleType([]); -} - -function <> meta::pure::router::preeval::isSimpleType(c : Class[1], visitedClasses : Class[*]) : Boolean[1] -{ - let types = $c.properties->map(p|$p->functionReturnType().rawType) - ->distinct() - ->removeAll([String, Integer, Date, DateTime, StrictDate, LatestDate,Float]) - ->removeAll($visitedClasses) - ->filter(x|!$x->instanceOf(Enumeration)); - - let newClassTypes = $types->filter(x|$x->instanceOf(Class))->cast(@Class); - let unsupporedBasicTypes = $types->removeAll($newClassTypes); - - $unsupporedBasicTypes->isEmpty() && $newClassTypes->forAll(newC|$newC->isSimpleType($visitedClasses->concatenate($newClassTypes))); -} - - -// ==================================================================================================================================================================== -// Utility Functions -// ==================================================================================================================================================================== - -function <> meta::pure::router::preeval::resolveGenericType(in : GenericType[1], state : meta::pure::router::preeval::State[1]):PrevalWrapper[1] -{ - if($in.typeParameter->isEmpty(), - | - let newTypeArgsWrappers = $in.typeArguments->map(ta|$ta->resolveGenericType($state)); - if(!$newTypeArgsWrappers->anyModified(), - | ^PrevalWrapper(value = $in, canPreval=true, modified = false), - | let value = ^$in(typeArguments = $newTypeArgsWrappers.value); - ^PrevalWrapper(value = $value, canPreval=$newTypeArgsWrappers->canPreval(), modified = $newTypeArgsWrappers->anyModified()); - );, - | - let value = $state.inScopeTypeParams->get($in.typeParameter->toOne().name); - if($value->isEmpty(), - | ^PrevalWrapper(value = $in, canPreval=true, modified = false), - | ^PrevalWrapper(value = $value->toOne(), canPreval=true, modified = true) - ); - ); -} - -function <> meta::pure::router::preeval::shouldInline(f:Function[1], state : meta::pure::router::preeval::State[1]):Boolean[1] -{ - ($f->instanceOf(FunctionDefinition) && ($f->cast(@FunctionDefinition).expressionSequence->size() == 1 )) - && !($f->instanceOf(QualifiedProperty) && $f->cast(@QualifiedProperty).owner == TDSRow) - && !($f->instanceOf(AbstractProperty) && $f->cast(@AbstractProperty)->meta::pure::milestoning::hasGeneratedMilestoningPropertyStereotype()) - && ($state.shouldInlineFxn->eval($f)) - && ($state.path->filter(x|$x == $f)->isEmpty()) -} - -function <> meta::pure::router::preeval::canInlineEvalFunctionExpression(fe:SimpleFunctionExpression[1]):Boolean[1] -{ - $fe.func->in([eval_Function_1__V_m_, - eval_Function_1__T_n__V_m_, - eval_Function_1__T_n__U_p__V_m_, - eval_Function_1__T_n__U_p__W_q__V_m_, - eval_Function_1__T_n__U_p__W_q__X_r__V_m_, - eval_Function_1__T_n__U_p__W_q__X_r__Y_s__V_m_, - eval_Function_1__T_n__U_p__W_q__X_r__Y_s__Z_t__V_m_, - eval_Function_1__S_n__T_o__U_p__W_q__X_r__Y_s__Z_t__V_m_]) - && $fe.parametersValues->at(0)->match([ - iv:InstanceValue[1]|$iv.values->toOne(), - vs:ValueSpecification[1]|$vs - ])->instanceOf(FunctionDefinition) - && $fe.parametersValues->at(0)->match([ - iv:InstanceValue[1]|$iv.values->toOne(), - vs:ValueSpecification[1]|$vs - ])->cast(@FunctionDefinition).expressionSequence->size() == 1 -} - -function <> meta::pure::router::preeval::toMultiplicity(size : Integer[1]) : Multiplicity[1] -{ - if($size == 0, - | PureZero, - | if($size == 1, - | PureOne, - | ^Multiplicity(lowerBound = ^MultiplicityValue(value=$size), upperBound = ^MultiplicityValue(value=$size)) - ); - ); -} - -function <> meta::pure::router::preeval::addToScope(sfe : SimpleFunctionExpression[1], state : meta::pure::router::preeval::State[1]):meta::pure::router::preeval::State[1] -{ - addToScope($state, $sfe.func, $sfe.resolvedTypeParameters, $sfe.parametersValues, true); -} - -function <> meta::pure::router::preeval::addToScope(state : meta::pure::router::preeval::State[1], func : Function[1],resolvedTypeParameters : GenericType[*], parametersValues : ValueSpecification[*]):meta::pure::router::preeval::State[1] -{ - addToScope($state, $func, $resolvedTypeParameters, $parametersValues, false) -} - -function <> meta::pure::router::preeval::addToScope(state : meta::pure::router::preeval::State[1], func : Function[1],resolvedTypeParameters : GenericType[*], parametersValues : ValueSpecification[*], cleanupInScopeVarsAndTypeParams:Boolean[1]):meta::pure::router::preeval::State[1] -{ - let typeParams = $func->functionType().typeParameters.name->zip($resolvedTypeParameters->map(x|$x->resolveGenericType($state).value)); - - let newParams = $parametersValues; - - let funcParamNames = $func->functionType().parameters.name; - let additionalInScopeVars = $funcParamNames - ->zip($newParams->map(pv|list($pv))) - ->map({p| let paramName = $p.first; - let paramValue = $p.second; - - $paramValue.values->match([ - ve:VariableExpression[1]| - if($ve.name == $paramName, - | [], //We drop easy "circular" / self-reference parameter variables - | $p - ), - vs:ValueSpecification[*]| - $p; - ]); - }); - - if($cleanupInScopeVarsAndTypeParams, |^$state(inScopeVars=newMap([]), inScopeTypeParams=newMap([])), |$state)->addToScope($additionalInScopeVars, $typeParams); -} - -function <> meta::pure::router::preeval::addToScope(state : meta::pure::router::preeval::State[1], additionalInScopeVars : Pair>[*], typeParams : Pair[*]):meta::pure::router::preeval::State[1] -{ - $state->printDebugWithDepth(|'Adding variables to scope: ' + $additionalInScopeVars.first->joinStrings('[', ',', ']')); - $state->printDebugWithDepth(|'Adding type params to scope: ' + $typeParams.first->joinStrings('[', ',', ']')); - - let newInScopeVars = $state.inScopeVars->putAll($additionalInScopeVars); - let newInScopeTypeParams = $state.inScopeTypeParams->putAll($typeParams); - - ^$state(inScopeVars = $newInScopeVars, inScopeTypeParams = $newInScopeTypeParams); -} - -function <> meta::pure::router::preeval::resolveVariable(name : String[1], state : meta::pure::router::preeval::State[1]):String[1] -{ - $state.inScopeVars->get($name).values->match([ - ve:VariableExpression[1]|assert($name != $ve.name, 'Circular variable reference: ' + $name); $ve.name->resolveVariable($state);, - a:Any[*]|$name - ]); -} - -function <> meta::pure::router::preeval::canPreval(items : PrevalWrapper[*]):Boolean[1] -{ - $items->forAll(p | $p.canPreval); -} - -function <> meta::pure::router::preeval::markModified(item : PrevalWrapper[1]):PrevalWrapper[1] -{ - if($item.modified, - | $item, - | ^$item(modified = true) - ); -} - -function <> meta::pure::router::preeval::anyModified(items : PrevalWrapper[*]):Boolean[1] -{ - $items.modified->contains(true); -} - -function <> meta::pure::router::preeval::openVars(items : PrevalWrapper[*], inScopeVars:Map>[1]):String[*] -{ - $items.openVars->openVars($inScopeVars); -} - -function <> meta::pure::router::preeval::openVars(vars : String[*], inScopeVars:Map>[1]):String[*] -{ - $vars->distinct()->filter(v|!$v->areAllInScope($inScopeVars)) -} - -function <> meta::pure::router::preeval::areAllInScope(items : PrevalWrapper[*], inScopeVars:Map>[1]):Boolean[1] -{ - $items.openVars->distinct()->areAllInScope($inScopeVars); -} - -function <> meta::pure::router::preeval::areAllInScope(vars : String[*], inScopeVars:Map>[1]):Boolean[1] -{ - if($vars->isEmpty(), - | true, - | let unresolvedVars = $vars->map(v|let varVal = $inScopeVars->get($v); - - if($varVal->isEmpty(), - | $v, - | $varVal.values->match([ - varExp : VariableExpression[1]|$varExp.name, - a : Any[*]|[] - ]) - ); - )->distinct(); - - let newVars = $unresolvedVars->removeAll($vars); - !$unresolvedVars->containsAny($vars) && $newVars->areAllInScope($inScopeVars); - ); -} - -function <> meta::pure::router::preeval::isInstanceValue(v: Any[1], inScopeVars:Map>[1]):Boolean[1] -{ - $v->match([ - iv : InstanceValue[1]| - $iv.values->forAll(subV|$subV->match([ - subVS : ValueSpecification[1]|$subVS->isInstanceValue($inScopeVars), - a : Any[1]| true; - ]));, - ve : VariableExpression[1]|$inScopeVars->get($ve.name)->isNotEmpty(), - sfe : SimpleFunctionExpression[1]|false; - ]) -} - -function <> meta::pure::router::preeval::printDebugWithDepth(state: meta::pure::router::preeval::State[1], func:Function<{->String[1]}>[1]):Nil[0] -{ - printDebug($state, | ('[' + toString($state.depth->orElse(''))->map(s|'0000'->substring(0, max(0, 3-$s->length())) + $s) + ']' + range($state.depth->orElse(0) + 1)->map(d|'')->joinStrings('', '-', ' ') + $func->eval())); -} - -function <> meta::pure::router::preeval::printDebug(state: meta::pure::router::preeval::State[1], func:Function<{->Any[*]}>[1]):Nil[0] -{ - printDebug($state.debug, $func); -} - -function <> meta::pure::router::preeval::printDebug(debugContext : DebugContext[1], func:Function<{->Any[*]}>[1]):Nil[0] -{ - if(!$debugContext.debug, - | print(''), - | printDebug($debugContext, $func->eval()); - ); -} - -function <> meta::pure::router::preeval::printDebug(state: meta::pure::router::preeval::State[1], param:Any[*]):Nil[0] -{ - printDebug($state.debug, $param); -} - -function <> meta::pure::router::preeval::printDebug(debugContext : DebugContext[1], param:Any[*]):Nil[0] -{ - if(!$debugContext.debug, - | print(''), - | println($param); - ); +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::router::preeval::*; +import meta::pure::extension::*; +import meta::pure::metamodel::path::*; +import meta::pure::router::extension::*; +import meta::pure::router::utils::*; + +Class meta::pure::router::preeval::State +{ + inScopeVars : Map>[1]; + rollingInScopeVars : Map>[1]; + inScopeTypeParams : Map[1]; + + debug : DebugContext[1]; + depth : Integer[0..1]; + path : FunctionDefinition[*]; + + shouldInlineFxn : FunctionDefinition<{Function[1] -> Boolean[1]}>[1]; + stopPreeval : Function<{Any[1] -> Boolean[1]}>[1]; + + toString() { $this->simpleToString() }:String[1]; +} + +Class meta::pure::router::preeval::PrevalWrapper +{ + canPreval : Boolean[1]; + value : T[1]; + openVars : String[*]; + modified : Boolean[1]; + + toString() { $this->simpleToString() }:String[1]; +} + + +// ==================================================================================================================================================================== +// Public Apis +// ==================================================================================================================================================================== + +function <> meta::pure::router::preeval::preval(f:FunctionDefinition[1], extensions:Extension[*]):FunctionDefinition[1] +{ + preval($f, $extensions, noDebug()); +} + +function <> meta::pure::router::preeval::preval(f:FunctionDefinition[1], extensions:Extension[*], debug:DebugContext[1]):FunctionDefinition[1] +{ + preval($f, newMap([]->cast(@Pair)), $f->openVariableValues(), $extensions, $debug).value->toOne()->cast($f); +} + +function <> meta::pure::router::preeval::preval(f : FunctionDefinition[1], vars:Map[1], inScopeVars:Map>[1], extensions:Extension[*]):PrevalWrapper>[1] +{ + preval($f, $vars, $inScopeVars, $extensions, noDebug()); +} + +function <> meta::pure::router::preeval::preval(f : FunctionDefinition[1], vars:Map[1], inScopeVars:Map>[1], extensions:Extension[*], debug:DebugContext[1]):PrevalWrapper>[1] +{ + let state = ^meta::pure::router::preeval::State(inScopeVars = $inScopeVars, + shouldInlineFxn = defaultFunctionInlineStrategy($extensions), + stopPreeval = defaultPreevalStopStrategy($extensions), + rollingInScopeVars = $inScopeVars, + debug = $debug, + inScopeTypeParams = ^Map()); + preval($f, $state, $extensions); +} + +function <> meta::pure::router::preeval::preval(f : FunctionDefinition[1], state:State[1], extensions:Extension[*]):PrevalWrapper>[1] +{ + let r = prevalInternal($f->evaluateAndDeactivate(), $state, $extensions)->toOne()->cast(@PrevalWrapper>); + + if(!$r->anyModified(), + | + $state->printDebug(|'No changes made for: '); + $state->printDebug(|$f->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::toPureGrammar::toPure($extensions));, + | + let res = $r.value->toOne(); + + // $state->printDebug('Transformed From:'); + // $state->printDebug(|$f->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::json::toJSON(50000)); + // $state->printDebug(|$f->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::protocols::pure::vX_X_X::transformation::toPureGrammar::toPure($extensions)); + + $state->printDebug('Transformed To:'); + $state->printDebug(|$res->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::json::toJSON(50000)); + $state->printDebug(|$res->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::transformFunctionBody($extensions)->meta::protocols::pure::vX_X_X::transformation::fromPureGraph::toPureGrammar::toPure($extensions)); + + $state->printDebug($res); + ); + + $r; +} + +function <> meta::pure::router::preeval::preval(f : FunctionExpression[1], inScopeVars:Map>[1], extensions:Extension[*], debug:DebugContext[1]):ValueSpecification[1] +{ + let state = ^meta::pure::router::preeval::State(inScopeVars = $inScopeVars, + shouldInlineFxn = defaultFunctionInlineStrategy($extensions), + stopPreeval = defaultPreevalStopStrategy($extensions), + rollingInScopeVars = $inScopeVars, + debug = $debug, + inScopeTypeParams = ^Map()); + prevalInternal($f, $state, $extensions).value->cast(@ValueSpecification); +} + +// ==================================================================================================================================================================== +// Processor Functions +// ==================================================================================================================================================================== + +function <> meta::pure::router::preeval::prevalInternal(item : Any[1], origState : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] +{ + let state = ^$origState(depth = if($origState.depth->isEmpty(), | 0, | $origState.depth->toOne() + 1), + path = if(!$item->instanceOf(FunctionDefinition), | $origState.path, | $origState.path->concatenate($item->cast(@FunctionDefinition)))); + + + $state->printDebugWithDepth(|'Processing ' + $item->type()->makeString() + ' ' + + $item->evaluateAndDeactivate() + ->match([ + iv:InstanceValue[1] | $iv.values->map(x|$x->type())->makeString(), + sfe:SimpleFunctionExpression[1] | '(' + $sfe.func->elementToPath() + ')', + ve:VariableExpression[1] | '(' + $ve.name + ')', + a:Any[*]|''] + ) + + ', inScopeVars: ' + $state.inScopeVars->keys()->joinStrings('[', ',', ']')); + + let newItem = $item->evaluateAndDeactivate() + ->match([ + {lambda : LambdaFunction[1]| + let fnArgNames = $lambda->functionType().parameters.name; + let fnOpenVariables = $lambda->openVariableValues()->putAll($state.inScopeVars->keyValues()->filter(kv | !$kv.first->in($fnArgNames))->newMap()); + + $lambda->prevalFunctionDefinition(^$state(inScopeVars = $fnOpenVariables), $extensions); + }, + {fd : FunctionDefinition[1]| + $fd->prevalFunctionDefinition(^$state(inScopeVars = newMap([])), $extensions); // Assigning empty map to inScopeVars as function definition can't have openVars + }, + {sfe : SimpleFunctionExpression[1]| + let handlers = [ + // Avoid handling known 'false' case (which may not be evalulatable / have runtime issues, e.g. a toOne() after an isNotEmpty condition) + ^Pair, LambdaFunction<{->PrevalWrapper[1]}>>(first = if_Boolean_1__Function_1__Function_1__T_m_, second = {| + let conditionPreVal = $sfe.parametersValues->at(0)->prevalInternal($state, $extensions); + if($conditionPreVal.value->toOne()->cast(@ValueSpecification)->isInstanceValue($state.inScopeVars), + | let valueValueSpecification = if($conditionPreVal.value->cast(@InstanceValue).values == true, + | $sfe.parametersValues->at(1), + | $sfe.parametersValues->at(2) + ); + let newSfe = ^$sfe(func = eval_Function_1__V_m_, parametersValues = $valueValueSpecification); + + $newSfe->prevalInternal($state, $extensions)->markModified();, + | prevalGenericSimpleFunctionExpression($sfe, $state, $extensions); + ); + }), + + // Avoid handling conditions after the first false (which may not be evalulatable / have runtime issues, e.g. a toOne() after an isNotEmpty condition) + ^Pair, LambdaFunction<{->PrevalWrapper[1]}>>(first = meta::pure::functions::boolean::and_Boolean_1__Boolean_1__Boolean_1_, second = {| + prevalBooleanSimpleFunctionExpression($sfe, $state, $extensions); + }), + + // Avoid handling conditions after the first true (which may not be evalulatable / have runtime issues, e.g. a toOne() after an isEmpty condition) + ^Pair, LambdaFunction<{->PrevalWrapper[1]}>>(first = meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_, second = {| + prevalBooleanSimpleFunctionExpression($sfe, $state, $extensions); + }) + + // TODO: + // 1. Optimise and_Boolean_MANY__Boolean_1_ and or_Boolean_$1_MANY$__Boolean_1_ + // 1. Optimise plus(many), minus(many) and times(many) (plus_String_MANY__String_1_, plus_Float_MANY__Float_1_, times_Number_MANY__Number_1_, minus_Float_MANY__Float_1_) + ]; + + let handler = $handlers->filter(p|$p.first == $sfe.func).second; + + if($handler->isNotEmpty(), + | $handler->toOne()->eval(), + | prevalGenericSimpleFunctionExpression($sfe, $state, $extensions)); + }, + {ve : VariableExpression[1]| + let varValue = $state.inScopeVars->get($ve.name).values; + let iv = if($state.inScopeVars->get($ve.name)->isEmpty(), + | ^PrevalWrapper(value = $ve, canPreval = true, modified = false, openVars = [$ve.name]), + | $varValue->match([ + n : Nil[0]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), + iv : InstanceValue[1]| $iv, + ve : VariableExpression[1]| $ve->evaluateAndDeactivate(), + sfe : SimpleFunctionExpression[1]| $sfe, + a : meta::pure::tds::AggregateValue[*]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), + a : meta::pure::functions::collection::AggregateValue[*]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), + a : meta::pure::tds::BasicColumnSpecification[*]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(), + r : meta::pure::graphFetch::RootGraphFetchTree[1]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate();, + b : meta::external::format::shared::binding::Binding[1]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate();, + sc : meta::pure::tds::schema::SchemaState[1]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $sc)->evaluateAndDeactivate(), + s : meta::pure::store::Store[1]| ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate();, + a : Any[*]| assert($state.stopPreeval->eval($a), | 'Unsupported type: ' + $a->type()->match([pe:PackageableElement[1]|$pe->elementToPath(), t:Type[1]|$t->makeString()])); + ^InstanceValue(multiplicity = $ve.multiplicity, genericType = $ve.genericType, values = $varValue)->evaluateAndDeactivate(); + ]) + ->map(x| + let newState = ^$state(inScopeVars = $state.inScopeVars->keyValues()->filter(p|$p.first != $ve.name)->newMap()); + $x->prevalInternal($newState, $extensions); + ) + ->markModified() + ); + }, + {iv : InstanceValue[1]| + let subVals = $iv.values->map(x|$x->prevalInternal($state, $extensions)); + let r = if(!$subVals->anyModified(), + | pair(false, $iv), + | let cleanIv = ^$iv(values = $subVals.value->map(x|$x->match([subIv:InstanceValue[1]|if($subIv.values->size() == 1, |$subIv.values, |$subIv), a:Any[*]|$a]))); + let cleanIv2 = ^$cleanIv(genericType = $cleanIv.genericType->resolveGenericType($state).value, + multiplicity = if($cleanIv.multiplicity == PureOne && $cleanIv.values->isEmpty(), + | $cleanIv.multiplicity, //This seems to be a quirk of Cast(@ABC) + | $cleanIv.values->size()->toMultiplicity()) + ); + pair(true, $cleanIv2); + ); + ^PrevalWrapper(value = $r.second, canPreval = $subVals->canPreval(), openVars = $subVals->openVars($state.inScopeVars), modified = $r.first); + }, + {k : KeyExpression[1]| + let e = $k.expression->prevalInternal($state, $extensions)->toOne(); + let ke = if(!$e->anyModified(), + | pair(false, $k), + | pair(true, ^$k(expression = $e.value->cast(@ValueSpecification)->toOne()))); + + ^PrevalWrapper(value = $ke.second, canPreval = $e.canPreval, openVars = $e->openVars($state.inScopeVars), modified = $ke.first); + }, + {p : ColumnSpecification[1]| + $p->match([ + bcs : BasicColumnSpecification[1]| + let f = $bcs.func->evaluateAndDeactivate(); + let newFuncWrapper = prevalInternal($f, ^$state(inScopeTypeParams = ^Map(), inScopeVars = $state.inScopeVars->putAll($f->openVariableValues())), $extensions); + + if(!$newFuncWrapper->anyModified(), + | ^PrevalWrapper(value = $bcs, canPreval = true, modified = false), + | ^PrevalWrapper(value = ^$bcs(func = $newFuncWrapper.value->cast($f)), canPreval = true, modified = true); + );, + o:ColumnSpecification[1] | ^PrevalWrapper(value = $o, canPreval = true, modified = false); + ]); + + }, + {av : meta::pure::tds::AggregateValue[1]| + let mapFn = $av.mapFn->evaluateAndDeactivate(); + let aggFn = $av.aggregateFn->evaluateAndDeactivate(); + + let newFuncWrapper_map = prevalInternal($mapFn, $state, $extensions); + let newFuncWrapper_agg = prevalInternal($aggFn, $state, $extensions); + + if(![$newFuncWrapper_map, $newFuncWrapper_agg]->anyModified(), + | ^PrevalWrapper(value = $av, canPreval = true, modified = false), + | ^PrevalWrapper(value = ^$av(mapFn = $newFuncWrapper_map.value->cast($mapFn), aggregateFn = $newFuncWrapper_agg.value->cast($aggFn)), canPreval = true, modified = true); + ); + }, + {av : meta::pure::functions::collection::AggregateValue[1]| + let mapFn = $av.mapFn->evaluateAndDeactivate(); + let aggFn = $av.aggregateFn->evaluateAndDeactivate(); + + let newFuncWrapper_map = prevalInternal($mapFn, $state, $extensions); + let newFuncWrapper_agg = prevalInternal($aggFn, $state, $extensions); + + if(![$newFuncWrapper_map, $newFuncWrapper_agg]->anyModified(), + | ^PrevalWrapper(value = $av, canPreval = true, modified = false), + | ^PrevalWrapper(value = ^$av(mapFn = $newFuncWrapper_map.value->cast($mapFn), aggregateFn = $newFuncWrapper_agg.value->cast($aggFn)), canPreval = true, modified = true); + ); + }, + {s: meta::pure::tds::schema::SchemaState[1] | ^PrevalWrapper(value = $s, canPreval = true, modified = false);}, + {r: meta::pure::graphFetch::RootGraphFetchTree[1] | ^PrevalWrapper(value = $r, canPreval = true, modified = false);}, + {b: meta::external::format::shared::binding::Binding[1] | ^PrevalWrapper(value = $b, canPreval = true, modified = false);}, + {s: meta::pure::store::Store[1] | ^PrevalWrapper(value = $s, canPreval = true, modified = false);}, + {a : Any[1]| + assert($state.stopPreeval->eval($a), | 'Unsupported type: ' + $a->type()->match([pe:PackageableElement[1]|$pe->elementToPath(), t:Type[1]|$t->makeString()])); + ^PrevalWrapper(value = $a, canPreval = true, modified = false); + } + ]); + + $state->printDebugWithDepth(|'Returning ' + $newItem.value->type()->makeString() + ' ' + $newItem.value->match([iv:InstanceValue[1]|$iv.values->map(x|$x->type())->makeString(), sfe:SimpleFunctionExpression[1]|'(' + $sfe.func->elementToPath() + ')', ve : VariableExpression[1]|'(' + $ve.name + ')', a:Any[*]|'']) + ' ' + $newItem.toString()); + $newItem; +} + +function <> meta::pure::router::preeval::prevalFunctionDefinition(fd : FunctionDefinition[1], origState : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] +{ + let items = $fd.expressionSequence; + + let r = range($items->size()) + ->fold({index, p| + let item = $items->at($index); + let state = $p.first; + + let isLastExpression = ($index == ($items->size()-1)); + + if(!$isLastExpression && !$item->isLetFunction(), + | + $state->printDebugWithDepth(|'Dropping non-variable assignement non-return statement: ' + $item->type()->makeString() + ' ' + + $item->match([ + iv:InstanceValue[1]|$iv.values->map(x|$x->type())->makeString(), + sfe:SimpleFunctionExpression[1]|'(' + $sfe.func->elementToPath() + ')', + ve : VariableExpression[1]|'(' + $ve.name + ')', + a:Any[*]|'' + ]) + + ', inScopeVars: ' + $state.inScopeVars->keys()->joinStrings('[', ',', ']')); + + pair($state, $p.second);, + | + let r = $item->prevalInternal($state, $extensions); + + let isLetFunctionValueSpecification = $r.value->instanceOf(ValueSpecification) && $r.value->cast(@ValueSpecification)->isLetFunction(); + + if(!$isLetFunctionValueSpecification, + | pair($state, list(if($p.second.values->isEmpty(), | $r, | $p.second.values->concatenate($r)))), + | + let fe = $r.value->cast(@FunctionExpression); + + let varName = $fe.parametersValues->at(0)->reactivate($state.inScopeVars)->cast(@String)->toOne(); + let varValue = $fe.parametersValues->at(1) + ->match([ + iv:InstanceValue[1]|$iv.values, + fe:ValueSpecification[1]|$fe + ]); + + $state->printDebugWithDepth(|'adding variable ' + $varName + ' to rolling scope, with value ' + $varValue->makeString()); + + let shouldInlineVariableExpression = ($varValue->match([v:VariableExpression[1]|true, a: Any[*]|false])) + || ($r->canPreval() && $r->areAllInScope($state.inScopeVars) && !$varValue->match([v:LambdaFunction[1]|true, a: Any[*]|false])); + + let newRollingInScopeVars = $state.rollingInScopeVars->put($varName, list($varValue)); + let newInScopeVars = if(!$shouldInlineVariableExpression, + | $state.inScopeVars, + | $state.inScopeVars->put($varName, list($varValue)) + ); + let newState = ^$state(inScopeVars = $newInScopeVars, rollingInScopeVars = $newRollingInScopeVars); + + let expValue = if($isLastExpression, + | ^PrevalWrapper(canPreval = $r.canPreval, modified = true, openVars =$r.openVars, value = $varValue->match([vs:ValueSpecification[1]|$vs, a:Any[*]|^InstanceValue(genericType = $fe.genericType, multiplicity = $a->size()->toMultiplicity(), values = $a)])), + | if($shouldInlineVariableExpression, + | [], + | $r + ) + ); + if($expValue->isEmpty(), + | pair($newState, $p.second), + | pair($newState, list(if($p.second.values->isEmpty(), | $expValue, | $p.second.values->concatenate($expValue)))) + ); + ); + ); + }, [pair($origState, list([]->cast(@PrevalWrapper)))] + ).second.values; + + + /* + This should not be necessary, but seems to be required otherwise fold can end up wrapping InstanceValues in InstanceValues, + which map doesn't seem to + */ + let newExpressionSequncePrevalWrappers = $r->evaluateAndDeactivate() + ->match([ + iv:InstanceValue[1]|$iv.values->cast(@PrevalWrapper), + a:PrevalWrapper[*]|$a + ]); + + let value = if(($newExpressionSequncePrevalWrappers->size() == $items->size()) && !$newExpressionSequncePrevalWrappers->anyModified(), + | pair(false, $fd), + | let newFd = ^$fd(expressionSequence = $newExpressionSequncePrevalWrappers.value->cast(@ValueSpecification)->toOneMany()) + ->match([ + lf : LambdaFunction[1]| ^$lf(openVariables = $lf.openVariables->map(v|$v->resolveVariable($origState)) + ->distinct() + ->intersection($newExpressionSequncePrevalWrappers.openVars->map(v|$v->resolveVariable($origState)))), + fd2:FunctionDefinition[1]| $fd2 + ]); + pair(true, $newFd); + ); + + ^PrevalWrapper>( + value = $value.second, + canPreval = $newExpressionSequncePrevalWrappers->canPreval(), + openVars = $fd->match([lf : LambdaFunction[1]|$value.second->cast(@LambdaFunction).openVariables, fd:FunctionDefinition[1]|[]]), + modified = $value.first + ); +} + +function <> meta::pure::router::preeval::prevalBooleanSimpleFunctionExpression(sfe : SimpleFunctionExpression[1], state : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] +{ + assert($sfe.func->in([and_Boolean_1__Boolean_1__Boolean_1_, or_Boolean_1__Boolean_1__Boolean_1_])); + assertEquals(2, $sfe.parametersValues->size()); + + let shortcutVal = !($sfe.func == and_Boolean_1__Boolean_1__Boolean_1_); + + let newParam1Wrapper = $sfe.parametersValues->at(0)->prevalInternal($state, $extensions); + + if($newParam1Wrapper.value->toOne()->isInstanceValue($state.inScopeVars), + | if(($newParam1Wrapper.value->cast(@InstanceValue).values == $shortcutVal), + | $newParam1Wrapper->markModified(), + | let newParam2Wrapper = $sfe.parametersValues->at(1)->prevalInternal($state, $extensions); + $newParam2Wrapper->markModified();), + | let newParam2Wrapper = $sfe.parametersValues->at(1)->prevalInternal($state, $extensions); + + if($newParam2Wrapper.value->toOne()->isInstanceValue($state.inScopeVars), + | if($newParam2Wrapper.value->cast(@InstanceValue).values != $shortcutVal, + | $newParam1Wrapper->markModified(), + | $newParam2Wrapper->markModified()), + | let newParamWrappers = [$newParam1Wrapper, $newParam2Wrapper]; + ^PrevalWrapper( + value = ^$sfe(parametersValues = $newParamWrappers.value->cast(@ValueSpecification)), + canPreval = $newParamWrappers->canPreval(), + openVars = $newParamWrappers->openVars($state.inScopeVars), + modified = $newParamWrappers->size() != $sfe.parametersValues->size() || $newParamWrappers->anyModified() + ); + ); + ); +} + +function <> meta::pure::router::preeval::prevalGenericSimpleFunctionExpression(sfe : SimpleFunctionExpression[1], state : meta::pure::router::preeval::State[1], extensions:Extension[*]):PrevalWrapper[1] +{ + let newParamWrappers = if($sfe.parametersValues->isEmpty(), + | [], + | $sfe.func->match([p:Property[1]|'this', f:Function[1]|$f->functionType().parameters.name]) + ->zip($sfe.parametersValues) + ->evaluateAndDeactivate() + ->map(p| $state->printDebugWithDepth(|'Processing parameter: ' + $p.first); + let r = $p.second->prevalInternal($state, $extensions); + $state->printDebugWithDepth(|'Completed processing parameter: ' + $p.first); + $r;) + ); + let newSfeModified = $newParamWrappers->anyModified(); + + let newSfe = if($newSfeModified, + | ^$sfe(parametersValues = $newParamWrappers.value->cast(@ValueSpecification) + ->map(pv|^$pv(genericType = $pv.genericType->resolveGenericType($state).value)), + genericType = $sfe.genericType->resolveGenericType($state).value), + | $sfe); + + let canPrevalFunc = !($newSfe.func->hasStereotype('SideEffectFunction', functionType) || $newSfe.func->hasStereotype('NotImplementedFunction', functionType) || $state.stopPreeval->eval($newSfe)); + + if(!$canPrevalFunc, + | $state->printDebugWithDepth(|'Unable to perform preval: ' + $newSfe.func->elementToPath() + ' (' + $newSfe.func->instanceOf(NativeFunction)->makeString() + ')'); + + ^PrevalWrapper( + value = $newSfe, + canPreval = ($canPrevalFunc || ($newSfe.func == letFunction_String_1__T_m__T_m_)) && $newParamWrappers->canPreval(), + openVars = $newParamWrappers->openVars($state.inScopeVars), + modified = $newSfeModified + );, + | if(shouldInline($newSfe.func, $state), + | + $state->printDebugWithDepth(|'Inlining: ' + $newSfe.func->elementToPath()); + assert($newSfe.func->instanceOf(FunctionDefinition) && $newSfe.func->cast(@FunctionDefinition).expressionSequence->size() == 1); + + let newState = $newSfe->addToScope($state)->map(s|^$s(path = $state.path->concatenate($newSfe.func->cast(@FunctionDefinition)))); + + let subSfe1 = $newSfe.func->cast(@FunctionDefinition).expressionSequence->toOne()->evaluateAndDeactivate(); + let subSfe = ^$subSfe1(genericType = $subSfe1.genericType->resolveGenericType($newState).value); + + let sPreValWrapper = $subSfe->prevalInternal($newState, $extensions); + + $sPreValWrapper->markModified();, + | if($newSfe->canInlineEvalFunctionExpression(), + | + $state->printDebugWithDepth(|'Expanding eval: ' + $newSfe.func->elementToPath()); + + let func = $newSfe.parametersValues->at(0)->match([ + iv:InstanceValue[1]|$iv.values->toOne(), + vs:ValueSpecification[*]|$vs + ])->cast(@FunctionDefinition)->toOne(); + + let newState = $newSfe->addToScope($state)->addToScope($func, $sfe.resolvedTypeParameters, $sfe.parametersValues->drop(1)); + + let subSfe1 = $func.expressionSequence->toOne()->evaluateAndDeactivate(); + let subSfe = ^$subSfe1(genericType = $subSfe1.genericType->resolveGenericType($newState).value); + + let sPreValWrapper = $subSfe->prevalInternal($newState, $extensions); + $sPreValWrapper->markModified();, + | + let notPrevalReason = if($newSfe.parametersValues->exists(pv|!$pv->isInstanceValue($state.inScopeVars)), + | 'params are not instance values';, + | if (!$newParamWrappers->canPreval(), + | 'params can not preval';, + | if(!$newParamWrappers->areAllInScope($state.inScopeVars), + | 'open variables not in scope', + | if($newSfe.func == meta::pure::functions::lang::cast_Any_m__T_1__T_m_ && $newParamWrappers.value->at(0)->map(x|$x->instanceOf(InstanceValue) && $x->cast(@InstanceValue).values->isEmpty()), + | 'cast of empty collection', + | [] + ) + ); + ) + ); + if($notPrevalReason->isNotEmpty(), + | + let handlers = [] + ->cast(@PairBoolean[1]}>, Function<{SimpleFunctionExpression[1]->PrevalWrapper[1]}>>) + ->concatenate([ + pair( + {theSfe:SimpleFunctionExpression[1]| + $newSfe.func.name == 'columns' + && $newSfe.func->instanceOf(AbstractProperty) + && $newSfe.func->cast(@AbstractProperty).owner == TabularDataSet + && $newParamWrappers->areAllInScope($state.rollingInScopeVars) + }, + {theSfe:SimpleFunctionExpression[1]| + /* + Simplistically we could just re-activate the expression here as well, however there are some + TDS functions that are not implemented in the "Pure" implementation and only work in relational + queries. Therefore we have to specifically redirect to a helper function to resolve the schema + instead + */ + let val = meta::pure::tds::schema::resolveSchema($newSfe.parametersValues->toOne(), $state.rollingInScopeVars, $extensions); + + let iv = ^InstanceValue(genericType = $newSfe.genericType,multiplicity = $val->size()->toMultiplicity(), values = $val)->evaluateAndDeactivate(); + ^PrevalWrapper(value = $iv, canPreval = true, modified = true); + }), + pair( + {theSfe:SimpleFunctionExpression[1]|$newSfe.func->in([map_T_m__Function_1__V_m_, map_T_$0_1$__Function_1__V_$0_1$_, map_T_MANY__Function_1__V_MANY_]) && $newSfe.parametersValues->forAll(pv|$pv->isInstanceValue($state.inScopeVars))}, + {theSfe:SimpleFunctionExpression[1]| + // This special handling is required to cover cases where 'fixed'/ InstanceValue inputs are being mapped + // using a function that can't be pre-evalled (because it has external input), so wouldn't normally be modified. + // However we can actually just expand the lambda applied to each of the inputs, to elimniate the Map. + + $state->printDebugWithDepth(|'Expanding: ' + $newSfe.func->elementToPath()); + + let inputVals = $newSfe.parametersValues->at(0)->reactivate($state.inScopeVars) + ->map(v|$v->match([ + vs:ValueSpecification[1]|$vs, + a:Any[*]| ^InstanceValue( + genericType = $newSfe.parametersValues->at(0).genericType, + multiplicity = PureOne, + values = $v->toOne() + )->evaluateAndDeactivate() + ])); + + let lambda = $newSfe.parametersValues->at(1)->cast(@InstanceValue).values->toOne()->cast(@FunctionDefinition); + //TODO: This should really be pre-checked as part of the If condition to get here (so that the assertion never fails) + assert($lambda->instanceOf(FunctionDefinition) && $lambda->cast(@FunctionDefinition).expressionSequence->size() == 1); + + let paramName = $lambda->functionType().parameters->toOne().name; + + let resultValWrappers = $inputVals->map(v| + let newState = $state->addToScope($lambda, [], $v); + + let subSfe1 = $lambda.expressionSequence->toOne()->evaluateAndDeactivate(); + let subSfe = ^$subSfe1(genericType = $sfe.genericType->resolveGenericType($newState).value); + + $subSfe->prevalInternal($newState, $extensions); + ); + + let resultVals = $resultValWrappers.value->map(v|$v->match([iv:InstanceValue[0..1]|$iv->evaluateAndDeactivate().values, a:Any[*]|$a])); + + let r = ^InstanceValue( + genericType = $newSfe.genericType, + multiplicity = $resultVals->size()->toMultiplicity(), + values = $resultVals + )->evaluateAndDeactivate(); + + ^PrevalWrapper( + value = $r, + canPreval = $resultValWrappers->canPreval(), + openVars = $resultValWrappers->openVars($state.inScopeVars), + modified = true + ); + }), + pair( + {theSfe:SimpleFunctionExpression[1]|$newSfe.func == fold_T_MANY__Function_1__V_m__V_m_ && $newSfe.parametersValues->forAll(pv|$pv->isInstanceValue($state.inScopeVars))}, + {theSfe:SimpleFunctionExpression[1]| + // This special handling is required to cover cases where 'fixed'/ InstanceValue inputs are being folded + // using a function that can't be pre-evalled (because it has external input), so wouldn't normally be modified. + // However we can actually just repeat apply the lambda applied to each of the inputs, to elimniate the Fold. + $state->printDebugWithDepth(|'Expanding: ' + $newSfe.func->elementToPath()); + + let inputVals = $newSfe.parametersValues->at(0)->reactivate($state.inScopeVars) + ->map(v| $v->match([ + vs:ValueSpecification[1]|$vs, + a:Any[1]| + ^InstanceValue( + genericType = $newSfe.parametersValues->at(0).genericType, + multiplicity = PureOne, + values = $v->toOne() + ) + ])); + + let lambda = $newSfe.parametersValues->at(1)->cast(@InstanceValue).values->toOne()->cast(@FunctionDefinition);//->evaluateAndDeactivate(); + //TODO: This should really be pre-checked as part of the If condition to get here (so that the assertion never fails) + assert($lambda->instanceOf(FunctionDefinition) && $lambda->cast(@FunctionDefinition).expressionSequence->size() == 1); + + let intialAccumlatorVal = $newSfe.parametersValues->at(2); + + let resultVal = $inputVals->fold({v,a| + let newState = $state->addToScope($lambda, [], $v->concatenate($a.value->cast(@ValueSpecification))->evaluateAndDeactivate()); + + let subSfe1 = $lambda.expressionSequence->toOne(); + let subSfe = ^$subSfe1(genericType = $sfe.genericType->resolveGenericType($newState).value); + + $subSfe->prevalInternal($newState, $extensions); + }, $intialAccumlatorVal->prevalInternal($state, $extensions)); + + $resultVal->markModified(); + }), + pair( + {theSfe:SimpleFunctionExpression[1]| + ($newSfe.func == concatenate_T_MANY__T_MANY__T_MANY_) + && ($newSfe.parametersValues.multiplicity->forAll(m|$m->in([PureOne, PureZero]) || $m->isMultiplicityConcrete() && $m->hasUpperBound() && $m->hasLowerBound() && ($m.lowerBound.value == $m.upperBound.value))) + /* + This last condition should not be needed, but there are circumstances where the inferred type of some lambdas is incorrect. + So this is guard to avoid trying to pre-optimise in those situations (where in the example below, we'd optimised to the lambda + simply return $index because accumulator is detected as a PureZero) + + let input = {| + [1,2]->fold({index, accumulator| + $accumulator->concatenate($index); + }, []) + }; + + let foldLambda = $input->evaluateAndDeactivate().expressionSequence->cast(@SimpleFunctionExpression).parametersValues->at(1)->cast(@InstanceValue).values->cast(@FunctionDefinition)->toOne(); + let foldLambdaAccumulatorParam = $foldLambda->functionType().parameters->at(1); + + assertEquals(ZeroMany, $input->functionReturnMultiplicity(), 'Original lambda return multiplicty expected (zero many)'); + assertEquals($foldLambda->functionReturnMultiplicity(), $input->functionReturnMultiplicity(), | 'Fold lambda return multiplicity matches the original function multiplicity'); + //fails + assertEquals($foldLambda->functionReturnMultiplicity(), $foldLambdaAccumulatorParam.multiplicity, 'Accumulator parameter multiplicity matches the return multiplicty of the fold lambda: %r vs %r', [$foldLambda->functionReturnMultiplicity(), $foldLambdaAccumulatorParam.multiplicity]); + + assertEquals(Integer, $input->functionReturnType().rawType, | 'Original lambda return type expected (integer)'); + assertEquals($foldLambda->functionReturnType().rawType, $input->functionReturnType().rawType, | 'Fold lambda matchs original lambda return type'); + //fails + assertEquals($foldLambda->functionReturnType().rawType, $foldLambdaAccumulatorParam.genericType.rawType, 'Accumulator parameter matches lambda return type: %r vs %r', [$foldLambda->functionReturnType().rawType->toOne(), $foldLambdaAccumulatorParam.genericType.rawType->toOne()]); + + */ + && $newParamWrappers->filter(pw|$pw.value->match([v:VariableExpression[1]|$v.multiplicity == PureZero, a:Any[*]|false]))->areAllInScope($state.inScopeVars) + }, + {theSfe:SimpleFunctionExpression[1]| + $state->printDebugWithDepth(|'Handling expand: ' + $newSfe.func->elementToPath()); + $state->printDebugWithDepth(|'Parameter multiplicities: ' + $newSfe.parametersValues.multiplicity->map(x|$x->makeString())->joinStrings('[',',',']')); + + let r = if($newSfe.parametersValues->at(0).multiplicity == PureZero, + | $newSfe.parametersValues->at(1);, + | if($newSfe.parametersValues->at(1).multiplicity == PureZero, + | $newSfe.parametersValues->at(0);, + | let parametersValues = $newSfe.parametersValues + ->map(x|$x->match([iv:InstanceValue[*]| $iv.values;, a:Any[*]| $a])); + + ^InstanceValue( + genericType = $newSfe.genericType->resolveGenericType($state).value, + multiplicity = $parametersValues->size()->toMultiplicity(), + values = $parametersValues + )->evaluateAndDeactivate() + ->map(x|^$x(values = $parametersValues)); + ) + ); + + ^PrevalWrapper( + value = $r, + canPreval = $newParamWrappers->canPreval(), + openVars = $newParamWrappers->openVars($state.inScopeVars), + modified = true + ); + }), + pair( + {theSfe:SimpleFunctionExpression[1]|$newSfe.func == meta::pure::functions::lang::cast_Any_m__T_1__T_m_ && $newParamWrappers.value->at(0)->map(x|$x->instanceOf(InstanceValue) && $x->cast(@InstanceValue).values->isEmpty())}, + {theSfe:SimpleFunctionExpression[1]| + //This special handling should not be required but []->cast(@MyType) seems to fail in compiled mode + + $state->printDebugWithDepth(|'Handling cast of empty collection: ' + $newSfe.func->elementToPath()); + + + let r = ^InstanceValue( + genericType = $newSfe.genericType->resolveGenericType($state).value, + multiplicity = $newSfe.multiplicity, + values = [] + )->evaluateAndDeactivate(); + + ^PrevalWrapper( + value = $r, + canPreval = true, + openVars = [], + modified = true + ); + }), + pair( + {theSfe:SimpleFunctionExpression[1] | + //TODO getAll check is only needed as router does not route FunctionDefinition wth constant result. + isFilterFunctionReturningConstant($theSfe, false) && !isGetAll($theSfe.parametersValues->at(0))}, + {theSfe:SimpleFunctionExpression[1] | + $state->printDebugWithDepth(|'Handling filter which returns false: ' + $newSfe.func->elementToPath()); + + let r = ^InstanceValue( + genericType = ^GenericType(rawType = Nil), + multiplicity = PureZero + )->evaluateAndDeactivate(); + + ^PrevalWrapper( + value = $r, + canPreval = true, + openVars = [], + modified = true + ); + } + ), + pair( + {theSfe:SimpleFunctionExpression[1] | isFilterFunctionReturningConstant($theSfe, true)}, + {theSfe:SimpleFunctionExpression[1] | + $state->printDebugWithDepth(|'Handling filter which returns true: ' + $newSfe.func->elementToPath()); + + ^PrevalWrapper( + value = $theSfe.parametersValues->at(0), + canPreval = true, + openVars = [], + modified = true + ); + } + ), + pair( + {theSfe:SimpleFunctionExpression[1] | ($theSfe.func == toOneMany_T_MANY__T_$1_MANY$_) && + $theSfe.parametersValues->match([ + vs:ValueSpecification[1]|$vs.multiplicity->isMultiplicityConcrete() && $vs.multiplicity->hasLowerBound() + && ($vs.multiplicity->getLowerBound() > 0), + a:Any[*]|false + ])}, + {theSfe:SimpleFunctionExpression[1] | + $state->printDebugWithDepth(|'Handling toOneMany'); + + $newParamWrappers->toOne()->map(pw|^$pw(modified=true)); + } + ), + pair( + {theSfe:SimpleFunctionExpression[1] | ($theSfe.func == toOne_T_MANY__T_1_) && + $theSfe.parametersValues->match([ + vs:VariableExpression[1]|$vs.multiplicity->isMultiplicityConcrete() + && $vs.multiplicity == PureOne, + a:Any[*]|false + ])}, + {theSfe:SimpleFunctionExpression[1] | + $state->printDebugWithDepth(|'Handling toOne'); + + $newParamWrappers->toOne()->map(pw|^$pw(modified=true)); + } + ) + ]); + + let handler = $handlers->filter(p|$p.first->eval($newSfe))->first(); + + if($handler->isNotEmpty(), + | $handler->toOne().second->eval($newSfe), + | $state->printDebugWithDepth(|'Not prevalling: ' + $newSfe.func->elementToPath() + ' (' + $notPrevalReason->toOne() + ')'); + + ^PrevalWrapper( + value = $newSfe, + canPreval = $newParamWrappers->canPreval(), + openVars = $newParamWrappers->openVars($state.inScopeVars), + modified = $newSfeModified + ); + );, + | + $state->printDebugWithDepth(|'Performing preval: ' + $newSfe.func->elementToPath() + ' (can recativate dynamically: '+ $newSfe->canReactivateDynamically()->makeString() + ')'); + + let valX = $newSfe->reactivate($state.inScopeVars)->cast(@Any); + + let val = $valX->map(x|$x->match([ + {bcs : BasicColumnSpecification[1]| prevalInternal($bcs, ^$state(inScopeTypeParams = ^Map(), inScopeVars = ^Map>()), $extensions).value}, + {av : meta::pure::functions::collection::AggregateValue[1]| prevalInternal($av, ^$state(inScopeTypeParams = ^Map(), inScopeVars = ^Map>()), $extensions).value}, + {av : meta::pure::tds::AggregateValue[1]| prevalInternal($av, ^$state(inScopeTypeParams = ^Map(), inScopeVars = ^Map>()), $extensions).value}, + {a:Any[1]|$a} + ])); + + let iv = $val->match([ + x : InstanceValue[1]| $x, + a : Any[*]| ^InstanceValue(genericType = $newSfe.genericType,multiplicity = $val->size()->toMultiplicity(), values = $val); + ])->evaluateAndDeactivate(); + ^PrevalWrapper(value = $iv, canPreval = true, modified = true); + ); + ); + ); + ); +} + +function meta::pure::router::preeval::isGetAll(v:ValueSpecification[0..1]):Boolean[1] +{ + $v->match([ + s:SimpleFunctionExpression[1] | meta::pure::router::routing::isGetAllFunction($s.func) || $s.parametersValues->first()->isGetAll(), + a:Any[*] | false; + ]); +} + +function <> meta::pure::router::preeval::isFilterFunctionReturningConstant(sfe:SimpleFunctionExpression[1], value:Boolean[1]) : Boolean[1] +{ + let isFilter = $sfe.func == filter_T_MANY__Function_1__T_MANY_; + + if ($isFilter, + | + let es = $sfe.parametersValues->last()->cast(@InstanceValue).values->cast(@LambdaFunction).expressionSequence->evaluateAndDeactivate(); + $es->last()->toOne()->instanceOf(InstanceValue) && $es->last()->toOne()->cast(@InstanceValue).values == $value;, + | false); +} + +// ==================================================================================================================================================================== +// State Building Functions +// ==================================================================================================================================================================== + +function <> meta::pure::router::preeval::defaultFunctionInlineStrategy(extensions:Extension[*]) : FunctionDefinition<{Function[1]->Boolean[1]}>[1] +{ + {f:Function[1] | $f->in([col_Function_1__String_1__BasicColumnSpecification_1_, + col_Function_1__String_1__String_1__BasicColumnSpecification_1_, + // TODO: We should confirm presence of below 2 functions + meta::pure::functions::date::mostRecentDayOfWeek_DayOfWeek_1__Date_1_, + meta::pure::functions::date::previousDayOfWeek_DayOfWeek_1__Date_1_]) + || (!meta::pure::router::routing::shouldStop($f, $extensions) && !($f->instanceOf(NativeFunction)))} +} + +function <> meta::pure::router::preeval::defaultPreevalStopStrategy(extensions:Extension[*]) : FunctionDefinition<{Any[1]->Boolean[1]}>[1] +{ + {a:Any[1] | stopPreeval($a, $extensions)}; +} + +function meta::pure::router::preeval::getPreevalStateWithAdditionalStopInlineFunc(inScopeVars:Map>[1], extensions:meta::pure::extension::Extension[*], stopInlineFunctions:Function[*]):meta::pure::router::preeval::State[1] +{ + ^meta::pure::router::preeval::State(inScopeVars = $inScopeVars, + shouldInlineFxn = {f:Function[1] | meta::pure::router::preeval::defaultFunctionInlineStrategy($extensions)->eval($f) + && !$f->in($stopInlineFunctions)}, + stopPreeval = meta::pure::router::preeval::defaultPreevalStopStrategy($extensions), + rollingInScopeVars = $inScopeVars, + debug = noDebug(), + inScopeTypeParams = ^Map()); +} + +function <> meta::pure::router::preeval::stopPreeval(value : Any[*], extensions:Extension[*]) : Boolean[1] +{ + $extensions.routerExtensions().shouldStopPreeval->exists(p | $p->eval($value)) || + $value->match([ + p : String[*]| true, + p : Float[*]| true, + p : Decimal[*] | true, + p : Enum[*]| true, + p : StrictDate[*]| true, + p : LatestDate[*]| true, + p : Integer[*]| true, + p : Boolean[*]| true, + p : DateTime[*]| true, + p : SortInformation[*]| true, + p : Enumeration[*]| true, + p : Class[*]| true, + p : Pair[*]| true, + p : Property[*]| true, + p : Type[*]| true, + p : Path[*]|true, + p : meta::pure::mapping::Mapping[1]| true, + p : meta::pure::runtime::PackageableRuntime[1]| true, + p : meta::pure::runtime::Runtime[1]| true, + p : meta::pure::runtime::Connection[1] | true, + p : meta::pure::runtime::ExecutionContext[1]| true, + p : meta::pure::tds::TDSColumn[*]| true, + p : TdsOlapRank[*]| true, + p : TdsOlapAggregation[*]| true, + p : meta::pure::graphFetch::execution::AlloySerializationConfig[1]| true, + f : SimpleFunctionExpression[1]| $f.func->in([letFunction_String_1__T_m__T_m_]), + a : Any[*]| $a->type() + ->match([ + c:Class[1]|$c->isSimpleType(), + o:Any[1]|false + ]) + ]); +} + +function <> meta::pure::router::preeval::isSimpleType(c : Class[1]) : Boolean[1] +{ + $c->isSimpleType([]); +} + +function <> meta::pure::router::preeval::isSimpleType(c : Class[1], visitedClasses : Class[*]) : Boolean[1] +{ + let types = $c.properties->map(p|$p->functionReturnType().rawType) + ->distinct() + ->removeAll([String, Integer, Date, DateTime, StrictDate, LatestDate,Float]) + ->removeAll($visitedClasses) + ->filter(x|!$x->instanceOf(Enumeration)); + + let newClassTypes = $types->filter(x|$x->instanceOf(Class))->cast(@Class); + let unsupporedBasicTypes = $types->removeAll($newClassTypes); + + $unsupporedBasicTypes->isEmpty() && $newClassTypes->forAll(newC|$newC->isSimpleType($visitedClasses->concatenate($newClassTypes))); +} + + +// ==================================================================================================================================================================== +// Utility Functions +// ==================================================================================================================================================================== + +function <> meta::pure::router::preeval::resolveGenericType(in : GenericType[1], state : meta::pure::router::preeval::State[1]):PrevalWrapper[1] +{ + if($in.typeParameter->isEmpty(), + | + let newTypeArgsWrappers = $in.typeArguments->map(ta|$ta->resolveGenericType($state)); + if(!$newTypeArgsWrappers->anyModified(), + | ^PrevalWrapper(value = $in, canPreval=true, modified = false), + | let value = ^$in(typeArguments = $newTypeArgsWrappers.value); + ^PrevalWrapper(value = $value, canPreval=$newTypeArgsWrappers->canPreval(), modified = $newTypeArgsWrappers->anyModified()); + );, + | + let value = $state.inScopeTypeParams->get($in.typeParameter->toOne().name); + if($value->isEmpty(), + | ^PrevalWrapper(value = $in, canPreval=true, modified = false), + | ^PrevalWrapper(value = $value->toOne(), canPreval=true, modified = true) + ); + ); +} + +function <> meta::pure::router::preeval::shouldInline(f:Function[1], state : meta::pure::router::preeval::State[1]):Boolean[1] +{ + ($f->instanceOf(FunctionDefinition) && ($f->cast(@FunctionDefinition).expressionSequence->size() == 1 )) + && !($f->instanceOf(QualifiedProperty) && $f->cast(@QualifiedProperty).owner == TDSRow) + && !($f->instanceOf(AbstractProperty) && $f->cast(@AbstractProperty)->meta::pure::milestoning::hasGeneratedMilestoningPropertyStereotype()) + && ($state.shouldInlineFxn->eval($f)) + && ($state.path->filter(x|$x == $f)->isEmpty()) +} + +function <> meta::pure::router::preeval::canInlineEvalFunctionExpression(fe:SimpleFunctionExpression[1]):Boolean[1] +{ + $fe.func->in([eval_Function_1__V_m_, + eval_Function_1__T_n__V_m_, + eval_Function_1__T_n__U_p__V_m_, + eval_Function_1__T_n__U_p__W_q__V_m_, + eval_Function_1__T_n__U_p__W_q__X_r__V_m_, + eval_Function_1__T_n__U_p__W_q__X_r__Y_s__V_m_, + eval_Function_1__T_n__U_p__W_q__X_r__Y_s__Z_t__V_m_, + eval_Function_1__S_n__T_o__U_p__W_q__X_r__Y_s__Z_t__V_m_]) + && $fe.parametersValues->at(0)->match([ + iv:InstanceValue[1]|$iv.values->toOne(), + vs:ValueSpecification[1]|$vs + ])->instanceOf(FunctionDefinition) + && $fe.parametersValues->at(0)->match([ + iv:InstanceValue[1]|$iv.values->toOne(), + vs:ValueSpecification[1]|$vs + ])->cast(@FunctionDefinition).expressionSequence->size() == 1 +} + +function <> meta::pure::router::preeval::toMultiplicity(size : Integer[1]) : Multiplicity[1] +{ + if($size == 0, + | PureZero, + | if($size == 1, + | PureOne, + | ^Multiplicity(lowerBound = ^MultiplicityValue(value=$size), upperBound = ^MultiplicityValue(value=$size)) + ); + ); +} + +function <> meta::pure::router::preeval::addToScope(sfe : SimpleFunctionExpression[1], state : meta::pure::router::preeval::State[1]):meta::pure::router::preeval::State[1] +{ + addToScope($state, $sfe.func, $sfe.resolvedTypeParameters, $sfe.parametersValues, true); +} + +function <> meta::pure::router::preeval::addToScope(state : meta::pure::router::preeval::State[1], func : Function[1],resolvedTypeParameters : GenericType[*], parametersValues : ValueSpecification[*]):meta::pure::router::preeval::State[1] +{ + addToScope($state, $func, $resolvedTypeParameters, $parametersValues, false) +} + +function <> meta::pure::router::preeval::addToScope(state : meta::pure::router::preeval::State[1], func : Function[1],resolvedTypeParameters : GenericType[*], parametersValues : ValueSpecification[*], cleanupInScopeVarsAndTypeParams:Boolean[1]):meta::pure::router::preeval::State[1] +{ + let typeParams = $func->functionType().typeParameters.name->zip($resolvedTypeParameters->map(x|$x->resolveGenericType($state).value)); + + let newParams = $parametersValues; + + let funcParamNames = $func->functionType().parameters.name; + let additionalInScopeVars = $funcParamNames + ->zip($newParams->map(pv|list($pv))) + ->map({p| let paramName = $p.first; + let paramValue = $p.second; + + $paramValue.values->match([ + ve:VariableExpression[1]| + if($ve.name == $paramName, + | [], //We drop easy "circular" / self-reference parameter variables + | $p + ), + vs:ValueSpecification[*]| + $p; + ]); + }); + + if($cleanupInScopeVarsAndTypeParams, |^$state(inScopeVars=newMap([]), inScopeTypeParams=newMap([])), |$state)->addToScope($additionalInScopeVars, $typeParams); +} + +function <> meta::pure::router::preeval::addToScope(state : meta::pure::router::preeval::State[1], additionalInScopeVars : Pair>[*], typeParams : Pair[*]):meta::pure::router::preeval::State[1] +{ + $state->printDebugWithDepth(|'Adding variables to scope: ' + $additionalInScopeVars.first->joinStrings('[', ',', ']')); + $state->printDebugWithDepth(|'Adding type params to scope: ' + $typeParams.first->joinStrings('[', ',', ']')); + + let newInScopeVars = $state.inScopeVars->putAll($additionalInScopeVars); + let newInScopeTypeParams = $state.inScopeTypeParams->putAll($typeParams); + + ^$state(inScopeVars = $newInScopeVars, inScopeTypeParams = $newInScopeTypeParams); +} + +function <> meta::pure::router::preeval::resolveVariable(name : String[1], state : meta::pure::router::preeval::State[1]):String[1] +{ + $state.inScopeVars->get($name).values->match([ + ve:VariableExpression[1]|assert($name != $ve.name, 'Circular variable reference: ' + $name); $ve.name->resolveVariable($state);, + a:Any[*]|$name + ]); +} + +function <> meta::pure::router::preeval::canPreval(items : PrevalWrapper[*]):Boolean[1] +{ + $items->forAll(p | $p.canPreval); +} + +function <> meta::pure::router::preeval::markModified(item : PrevalWrapper[1]):PrevalWrapper[1] +{ + if($item.modified, + | $item, + | ^$item(modified = true) + ); +} + +function <> meta::pure::router::preeval::anyModified(items : PrevalWrapper[*]):Boolean[1] +{ + $items.modified->contains(true); +} + +function <> meta::pure::router::preeval::openVars(items : PrevalWrapper[*], inScopeVars:Map>[1]):String[*] +{ + $items.openVars->openVars($inScopeVars); +} + +function <> meta::pure::router::preeval::openVars(vars : String[*], inScopeVars:Map>[1]):String[*] +{ + $vars->distinct()->filter(v|!$v->areAllInScope($inScopeVars)) +} + +function <> meta::pure::router::preeval::areAllInScope(items : PrevalWrapper[*], inScopeVars:Map>[1]):Boolean[1] +{ + $items.openVars->distinct()->areAllInScope($inScopeVars); +} + +function <> meta::pure::router::preeval::areAllInScope(vars : String[*], inScopeVars:Map>[1]):Boolean[1] +{ + if($vars->isEmpty(), + | true, + | let unresolvedVars = $vars->map(v|let varVal = $inScopeVars->get($v); + + if($varVal->isEmpty(), + | $v, + | $varVal.values->match([ + varExp : VariableExpression[1]|$varExp.name, + a : Any[*]|[] + ]) + ); + )->distinct(); + + let newVars = $unresolvedVars->removeAll($vars); + !$unresolvedVars->containsAny($vars) && $newVars->areAllInScope($inScopeVars); + ); +} + +function <> meta::pure::router::preeval::isInstanceValue(v: Any[1], inScopeVars:Map>[1]):Boolean[1] +{ + $v->match([ + iv : InstanceValue[1]| + $iv.values->forAll(subV|$subV->match([ + subVS : ValueSpecification[1]|$subVS->isInstanceValue($inScopeVars), + a : Any[1]| true; + ]));, + ve : VariableExpression[1]|$inScopeVars->get($ve.name)->isNotEmpty(), + sfe : SimpleFunctionExpression[1]|false; + ]) +} + +function <> meta::pure::router::preeval::printDebugWithDepth(state: meta::pure::router::preeval::State[1], func:Function<{->String[1]}>[1]):Nil[0] +{ + printDebug($state, | ('[' + toString($state.depth->orElse(''))->map(s|'0000'->substring(0, max(0, 3-$s->length())) + $s) + ']' + range($state.depth->orElse(0) + 1)->map(d|'')->joinStrings('', '-', ' ') + $func->eval())); +} + +function <> meta::pure::router::preeval::printDebug(state: meta::pure::router::preeval::State[1], func:Function<{->Any[*]}>[1]):Nil[0] +{ + printDebug($state.debug, $func); +} + +function <> meta::pure::router::preeval::printDebug(debugContext : DebugContext[1], func:Function<{->Any[*]}>[1]):Nil[0] +{ + if(!$debugContext.debug, + | print(''), + | printDebug($debugContext, $func->eval()); + ); +} + +function <> meta::pure::router::preeval::printDebug(state: meta::pure::router::preeval::State[1], param:Any[*]):Nil[0] +{ + printDebug($state.debug, $param); +} + +function <> meta::pure::router::preeval::printDebug(debugContext : DebugContext[1], param:Any[*]):Nil[0] +{ + if(!$debugContext.debug, + | print(''), + | println($param); + ); } \ No newline at end of file diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/tests.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/tests.pure index 22b16d67d19..39515a0b3e7 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/tests.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/preeval/tests.pure @@ -1,1597 +1,1603 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Press F9 to execute the 'go' function... -// Press F10 to run the full test suite - -import meta::external::format::shared::binding::*; -import meta::external::format::shared::functions::*; -import meta::json::*; -import meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::*; -import meta::protocols::pure::vX_X_X::transformation::fromPureGraph::toPureGrammar::*; -import meta::pure::extension::*; -import meta::pure::graphFetch::execution::*; -import meta::pure::mapping::*; -import meta::pure::mapping::modelToModel::test::simple::*; -import meta::pure::mapping::modelToModel::test::shared::dest::*; -import meta::pure::mapping::modelToModel::test::shared::src::*; -import meta::pure::router::preeval::*; -import meta::pure::router::preeval::tests::*; -import meta::pure::runtime::*; - -Class meta::pure::router::preeval::tests::Person -{ - firstName : String[1]; - lastName : String[1]; - otherNames : String[*]; - - name(){$this.firstName+' '+$this.lastName}:String[1]; - nameWithTitle(title:String[1]){$title+' '+$this.firstName+' '+$this.lastName}:String[1]; - nameWithPrefixAndSuffix(prefix:String[0..1], suffixes:String[*]) - { - if($prefix->isEmpty(), - | if($suffixes->isEmpty(), - | $this.firstName + ' ' + $this.lastName, - | $this.firstName + ' ' + $this.lastName + ', ' + $suffixes->joinStrings(', ')), - | if($suffixes->isEmpty(), - | $prefix->toOne() + ' ' + $this.firstName + ' ' + $this.lastName, - | $prefix->toOne() + ' ' + $this.firstName + ' ' + $this.lastName + ', ' + $suffixes->joinStrings(', '))) - }:String[1]; - - fullName(lastNameFirst:Boolean[1]) - { - if($lastNameFirst, | $this.lastName + ', ' + $this.firstName, | $this.firstName + ' ' + $this.lastName) - }:String[1]; - extraInformation : String[0..1]; - manager : meta::pure::router::preeval::tests::Person[0..1]; - age : Integer[0..1]; - constant() { 'constant' } : String[1]; - nickName : String[0..1]; - activeEmployment: Boolean[0..1]; -} - -Class meta::pure::router::preeval::tests::FinancialInstrument -{ - id : Integer[1]; - price : Decimal[1]; -} - -Class <> meta::pure::router::preeval::tests::BaseAccount -{ - id: Integer[1]; - accountId(){ - $this.id - }:Integer[1]; -} - -Class <> meta::pure::router::preeval::tests::Position -{ - account:BaseAccount[1]; - id: Integer[1]; - quantity: Float[1]; -} - -Class <> meta::pure::router::preeval::tests::BaseOrganizationalEntity -{ - name: String[1]; - id: Integer[1]; -} - -Class <> meta::pure::router::preeval::tests::OrganizationalEntity extends BaseOrganizationalEntity -{ - isActive: Boolean[1]; - parent: OrganizationalEntity[1]; - parentAtDate(date:Date[1]) - { - $this.parent($date); - }:OrganizationalEntity[1]; - ceo: meta::pure::router::preeval::tests::Person[1]; - - latestPositions() - { - $this.positions(now()); - }:Position[*]; -} - -Association meta::pure::router::preeval::tests::PositionClient -{ - positions:Position[*]; - client:OrganizationalEntity[1]; -} - -Class <> meta::pure::router::preeval::tests::ClassA -{ - myPropertyB : meta::pure::router::preeval::tests::ClassB[*]; -} - -Class <> meta::pure::router::preeval::tests::ClassB -{ - myLeafProperty : String[1]; -} - - -function <> meta::pure::router::preeval::tests::testPrerouting1():Boolean[1] -{ - let input = {| let x = 'hello' + ' world'; $x + (' ' + (5->toString()));}; - - let expected = {| 'hello world 5';}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting2():Boolean[1] -{ - let input = {| let x = today()->adjust(1, DurationUnit.DAYS); let y = %2017-01-01->adjust(1, DurationUnit.DAYS); }; - - let expected = {| let x = today()->adjust(1, DurationUnit.DAYS); %2017-01-02; }; - - assertRoundTrip($input, $expected); -} - - -function <> meta::pure::router::preeval::tests::testPrerouting2b():Boolean[1] -{ - let input = {| - let x = firstDayOfThisYear()->adjust(1, DurationUnit.DAYS); - let y = firstDayOfThisMonth(); - let z = firstDayOfThisQuarter()->adjust(2, DurationUnit.DAYS); - - let a = mostRecentDayOfWeek(DayOfWeek.Monday); - let b= previousDayOfWeek(DayOfWeek.Monday); - - true; - }; - - let expected = {| - let x = today()->firstDayOfYear()->adjust(1, DurationUnit.DAYS); - let y = today()->firstDayOfMonth(); - let z = today()->firstDayOfQuarter()->adjust(2, DurationUnit.DAYS); - - let a = today()->mostRecentDayOfWeek(DayOfWeek.Monday); - let b = today()->previousDayOfWeek(DayOfWeek.Monday); - - true; - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting3():Boolean[1] -{ - let input = {| let y = 1; $y + (6 + 10);}; - - let expected = {| 17;}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting4():Boolean[1] -{ - let value = 'hello world'; - - let input = {| let y = $value->substring(0, 5); }; - - let expected = {| 'hello'}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting5():Boolean[1] -{ - let input = {| currentUserId() + '_user' ;}; - - let expected = $input; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting6():Boolean[1] -{ - let value = 10; - - let input = {| range(5)->map(x|$x+$value); }; - - let expected = {| [10, 11, 12,13,14]}; - - assertRoundTrip($input, $expected); -} - - -function <> meta::pure::router::preeval::tests::testPrerouting7():Boolean[1] -{ - let input = {| %2018-01-02->toString()->myExpandableFunc(5) ;}; - - let expected = {| '2018-01-02|2018-01-02|2018-01-02|2018-01-02|2018-01-02'}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting8():Boolean[1] -{ - let input = {| today()->toString()->myNonExpandableMultiExpressionFunc(5) ;}; - - let expected = $input; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting9():Boolean[1] -{ - let input = {| {x:String[1]| $x + 'hello'} ;}; - - let expected = $input; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting10():Boolean[1] -{ - let input = {| range(3)->map(x|$x->toString());}; - - let expected = {| ['0','1','2']}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting11():Boolean[1] -{ - let input = {| range(3)->map(x| $x->map(index|$index->toString()));}; - - let expected = {| ['0','1','2']}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting12():Boolean[1] -{ - let input = {| range(3)->map(x| $x->toString()->map(index|$index));}; - - let expected = {| ['0','1','2']}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting13():Boolean[1] -{ - let input = {| range(4)->map(x| [1,2,3,4,5]->take($x)->map(index|$index));}; - - let expected = {| [1,1,2,1,2,3]}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting14():Boolean[1] -{ - let input = {| range(3)->map(x| [1,2,3,4,5]->take($x + 1)->map(index|$index));}; - - let expected = {| [1,1,2,1,2,3]}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting15():Boolean[1] -{ - // ->cast(@Integer) added to help with compile mode java generation, this should not be needed but java compiler fails on java generated code - let input = {| range(3)->map(x| range(5)->map(k|$k+1)->cast(@Integer)->take($x + 1)->map(index|$index->cast(@Integer)));}; - - let expected = {| [1,1,2,1,2,3]}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting16():Boolean[1] -{ - let input = {| range(3)->map(x|$x+1)->map(index|'i' + $index->toString()) ;}; - - let expected = {|['i1', 'i2', 'i3'] }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting17():Boolean[1] -{ - let c = 3; - - let input = {| range($c)->map(x|$x+1)->map(index|'i' + $index->toString()) ;}; - - let expected = {|['i1', 'i2', 'i3'] }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting18():Boolean[1] -{ - let c = 3; - - let input = {| range($c)->map(x|$x+1)->map(index|('i' + $c->makeString()) + $index->toString()) ;}; - - let expected = {|['i31', 'i32', 'i33'] }; - - assertRoundTrip($input, $expected); -} - - -function <> meta::pure::router::preeval::tests::testPrerouting19():Boolean[1] -{ - let input = {|['a']->fold({p,x|$x->concatenate(range(2)->map(index|$p)->joinStrings('|'))}, [])}; - - let expected = {|['a|a']}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting20():Boolean[1] -{ - let input = {|['hello']->map(a|$a->f1())}; - - let expected = {|'hello'}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::f1(zzz : String[1]):String[1] -{ - $zzz->map(k|$zzz->substring(0,5)); -} - -function <> {test.excludePlatform = 'Java compiled'} meta::pure::router::preeval::tests::testPrerouting20a():Boolean[1] -{ - let input = {|'hello'->map(x|$x->myFirst1())}; - - let expected = {|'hello'}; - - assertRoundTrip($input, $expected); -} - -function meta::pure::router::preeval::tests::myFirst1(theFirstSet:T[*]):T[0..1] -{ - myFirst2($theFirstSet); -} - -function meta::pure::router::preeval::tests::myFirst2(theSecondSet:T[*]):T[0..1] -{ - $theSecondSet->first(); -} - -function <> meta::pure::router::preeval::tests::testPrerouting21():Boolean[1] -{ - let input = {|['a']->fold({p,x|$x->concatenate(myExpandableFunc2($p, 3))}, [])}; - - let expected = {|'a|a|a' }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::myExpandableFunc2(val : String[1], count : Integer[1]):String[*] -{ - range(0, $count, 1)->map(index|$val)->joinStrings('|'); -// $val->myFakeJoin('|'); -} - - -function <> meta::pure::router::preeval::tests::testPrerouting_mapOnInstanceValuesExpanded():Boolean[1] -{ - let c = 3; - - let input = {| range(1, $c+1, 1)->map(index|$index->toString() + currentUserId()->toString()) ;}; - - let expected = {|[ - ('1' + meta::pure::runtime::currentUserId()->toString()), - ('2' + meta::pure::runtime::currentUserId()->toString()), - ('3' + meta::pure::runtime::currentUserId()->toString()) - ] }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_foldOnInstanceValuesExpanded():Boolean[1] -{ - let c = 2; - - let input = {| - range(1, $c+1, 1)->fold({index, r| - $r + ($index->toString() + currentUserId() + ';'); - }, 'Values: ') - }; - - let expected = {| - 'Values: ' + '1' + meta::pure::runtime::currentUserId() + ';' + '2' + meta::pure::runtime::currentUserId() + ';' - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_foldOnInstanceValuesExpanded2():Boolean[1] -{ - let c = 2; - - let input = {| - range(1, $c+1, 1)->fold({index, r| - $r->concatenate(($index->toString() + currentUserId())); - }, []) - }; - - let expected = {| - [('1' + currentUserId()), ('2' + currentUserId())] - ///[('1' + meta::pure::runtime::currentUserId())] ->concatenate('2' + meta::pure::runtime::currentUserId()) - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_concatenateInstanceValuesExpanded_Basic():Boolean[1] -{ - let c = 2; - - let input = {| - ['hello']->concatenate(currentUserId()); - }; - - let expected = {| - ['hello', currentUserId()]; - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_concatenateInstanceValuesExpanded_Complex():Boolean[1] -{ - let x = 2; - - let input = {| - let a = []->concatenate(currentUserId()); - let b = []->concatenate(['hello', currentUserId()]); - let c = []->concatenate(['hello', currentUserId(), (currentUserId() + $x->toString())]); - - let d = currentUserId()->concatenate([]); - let e = ['hello', currentUserId()]->concatenate([]); - let f = ['hello', currentUserId(), (currentUserId() + $x->toString())]->concatenate([]); - - let g = currentUserId()->concatenate(currentUserId()); - let h = ['hello', currentUserId()]->concatenate(['hello', currentUserId()]); - let i = ['hello', currentUserId(), (currentUserId() + $x->toString())]->concatenate(['hello', currentUserId(), (currentUserId() + $x->toString())]); - - let j = myCollectionFunc($x->toString())->concatenate(currentUserId()); - - let k = {a:Integer[1], b:Integer[1]|$a->concatenate($b)}; - let l = {a:Integer[*], b:Integer[1]|$a->concatenate($b)}; - let m = {a:Integer[1], b:Integer[*]|$a->concatenate($b)}; - let n = {a:Integer[*], b:Integer[*]|$a->concatenate($b)}; - - true; - }; - - let expected = {| - let a = currentUserId(); - let b = ['hello', currentUserId()]; - let c = ['hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2')]; - - let d = currentUserId(); - let e = ['hello', currentUserId()]; - let f = ['hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2')]; - - let g = [meta::pure::runtime::currentUserId(),meta::pure::runtime::currentUserId()]; - let h = ['hello',meta::pure::runtime::currentUserId(),'hello',meta::pure::runtime::currentUserId()]; - let i = ['hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2'),'hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2')]; - - let j= ['2', currentUserId()]; - - let k = {a:Integer[1], b:Integer[1]|[$a,$b]}; - let l = {a:Integer[*], b:Integer[1]|$a->concatenate($b)}; - let m = {a:Integer[1], b:Integer[*]|$a->concatenate($b)}; - let n = {a:Integer[*], b:Integer[*]|$a->concatenate($b)}; - - true; - }; - - assertRoundTrip($input, $expected); -} - -function meta::pure::router::preeval::tests::myCollectionFunc(values:T[m]):T[m] -{ - $values; -} - -function <> meta::pure::router::preeval::tests::testPrerouting23a():Boolean[1] -{ - let input = {| if(false, | []->toOne(), | 'hello ' + 'world') ;}; - - let expected = {| 'hello world' }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting23b():Boolean[1] -{ - let input = {| if(true, | 'hello ' + 'world', | []->toOne() ) ;}; - - let expected = {| 'hello world' }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting24a():Boolean[1] -{ - let x = []; - - let input = {| - $x->isEmpty() || ($x->at(0) == 'hello'); - }; - - let expected = {| true }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting24b():Boolean[1] -{ - let x = []; - - let input = {| - $x->isNotEmpty() && ($x->at(0) == 'hello'); - }; - - let expected = {| false }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting24c():Boolean[1] -{ - let x = []; - - let input = {| - (today() > %2018-01-01) && (1 == 1); - }; - - let expected = {| today() > %2018-01-01 }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting24d():Boolean[1] -{ - let x = []; - - let input = {| - (today() > %2018-01-01) || (1 == 1); - }; - - let expected = {| true }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting24e():Boolean[1] -{ - let x = []; - - let input = {| - (1 == 1) && (today() > %2018-01-01); - }; - - let expected = {| today() > %2018-01-01 }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting24f():Boolean[1] -{ - let x = []; - - let input = {| - (1 == 1) || (today() > %2018-01-01); - }; - - let expected = {| true }; - - assertRoundTrip($input, $expected); -} - - -function <> meta::pure::router::preeval::tests::testPrerouting25a():Boolean[1] -{ - let input = {|meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}, 'lastName'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age'), - col({p:meta::pure::router::preeval::tests::Person[1]|today()}, 'today') - ]) - ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}))} ; - - let expected = {| meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}), - ^BasicColumnSpecification(name = 'lastName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}), - ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}), - ^BasicColumnSpecification(name = 'today', func = {p:meta::pure::router::preeval::tests::Person[1]|today()}) - ]) - ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age') ->toString()}))}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting25b():Boolean[1] -{ - let input = {|meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + ('hello' + '_world')}, 'firstName'), - col({p:meta::pure::router::preeval::tests::Person[1]|'world' + ' blue'}, 'const'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') - ]) - ->extend(['e']->map(x|col({row:TDSRow[1]|$row.getInteger('ag' + $x) ->toString()}, 'ageString')))}; - - let expected = {| meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + 'hello_world'}), - ^BasicColumnSpecification(name = 'const', func = {p:meta::pure::router::preeval::tests::Person[1]|'world blue'}), - ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age})]) - ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}))}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting25c():Boolean[1] -{ - let blue = 'blue'; - - let cols = [ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + ('hello' + '_world')}, 'firstName'), - col({p:meta::pure::router::preeval::tests::Person[1]|'world' + ' ' + $blue}, 'const'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') - ]; - - let input = {|meta::pure::router::preeval::tests::Person.all() - ->project($cols) - ->extend(['e']->map(x|col({row:TDSRow[1]|$row.getInteger('ag' + $x) ->toString()}, 'ageString'))) - }; - - let expected = {| meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + 'hello_world'}), - ^BasicColumnSpecification(name = 'const', func = {p:meta::pure::router::preeval::tests::Person[1]|'world blue'}), - ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age})]) - ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}))}; - - assertRoundTrip($input, $expected); -} - - - -function <> meta::pure::router::preeval::tests::testPrerouting26():Boolean[1] -{ - let input = {| - pair('hello', today()); - }; - - let expected = {| ^Pair(first='hello', second = today()) }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting27a():Boolean[1] -{ - let input = {| - [1,2,3]->map(v| ^Pair(first = 'hello', second = $v)); - }; - - let expected = {| [^Pair(first = 'hello', second = 1),^Pair(first = 'hello', second = 2),^Pair(first = 'hello', second = 3)] }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting27b():Boolean[1] -{ - let input = {| - [1,2,3]->map(v| pair('hello', $v)); - }; - - let expected = {| [^Pair(first = 'hello', second = 1),^Pair(first = 'hello', second = 2),^Pair(first = 'hello', second = 3)] }; - - assertRoundTrip($input, $expected); -} - - -// function <> meta::pure::router::preeval::tests::testPrerouting27c():Boolean[1] -// { -// let input = {| -// [today(),today(),today()]->map(v| pair('hello', $v)); -// }; - -// let expected = {| [today(),today(),today()]->map(v|^Pair(first = 'hello', second = $v)) }; - -// assertRoundTrip($input, $expected); -// } - -function <> meta::pure::router::preeval::tests::testPrerouting28():Boolean[1] -{ - let input = {| - [1,2,3]->map(v| $v->map(x|pair('hello' + '*' + $x->makeString(), $v))); - }; - - let expected = {| [^Pair(first = 'hello*1', second = 1),^Pair(first = 'hello*2', second = 2),^Pair(first = 'hello*3', second = 3)] }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting29a():Boolean[1] -{ - let input = {r:String[1]| ['a', 'b', 'c']->map(v| $r + ('_' + $v))}; - - let expected = {r:String[1]|[ ($r + '_a'),($r + '_b'),($r + '_c')]}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::myPair(firstVal : String[1], secondVal : StrictDate[1]):Pair[1] -{ - ^Pair(first = $firstVal, second=$secondVal); -} - -function <> meta::pure::router::preeval::tests::testPrerouting29b():Boolean[1] -{ - let input = {| range(3)->map(index|today()) ;}; - - let expected = {|[today(),today(),today()] }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting29c():Boolean[1] -{ - let input = {| myNonFullyExpandableDateFunc('a', 5) ;}; - - let expected = {|[ - ('a' + (today()->toString())), - ('a' + (today()->toString())), - ('a' + (today()->toString())), - ('a' + (today()->toString())), - ('a' + (today()->toString())) - ]->joinStrings('|'); - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::myExpandableFunc(val : String[1], count : Integer[1]):String[1] -{ - range($count)->map(index|$val)->joinStrings('|'); -} - -function <> meta::pure::router::preeval::tests::testPrerouting30():Boolean[1] -{ - let input = {| - range(3)->map(index|[$index, today()]); - }; - - let expected = {| - [0,today(), 1,today(), 2,today()] - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting31a():Boolean[1] -{ - let input = {| - range(2)->map(a|^BasicColumnSpecification(name = 'age_' + $a->makeString(), func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + ($a * 10)})) - }; - - let expected = {|[^BasicColumnSpecification(name = 'age_0', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 0}),^BasicColumnSpecification(name = 'age_1', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 10})] }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting31b():Boolean[1] -{ - let input = {| meta::pure::router::preeval::tests::Person.all() - ->project( - range(2)->map(a| - [col({p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + ($a * 10)}, 'age_' + makeString($a*10)),col({p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + (($a*10) + 1)}, 'age_' + makeString(($a*10) + 1))] - ) - ) - }; - - let expected = {|meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'age_0', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 0}), - ^BasicColumnSpecification(name = 'age_1', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 1}), - ^BasicColumnSpecification(name = 'age_10', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 10}), - ^BasicColumnSpecification(name = 'age_11', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 11}) - ]) - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting32a():Boolean[1] -{ - let input = {| ['e']->map(x|col({row:TDSRow[1]|$row.getInteger('ag' + $x)->toString()}, 'ageString'));}; - - let expected = {|^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}) }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting32b():Boolean[1] -{ - let input = {| let x = 'e'; col({row:TDSRow[1]|$row.getInteger('ag' + $x)->toString()}, 'ageString');}; - - let expected = {|^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}); }; - - assertRoundTrip($input, $expected); -} - - - -function <> meta::pure::router::preeval::tests::myFakeJoin(vals : String[*], char : String[1]):String[1] -{ - let x = 'hello'; - $vals->drop(1)->fold({x,n| $x + $char + $n}, $vals->at(0)); - //$vals->at(0); -} - -function <> meta::pure::router::preeval::tests::myComplexExpandableFunc(vals : String[*]):String[1] -{ - $vals->map(x|$x->myExpandableFunc(1))->joinStrings('hello'); -} - - - -function meta::pure::router::preeval::tests::myNonExpandableMultiExpressionFunc(val : String[1], count : Integer[1]):String[1] -{ - let x = range($count); - $x->map(index|$val)->joinStrings('|'); -} - -function <> meta::pure::router::preeval::tests::myNonFullyExpandableDateFunc(val : String[1], count : Integer[1]):String[1] -{ - range($count)->map(index|$val + today()->toString())->joinStrings('|'); -} - - - -function <> meta::pure::router::preeval::tests::testPrerouting33():Boolean[1] -{ - let input = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}, 'lastName'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age'), - col({p:meta::pure::router::preeval::tests::Person[1]|today()}, 'today') - ]) - ->filterToStringColumns() - }; - - let expected = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}), - ^BasicColumnSpecification(name = 'lastName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}), - ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}), - ^BasicColumnSpecification(name = 'today', func = {p:meta::pure::router::preeval::tests::Person[1]|today()}) - ]) - ->restrict(['firstName','lastName']) - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::filterToStringColumns(input : TabularDataSet[1]):TabularDataSet[1] -{ - $input->restrict($input.columns->filter(c|$c.type == String).name); -} - -function <> meta::pure::router::preeval::tests::extendColumns(input : TabularDataSet[1], cols : String[*]):TabularDataSet[1] -{ - $input->extend($input.columns->filter(c|$c.name->in($cols))->sortBy(c|$cols->indexOf($c.name))->map(c| - if($c.type == String, - | col({row:TDSRow[1]|$row.getString($c.name)}, $c.name + '_ext'), - | if($c.type == Integer, - | col({row:TDSRow[1]|$row.getInteger($c.name)}, $c.name + '_ext'), - | col({row:TDSRow[1]|$row.getDate($c.name)}, $c.name + '_ext') - ) - ) - )); -} - -function <> meta::pure::router::preeval::tests::testPrerouting34():Boolean[1] -{ - let input = {| - let businessDate = now(); - let processingDate = today()->adjust(-1, DurationUnit.DAYS); - - meta::pure::router::preeval::tests::Position.all($processingDate) - ->project([ - x | $x.quantity, - x | $x.client($businessDate).name], - ['Quantity', - 'Client/Name'] - ) - ->extendColumns('Quantity'); - }; - - let expected = {| - let businessDate = now(); - let processingDate = today()->adjust(-1, DurationUnit.DAYS); - - meta::pure::router::preeval::tests::Position - ->getAll($processingDate) - ->project([{x|$x.quantity},{x|$x.client($businessDate).name}], ['Quantity','Client/Name']) - ->extend(^BasicColumnSpecification(name = 'Quantity_ext', func = {row:TDSRow[1]|$row.getDate('Quantity')})); - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting35():Boolean[1] -{ - let input = {| - let x = 'world'; - - {|('hello' + ' ') + $x}->eval(); - }; - - let expected = {| - 'hello world'; - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_parameterAssignedToVariable():Boolean[1] -{ - let input = {item:Pair[1]| - let other = $item; - $other.first->isNotEmpty(); - }; - - let expected = {item:Pair[1]| - $item.first->isNotEmpty(); - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting36():Boolean[1] -{ - let input = {a:meta::pure::router::preeval::tests::ClassA[1]| - $a.myPropertyB.myLeafProperty->map(x|now())->map(d|$d->adjust(-1, DurationUnit.DAYS)) - }; - - assertRoundTrip($input, $input); -} - -function <> meta::pure::router::preeval::tests::testPrerouting37():Boolean[1] -{ - let input = {x:Integer[*]| - $x->otherFunc2(y|$y+1) - }; - - let expected = {x:Integer[*]| - $x->map({y|$y + 1}) - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::otherFunc2(x : Integer[*], l : FunctionDefinition<{Integer[1]->Integer[1]}>[1]):Integer[*] -{ - $x->map($l); -} - -function <> meta::pure::router::preeval::tests::testPrerouting38():Boolean[1] -{ - let input = {| - let x = myPair('hello', 'world'); - let y = myPair(1, $x.first + ' ' + $x.second); - }; - - let expected = {| - ^Pair(first = 1, second = 'hello world'); - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::myPair(v1 : X[1], v2 : Y[1]) : Pair[1] -{ - pair($v1, $v2); -} - -function <> meta::pure::router::preeval::tests::myPair2(v1 : U[1], v2 : V[1]) : Pair[1] -{ - pair($v1, $v2); -} - -function <> meta::pure::router::preeval::tests::testPrerouting39():Boolean[1] -{ - let input = {| - let x = 123; - $x->cast(@Integer); - }; - - let expected = {| - 123 - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting40a():Boolean[1] -{ - let input = {| - let x = 123; - $x->toOne()->myToOne(); - }; - - let expected = {| - 123; - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting40b():Boolean[1] -{ - let input = {| - let x = today(); - $x->toOne()->myToOne(); - }; - - let expected = {| - let x = today(); - $x; - }; - - assertRoundTrip($input, $expected); -} - - -function <> meta::pure::router::preeval::tests::myToOne(x : T[1]) : T[1] -{ - $x->toOne(); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_PropertyValue():Boolean[1] -{ - let prop = meta::pure::router::preeval::tests::Person.properties->at(0); - let name = $prop.name; - - let input = {| $prop.name;}; - - let expected = {| $name}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_OptionalLimit1():Boolean[1] -{ - let limit = 5; - - let input = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') - ]) - ->meta::pure::router::preeval::tests::optionalLimit($limit) - }; - - let expected = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) - ]) - ->limit(5); - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_OptionalLimit2():Boolean[1] -{ - let limit = []; - - let input = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') - ]) - ->meta::pure::router::preeval::tests::optionalLimit($limit) - }; - - let expected = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) - ]); - }; - - assertRoundTrip($input, $expected); -} - -function meta::pure::router::preeval::tests::optionalLimit(t: TabularDataSet[1], limit:Integer[0..1]) : TabularDataSet[1] -{ - if($limit->isEmpty() || ($limit->toOne()<=0), | $t, | $t->limit($limit)); -} - - -function <> meta::pure::router::preeval::tests::testPrerouting_mappedTdsAgg():Boolean[1] -{ - let v = [pair('name', '6')]; - - let input = {|meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') - ]) - ->groupBy([], $v->map(pv|agg('sumAgg_' + $pv.first->toString(), {row:TDSRow[1]|if($row.getInteger('age')->toString() == $pv.second, | 1, | 0)}, y|$y->sum()))) - }; - - let expected = {| meta::pure::router::preeval::tests::Person.all() - ->project(^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age})) - ->groupBy([], agg('sumAgg_name', {row:TDSRow[1]|if($row.getInteger('age')->toString() == '6', |1, |0)}, {y|$y->sum()})) - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_mappedModelAgg():Boolean[1] -{ - let v = [pair('name', '6')]; - - let input = {|meta::pure::router::preeval::tests::Person.all() - ->groupBy([], - $v->map(pv|agg({p:meta::pure::router::preeval::tests::Person[1]|if($p.age->toOne()->toString() == $pv.second, | 1, | 0)}, y|$y->sum())), - $v->map(pv|'sumAgg_' + $pv.first) - ) - }; - - let expected = {|meta::pure::router::preeval::tests::Person.all() - ->groupBy([], agg({p:meta::pure::router::preeval::tests::Person[1]|if($p.age->toOne()->toString() == '6', |1, |0)}, {y|$y->sum()}), 'sumAgg_name') - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_constantInAgg():Boolean[1] -{ - let lambda = {| - let businessDate = %2023-01-01; - let x = 'a'; - meta::pure::router::preeval::tests::Position.all($businessDate)->groupBy( - [], - agg(x|$x.client($businessDate).name + 'b', y|$y->count()), - 'ID Count' - ); - }; - - //we re-write the lambda to inline an AggregateValue with businessDate variable - let aggregateValue = ^meta::pure::functions::collection::AggregateValue(mapFn={x:meta::pure::router::preeval::tests::Position[1] | $x.client(%2023-01-01).name + $x->toString()},aggregateFn={y:String[*] | $y->count()}); - - let expressionSequence = $lambda.expressionSequence->evaluateAndDeactivate(); - let groupBy = $expressionSequence->at(2)->cast(@SimpleFunctionExpression); - - let mapFn = $aggregateValue.mapFn; - let mapFnExpressionSequnce = $mapFn.expressionSequence->evaluateAndDeactivate()->cast(@SimpleFunctionExpression)->toOne(); - - let plus = $mapFnExpressionSequnce.parametersValues->at(0)->cast(@InstanceValue); - let plusParams = $plus.values; - let plusLeft = $plusParams->at(0)->cast(@SimpleFunctionExpression); - let client = $plusLeft.parametersValues->at(0)->cast(@SimpleFunctionExpression); - let newClient = ^$client(parametersValues = [$client.parametersValues->at(0), ^VariableExpression(name='businessDate',genericType=^GenericType(rawType = StrictDate),multiplicity=PureOne)]); - - let newPlusLeft = ^$plusLeft(parametersValues = $newClient); - let newPlus = ^$plus(values = [$newPlusLeft,$plus.values->at(1)]); - - let newAggregateValue = ^$aggregateValue(mapFn = ^$mapFn(expressionSequence = ^$mapFnExpressionSequnce(parametersValues = [$newPlus]))); - let newAggregateValueInstance = ^InstanceValue(genericType=^GenericType(rawType = meta::pure::functions::collection::AggregateValue),multiplicity=PureOne, values = $newAggregateValue); - let newGroupBy = ^$groupBy(parametersValues = [$groupBy.parametersValues->at(0), $groupBy.parametersValues->at(1), $newAggregateValueInstance, $groupBy.parametersValues->at(3)]); - let newLambda = ^$lambda(expressionSequence = [$expressionSequence->at(0), $expressionSequence->at(1), $newGroupBy]); - - - let expected = {| - meta::pure::router::preeval::tests::Position.all(%2023-01-01)->groupBy([], ^meta::pure::functions::collection::AggregateValue(mapFn={x:meta::pure::router::preeval::tests::Position[1] | $x.client(%2023-01-01).name + $x->toString()},aggregateFn={y:String[*]|$y->count()}), ['ID Count']) - }; - - assertRoundTrip($newLambda, $expected); -} - -function <> meta::pure::router::preeval::tests::testDecimalType():Boolean[1] -{ - let input = {|meta::pure::router::preeval::tests::FinancialInstrument.all() - ->project([ - col(f:meta::pure::router::preeval::tests::FinancialInstrument[1]|$f.price, 'price'), - col(f:meta::pure::router::preeval::tests::FinancialInstrument[1]|parseDecimal('3.14D'), 'decimal_constant_col') - ]) - }; - - let expected = {|meta::pure::router::preeval::tests::FinancialInstrument.all() - ->project([ - ^BasicColumnSpecification(name = 'price', func={f:meta::pure::router::preeval::tests::FinancialInstrument[1]| $f.price }), - ^BasicColumnSpecification(name = 'decimal_constant_col', func={f:meta::pure::router::preeval::tests::FinancialInstrument[1]| 3.14}) - ]) - }; - - assertRoundTrip($input,$expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_castEmptyCollection():Boolean[1] -{ - let input = {b:Integer[*]|$b->uniqueValueOnly()}; - - let expected = {b:Integer[*]|if($b->distinct()->size() == 1, {|$b->max()}, {|[]})}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testEvalWithArgs1():Boolean[1] -{ - let input = {| - {x:Integer[1]|$x+1}->eval(5); - }; - - let expected = {|6}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testEvalWithArgs2():Boolean[1] -{ - let input = {|{t:TabularDataSet[1]|$t->filter(r:TDSRow[1]|$r.getString('firstName') == 'firstName')} - ->eval({|meta::pure::router::preeval::tests::Person.all()->project(col(p:meta::pure::router::preeval::tests::Person[1]|$p.firstName, 'firstName'))}->eval())}; - - let expected = {|meta::pure::router::preeval::tests::Person.all() - ->project(^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName})) - ->filter(r:TDSRow[1]|$r.getString('firstName') == 'firstName') - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testEvalWithArgs3():Boolean[1] -{ - let input = {a:Integer[1]|{b:Integer[1]|$b+1}->eval($a)}; - - let expected = {a:Integer[1]| $a+1 }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testProjectWithInferredParameterType():Boolean[1] -{ - let input = {|meta::pure::router::preeval::tests::myFunc888()}; - - let expected = {| meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}) - ]) - }; - - assertRoundTrip($input, $expected); -} - -function meta::pure::router::preeval::tests::myFunc888() : TabularDataSet[1] -{ - meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') - ]) -} - - - - -function <> meta::pure::router::preeval::tests::testPreroutingRemoveUnnecessaryStatements():Boolean[1] -{ - let input = {| - range(5)->map(x|$x+1)->map(index|'i' + $index->toString()); - 123; - }; - - let expected = {| - 123; - }; - - assertRoundTrip($input, $expected); -} - - -function meta::pure::router::preeval::tests::myStopFunction(b: Integer[1]) : Integer[1] -{ - $b+1; -} - -function <> meta::pure::router::preeval::tests::testAdditionalStopFunction():Boolean[1] -{ - let input = {a:Integer[1]|myStopFunction($a)}; - - let expected = {a:Integer[1]| $a+1 }; - - assertRoundTrip($input, $expected); - assertRoundTrip($input, $input, myStopFunction_Integer_1__Integer_1_, [], noDebug()); -} - -function <> meta::pure::router::preeval::tests::testFilterFalseSimplification():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | false).id}; - - let expected = {a:OrganizationalEntity[1]| []}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testFilterFalseSimplificationAll():Boolean[1] -{ - let input = {s:String[0..1]| OrganizationalEntity.all(%2023-01-01)->filter(c | 1 == 2)->project(c | $c.name, 'name')}; - - let expected = {s:String[0..1]| OrganizationalEntity.all(%2023-01-01)->filter(c | false)->project(c | $c.name, 'name')}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testFilterFalseSimplification2():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | 1 == 2).id}; - - let expected = {o:OrganizationalEntity[1]| []}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testFilterFalseConstantSimplification():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| let bool = false; let date = %2023-01-01; $o.positions($date)->filter(p | $bool).id;}; - - let expected = {o:OrganizationalEntity[1]| []}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testFilterFalseSimplificationReturnType():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | false).id;}; - let result = $input->preval([]); - - assertEquals(Integer, $result->functionReturnType().rawType); -} - -function <> meta::pure::router::preeval::tests::testFilterTrueSimplification():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | false; true;).id}; - - let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01).id}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testFilterTrueSimplification2():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | 1 == 1).id}; - - let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01).id}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testFilterTrueConstantSimplification():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| let bool = true; let date = %2023-01-01; $o.positions($date)->filter(p | $bool).id;}; - - let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01).id}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testFilterNoSimplification():Boolean[1] -{ - let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | $p.id == true).id}; - - let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | $p.id == true).id}; - - assertRoundTrip($input, $expected); -} - - -function <> meta::pure::router::preeval::tests::testToOneManyElimination1():Boolean[1] -{ - let input = {| 'hello'->toOneMany()}; - - let expected = {| 'hello'}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testToOneManyElimination2():Boolean[1] -{ - let input = {x:String[*]| $x->toOneMany()}; - - let expected = $input; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testToOneManyElimination3():Boolean[1] -{ - let input = {x:String[1..*]| $x->toOneMany()}; - - let expected = {x:String[1..*]| $x}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testToOneElimination1():Boolean[1] -{ - let input = {| 'hello'->toOne()}; - - let expected = {| 'hello'}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testToOneElimination2():Boolean[1] -{ - let input = {x:String[*]| $x->toOne()}; - - let expected = $input; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testToOneElimination3():Boolean[1] -{ - let input = {x:String[1]| $x->toOne()}; - - let expected = {x:String[1]| $x}; - - assertRoundTrip($input, $expected); -} - - -function <> meta::pure::router::preeval::tests::testRecursiveSimpleConcreteFunctionDefinition():Boolean[1] -{ - // This is specially selected to be a single function that is recursive on itself - // The test is to validate that we don't keep trying to inline ad infinitum - - let input = {s:Class[1]|meta::pure::functions::meta::hierarchicalProperties($s)}; - - let expected = {s:Class[1]| - if($s == Any, {|[]}, {|$s.properties->concatenate($s.generalizations->map({g|$g.general.rawType->cast(Any) - ->toOne() - ->hierarchicalProperties()})) - ->removeDuplicates([], [])}) - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testLetOnlyStatement():Boolean[1] -{ - let input = {|myFunc2()}; - - let expected = {| 'hello'}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testLambdaParamOverride():Boolean[1] -{ - let input = { | let val = 1; - {val:Integer[1]| $val};}; - - let expected = { | {val:Integer[1]| $val};}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testLambdaParamOverride2():Boolean[1] -{ - let val = 1; - let input = { | let x = 1; - {p:Integer[1]|$p->map(x|$x+$val)};}; - let expected = { | {p:Integer[1]|$p->map(x|$x+1)}}; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::myFunc2():String[1] -{ - let x = 'hello'; -} - -function meta::pure::router::preeval::tests::getTestRuntimeWithModelConnection(sourceClass:Class[1], inputs:Any[*]):Runtime[1] -{ - // Need element as string to allow roundtrip to pass. When using ^ModelStore(), preeval evaluates it to a instance, however expected as it as a new function - ^Runtime(connections=^meta::pure::mapping::modelToModel::ModelConnection(element = 'DummyStore', instances=newMap(pair($sourceClass, list($inputs))))); -} - -function <> meta::pure::router::preeval::tests::assertRoundTrip(input : FunctionDefinition[1], expectedOrig : FunctionDefinition[1]):Boolean[1] -{ - assertRoundTrip($input, $expectedOrig, noDebug()) -} - -function <> meta::pure::router::preeval::tests::assertRoundTrip(input : FunctionDefinition[1], expectedOrig : FunctionDefinition[1], debug:meta::pure::tools::DebugContext[1]):Boolean[1] -{ - meta::pure::router::preeval::tests::assertRoundTrip($input, $expectedOrig, [], [], $debug); -} - -function <> meta::pure::router::preeval::tests::assertRoundTrip(input : FunctionDefinition[1], expectedOrig : FunctionDefinition[1], stopInlineFunctions:Function[*], extensions:Extension[*], debug:meta::pure::tools::DebugContext[1]):Boolean[1] -{ - let expected = $expectedOrig; - - let result = if($stopInlineFunctions->isEmpty(), - | $input->preval(newMap([]->cast(@Pair)), $input->openVariableValues(), [], $debug).value->toOne(); , - | let inScopeVars = $input->openVariableValues(); - let state = meta::pure::router::preeval::getPreevalStateWithAdditionalStopInlineFunc($inScopeVars, $extensions, $stopInlineFunctions); - $input->preval($state, $extensions).value->toOne();); - - let tExpected = $expected->transformFunctionBody($extensions); - let tResult = $result->transformFunctionBody($extensions); - - let jExpected = $tExpected->toJSON(50000); - let jResult = $tResult->toJSON(50000); - - let pjExpected = $jExpected->parseJSON()->toPrettyJSONString(); - let pjResult = $jResult->parseJSON()->toPrettyJSONString(); - - if($pjExpected == $pjResult, - | true, - | - println($pjExpected); - println($pjResult); - - let pExpected = $tExpected->toPure($extensions); - let pResult = $tResult->toPure($extensions); - - assertEquals($pExpected, $pResult, 'Mismatch between expected and actual result content.\nexpected pure:\n%r\n\nactual pure:\n%r', [$pExpected->toOne(), $pResult->toOne()]); - ); - - // Open variables on lambdas should have been inlined / eliminated - if (!$result->instanceOf(LambdaFunction), - | true, - | assertEmpty($result->cast(@LambdaFunction).openVariables) - ); - - if($input->cast(@Function)->functionType().parameters->isNotEmpty(), - | true, - | - let inputEvalResult = $input->evaluate([]); - let expectedEvalResult = $expected->evaluate([]); - let resultEvalResult = $result->evaluate([]); - - let open = $input->openVariableValues(); - - //println($input->cast(@Function)->functionReturnMultiplicity()); - - let mInput = $input->cast(@Function)->functionReturnMultiplicity(); - let mExpected = $expected->cast(@Function)->functionReturnMultiplicity(); - let mResult = $result->cast(@Function)->functionReturnMultiplicity(); - - let jInputEvalResult = $inputEvalResult->transformAny([], $open, $mExpected, $extensions)->toJSON(50000); - let jExpectedEvalResult = $expectedEvalResult->transformAny([], $open, $mExpected, $extensions)->toJSON(50000); - println($jExpectedEvalResult); - let jResultEvalResult = $resultEvalResult->transformAny([], $open, $mExpected, $extensions)->toJSON(50000); - - - assertEquals($jExpectedEvalResult, $jResultEvalResult, '\nexpected eval result: %r\nactual eval result: %r', [$jExpectedEvalResult->toOne(), $jResultEvalResult->toOne()]); - - if(!$inputEvalResult->exists(x|$x->match([b:BasicColumnSpecification[*]|true, a:Any[*]|false])), - | assertEquals($jInputEvalResult, $jExpectedEvalResult, '\nexpected input eval: %r\nactual expected eval: %r', [$jInputEvalResult->toOne(), $jExpectedEvalResult->toOne()]), - | true //The lambda on the BCS can get confused for assertion checking (as it can be "un-optimised") - ); - ); - - true; +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Press F9 to execute the 'go' function... +// Press F10 to run the full test suite + +import meta::json::*; +import meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::*; +import meta::protocols::pure::vX_X_X::transformation::fromPureGraph::toPureGrammar::*; +import meta::pure::extension::*; +import meta::pure::graphFetch::execution::*; +import meta::pure::mapping::*; +import meta::pure::mapping::modelToModel::test::simple::*; +import meta::pure::mapping::modelToModel::test::shared::dest::*; +import meta::pure::mapping::modelToModel::test::shared::src::*; +import meta::pure::router::preeval::*; +import meta::pure::router::preeval::tests::*; +import meta::pure::runtime::*; + +Class meta::pure::router::preeval::tests::Person +{ + firstName : String[1]; + lastName : String[1]; + otherNames : String[*]; + + name(){$this.firstName+' '+$this.lastName}:String[1]; + nameWithTitle(title:String[1]){$title+' '+$this.firstName+' '+$this.lastName}:String[1]; + nameWithPrefixAndSuffix(prefix:String[0..1], suffixes:String[*]) + { + if($prefix->isEmpty(), + | if($suffixes->isEmpty(), + | $this.firstName + ' ' + $this.lastName, + | $this.firstName + ' ' + $this.lastName + ', ' + $suffixes->joinStrings(', ')), + | if($suffixes->isEmpty(), + | $prefix->toOne() + ' ' + $this.firstName + ' ' + $this.lastName, + | $prefix->toOne() + ' ' + $this.firstName + ' ' + $this.lastName + ', ' + $suffixes->joinStrings(', '))) + }:String[1]; + + fullName(lastNameFirst:Boolean[1]) + { + if($lastNameFirst, | $this.lastName + ', ' + $this.firstName, | $this.firstName + ' ' + $this.lastName) + }:String[1]; + extraInformation : String[0..1]; + manager : meta::pure::router::preeval::tests::Person[0..1]; + age : Integer[0..1]; + constant() { 'constant' } : String[1]; + nickName : String[0..1]; + activeEmployment: Boolean[0..1]; +} + +Class meta::pure::router::preeval::tests::FinancialInstrument +{ + id : Integer[1]; + price : Decimal[1]; +} + +Class <> meta::pure::router::preeval::tests::BaseAccount +{ + id: Integer[1]; + accountId(){ + $this.id + }:Integer[1]; +} + +Class <> meta::pure::router::preeval::tests::Position +{ + account:BaseAccount[1]; + id: Integer[1]; + quantity: Float[1]; +} + +Class <> meta::pure::router::preeval::tests::BaseOrganizationalEntity +{ + name: String[1]; + id: Integer[1]; +} + +Class <> meta::pure::router::preeval::tests::OrganizationalEntity extends BaseOrganizationalEntity +{ + isActive: Boolean[1]; + parent: OrganizationalEntity[1]; + parentAtDate(date:Date[1]) + { + $this.parent($date); + }:OrganizationalEntity[1]; + ceo: meta::pure::router::preeval::tests::Person[1]; + + latestPositions() + { + $this.positions(now()); + }:Position[*]; +} + +Association meta::pure::router::preeval::tests::PositionClient +{ + positions:Position[*]; + client:OrganizationalEntity[1]; +} + +Class <> meta::pure::router::preeval::tests::ClassA +{ + myPropertyB : meta::pure::router::preeval::tests::ClassB[*]; +} + +Class <> meta::pure::router::preeval::tests::ClassB +{ + myLeafProperty : String[1]; +} + + +function <> meta::pure::router::preeval::tests::testPrerouting1():Boolean[1] +{ + let input = {| let x = 'hello' + ' world'; $x + (' ' + (5->toString()));}; + + let expected = {| 'hello world 5';}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting2():Boolean[1] +{ + let input = {| let x = today()->adjust(1, DurationUnit.DAYS); let y = %2017-01-01->adjust(1, DurationUnit.DAYS); }; + + let expected = {| let x = today()->adjust(1, DurationUnit.DAYS); %2017-01-02; }; + + assertRoundTrip($input, $expected); +} + + +function <> meta::pure::router::preeval::tests::testPrerouting2b():Boolean[1] +{ + let input = {| + let x = firstDayOfThisYear()->adjust(1, DurationUnit.DAYS); + let y = firstDayOfThisMonth(); + let z = firstDayOfThisQuarter()->adjust(2, DurationUnit.DAYS); + + let a = mostRecentDayOfWeek(DayOfWeek.Monday); + let b= previousDayOfWeek(DayOfWeek.Monday); + + true; + }; + + let expected = {| + let x = today()->firstDayOfYear()->adjust(1, DurationUnit.DAYS); + let y = today()->firstDayOfMonth(); + let z = today()->firstDayOfQuarter()->adjust(2, DurationUnit.DAYS); + + let a = today()->mostRecentDayOfWeek(DayOfWeek.Monday); + let b = today()->previousDayOfWeek(DayOfWeek.Monday); + + true; + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting3():Boolean[1] +{ + let input = {| let y = 1; $y + (6 + 10);}; + + let expected = {| 17;}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting4():Boolean[1] +{ + let value = 'hello world'; + + let input = {| let y = $value->substring(0, 5); }; + + let expected = {| 'hello'}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting5():Boolean[1] +{ + let input = {| currentUserId() + '_user' ;}; + + let expected = $input; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting6():Boolean[1] +{ + let value = 10; + + let input = {| range(5)->map(x|$x+$value); }; + + let expected = {| [10, 11, 12,13,14]}; + + assertRoundTrip($input, $expected); +} + + +function <> meta::pure::router::preeval::tests::testPrerouting7():Boolean[1] +{ + let input = {| %2018-01-02->toString()->myExpandableFunc(5) ;}; + + let expected = {| '2018-01-02|2018-01-02|2018-01-02|2018-01-02|2018-01-02'}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting8():Boolean[1] +{ + let input = {| today()->toString()->myNonExpandableMultiExpressionFunc(5) ;}; + + let expected = $input; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting9():Boolean[1] +{ + let input = {| {x:String[1]| $x + 'hello'} ;}; + + let expected = $input; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting10():Boolean[1] +{ + let input = {| range(3)->map(x|$x->toString());}; + + let expected = {| ['0','1','2']}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting11():Boolean[1] +{ + let input = {| range(3)->map(x| $x->map(index|$index->toString()));}; + + let expected = {| ['0','1','2']}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting12():Boolean[1] +{ + let input = {| range(3)->map(x| $x->toString()->map(index|$index));}; + + let expected = {| ['0','1','2']}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting13():Boolean[1] +{ + let input = {| range(4)->map(x| [1,2,3,4,5]->take($x)->map(index|$index));}; + + let expected = {| [1,1,2,1,2,3]}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting14():Boolean[1] +{ + let input = {| range(3)->map(x| [1,2,3,4,5]->take($x + 1)->map(index|$index));}; + + let expected = {| [1,1,2,1,2,3]}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting15():Boolean[1] +{ + // ->cast(@Integer) added to help with compile mode java generation, this should not be needed but java compiler fails on java generated code + let input = {| range(3)->map(x| range(5)->map(k|$k+1)->cast(@Integer)->take($x + 1)->map(index|$index->cast(@Integer)));}; + + let expected = {| [1,1,2,1,2,3]}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting16():Boolean[1] +{ + let input = {| range(3)->map(x|$x+1)->map(index|'i' + $index->toString()) ;}; + + let expected = {|['i1', 'i2', 'i3'] }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting17():Boolean[1] +{ + let c = 3; + + let input = {| range($c)->map(x|$x+1)->map(index|'i' + $index->toString()) ;}; + + let expected = {|['i1', 'i2', 'i3'] }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting18():Boolean[1] +{ + let c = 3; + + let input = {| range($c)->map(x|$x+1)->map(index|('i' + $c->makeString()) + $index->toString()) ;}; + + let expected = {|['i31', 'i32', 'i33'] }; + + assertRoundTrip($input, $expected); +} + + +function <> meta::pure::router::preeval::tests::testPrerouting19():Boolean[1] +{ + let input = {|['a']->fold({p,x|$x->concatenate(range(2)->map(index|$p)->joinStrings('|'))}, [])}; + + let expected = {|['a|a']}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting20():Boolean[1] +{ + let input = {|['hello']->map(a|$a->f1())}; + + let expected = {|'hello'}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::f1(zzz : String[1]):String[1] +{ + $zzz->map(k|$zzz->substring(0,5)); +} + +function <> {test.excludePlatform = 'Java compiled'} meta::pure::router::preeval::tests::testPrerouting20a():Boolean[1] +{ + let input = {|'hello'->map(x|$x->myFirst1())}; + + let expected = {|'hello'}; + + assertRoundTrip($input, $expected); +} + +function meta::pure::router::preeval::tests::myFirst1(theFirstSet:T[*]):T[0..1] +{ + myFirst2($theFirstSet); +} + +function meta::pure::router::preeval::tests::myFirst2(theSecondSet:T[*]):T[0..1] +{ + $theSecondSet->first(); +} + +function <> meta::pure::router::preeval::tests::testPrerouting21():Boolean[1] +{ + let input = {|['a']->fold({p,x|$x->concatenate(myExpandableFunc2($p, 3))}, [])}; + + let expected = {|'a|a|a' }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::myExpandableFunc2(val : String[1], count : Integer[1]):String[*] +{ + range(0, $count, 1)->map(index|$val)->joinStrings('|'); +// $val->myFakeJoin('|'); +} + + +function <> meta::pure::router::preeval::tests::testPrerouting_mapOnInstanceValuesExpanded():Boolean[1] +{ + let c = 3; + + let input = {| range(1, $c+1, 1)->map(index|$index->toString() + currentUserId()->toString()) ;}; + + let expected = {|[ + ('1' + meta::pure::runtime::currentUserId()->toString()), + ('2' + meta::pure::runtime::currentUserId()->toString()), + ('3' + meta::pure::runtime::currentUserId()->toString()) + ] }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_foldOnInstanceValuesExpanded():Boolean[1] +{ + let c = 2; + + let input = {| + range(1, $c+1, 1)->fold({index, r| + $r + ($index->toString() + currentUserId() + ';'); + }, 'Values: ') + }; + + let expected = {| + 'Values: ' + '1' + meta::pure::runtime::currentUserId() + ';' + '2' + meta::pure::runtime::currentUserId() + ';' + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_foldOnInstanceValuesExpanded2():Boolean[1] +{ + let c = 2; + + let input = {| + range(1, $c+1, 1)->fold({index, r| + $r->concatenate(($index->toString() + currentUserId())); + }, []) + }; + + let expected = {| + [('1' + currentUserId()), ('2' + currentUserId())] + ///[('1' + meta::pure::runtime::currentUserId())] ->concatenate('2' + meta::pure::runtime::currentUserId()) + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_concatenateInstanceValuesExpanded_Basic():Boolean[1] +{ + let c = 2; + + let input = {| + ['hello']->concatenate(currentUserId()); + }; + + let expected = {| + ['hello', currentUserId()]; + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_concatenateInstanceValuesExpanded_Complex():Boolean[1] +{ + let x = 2; + + let input = {| + let a = []->concatenate(currentUserId()); + let b = []->concatenate(['hello', currentUserId()]); + let c = []->concatenate(['hello', currentUserId(), (currentUserId() + $x->toString())]); + + let d = currentUserId()->concatenate([]); + let e = ['hello', currentUserId()]->concatenate([]); + let f = ['hello', currentUserId(), (currentUserId() + $x->toString())]->concatenate([]); + + let g = currentUserId()->concatenate(currentUserId()); + let h = ['hello', currentUserId()]->concatenate(['hello', currentUserId()]); + let i = ['hello', currentUserId(), (currentUserId() + $x->toString())]->concatenate(['hello', currentUserId(), (currentUserId() + $x->toString())]); + + let j = myCollectionFunc($x->toString())->concatenate(currentUserId()); + + let k = {a:Integer[1], b:Integer[1]|$a->concatenate($b)}; + let l = {a:Integer[*], b:Integer[1]|$a->concatenate($b)}; + let m = {a:Integer[1], b:Integer[*]|$a->concatenate($b)}; + let n = {a:Integer[*], b:Integer[*]|$a->concatenate($b)}; + + true; + }; + + let expected = {| + let a = currentUserId(); + let b = ['hello', currentUserId()]; + let c = ['hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2')]; + + let d = currentUserId(); + let e = ['hello', currentUserId()]; + let f = ['hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2')]; + + let g = [meta::pure::runtime::currentUserId(),meta::pure::runtime::currentUserId()]; + let h = ['hello',meta::pure::runtime::currentUserId(),'hello',meta::pure::runtime::currentUserId()]; + let i = ['hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2'),'hello',meta::pure::runtime::currentUserId(),(meta::pure::runtime::currentUserId() + '2')]; + + let j= ['2', currentUserId()]; + + let k = {a:Integer[1], b:Integer[1]|[$a,$b]}; + let l = {a:Integer[*], b:Integer[1]|$a->concatenate($b)}; + let m = {a:Integer[1], b:Integer[*]|$a->concatenate($b)}; + let n = {a:Integer[*], b:Integer[*]|$a->concatenate($b)}; + + true; + }; + + assertRoundTrip($input, $expected); +} + +function meta::pure::router::preeval::tests::myCollectionFunc(values:T[m]):T[m] +{ + $values; +} + +function <> meta::pure::router::preeval::tests::testPrerouting23a():Boolean[1] +{ + let input = {| if(false, | []->toOne(), | 'hello ' + 'world') ;}; + + let expected = {| 'hello world' }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting23b():Boolean[1] +{ + let input = {| if(true, | 'hello ' + 'world', | []->toOne() ) ;}; + + let expected = {| 'hello world' }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting24a():Boolean[1] +{ + let x = []; + + let input = {| + $x->isEmpty() || ($x->at(0) == 'hello'); + }; + + let expected = {| true }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting24b():Boolean[1] +{ + let x = []; + + let input = {| + $x->isNotEmpty() && ($x->at(0) == 'hello'); + }; + + let expected = {| false }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting24c():Boolean[1] +{ + let x = []; + + let input = {| + (today() > %2018-01-01) && (1 == 1); + }; + + let expected = {| today() > %2018-01-01 }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting24d():Boolean[1] +{ + let x = []; + + let input = {| + (today() > %2018-01-01) || (1 == 1); + }; + + let expected = {| true }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting24e():Boolean[1] +{ + let x = []; + + let input = {| + (1 == 1) && (today() > %2018-01-01); + }; + + let expected = {| today() > %2018-01-01 }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting24f():Boolean[1] +{ + let x = []; + + let input = {| + (1 == 1) || (today() > %2018-01-01); + }; + + let expected = {| true }; + + assertRoundTrip($input, $expected); +} + + +function <> meta::pure::router::preeval::tests::testPrerouting25a():Boolean[1] +{ + let input = {|meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}, 'lastName'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age'), + col({p:meta::pure::router::preeval::tests::Person[1]|today()}, 'today') + ]) + ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}))} ; + + let expected = {| meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}), + ^BasicColumnSpecification(name = 'lastName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}), + ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}), + ^BasicColumnSpecification(name = 'today', func = {p:meta::pure::router::preeval::tests::Person[1]|today()}) + ]) + ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age') ->toString()}))}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting25b():Boolean[1] +{ + let input = {|meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + ('hello' + '_world')}, 'firstName'), + col({p:meta::pure::router::preeval::tests::Person[1]|'world' + ' blue'}, 'const'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') + ]) + ->extend(['e']->map(x|col({row:TDSRow[1]|$row.getInteger('ag' + $x) ->toString()}, 'ageString')))}; + + let expected = {| meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + 'hello_world'}), + ^BasicColumnSpecification(name = 'const', func = {p:meta::pure::router::preeval::tests::Person[1]|'world blue'}), + ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age})]) + ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}))}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting25c():Boolean[1] +{ + let blue = 'blue'; + + let cols = [ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + ('hello' + '_world')}, 'firstName'), + col({p:meta::pure::router::preeval::tests::Person[1]|'world' + ' ' + $blue}, 'const'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') + ]; + + let input = {|meta::pure::router::preeval::tests::Person.all() + ->project($cols) + ->extend(['e']->map(x|col({row:TDSRow[1]|$row.getInteger('ag' + $x) ->toString()}, 'ageString'))) + }; + + let expected = {| meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName + 'hello_world'}), + ^BasicColumnSpecification(name = 'const', func = {p:meta::pure::router::preeval::tests::Person[1]|'world blue'}), + ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age})]) + ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}))}; + + assertRoundTrip($input, $expected); +} + + + +function <> meta::pure::router::preeval::tests::testPrerouting26():Boolean[1] +{ + let input = {| + pair('hello', today()); + }; + + let expected = {| ^Pair(first='hello', second = today()) }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting27a():Boolean[1] +{ + let input = {| + [1,2,3]->map(v| ^Pair(first = 'hello', second = $v)); + }; + + let expected = {| [^Pair(first = 'hello', second = 1),^Pair(first = 'hello', second = 2),^Pair(first = 'hello', second = 3)] }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting27b():Boolean[1] +{ + let input = {| + [1,2,3]->map(v| pair('hello', $v)); + }; + + let expected = {| [^Pair(first = 'hello', second = 1),^Pair(first = 'hello', second = 2),^Pair(first = 'hello', second = 3)] }; + + assertRoundTrip($input, $expected); +} + + +// function <> meta::pure::router::preeval::tests::testPrerouting27c():Boolean[1] +// { +// let input = {| +// [today(),today(),today()]->map(v| pair('hello', $v)); +// }; + +// let expected = {| [today(),today(),today()]->map(v|^Pair(first = 'hello', second = $v)) }; + +// assertRoundTrip($input, $expected); +// } + +function <> meta::pure::router::preeval::tests::testPrerouting28():Boolean[1] +{ + let input = {| + [1,2,3]->map(v| $v->map(x|pair('hello' + '*' + $x->makeString(), $v))); + }; + + let expected = {| [^Pair(first = 'hello*1', second = 1),^Pair(first = 'hello*2', second = 2),^Pair(first = 'hello*3', second = 3)] }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting29a():Boolean[1] +{ + let input = {r:String[1]| ['a', 'b', 'c']->map(v| $r + ('_' + $v))}; + + let expected = {r:String[1]|[ ($r + '_a'),($r + '_b'),($r + '_c')]}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::myPair(firstVal : String[1], secondVal : StrictDate[1]):Pair[1] +{ + ^Pair(first = $firstVal, second=$secondVal); +} + +function <> meta::pure::router::preeval::tests::testPrerouting29b():Boolean[1] +{ + let input = {| range(3)->map(index|today()) ;}; + + let expected = {|[today(),today(),today()] }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting29c():Boolean[1] +{ + let input = {| myNonFullyExpandableDateFunc('a', 5) ;}; + + let expected = {|[ + ('a' + (today()->toString())), + ('a' + (today()->toString())), + ('a' + (today()->toString())), + ('a' + (today()->toString())), + ('a' + (today()->toString())) + ]->joinStrings('|'); + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::myExpandableFunc(val : String[1], count : Integer[1]):String[1] +{ + range($count)->map(index|$val)->joinStrings('|'); +} + +function <> meta::pure::router::preeval::tests::testPrerouting30():Boolean[1] +{ + let input = {| + range(3)->map(index|[$index, today()]); + }; + + let expected = {| + [0,today(), 1,today(), 2,today()] + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting31a():Boolean[1] +{ + let input = {| + range(2)->map(a|^BasicColumnSpecification(name = 'age_' + $a->makeString(), func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + ($a * 10)})) + }; + + let expected = {|[^BasicColumnSpecification(name = 'age_0', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 0}),^BasicColumnSpecification(name = 'age_1', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 10})] }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting31b():Boolean[1] +{ + let input = {| meta::pure::router::preeval::tests::Person.all() + ->project( + range(2)->map(a| + [col({p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + ($a * 10)}, 'age_' + makeString($a*10)),col({p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + (($a*10) + 1)}, 'age_' + makeString(($a*10) + 1))] + ) + ) + }; + + let expected = {|meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'age_0', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 0}), + ^BasicColumnSpecification(name = 'age_1', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 1}), + ^BasicColumnSpecification(name = 'age_10', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 10}), + ^BasicColumnSpecification(name = 'age_11', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age->toOne() + 11}) + ]) + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting32a():Boolean[1] +{ + let input = {| ['e']->map(x|col({row:TDSRow[1]|$row.getInteger('ag' + $x)->toString()}, 'ageString'));}; + + let expected = {|^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}) }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting32b():Boolean[1] +{ + let input = {| let x = 'e'; col({row:TDSRow[1]|$row.getInteger('ag' + $x)->toString()}, 'ageString');}; + + let expected = {|^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age')->toString()}); }; + + assertRoundTrip($input, $expected); +} + + + +function <> meta::pure::router::preeval::tests::myFakeJoin(vals : String[*], char : String[1]):String[1] +{ + let x = 'hello'; + $vals->drop(1)->fold({x,n| $x + $char + $n}, $vals->at(0)); + //$vals->at(0); +} + +function <> meta::pure::router::preeval::tests::myComplexExpandableFunc(vals : String[*]):String[1] +{ + $vals->map(x|$x->myExpandableFunc(1))->joinStrings('hello'); +} + + + +function meta::pure::router::preeval::tests::myNonExpandableMultiExpressionFunc(val : String[1], count : Integer[1]):String[1] +{ + let x = range($count); + $x->map(index|$val)->joinStrings('|'); +} + +function <> meta::pure::router::preeval::tests::myNonFullyExpandableDateFunc(val : String[1], count : Integer[1]):String[1] +{ + range($count)->map(index|$val + today()->toString())->joinStrings('|'); +} + + + +function <> meta::pure::router::preeval::tests::testPrerouting33():Boolean[1] +{ + let input = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}, 'lastName'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age'), + col({p:meta::pure::router::preeval::tests::Person[1]|today()}, 'today') + ]) + ->filterToStringColumns() + }; + + let expected = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}), + ^BasicColumnSpecification(name = 'lastName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}), + ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}), + ^BasicColumnSpecification(name = 'today', func = {p:meta::pure::router::preeval::tests::Person[1]|today()}) + ]) + ->restrict(['firstName','lastName']) + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::filterToStringColumns(input : TabularDataSet[1]):TabularDataSet[1] +{ + $input->restrict($input.columns->filter(c|$c.type == String).name); +} + +function <> meta::pure::router::preeval::tests::extendColumns(input : TabularDataSet[1], cols : String[*]):TabularDataSet[1] +{ + $input->extend($input.columns->filter(c|$c.name->in($cols))->sortBy(c|$cols->indexOf($c.name))->map(c| + if($c.type == String, + | col({row:TDSRow[1]|$row.getString($c.name)}, $c.name + '_ext'), + | if($c.type == Integer, + | col({row:TDSRow[1]|$row.getInteger($c.name)}, $c.name + '_ext'), + | col({row:TDSRow[1]|$row.getDate($c.name)}, $c.name + '_ext') + ) + ) + )); +} + +function <> meta::pure::router::preeval::tests::testPrerouting34():Boolean[1] +{ + let input = {| + let businessDate = now(); + let processingDate = today()->adjust(-1, DurationUnit.DAYS); + + meta::pure::router::preeval::tests::Position.all($processingDate) + ->project([ + x | $x.quantity, + x | $x.client($businessDate).name], + ['Quantity', + 'Client/Name'] + ) + ->extendColumns('Quantity'); + }; + + let expected = {| + let businessDate = now(); + let processingDate = today()->adjust(-1, DurationUnit.DAYS); + + meta::pure::router::preeval::tests::Position + ->getAll($processingDate) + ->project([{x|$x.quantity},{x|$x.client($businessDate).name}], ['Quantity','Client/Name']) + ->extend(^BasicColumnSpecification(name = 'Quantity_ext', func = {row:TDSRow[1]|$row.getDate('Quantity')})); + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting35():Boolean[1] +{ + let input = {| + let x = 'world'; + + {|('hello' + ' ') + $x}->eval(); + }; + + let expected = {| + 'hello world'; + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_parameterAssignedToVariable():Boolean[1] +{ + let input = {item:Pair[1]| + let other = $item; + $other.first->isNotEmpty(); + }; + + let expected = {item:Pair[1]| + $item.first->isNotEmpty(); + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting36():Boolean[1] +{ + let input = {a:meta::pure::router::preeval::tests::ClassA[1]| + $a.myPropertyB.myLeafProperty->map(x|now())->map(d|$d->adjust(-1, DurationUnit.DAYS)) + }; + + assertRoundTrip($input, $input); +} + +function <> meta::pure::router::preeval::tests::testPrerouting37():Boolean[1] +{ + let input = {x:Integer[*]| + $x->otherFunc2(y|$y+1) + }; + + let expected = {x:Integer[*]| + $x->map({y|$y + 1}) + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::otherFunc2(x : Integer[*], l : FunctionDefinition<{Integer[1]->Integer[1]}>[1]):Integer[*] +{ + $x->map($l); +} + +function <> meta::pure::router::preeval::tests::testPrerouting38():Boolean[1] +{ + let input = {| + let x = myPair('hello', 'world'); + let y = myPair(1, $x.first + ' ' + $x.second); + }; + + let expected = {| + ^Pair(first = 1, second = 'hello world'); + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::myPair(v1 : X[1], v2 : Y[1]) : Pair[1] +{ + pair($v1, $v2); +} + +function <> meta::pure::router::preeval::tests::myPair2(v1 : U[1], v2 : V[1]) : Pair[1] +{ + pair($v1, $v2); +} + +function <> meta::pure::router::preeval::tests::testPrerouting39():Boolean[1] +{ + let input = {| + let x = 123; + $x->cast(@Integer); + }; + + let expected = {| + 123 + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting40a():Boolean[1] +{ + let input = {| + let x = 123; + $x->toOne()->myToOne(); + }; + + let expected = {| + 123; + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting40b():Boolean[1] +{ + let input = {| + let x = today(); + $x->toOne()->myToOne(); + }; + + let expected = {| + let x = today(); + $x; + }; + + assertRoundTrip($input, $expected); +} + + +function <> meta::pure::router::preeval::tests::myToOne(x : T[1]) : T[1] +{ + $x->toOne(); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_PropertyValue():Boolean[1] +{ + let prop = meta::pure::router::preeval::tests::Person.properties->at(0); + let name = $prop.name; + + let input = {| $prop.name;}; + + let expected = {| $name}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_OptionalLimit1():Boolean[1] +{ + let limit = 5; + + let input = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') + ]) + ->meta::pure::router::preeval::tests::optionalLimit($limit) + }; + + let expected = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) + ]) + ->limit(5); + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_OptionalLimit2():Boolean[1] +{ + let limit = []; + + let input = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') + ]) + ->meta::pure::router::preeval::tests::optionalLimit($limit) + }; + + let expected = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) + ]); + }; + + assertRoundTrip($input, $expected); +} + +function meta::pure::router::preeval::tests::optionalLimit(t: TabularDataSet[1], limit:Integer[0..1]) : TabularDataSet[1] +{ + if($limit->isEmpty() || ($limit->toOne()<=0), | $t, | $t->limit($limit)); +} + + +function <> meta::pure::router::preeval::tests::testPrerouting_mappedTdsAgg():Boolean[1] +{ + let v = [pair('name', '6')]; + + let input = {|meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') + ]) + ->groupBy([], $v->map(pv|agg('sumAgg_' + $pv.first->toString(), {row:TDSRow[1]|if($row.getInteger('age')->toString() == $pv.second, | 1, | 0)}, y|$y->sum()))) + }; + + let expected = {| meta::pure::router::preeval::tests::Person.all() + ->project(^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age})) + ->groupBy([], agg('sumAgg_name', {row:TDSRow[1]|if($row.getInteger('age')->toString() == '6', |1, |0)}, {y|$y->sum()})) + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_mappedModelAgg():Boolean[1] +{ + let v = [pair('name', '6')]; + + let input = {|meta::pure::router::preeval::tests::Person.all() + ->groupBy([], + $v->map(pv|agg({p:meta::pure::router::preeval::tests::Person[1]|if($p.age->toOne()->toString() == $pv.second, | 1, | 0)}, y|$y->sum())), + $v->map(pv|'sumAgg_' + $pv.first) + ) + }; + + let expected = {|meta::pure::router::preeval::tests::Person.all() + ->groupBy([], agg({p:meta::pure::router::preeval::tests::Person[1]|if($p.age->toOne()->toString() == '6', |1, |0)}, {y|$y->sum()}), 'sumAgg_name') + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_constantInAgg():Boolean[1] +{ + let lambda = {| + let businessDate = %2023-01-01; + let x = 'a'; + meta::pure::router::preeval::tests::Position.all($businessDate)->groupBy( + [], + agg(x|$x.client($businessDate).name + 'b', y|$y->count()), + 'ID Count' + ); + }; + + //we re-write the lambda to inline an AggregateValue with businessDate variable + let aggregateValue = ^meta::pure::functions::collection::AggregateValue(mapFn={x:meta::pure::router::preeval::tests::Position[1] | $x.client(%2023-01-01).name + $x->toString()},aggregateFn={y:String[*] | $y->count()}); + + let expressionSequence = $lambda.expressionSequence->evaluateAndDeactivate(); + let groupBy = $expressionSequence->at(2)->cast(@SimpleFunctionExpression); + + let mapFn = $aggregateValue.mapFn; + let mapFnExpressionSequnce = $mapFn.expressionSequence->evaluateAndDeactivate()->cast(@SimpleFunctionExpression)->toOne(); + + let plus = $mapFnExpressionSequnce.parametersValues->at(0)->cast(@InstanceValue); + let plusParams = $plus.values; + let plusLeft = $plusParams->at(0)->cast(@SimpleFunctionExpression); + let client = $plusLeft.parametersValues->at(0)->cast(@SimpleFunctionExpression); + let newClient = ^$client(parametersValues = [$client.parametersValues->at(0), ^VariableExpression(name='businessDate',genericType=^GenericType(rawType = StrictDate),multiplicity=PureOne)]); + + let newPlusLeft = ^$plusLeft(parametersValues = $newClient); + let newPlus = ^$plus(values = [$newPlusLeft,$plus.values->at(1)]); + + let newAggregateValue = ^$aggregateValue(mapFn = ^$mapFn(expressionSequence = ^$mapFnExpressionSequnce(parametersValues = [$newPlus]))); + let newAggregateValueInstance = ^InstanceValue(genericType=^GenericType(rawType = meta::pure::functions::collection::AggregateValue),multiplicity=PureOne, values = $newAggregateValue); + let newGroupBy = ^$groupBy(parametersValues = [$groupBy.parametersValues->at(0), $groupBy.parametersValues->at(1), $newAggregateValueInstance, $groupBy.parametersValues->at(3)]); + let newLambda = ^$lambda(expressionSequence = [$expressionSequence->at(0), $expressionSequence->at(1), $newGroupBy]); + + + let expected = {| + meta::pure::router::preeval::tests::Position.all(%2023-01-01)->groupBy([], ^meta::pure::functions::collection::AggregateValue(mapFn={x:meta::pure::router::preeval::tests::Position[1] | $x.client(%2023-01-01).name + $x->toString()},aggregateFn={y:String[*]|$y->count()}), ['ID Count']) + }; + + assertRoundTrip($newLambda, $expected); +} + +function <> meta::pure::router::preeval::tests::testDecimalType():Boolean[1] +{ + let input = {|meta::pure::router::preeval::tests::FinancialInstrument.all() + ->project([ + col(f:meta::pure::router::preeval::tests::FinancialInstrument[1]|$f.price, 'price'), + col(f:meta::pure::router::preeval::tests::FinancialInstrument[1]|parseDecimal('3.14D'), 'decimal_constant_col') + ]) + }; + + let expected = {|meta::pure::router::preeval::tests::FinancialInstrument.all() + ->project([ + ^BasicColumnSpecification(name = 'price', func={f:meta::pure::router::preeval::tests::FinancialInstrument[1]| $f.price }), + ^BasicColumnSpecification(name = 'decimal_constant_col', func={f:meta::pure::router::preeval::tests::FinancialInstrument[1]| 3.14}) + ]) + }; + + assertRoundTrip($input,$expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_castEmptyCollection():Boolean[1] +{ + let input = {b:Integer[*]|$b->uniqueValueOnly()}; + + let expected = {b:Integer[*]|if($b->distinct()->size() == 1, {|$b->max()}, {|[]})}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testEvalWithArgs1():Boolean[1] +{ + let input = {| + {x:Integer[1]|$x+1}->eval(5); + }; + + let expected = {|6}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testEvalWithArgs2():Boolean[1] +{ + let input = {|{t:TabularDataSet[1]|$t->filter(r:TDSRow[1]|$r.getString('firstName') == 'firstName')} + ->eval({|meta::pure::router::preeval::tests::Person.all()->project(col(p:meta::pure::router::preeval::tests::Person[1]|$p.firstName, 'firstName'))}->eval())}; + + let expected = {|meta::pure::router::preeval::tests::Person.all() + ->project(^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName})) + ->filter(r:TDSRow[1]|$r.getString('firstName') == 'firstName') + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testEvalWithArgs3():Boolean[1] +{ + let input = {a:Integer[1]|{b:Integer[1]|$b+1}->eval($a)}; + + let expected = {a:Integer[1]| $a+1 }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testProjectWithInferredParameterType():Boolean[1] +{ + let input = {|meta::pure::router::preeval::tests::myFunc888()}; + + let expected = {| meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}) + ]) + }; + + assertRoundTrip($input, $expected); +} + +function meta::pure::router::preeval::tests::myFunc888() : TabularDataSet[1] +{ + meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age') + ]) +} + + + + +function <> meta::pure::router::preeval::tests::testPreroutingRemoveUnnecessaryStatements():Boolean[1] +{ + let input = {| + range(5)->map(x|$x+1)->map(index|'i' + $index->toString()); + 123; + }; + + let expected = {| + 123; + }; + + assertRoundTrip($input, $expected); +} + + +function meta::pure::router::preeval::tests::myStopFunction(b: Integer[1]) : Integer[1] +{ + $b+1; +} + +function <> meta::pure::router::preeval::tests::testAdditionalStopFunction():Boolean[1] +{ + let input = {a:Integer[1]|myStopFunction($a)}; + + let expected = {a:Integer[1]| $a+1 }; + + assertRoundTrip($input, $expected); + assertRoundTrip($input, $input, myStopFunction_Integer_1__Integer_1_, [], noDebug()); +} + +function <> meta::pure::router::preeval::tests::testFilterFalseSimplification():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | false).id}; + + let expected = {a:OrganizationalEntity[1]| []}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testFilterFalseSimplificationAll():Boolean[1] +{ + let input = {s:String[0..1]| OrganizationalEntity.all(%2023-01-01)->filter(c | 1 == 2)->project(c | $c.name, 'name')}; + + let expected = {s:String[0..1]| OrganizationalEntity.all(%2023-01-01)->filter(c | false)->project(c | $c.name, 'name')}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testFilterFalseSimplification2():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | 1 == 2).id}; + + let expected = {o:OrganizationalEntity[1]| []}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testFilterFalseConstantSimplification():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| let bool = false; let date = %2023-01-01; $o.positions($date)->filter(p | $bool).id;}; + + let expected = {o:OrganizationalEntity[1]| []}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testFilterFalseSimplificationReturnType():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | false).id;}; + let result = $input->preval([]); + + assertEquals(Integer, $result->functionReturnType().rawType); +} + +function <> meta::pure::router::preeval::tests::testFilterTrueSimplification():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | false; true;).id}; + + let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01).id}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testFilterTrueSimplification2():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | 1 == 1).id}; + + let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01).id}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testFilterTrueConstantSimplification():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| let bool = true; let date = %2023-01-01; $o.positions($date)->filter(p | $bool).id;}; + + let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01).id}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testFilterNoSimplification():Boolean[1] +{ + let input = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | $p.id == true).id}; + + let expected = {o:OrganizationalEntity[1]| $o.positions(%2023-01-01)->filter(p | $p.id == true).id}; + + assertRoundTrip($input, $expected); +} + + +function <> meta::pure::router::preeval::tests::testToOneManyElimination1():Boolean[1] +{ + let input = {| 'hello'->toOneMany()}; + + let expected = {| 'hello'}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testToOneManyElimination2():Boolean[1] +{ + let input = {x:String[*]| $x->toOneMany()}; + + let expected = $input; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testToOneManyElimination3():Boolean[1] +{ + let input = {x:String[1..*]| $x->toOneMany()}; + + let expected = {x:String[1..*]| $x}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testToOneElimination1():Boolean[1] +{ + let input = {| 'hello'->toOne()}; + + let expected = {| 'hello'}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testToOneElimination2():Boolean[1] +{ + let input = {x:String[*]| $x->toOne()}; + + let expected = $input; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testToOneElimination3():Boolean[1] +{ + let input = {x:String[1]| $x->toOne()}; + + let expected = {x:String[1]| $x}; + + assertRoundTrip($input, $expected); +} + + +function <> meta::pure::router::preeval::tests::testRecursiveSimpleConcreteFunctionDefinition():Boolean[1] +{ + // This is specially selected to be a single function that is recursive on itself + // The test is to validate that we don't keep trying to inline ad infinitum + + let input = {s:Class[1]|meta::pure::functions::meta::hierarchicalProperties($s)}; + + let expected = {s:Class[1]| + if($s == Any, {|[]}, {|$s.properties->concatenate($s.generalizations->map({g|$g.general.rawType->cast(Any) + ->toOne() + ->hierarchicalProperties()})) + ->removeDuplicates([], [])}) + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testLetOnlyStatement():Boolean[1] +{ + let input = {|myFunc2()}; + + let expected = {| 'hello'}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testLambdaParamOverride():Boolean[1] +{ + let input = { | let val = 1; + {val:Integer[1]| $val};}; + + let expected = { | {val:Integer[1]| $val};}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testSchemaStateUnfurl():Boolean[1] +{ + let state = ^meta::pure::tds::schema::SchemaState(columns=[^meta::pure::tds::TDSColumn(name='column1'), ^meta::pure::tds::TDSColumn(name='column2')]); + let input = {|$state.columns.name}; + let expected = {|['column1', 'column2']}; + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testLambdaParamOverride2():Boolean[1] +{ + let val = 1; + let input = { | let x = 1; + {p:Integer[1]|$p->map(x|$x+$val)};}; + let expected = { | {p:Integer[1]|$p->map(x|$x+1)}}; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::myFunc2():String[1] +{ + let x = 'hello'; +} + +function meta::pure::router::preeval::tests::getTestRuntimeWithModelConnection(sourceClass:Class[1], inputs:Any[*]):Runtime[1] +{ + // Need element as string to allow roundtrip to pass. When using ^ModelStore(), preeval evaluates it to a instance, however expected as it as a new function + ^Runtime(connections=^meta::pure::mapping::modelToModel::ModelConnection(element = 'DummyStore', instances=newMap(pair($sourceClass, list($inputs))))); +} + +function <> meta::pure::router::preeval::tests::assertRoundTrip(input : FunctionDefinition[1], expectedOrig : FunctionDefinition[1]):Boolean[1] +{ + assertRoundTrip($input, $expectedOrig, noDebug()) +} + +function <> meta::pure::router::preeval::tests::assertRoundTrip(input : FunctionDefinition[1], expectedOrig : FunctionDefinition[1], debug:meta::pure::tools::DebugContext[1]):Boolean[1] +{ + meta::pure::router::preeval::tests::assertRoundTrip($input, $expectedOrig, [], [], $debug); +} + +function <> meta::pure::router::preeval::tests::assertRoundTrip(input : FunctionDefinition[1], expectedOrig : FunctionDefinition[1], stopInlineFunctions:Function[*], extensions:Extension[*], debug:meta::pure::tools::DebugContext[1]):Boolean[1] +{ + let expected = $expectedOrig; + + let result = if($stopInlineFunctions->isEmpty(), + | $input->preval(newMap([]->cast(@Pair)), $input->openVariableValues(), [], $debug).value->toOne(); , + | let inScopeVars = $input->openVariableValues(); + let state = meta::pure::router::preeval::getPreevalStateWithAdditionalStopInlineFunc($inScopeVars, $extensions, $stopInlineFunctions); + $input->preval($state, $extensions).value->toOne();); + + let tExpected = $expected->transformFunctionBody($extensions); + let tResult = $result->transformFunctionBody($extensions); + + let jExpected = $tExpected->toJSON(50000); + let jResult = $tResult->toJSON(50000); + + let pjExpected = $jExpected->parseJSON()->toPrettyJSONString(); + let pjResult = $jResult->parseJSON()->toPrettyJSONString(); + + if($pjExpected == $pjResult, + | true, + | + println($pjExpected); + println($pjResult); + + let pExpected = $tExpected->toPure($extensions); + let pResult = $tResult->toPure($extensions); + + assertEquals($pExpected, $pResult, 'Mismatch between expected and actual result content.\nexpected pure:\n%r\n\nactual pure:\n%r', [$pExpected->toOne(), $pResult->toOne()]); + ); + + // Open variables on lambdas should have been inlined / eliminated + if (!$result->instanceOf(LambdaFunction), + | true, + | assertEmpty($result->cast(@LambdaFunction).openVariables) + ); + + if($input->cast(@Function)->functionType().parameters->isNotEmpty(), + | true, + | + let inputEvalResult = $input->evaluate([]); + let expectedEvalResult = $expected->evaluate([]); + let resultEvalResult = $result->evaluate([]); + + let open = $input->openVariableValues(); + + //println($input->cast(@Function)->functionReturnMultiplicity()); + + let mInput = $input->cast(@Function)->functionReturnMultiplicity(); + let mExpected = $expected->cast(@Function)->functionReturnMultiplicity(); + let mResult = $result->cast(@Function)->functionReturnMultiplicity(); + + let jInputEvalResult = $inputEvalResult->transformAny([], $open, $mExpected, $extensions)->toJSON(50000); + let jExpectedEvalResult = $expectedEvalResult->transformAny([], $open, $mExpected, $extensions)->toJSON(50000); + println($jExpectedEvalResult); + let jResultEvalResult = $resultEvalResult->transformAny([], $open, $mExpected, $extensions)->toJSON(50000); + + + assertEquals($jExpectedEvalResult, $jResultEvalResult, '\nexpected eval result: %r\nactual eval result: %r', [$jExpectedEvalResult->toOne(), $jResultEvalResult->toOne()]); + + if(!$inputEvalResult->exists(x|$x->match([b:BasicColumnSpecification[*]|true, a:Any[*]|false])), + | assertEquals($jInputEvalResult, $jExpectedEvalResult, '\nexpected input eval: %r\nactual expected eval: %r', [$jInputEvalResult->toOne(), $jExpectedEvalResult->toOne()]), + | true //The lambda on the BCS can get confused for assertion checking (as it can be "un-optimised") + ); + ); + + true; } \ No newline at end of file diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure index 4bb260f1e66..816b1e27d55 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/router/routing/router_routing.pure @@ -766,6 +766,7 @@ function meta::pure::router::routing::shouldStopFunctions(extensions:meta::pure: or_Boolean_$1_MANY$__Boolean_1_, tdsContains_T_1__Function_MANY__TabularDataSet_1__Boolean_1_, tdsContains_T_1__Function_MANY__String_MANY__TabularDataSet_1__Function_1__Boolean_1_, + splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_, meta::pure::tds::extensions::firstNotNull_T_MANY__T_$0_1$_, meta::pure::functions::date::calendar::annualized_Date_1__String_1__Date_1__Number_$0_1$__Number_$0_1$_, meta::pure::functions::date::calendar::cme_Date_1__String_1__Date_1__Number_$0_1$__Number_$0_1$_, diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/tests/legend/milestoning/MilestonedM2M.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/tests/legend/milestoning/MilestonedM2M.pure new file mode 100644 index 00000000000..48ebd2e858e --- /dev/null +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/tests/legend/milestoning/MilestonedM2M.pure @@ -0,0 +1,44 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::executionPlan::profiles::*; +import meta::pure::executionPlan::*; +import meta::pure::mapping::*; +import meta::json::*; +import meta::pure::mapping::modelToModel::*; +import meta::pure::graphFetch::execution::*; +import meta::pure::runtime::*; + +function <> {serverVersion.start='v1_32_0'}meta::pure::mapping::modelToModel::test::alloy::milestoning::model::testM2MMappingWithMilestonedModelsWithAllVersions():Boolean[1] +{ + let tree = #{ meta::pure::mapping::modelToModel::test::alloy::milestoning::model::T_VehicleOwner{ + vehicleName, + personName + } + }#; + let query = {|meta::pure::mapping::modelToModel::test::alloy::milestoning::model::T_VehicleOwner.all(%2020-10-15)->graphFetch($tree)->serialize($tree)}; + + let mapping = meta::pure::mapping::modelToModel::test::alloy::milestoning::model::M2MMapping; + let runtime = ^Runtime(connections = ^JsonModelConnection( + element=^ModelStore(), + class=meta::pure::mapping::modelToModel::test::alloy::milestoning::model::S_VehicleOwner, + url='data:application/json,{"businessDate": "2020-10-15","name" : "Ram", "vehicleAllVersions": [{"name": "B", "businessDate" : "2020-10-15", "id" : "1"}]}' + ) + ); + let result = meta::pure::router::execute($query, $mapping, $runtime, meta::pure::extension::configuration::coreExtensions()); + + let json = $result.values->toOne(); + let expected = '{"personName":"Ram","vehicleName":"B"}'; + assert(jsonEquivalent($expected->parseJSON(), $json->parseJSON())); +} diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/tests/legend/milestoning/model.pure b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/tests/legend/milestoning/model.pure new file mode 100644 index 00000000000..3d9c05ad751 --- /dev/null +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/tests/legend/milestoning/model.pure @@ -0,0 +1,46 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::graphFetch::execution::*; +import meta::pure::mapping::modelToModel::test::alloy::milestoning::model::*; + +Class <> meta::pure::mapping::modelToModel::test::alloy::milestoning::model::Vehicle +{ + id : Integer[1]; + description: String[1]; + name : String[1]; +} + +Class <> meta::pure::mapping::modelToModel::test::alloy::milestoning::model::S_VehicleOwner +{ + vehicle : Vehicle[1]; + name : String[1]; +} + +Class <> meta::pure::mapping::modelToModel::test::alloy::milestoning::model::T_VehicleOwner +{ + vehicleName : String[1]; + personName : String[1]; +} + +###Mapping +Mapping meta::pure::mapping::modelToModel::test::alloy::milestoning::model::M2MMapping +( + *meta::pure::mapping::modelToModel::test::alloy::milestoning::model::T_VehicleOwner: Pure + { + ~src meta::pure::mapping::modelToModel::test::alloy::milestoning::model::S_VehicleOwner + vehicleName: $src.vehicle(%2020-10-15).name, + personName : $src.name + } +) diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-functions/pom.xml b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-functions/pom.xml index e7c730cdc29..76f6bd3ed93 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-functions/pom.xml +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-compiled-functions/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-code - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-code-compiled-functions diff --git a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-core-extension/pom.xml b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-core-extension/pom.xml index ee5206cf322..6a131f4ca6f 100644 --- a/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-core-extension/pom.xml +++ b/legend-engine-pure/legend-engine-pure-code/legend-engine-pure-code-core-extension/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-code - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-code-core-extension diff --git a/legend-engine-pure/legend-engine-pure-code/pom.xml b/legend-engine-pure/legend-engine-pure-code/pom.xml index f45154d3f0c..9c15bdbc4a3 100644 --- a/legend-engine-pure/legend-engine-pure-code/pom.xml +++ b/legend-engine-pure/legend-engine-pure-code/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-metadata-pure/pom.xml b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-metadata-pure/pom.xml index c4189b4a5e3..7d45b1ac42a 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-metadata-pure/pom.xml +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-metadata-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-pure-ide - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml index 08e2d0aff40..6b12ec98c50 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-pure-ide - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml index 91d7d96e36a..979bc2b5161 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-pure-ide - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java index 2eea4dbd2b9..ad32fdc40f1 100644 --- a/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java +++ b/legend-engine-pure/legend-engine-pure-ide/legend-engine-pure-ide-light/src/main/java/org/finos/legend/engine/ide/PureIDELight.java @@ -61,6 +61,7 @@ protected MutableList buildRepositories(SourceLocationCon .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/legend-engine-xt-relationalStore-presto-pure", "relational_presto")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/legend-engine-xt-relationalStore-sybase-pure", "relational_sybase")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/legend-engine-xt-relationalStore-sybaseiq-pure", "relational_sybaseiq")) + .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure", "relational_sparksql")) .with(this.buildCore("legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-pure", "relational_store_entitlement")) .with(this.buildCore("legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-pure", "servicestore")) .with(this.buildCore("legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-javaPlatformBinding-pure", "servicestore-java-platform-binding")) diff --git a/legend-engine-pure/legend-engine-pure-ide/pom.xml b/legend-engine-pure/legend-engine-pure-ide/pom.xml index 9e350faea99..c6b0e557310 100644 --- a/legend-engine-pure/legend-engine-pure-ide/pom.xml +++ b/legend-engine-pure/legend-engine-pure-ide/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-diagram-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-diagram-java/pom.xml index 9d844f9fcf9..98577137c99 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-diagram-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-diagram-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-dsl-diagram-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-graph-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-graph-java/pom.xml index bcf40be1211..933dc1ef6a4 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-graph-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-graph-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-dsl-graph-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-mapping-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-mapping-java/pom.xml index 3e7da03ed5e..1cbedd0bd6c 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-mapping-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-mapping-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-dsl-mapping-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-path-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-path-java/pom.xml index 2e22124621e..15b3ab7616d 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-path-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-dsl-path-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-dsl-path-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-java/pom.xml index b292b839da8..3d29bed6e6a 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-functions-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-json-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-json-java/pom.xml index da37a96c9e6..565edeb087c 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-json-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-functions-json-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-functions-json-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-java/pom.xml index b9b25df98da..fd09d96a635 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-store-relational-java/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-store-relational-java/pom.xml index 577cf13f3d4..d6cff1791be 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-store-relational-java/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/legend-engine-pure-platform-store-relational-java/pom.xml @@ -22,7 +22,7 @@ org.finos.legend.engine legend-engine-pure-platform-modular-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-pure-platform-store-relational-java diff --git a/legend-engine-pure/legend-engine-pure-platform-modular-generation/pom.xml b/legend-engine-pure/legend-engine-pure-platform-modular-generation/pom.xml index f9e1631e035..ef988597e55 100644 --- a/legend-engine-pure/legend-engine-pure-platform-modular-generation/pom.xml +++ b/legend-engine-pure/legend-engine-pure-platform-modular-generation/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-compiler/pom.xml b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-compiler/pom.xml index 70cbbbf76d8..13607722ff0 100644 --- a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-compiler/pom.xml +++ b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-compiler/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-pure-runtime - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-execution/pom.xml b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-execution/pom.xml index 139f0b72d0f..885ad5f6d3f 100644 --- a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-execution/pom.xml +++ b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-pure-runtime - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-extensions/pom.xml b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-extensions/pom.xml index c2dfbfbb542..63c1946e165 100644 --- a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-extensions/pom.xml +++ b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-pure-runtime-extensions/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-pure-runtime - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-xt-java-runtime-compiler/pom.xml b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-xt-java-runtime-compiler/pom.xml index 9fae6c0fdf3..884f63d4d03 100644 --- a/legend-engine-pure/legend-engine-pure-runtime/legend-engine-xt-java-runtime-compiler/pom.xml +++ b/legend-engine-pure/legend-engine-pure-runtime/legend-engine-xt-java-runtime-compiler/pom.xml @@ -20,7 +20,7 @@ org.finos.legend.engine legend-engine-pure-runtime - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-java-runtime-compiler diff --git a/legend-engine-pure/legend-engine-pure-runtime/pom.xml b/legend-engine-pure/legend-engine-pure-runtime/pom.xml index 617a9c6ccd2..2603bc254d1 100644 --- a/legend-engine-pure/legend-engine-pure-runtime/pom.xml +++ b/legend-engine-pure/legend-engine-pure-runtime/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-pure - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-pure/pom.xml b/legend-engine-pure/pom.xml index 6ff9cd92d05..66299a22fed 100644 --- a/legend-engine-pure/pom.xml +++ b/legend-engine-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-api/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-api/pom.xml index 0a76eecdeec..4a3be642405 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-api/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-api/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-analytics-lineage org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml index 478b8c7e95c..8e21753808d 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-analytics-lineage org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/pom.xml index 58e2520ecc9..055a46190cf 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-analytics - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-api/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-api/pom.xml index 651e980392f..f722409f2e0 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-api/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-api/pom.xml @@ -3,7 +3,7 @@ legend-engine-xts-analytics-mapping org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 Legend Engine - XT - Analytics - Mapping - API diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-protocol/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-protocol/pom.xml index 890b9660991..d902048384c 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-protocol/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-analytics-mapping - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 Legend Engine - XT - Analytics - Mapping - Protocol diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-pure/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-pure/pom.xml index 19c4e7102c8..b7250e5f323 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-pure/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/legend-engine-xt-analytics-mapping-pure/pom.xml @@ -3,7 +3,7 @@ legend-engine-xts-analytics-mapping org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/pom.xml index 37de6a6b36f..03d1d031b26 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-mapping/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-analytics - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-generation/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-generation/pom.xml index 3b5a5db5b3f..fe519d179e3 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-generation/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-generation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-analytics-search - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-pure/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-pure/pom.xml index 443f4142351..de71b84a8ab 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-pure/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-search/legend-engine-xt-analytics-search-pure/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-analytics-search org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-search/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-search/pom.xml index e21b8ae4401..a0c90b85da0 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-search/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-search/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-analytics - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement-api/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement-api/pom.xml index 2af8152b334..8426958ba66 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement-api/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement-api/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-analytics-store org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-analytics-store-entitlement-api diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement/pom.xml index db9aa02ff24..c6b251f6fa9 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-store/legend-engine-xt-analytics-store-entitlement/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-analytics-store org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-store/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-store/pom.xml index e51a961bf3c..f50f446a6ec 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-store/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-store/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-analytics - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-analytics/pom.xml b/legend-engine-xts-analytics/pom.xml index df2c2aa67fa..ee319565f6b 100644 --- a/legend-engine-xts-analytics/pom.xml +++ b/legend-engine-xts-analytics/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/pom.xml b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/pom.xml index f71e09dd7d5..9b3f1efc9a5 100644 --- a/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/pom.xml +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-arrow - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/pom.xml b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/pom.xml index 71b97b2f1a3..e64a8237fef 100644 --- a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/pom.xml +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/pom.xml @@ -3,7 +3,7 @@ org.finos.legend.engine legend-engine-xts-arrow - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/apache/arrow/adapter/jdbc/LegendArrowVectorIterator.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/apache/arrow/adapter/jdbc/LegendArrowVectorIterator.java new file mode 100644 index 00000000000..fa41a6de12e --- /dev/null +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/apache/arrow/adapter/jdbc/LegendArrowVectorIterator.java @@ -0,0 +1,229 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License + +package org.apache.arrow.adapter.jdbc; + +import static org.apache.arrow.adapter.jdbc.JdbcToArrowUtils.getJdbcFieldInfoForColumn; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.Iterator; + +import org.apache.arrow.adapter.jdbc.consumer.CompositeJdbcConsumer; +import org.apache.arrow.adapter.jdbc.consumer.JdbcConsumer; +import org.apache.arrow.util.AutoCloseables; +import org.apache.arrow.util.Preconditions; +import org.apache.arrow.vector.FieldVector; +import org.apache.arrow.vector.VectorSchemaRoot; +import org.apache.arrow.vector.types.pojo.ArrowType; +import org.apache.arrow.vector.types.pojo.Schema; +import org.apache.arrow.vector.util.ValueVectorUtility; + +/** + * temporary over ride of ArrowVectorIterator until https://github.com/apache/arrow/pull/37085 is released. This works around an long standing bug in H2 where ResultSetMetadata may incorrectly set a result column as not nullable + */ +public class LegendArrowVectorIterator implements Iterator, AutoCloseable +{ + + private final ResultSet resultSet; + private final JdbcToArrowConfig config; + + private final Schema schema; + private final ResultSetMetaData rsmd; + + private final JdbcConsumer[] consumers; + final CompositeJdbcConsumer compositeConsumer; + + // this is used only if resuing vector schema root is enabled. + private VectorSchemaRoot nextBatch; + + private final int targetBatchSize; + + // This is used to track whether the ResultSet has been fully read, and is needed spcifically for cases where there + // is a ResultSet having zero rows (empty): + private boolean readComplete = false; + + /** + * Construct an instance. + */ + private LegendArrowVectorIterator(ResultSet resultSet, JdbcToArrowConfig config) throws SQLException + { + this.resultSet = resultSet; + this.config = config; + this.schema = JdbcToArrowUtils.jdbcToArrowSchema(resultSet.getMetaData(), config); + this.targetBatchSize = config.getTargetBatchSize(); + + rsmd = resultSet.getMetaData(); + consumers = new JdbcConsumer[rsmd.getColumnCount()]; + this.compositeConsumer = new CompositeJdbcConsumer(consumers); + this.nextBatch = config.isReuseVectorSchemaRoot() ? createVectorSchemaRoot() : null; + } + + /** + * Create a ArrowVectorIterator to partially convert data. + */ + public static LegendArrowVectorIterator create( + ResultSet resultSet, + JdbcToArrowConfig config) + throws SQLException + { + LegendArrowVectorIterator iterator = null; + try + { + iterator = new LegendArrowVectorIterator(resultSet, config); + } + catch (Throwable e) + { + AutoCloseables.close(e, iterator); + throw new RuntimeException("Error occurred while creating iterator.", e); + } + return iterator; + } + + private void consumeData(VectorSchemaRoot root) + { + // consume data + try + { + int readRowCount = 0; + if (targetBatchSize == JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) + { + while (resultSet.next()) + { + ValueVectorUtility.ensureCapacity(root, readRowCount + 1); + compositeConsumer.consume(resultSet); + readRowCount++; + } + readComplete = true; + } + else + { + while ((readRowCount < targetBatchSize) && !readComplete) + { + if (resultSet.next()) + { + compositeConsumer.consume(resultSet); + readRowCount++; + } + else + { + readComplete = true; + } + } + } + + root.setRowCount(readRowCount); + } + catch (Throwable e) + { + compositeConsumer.close(); + throw new RuntimeException("Error occurred while consuming data.", e); + } + } + + private VectorSchemaRoot createVectorSchemaRoot() throws SQLException + { + VectorSchemaRoot root = null; + try + { + root = VectorSchemaRoot.create(schema, config.getAllocator()); + if (config.getTargetBatchSize() != JdbcToArrowConfig.NO_LIMIT_BATCH_SIZE) + { + ValueVectorUtility.preAllocate(root, config.getTargetBatchSize()); + } + } + catch (Throwable e) + { + if (root != null) + { + root.close(); + } + throw new RuntimeException("Error occurred while creating schema root.", e); + } + initialize(root); + return root; + } + + private void initialize(VectorSchemaRoot root) throws SQLException + { + for (int i = 1; i <= consumers.length; i++) + { + final JdbcFieldInfo columnFieldInfo = getJdbcFieldInfoForColumn(rsmd, i, config); + ArrowType arrowType = config.getJdbcToArrowTypeConverter().apply(columnFieldInfo); + consumers[i - 1] = JdbcToArrowUtils.getConsumer( + arrowType, i, true, root.getVector(i - 1), config); + } + } + + // Loads the next schema root or null if no more rows are available. + private void load(VectorSchemaRoot root) + { + for (int i = 0; i < consumers.length; i++) + { + FieldVector vec = root.getVector(i); + if (config.isReuseVectorSchemaRoot()) + { + // if we are reusing the vector schema root, + // we must reset the vector before populating it with data. + vec.reset(); + } + consumers[i].resetValueVector(vec); + } + + consumeData(root); + } + + @Override + public boolean hasNext() + { + return !readComplete; + } + + /** + * Gets the next vector. + * If {@link JdbcToArrowConfig#isReuseVectorSchemaRoot()} is false, + * the client is responsible for freeing its resources. + */ + @Override + public VectorSchemaRoot next() + { + Preconditions.checkArgument(hasNext()); + try + { + VectorSchemaRoot ret = config.isReuseVectorSchemaRoot() ? nextBatch : createVectorSchemaRoot(); + load(ret); + return ret; + } + catch (Exception e) + { + close(); + throw new RuntimeException("Error occurred while getting next schema root.", e); + } + } + + /** + * Clean up resources ONLY WHEN THE {@link VectorSchemaRoot} HOLDING EACH BATCH IS REUSED. If a new VectorSchemaRoot + * is created for each batch, each root must be closed manually by the client code. + */ + @Override + public void close() + { + if (config.isReuseVectorSchemaRoot()) + { + nextBatch.close(); + compositeConsumer.close(); + } + } +} diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowDataWriter.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowDataWriter.java index 012a4973a9e..330d78225c6 100644 --- a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowDataWriter.java +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/main/java/org/finos/legend/engine/external/format/arrow/ArrowDataWriter.java @@ -14,13 +14,16 @@ package org.finos.legend.engine.external.format.arrow; +import java.nio.charset.Charset; import java.util.Calendar; +import java.util.HashMap; import java.util.Locale; import java.util.TimeZone; -import org.apache.arrow.adapter.jdbc.ArrowVectorIterator; -import org.apache.arrow.adapter.jdbc.JdbcToArrow; + +import org.apache.arrow.adapter.jdbc.JdbcFieldInfo; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfig; import org.apache.arrow.adapter.jdbc.JdbcToArrowConfigBuilder; +import org.apache.arrow.adapter.jdbc.LegendArrowVectorIterator; import org.apache.arrow.memory.BufferAllocator; import org.apache.arrow.memory.RootAllocator; import org.apache.arrow.vector.VectorSchemaRoot; @@ -34,22 +37,23 @@ public class ArrowDataWriter extends ExternalFormatWriter implements AutoCloseable { - private final ArrowVectorIterator iterator; + private final LegendArrowVectorIterator iterator; private final BufferAllocator allocator; - public ArrowDataWriter(ResultSet resultSet) throws SQLException, IOException + public ArrowDataWriter(ResultSet resultSet) throws SQLException { + + HashMap map = new HashMap(); + this.allocator = new RootAllocator(); JdbcToArrowConfig config = new JdbcToArrowConfigBuilder(allocator, Calendar.getInstance(TimeZone.getDefault(), Locale.ROOT)).build(); - this.iterator = JdbcToArrow.sqlToArrowVectorIterator(resultSet, config); + this.iterator = LegendArrowVectorIterator.create(resultSet, config); } - @Override public void writeData(OutputStream outputStream) throws IOException { - try { while (this.iterator.hasNext()) @@ -71,6 +75,28 @@ public void writeData(OutputStream outputStream) throws IOException } + @Override + public void writeDataAsString(OutputStream outputStream) throws IOException + { + try + { + while (this.iterator.hasNext()) + { + try (VectorSchemaRoot vector = iterator.next()) + { + outputStream.write(vector.contentToTSVString().getBytes(Charset.forName("UTF-8"))); + } + + } + this.close(); + } + catch (Exception e) + { + this.close(); + throw e; + } + } + @Override public void close() { diff --git a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowNodeExecutor.java b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowNodeExecutor.java index a2cf57af287..0c388a95af5 100644 --- a/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowNodeExecutor.java +++ b/legend-engine-xts-arrow/legend-engine-xt-arrow-runtime/src/test/java/TestArrowNodeExecutor.java @@ -44,6 +44,8 @@ public class TestArrowNodeExecutor { + + @Test public void testExternalize() throws Exception { @@ -57,6 +59,47 @@ public void testExternalize() throws Exception Mockito.when(mockDatabaseConnection.accept(any())).thenReturn(false); try (Connection conn = DriverManager.getConnection("jdbc:h2:~/test", "sa", ""); ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) + { + //setup table + conn.createStatement().execute("DROP TABLE IF EXISTS testtable"); + conn.createStatement().execute("DROP TABLE IF EXISTS testtableJoin"); + + conn.createStatement().execute("Create Table testtable (testInt INTEGER, testString VARCHAR(255), testDate TIMESTAMP, testBool BOOLEAN)"); + conn.createStatement().execute("Create Table testtableJoin (testIntR INTEGER, testStringR VARCHAR(255) PRIMARY KEY )"); + + conn.createStatement().execute("INSERT INTO testtable (testInt, testString, testDate, testBool) VALUES(1,'A', '2020-01-01 00:00:00-05:00',true),( 2,null, '2020-01-01 00:00:00-02:00',false ),( 3,'B', '2020-01-01 00:00:00-05:00',false )"); + conn.createStatement().execute("INSERT INTO testtableJoin (testIntR, testStringR) VALUES(6,'A'), (1,'B')"); + + RelationalResult result = new RelationalResult(FastList.newListWith(new RelationalExecutionActivity("SELECT * FROM testtable left join testtableJoin on testtable.testInt=testtableJoin.testIntR", null)), mockExecutionNode, FastList.newListWith(new SQLResultColumn("testInt", "INTEGER"), new SQLResultColumn("testStringR", "VARCHAR"), new SQLResultColumn("testString", "VARCHAR"), new SQLResultColumn("testDate", "TIMESTAMP"), new SQLResultColumn("testBool", "TIMESTAMP")), null, "GMT", conn, null, null, null, new RequestContext()); + + ExternalFormatSerializeResult nodeExecute = (ExternalFormatSerializeResult) extension.executeExternalizeTDSExecutionNode(node, result, null, null); + + + nodeExecute.stream(outputStream, SerializationFormat.DEFAULT); + + String expected = "TESTINT\tTESTSTRING\tTESTDATE\tTESTBOOL\tTESTINTR\tTESTSTRINGR\n" + + "1\tA\t1577854800000\ttrue\t1\tB\n" + + "2\tnull\t1577844000000\tfalse\tnull\tnull\n" + + "3\tB\t1577854800000\tfalse\tnull\tnull\n"; + assertArrow(outputStream, expected); + } + + } + + @Test + public void testExternalizeAsString() throws Exception + { + ArrowRuntimeExtension extension = new ArrowRuntimeExtension(); + ExternalFormatExternalizeTDSExecutionNode node = new ExternalFormatExternalizeTDSExecutionNode(); + //create a real result from H2 + RelationalExecutionNode mockExecutionNode = Mockito.mock(RelationalExecutionNode.class); + DatabaseConnection mockDatabaseConnection = Mockito.mock(DatabaseConnection.class); + + mockExecutionNode.connection = mockDatabaseConnection; + Mockito.when(mockDatabaseConnection.accept(any())).thenReturn(false); + try (Connection conn = DriverManager.getConnection("jdbc:h2:~/test", "sa", ""); + ) + { //setup table conn.createStatement().execute("DROP TABLE IF EXISTS testtable"); @@ -66,14 +109,18 @@ public void testExternalize() throws Exception RelationalResult result = new RelationalResult(FastList.newListWith(new RelationalExecutionActivity("SELECT * FROM testtable", null)), mockExecutionNode, FastList.newListWith(new SQLResultColumn("testInt", "INTEGER"), new SQLResultColumn("testString", "VARCHAR"), new SQLResultColumn("testDate", "TIMESTAMP"), new SQLResultColumn("testBool", "TIMESTAMP")), null, "GMT", conn, null, null, null, new RequestContext()); ExternalFormatSerializeResult nodeExecute = (ExternalFormatSerializeResult) extension.executeExternalizeTDSExecutionNode(node, result, null, null); - Assert.assertEquals(nodeExecute.resultFormat, "application/x.arrow"); - nodeExecute.stream(outputStream, SerializationFormat.DEFAULT); - assertArrow(outputStream, "TESTINT\tTESTSTRING\tTESTDATE\tTESTBOOL\n" + + + String expected = "TESTINT\tTESTSTRING\tTESTDATE\tTESTBOOL\n" + "1\tA\t1577854800000\ttrue\n" + "2\tB\t1577844000000\tfalse\n" + - "3\tB\t1577854800000\tfalse\n"); + "3\tB\t1577854800000\tfalse\n"; + + String outputasString = nodeExecute.flush(nodeExecute.getSerializer(SerializationFormat.DEFAULT)); + + Assert.assertEquals(expected, outputasString); } + } private void assertArrow(ByteArrayOutputStream actualOutputStream, String expectedTSV) throws IOException //input a TSV String @@ -104,27 +151,4 @@ private void assertArrow(ByteArrayOutputStream actualOutputStream, String expect } - - /* - * This is a utility for generating arrow raw data to a file for test setup - */ - protected void arrowFileCreationUtility(VectorSchemaRoot vectorSchemaRoot, String file) - { - - try ( - FileOutputStream fileOutputStream = new FileOutputStream(file); - ArrowFileWriter writer = new ArrowFileWriter(vectorSchemaRoot, null, fileOutputStream.getChannel()) - ) - { - writer.start(); - writer.writeBatch(); - writer.end(); - } - catch (IOException e) - { - e.printStackTrace(); - } - - } - } diff --git a/legend-engine-xts-arrow/pom.xml b/legend-engine-xts-arrow/pom.xml index 2018fca00d6..367f2e200f4 100644 --- a/legend-engine-xts-arrow/pom.xml +++ b/legend-engine-xts-arrow/pom.xml @@ -4,7 +4,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xts-arrow diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/pom.xml b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/pom.xml index a90d5b692a3..444b1f80674 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/pom.xml +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-authentication - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -76,8 +76,13 @@ - junit - junit + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.jupiter + junit-jupiter-engine test diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismConfiguration.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismConfiguration.java new file mode 100644 index 00000000000..11ca50b9ce1 --- /dev/null +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismConfiguration.java @@ -0,0 +1,102 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection; + +import org.eclipse.collections.api.block.function.Function0; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ImmutableList; +import org.finos.legend.connection.protocol.AuthenticationConfiguration; +import org.finos.legend.connection.protocol.AuthenticationMechanism; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +public class AuthenticationMechanismConfiguration +{ + private final AuthenticationMechanism authenticationMechanism; + private final ImmutableList> authenticationConfigurationTypes; + private final Function0 defaultAuthenticationConfigurationGenerator; + + private AuthenticationMechanismConfiguration(AuthenticationMechanism authenticationMechanism, List> authenticationConfigurationTypes, Function0 defaultAuthenticationConfigurationGenerator) + { + this.authenticationMechanism = Objects.requireNonNull(authenticationMechanism, "Authentication mechanism is missing"); + this.authenticationConfigurationTypes = Lists.immutable.withAll(authenticationConfigurationTypes); + this.defaultAuthenticationConfigurationGenerator = defaultAuthenticationConfigurationGenerator; + } + + public AuthenticationMechanism getAuthenticationMechanism() + { + return authenticationMechanism; + } + + public ImmutableList> getAuthenticationConfigurationTypes() + { + return authenticationConfigurationTypes; + } + + public Function0 getDefaultAuthenticationConfigurationGenerator() + { + return defaultAuthenticationConfigurationGenerator; + } + + public static class Builder + { + private final AuthenticationMechanism authenticationMechanism; + private final Set> authenticationConfigurationTypes = new LinkedHashSet<>(); + private Function0 defaultAuthenticationConfigurationGenerator; + + public Builder(AuthenticationMechanism authenticationMechanism) + { + this.authenticationMechanism = authenticationMechanism; + } + + public Builder withAuthenticationConfigurationType(Class authenticationConfigurationType) + { + this.authenticationConfigurationTypes.add(authenticationConfigurationType); + return this; + } + + public Builder withAuthenticationConfigurationTypes(List> authenticationConfigurationTypes) + { + this.authenticationConfigurationTypes.addAll(authenticationConfigurationTypes); + return this; + } + + @SafeVarargs + public final Builder withAuthenticationConfigurationTypes(Class... authenticationConfigurationTypes) + { + this.authenticationConfigurationTypes.addAll(Lists.mutable.of(authenticationConfigurationTypes)); + return this; + } + + public Builder withDefaultAuthenticationConfigurationGenerator(Function0 defaultAuthenticationConfigurationGenerator) + { + this.defaultAuthenticationConfigurationGenerator = defaultAuthenticationConfigurationGenerator; + return this; + } + + public AuthenticationMechanismConfiguration build() + { + return new AuthenticationMechanismConfiguration( + this.authenticationMechanism, + new ArrayList<>(this.authenticationConfigurationTypes), + this.defaultAuthenticationConfigurationGenerator + ); + } + } +} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/Authenticator.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/Authenticator.java index 85120be4ab4..90e10882b38 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/Authenticator.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/Authenticator.java @@ -14,40 +14,46 @@ package org.finos.legend.connection; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ImmutableList; import org.finos.legend.connection.protocol.AuthenticationConfiguration; +import org.finos.legend.connection.protocol.AuthenticationMechanism; import org.finos.legend.engine.shared.core.identity.Credential; import org.finos.legend.engine.shared.core.identity.Identity; import java.util.List; import java.util.Optional; -public class Authenticator +public class Authenticator { - private final Identity identity; private final StoreInstance storeInstance; + private final AuthenticationMechanism authenticationMechanism; private final AuthenticationConfiguration authenticationConfiguration; private final Class sourceCredentialType; - - private final List credentialBuilders; + private final Class targetCredentialType; + private final ImmutableList credentialBuilders; private final ConnectionBuilder connectionBuilder; + private final LegendEnvironment environment; - public Authenticator(Identity identity, StoreInstance storeInstance, AuthenticationConfiguration authenticationConfiguration, Class sourceCredentialType, List credentialBuilders, ConnectionBuilder connectionBuilder) + public Authenticator(StoreInstance storeInstance, AuthenticationMechanism authenticationMechanism, AuthenticationConfiguration authenticationConfiguration, Class sourceCredentialType, Class targetCredentialType, List credentialBuilders, ConnectionBuilder connectionBuilder, LegendEnvironment environment) { - this.identity = identity; this.storeInstance = storeInstance; + this.authenticationMechanism = authenticationMechanism; this.authenticationConfiguration = authenticationConfiguration; this.sourceCredentialType = sourceCredentialType; - this.credentialBuilders = credentialBuilders; + this.targetCredentialType = targetCredentialType; + this.credentialBuilders = Lists.immutable.withAll(credentialBuilders); this.connectionBuilder = connectionBuilder; + this.environment = environment; } - public Credential makeCredential(EnvironmentConfiguration configuration) throws Exception + public CRED makeCredential(Identity identity) throws Exception { Credential credential = null; // no need to resolve the source credential if the flow starts with generic `Credential` node if (!this.sourceCredentialType.equals(Credential.class)) { - Optional credentialOptional = this.identity.getCredential((Class) this.sourceCredentialType); + Optional credentialOptional = identity.getCredential((Class) this.sourceCredentialType); if (!credentialOptional.isPresent()) { throw new RuntimeException(String.format("Can't resolve source credential of type '%s' from the specified identity", this.sourceCredentialType.getSimpleName())); @@ -59,14 +65,23 @@ public Credential makeCredential(EnvironmentConfiguration configuration) throws } for (CredentialBuilder credentialBuilder : this.credentialBuilders) { - credential = credentialBuilder.makeCredential(this.identity, this.authenticationConfiguration, credential, configuration); + credential = credentialBuilder.makeCredential(identity, this.authenticationConfiguration, credential, this.environment); + } + if (!this.targetCredentialType.equals(credential.getClass())) + { + throw new RuntimeException(String.format("Generated credential type is expected to be '%s' (found: %s)", this.targetCredentialType.getSimpleName(), credential.getClass().getSimpleName())); } - return credential; + return (CRED) credential; } - public ConnectionBuilder getConnectionBuilder() + public AuthenticationMechanism getAuthenticationMechanism() { - return connectionBuilder; + return authenticationMechanism; + } + + public AuthenticationConfiguration getAuthenticationConfiguration() + { + return authenticationConfiguration; } public StoreInstance getStoreInstance() @@ -79,8 +94,19 @@ public Class getSourceCredentialType() return sourceCredentialType; } - public List getCredentialBuilders() + + public Class getTargetCredentialType() + { + return targetCredentialType; + } + + public ImmutableList getCredentialBuilders() { return credentialBuilders; } + + public ConnectionBuilder getConnectionBuilder() + { + return connectionBuilder; + } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionBuilder.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionBuilder.java index 2119dd37648..49de64a0f9c 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionBuilder.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionBuilder.java @@ -16,14 +16,15 @@ import org.finos.legend.connection.protocol.ConnectionSpecification; import org.finos.legend.engine.shared.core.identity.Credential; +import org.finos.legend.engine.shared.core.identity.Identity; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; import java.util.Objects; -public abstract class ConnectionBuilder +public abstract class ConnectionBuilder { - public abstract T getConnection(CRED credential, SPEC connectionSpecification, StoreInstance storeInstance) throws Exception; + protected abstract CONNECTION getConnection(SPEC connectionSpecification, Authenticator authenticator, Identity identity) throws Exception; public Class getCredentialType() { @@ -35,13 +36,27 @@ public Class getConnectionSpecificationType() return (Class) actualTypeArguments()[2]; } - private Type[] actualTypeArguments() + protected Type[] actualTypeArguments() { Type genericSuperClass = this.getClass().getGenericSuperclass(); ParameterizedType parameterizedType = (ParameterizedType) genericSuperClass; return parameterizedType.getActualTypeArguments(); } + public Authenticator getAuthenticatorCompatible(Authenticator authenticator) + { + if (!this.getCredentialType().equals(authenticator.getTargetCredentialType())) + { + throw new RuntimeException(String.format("Authenticator target credential type is expected to be '%s' (found: %s)", this.getCredentialType().getSimpleName(), authenticator.getTargetCredentialType().getSimpleName())); + } + return (Authenticator) authenticator; + } + + public ConnectionManager getConnectionManager() + { + return null; + } + public static class Key { private final Class connectionSpecificationType; diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionFactory.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionFactory.java index 4f9a88dde27..5eca86a7c23 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionFactory.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionFactory.java @@ -14,6 +14,7 @@ package org.finos.legend.connection; +import org.eclipse.collections.api.block.function.Function0; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.connection.protocol.AuthenticationConfiguration; @@ -33,20 +34,19 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.ServiceLoader; import java.util.Set; -import java.util.stream.Collectors; public class ConnectionFactory { - private final EnvironmentConfiguration environmentConfiguration; + private final LegendEnvironment environment; + private final StoreInstanceProvider storeInstanceProvider; private final Map credentialBuildersIndex = new LinkedHashMap<>(); private final Map connectionBuildersIndex = new LinkedHashMap<>(); - private final Map storeInstancesIndex; - private ConnectionFactory(EnvironmentConfiguration environmentConfiguration, List credentialBuilders, List connectionBuilders, Map storeInstancesIndex) + private ConnectionFactory(LegendEnvironment environment, StoreInstanceProvider storeInstanceProvider, List credentialBuilders, List connectionBuilders) { - this.environmentConfiguration = environmentConfiguration; + this.environment = environment; + this.storeInstanceProvider = storeInstanceProvider; for (ConnectionBuilder builder : connectionBuilders) { this.connectionBuildersIndex.put(new ConnectionBuilder.Key(builder.getConnectionSpecificationType(), builder.getCredentialType()), builder); @@ -55,80 +55,91 @@ private ConnectionFactory(EnvironmentConfiguration environmentConfiguration, Lis { this.credentialBuildersIndex.put(new CredentialBuilder.Key(builder.getAuthenticationConfigurationType(), builder.getInputCredentialType(), builder.getOutputCredentialType()), builder); } - this.storeInstancesIndex = storeInstancesIndex; } - /** - * This method is meant for testing. - * The recommended usage is to include all the store instances during initialization - */ - public void injectStoreInstance(StoreInstance storeInstance) + public LegendEnvironment getEnvironment() { - if (this.storeInstancesIndex.containsKey(storeInstance.getIdentifier())) - { - throw new RuntimeException(String.format("Can't register store instance: found multiple store instances with identifier '%s'", storeInstance.getIdentifier())); - } - this.storeInstancesIndex.put(storeInstance.getIdentifier(), storeInstance); + return environment; + } + + public Authenticator getAuthenticator(Identity identity, String storeInstanceIdentifier, AuthenticationMechanism authenticationMechanism) + { + return this.getAuthenticator(identity, this.storeInstanceProvider.lookup(storeInstanceIdentifier), authenticationMechanism); } - private StoreInstance findStoreInstance(String identifier) + public Authenticator getAuthenticator(Identity identity, StoreInstance storeInstance, AuthenticationMechanism authenticationMechanism) { - return Objects.requireNonNull(this.storeInstancesIndex.get(identifier), String.format("Can't find store instance with identifier '%s'", identifier)); + AuthenticationMechanismConfiguration authenticationMechanismConfiguration = Objects.requireNonNull(storeInstance.getAuthenticationMechanismConfiguration(authenticationMechanism), String.format("Store '%s' does not support authentication mechanism '%s'. Supported mechanism(s):\n%s", + storeInstance.getIdentifier(), + authenticationMechanism.getLabel(), + ListIterate.collect(storeInstance.getAuthenticationMechanisms(), mechanism -> "- " + mechanism.getLabel()).makeString("\n") + )); + Function0 generator = authenticationMechanismConfiguration.getDefaultAuthenticationConfigurationGenerator(); + AuthenticationConfiguration authenticationConfiguration = Objects.requireNonNull(generator != null ? generator.get() : null, String.format("Can't auto-generate authentication configuration for store '%s' with authentication mechanism '%s'. Please provide a configuration of one of the following type(s):\n%s", + storeInstance.getIdentifier(), + authenticationMechanism.getLabel(), + authenticationMechanismConfiguration.getAuthenticationConfigurationTypes().collect(configType -> "- " + configType.getSimpleName()).makeString("\n") + )); + return this.getAuthenticator(identity, storeInstance, authenticationMechanism, authenticationConfiguration); } public Authenticator getAuthenticator(Identity identity, String storeInstanceIdentifier, AuthenticationConfiguration authenticationConfiguration) { - return this.getAuthenticator(identity, this.findStoreInstance(storeInstanceIdentifier), authenticationConfiguration); + return this.getAuthenticator(identity, this.storeInstanceProvider.lookup(storeInstanceIdentifier), authenticationConfiguration); } public Authenticator getAuthenticator(Identity identity, StoreInstance storeInstance, AuthenticationConfiguration authenticationConfiguration) { - AuthenticationMechanism authenticationMechanism = environmentConfiguration.findAuthenticationMechanismForConfiguration(authenticationConfiguration); - String authenticationMechanismLabel = authenticationMechanism != null ? ("authentication mechanism '" + authenticationMechanism.getLabel() + "'") : ("authentication mechanism with configuration '" + authenticationConfiguration.getClass().getSimpleName() + "'"); - if (!storeInstance.getAuthenticationConfigurationTypes().contains(authenticationConfiguration.getClass())) - { - throw new RuntimeException(String.format("Can't get authenticator: %s is not supported by store '%s'. Supported mechanism(s):\n%s", - authenticationMechanismLabel, - storeInstance.getIdentifier(), - storeInstance.getAuthenticationMechanisms().stream().map(mechanism -> "- " + mechanism.getLabel() + " (config: " + mechanism.getAuthenticationConfigurationType().getSimpleName() + ")").collect(Collectors.joining("\n"))) - ); - } - AuthenticationFlowResolver.ResolutionResult result = AuthenticationFlowResolver.run(this.credentialBuildersIndex, this.connectionBuildersIndex, identity, authenticationConfiguration, storeInstance.getConnectionSpecification()); + AuthenticationMechanism authenticationMechanism = Objects.requireNonNull(storeInstance.getAuthenticationMechanism(authenticationConfiguration.getClass()), String.format("Store '%s' does not accept authentication configuration type '%s'. Supported configuration type(s):\n%s", + storeInstance.getIdentifier(), + authenticationConfiguration.getClass().getSimpleName(), + ListIterate.collect(storeInstance.getAuthenticationConfigurationTypes(), configType -> "- " + configType.getSimpleName()).makeString("\n") + )); + return this.getAuthenticator(identity, storeInstance, authenticationMechanism, authenticationConfiguration); + } + + private Authenticator getAuthenticator(Identity identity, StoreInstance storeInstance, AuthenticationMechanism authenticationMechanism, AuthenticationConfiguration authenticationConfiguration) + { + AuthenticationFlowResolver.ResolutionResult result = AuthenticationFlowResolver.run(this.credentialBuildersIndex, this.connectionBuildersIndex, identity, authenticationMechanism, authenticationConfiguration, storeInstance.getConnectionSpecification()); if (result == null) { - throw new RuntimeException(String.format("Can't get authenticator: no authentication flow for store '%s' can be resolved for the specified identity using %s (authentication configuration: %s, connection specification: %s)", + throw new RuntimeException(String.format("No authentication flow for store '%s' can be resolved for the specified identity (authentication configuration: %s, connection specification: %s)", storeInstance.getIdentifier(), - authenticationMechanismLabel, authenticationConfiguration.getClass().getSimpleName(), - storeInstance.getConnectionSpecification().getClass().getSimpleName()) - ); + storeInstance.getConnectionSpecification().getClass().getSimpleName() + )); } - return new Authenticator(identity, storeInstance, authenticationConfiguration, result.sourceCredentialType, result.flow, connectionBuildersIndex.get(new ConnectionBuilder.Key(storeInstance.getConnectionSpecification().getClass(), result.targetCredentialType))); + return new Authenticator(storeInstance, authenticationMechanism, authenticationConfiguration, result.sourceCredentialType, result.targetCredentialType, result.flow, connectionBuildersIndex.get(new ConnectionBuilder.Key(storeInstance.getConnectionSpecification().getClass(), result.targetCredentialType)), this.environment); } public Authenticator getAuthenticator(Identity identity, String storeInstanceIdentifier) { - return this.getAuthenticator(identity, this.findStoreInstance(storeInstanceIdentifier)); + return this.getAuthenticator(identity, this.storeInstanceProvider.lookup(storeInstanceIdentifier)); } public Authenticator getAuthenticator(Identity identity, StoreInstance storeInstance) { - List authenticationConfigurations = ListIterate.collect(storeInstance.getAuthenticationMechanisms(), AuthenticationMechanism::generateConfiguration).select(Objects::nonNull); Authenticator authenticator = null; - for (AuthenticationConfiguration authenticationConfiguration : authenticationConfigurations) + for (AuthenticationMechanism authenticationMechanism : storeInstance.getAuthenticationMechanisms()) { - AuthenticationFlowResolver.ResolutionResult result = AuthenticationFlowResolver.run(this.credentialBuildersIndex, this.connectionBuildersIndex, identity, authenticationConfiguration, storeInstance.getConnectionSpecification()); - if (result != null) + AuthenticationMechanismConfiguration authenticationMechanismConfiguration = storeInstance.getAuthenticationMechanismConfiguration(authenticationMechanism); + Function0 generator = authenticationMechanismConfiguration.getDefaultAuthenticationConfigurationGenerator(); + AuthenticationConfiguration authenticationConfiguration = generator != null ? generator.get() : null; + if (authenticationConfiguration != null) { - authenticator = new Authenticator(identity, storeInstance, authenticationConfiguration, result.sourceCredentialType, result.flow, connectionBuildersIndex.get(new ConnectionBuilder.Key(storeInstance.getConnectionSpecification().getClass(), result.targetCredentialType))); - break; + AuthenticationFlowResolver.ResolutionResult result = AuthenticationFlowResolver.run(this.credentialBuildersIndex, this.connectionBuildersIndex, identity, authenticationMechanism, authenticationConfiguration, storeInstance.getConnectionSpecification()); + if (result != null) + { + authenticator = new Authenticator(storeInstance, authenticationMechanism, authenticationConfiguration, result.sourceCredentialType, result.targetCredentialType, result.flow, connectionBuildersIndex.get(new ConnectionBuilder.Key(storeInstance.getConnectionSpecification().getClass(), result.targetCredentialType)), this.environment); + break; + } } } if (authenticator == null) { - throw new RuntimeException(String.format("Can't get authenticator: no authentication flow for store '%s' can be resolved for the specified identity using auto-generated authentication configuration. Try specifying an authentication mechanism by providing a configuration of one of the following types:\n%s", + throw new RuntimeException(String.format("No authentication flow for store '%s' can be resolved for the specified identity. Try specifying an authentication mechanism or authentication configuration. Supported configuration type(s):\n%s", storeInstance.getIdentifier(), - ListIterate.select(storeInstance.getAuthenticationMechanisms(), mechanism -> mechanism.generateConfiguration() == null).collect(mechanism -> "- " + mechanism.getAuthenticationConfigurationType().getSimpleName() + " (mechanism: " + mechanism.getLabel() + ")").makeString("\n") + ListIterate.collect(storeInstance.getAuthenticationConfigurationTypes(), configType -> "- " + configType.getSimpleName() + " (" + storeInstance.getAuthenticationMechanism(configType).getLabel() + ")").makeString("\n") )); } return authenticator; @@ -156,7 +167,7 @@ private static class AuthenticationFlowResolver * NOTE: * - Since some credential builders do not require a specific input credential type, we added a generic `Credential` node * to Identity (start node) - * - We want to differentiate credential and credential-type nodes because we want to account for (short-circuit) case where + * - We want to differentiate credential and credential-type nodes because we want to account for (short-circuit) cases where * no resolution is needed: some credentials that belong to the identity is enough to build the connection (e.g. Kerberos). * We want to be very explicit about this case, we don't want this behavior to be generic for all types of credentials; for example, * just because an identity comes with a username-password credential, does not mean this credential is appropriate to be used to @@ -164,7 +175,7 @@ private static class AuthenticationFlowResolver *

* With this setup, we can use a basic graph search algorithm (e.g. BFS) to resolve the shortest path to build a connection */ - private AuthenticationFlowResolver(Map credentialBuildersIndex, Map connectionBuildersIndex, Identity identity, AuthenticationConfiguration authenticationConfiguration, ConnectionSpecification connectionSpecification) + private AuthenticationFlowResolver(Map credentialBuildersIndex, Map connectionBuildersIndex, Identity identity, AuthenticationConfiguration authenticationConfiguration, AuthenticationMechanism authenticationMechanism, ConnectionSpecification connectionSpecification) { // add start node (i.e. identity node) this.startNode = new FlowNode(identity); @@ -177,7 +188,7 @@ private AuthenticationFlowResolver(Map .filter(builder -> builder.getAuthenticationConfigurationType().equals(authenticationConfiguration.getClass())) .forEach(builder -> { - if (!(builder instanceof CredentialExtractor)) + if (!(builder.getInputCredentialType().equals(builder.getOutputCredentialType()))) { this.processEdge(new FlowNode(builder.getInputCredentialType()), new FlowNode(builder.getOutputCredentialType())); } @@ -192,7 +203,7 @@ private AuthenticationFlowResolver(Map .forEach(builder -> this.processEdge(new FlowNode(builder.getCredentialType()), this.endNode)); } - public static String createCredentialBuilderKey(String inputCredentialType, String outputCredentialType) + static String createCredentialBuilderKey(String inputCredentialType, String outputCredentialType) { return inputCredentialType + "__" + outputCredentialType; } @@ -217,10 +228,10 @@ private void processEdge(FlowNode node, FlowNode adjacentNode) /** * Resolves the authentication flow in order to build a connection for a specified identity */ - public static ResolutionResult run(Map credentialBuildersIndex, Map connectionBuildersIndex, Identity identity, AuthenticationConfiguration authenticationConfiguration, ConnectionSpecification connectionSpecification) + public static ResolutionResult run(Map credentialBuildersIndex, Map connectionBuildersIndex, Identity identity, AuthenticationMechanism authenticationMechanism, AuthenticationConfiguration authenticationConfiguration, ConnectionSpecification connectionSpecification) { // using BFS algo to search for the shortest (non-cyclic) path - AuthenticationFlowResolver state = new AuthenticationFlowResolver(credentialBuildersIndex, connectionBuildersIndex, identity, authenticationConfiguration, connectionSpecification); + AuthenticationFlowResolver state = new AuthenticationFlowResolver(credentialBuildersIndex, connectionBuildersIndex, identity, authenticationConfiguration, authenticationMechanism, connectionSpecification); boolean found = false; Set visitedNodes = new HashSet<>(); // Create a set to keep track of visited vertices @@ -362,131 +373,95 @@ public ResolutionResult(List flow, Class T getConnection(Identity identity, StoreInstance storeInstance, AuthenticationConfiguration authenticationConfiguration) throws Exception { - return this.getConnection(this.getAuthenticator(identity, storeInstance, authenticationConfiguration)); + return this.getConnection(identity, this.getAuthenticator(identity, storeInstance, authenticationConfiguration)); } public T getConnection(Identity identity, String storeInstanceIdentifier, AuthenticationConfiguration authenticationConfiguration) throws Exception { - return this.getConnection(this.getAuthenticator(identity, storeInstanceIdentifier, authenticationConfiguration)); + return this.getConnection(identity, this.getAuthenticator(identity, storeInstanceIdentifier, authenticationConfiguration)); } public T getConnection(Identity identity, StoreInstance storeInstance) throws Exception { - return this.getConnection(this.getAuthenticator(identity, storeInstance)); + return this.getConnection(identity, this.getAuthenticator(identity, storeInstance)); } public T getConnection(Identity identity, String storeInstanceIdentifier) throws Exception { - return this.getConnection(this.getAuthenticator(identity, storeInstanceIdentifier)); + return this.getConnection(identity, this.getAuthenticator(identity, storeInstanceIdentifier)); } - public T getConnection(Authenticator authenticator) throws Exception + public T getConnection(Identity identity, Authenticator authenticator) throws Exception { - Credential credential = authenticator.makeCredential(this.environmentConfiguration); ConnectionBuilder flow = (ConnectionBuilder) authenticator.getConnectionBuilder(); - return flow.getConnection(credential, authenticator.getStoreInstance().getConnectionSpecification(), authenticator.getStoreInstance()); + return flow.getConnection(authenticator.getStoreInstance().getConnectionSpecification(), flow.getAuthenticatorCompatible(authenticator), identity); } public static class Builder { - private final EnvironmentConfiguration environmentConfiguration; - private CredentialBuilderProvider credentialBuilderProvider; - private ConnectionBuilderProvider connectionBuilderProvider; - private final List> credentialBuilders = Lists.mutable.empty(); - private final List> connectionBuilders = Lists.mutable.empty(); - private final Map storeInstancesIndex = new HashMap<>(); - - public Builder(EnvironmentConfiguration environmentConfiguration) - { - this.environmentConfiguration = environmentConfiguration; - } - - public Builder withCredentialBuilderProvider(CredentialBuilderProvider provider) - { - this.credentialBuilderProvider = provider; - return this; - } + private final LegendEnvironment environment; + private final StoreInstanceProvider storeInstanceProvider; + private final List credentialBuilders = Lists.mutable.empty(); + private final List connectionBuilders = Lists.mutable.empty(); - public Builder withConnectionBuilderProvider(ConnectionBuilderProvider provider) + public Builder(LegendEnvironment environment, StoreInstanceProvider storeInstanceProvider) { - this.connectionBuilderProvider = provider; - return this; + this.environment = environment; + this.storeInstanceProvider = storeInstanceProvider; } - public Builder withCredentialBuilders(List> credentialBuilders) + public Builder withCredentialBuilders(List credentialBuilders) { this.credentialBuilders.addAll(credentialBuilders); return this; } - public Builder withCredentialBuilders(CredentialBuilder... credentialBuilders) + public Builder withCredentialBuilders(CredentialBuilder... credentialBuilders) { this.credentialBuilders.addAll(Lists.mutable.with(credentialBuilders)); return this; } - public Builder withCredentialBuilder(CredentialBuilder credentialBuilder) + public Builder withCredentialBuilder(CredentialBuilder credentialBuilder) { this.credentialBuilders.add(credentialBuilder); return this; } - public Builder withConnectionBuilders(List> connectionBuilders) + public Builder withConnectionBuilders(List connectionBuilders) { this.connectionBuilders.addAll(connectionBuilders); return this; } - public Builder withConnectionBuilders(ConnectionBuilder... connectionBuilders) + public Builder withConnectionBuilders(ConnectionBuilder... connectionBuilders) { this.connectionBuilders.addAll(Lists.mutable.with(connectionBuilders)); return this; } - public Builder withConnectionBuilder(ConnectionBuilder connectionBuilder) + public Builder withConnectionBuilder(ConnectionBuilder connectionBuilder) { this.connectionBuilders.add(connectionBuilder); return this; } - public Builder withStoreInstances(List storeInstances) - { - storeInstances.forEach(this::registerStoreInstance); - return this; - } - - public Builder withStoreInstance(StoreInstance storeInstance) - { - this.registerStoreInstance(storeInstance); - return this; - } - - private void registerStoreInstance(StoreInstance storeInstance) - { - if (this.storeInstancesIndex.containsKey(storeInstance.getIdentifier())) - { - throw new RuntimeException(String.format("Can't register store instance: found multiple store instances with identifier '%s'", storeInstance.getIdentifier())); - } - this.storeInstancesIndex.put(storeInstance.getIdentifier(), storeInstance); - } - public ConnectionFactory build() { - List credentialBuilders = this.credentialBuilderProvider != null ? this.credentialBuilderProvider.getBuilders() : Lists.mutable.empty(); - credentialBuilders.addAll(this.credentialBuilders); - List connectionBuilders = this.connectionBuilderProvider != null ? this.connectionBuilderProvider.getBuilders() : Lists.mutable.empty(); - connectionBuilders.addAll(this.connectionBuilders); - - for (ConnectionManager connectionManager : ServiceLoader.load(ConnectionManager.class)) + for (ConnectionBuilder connectionBuilder : connectionBuilders) { - connectionManager.initialize(); + ConnectionManager connectionManager = connectionBuilder.getConnectionManager(); + if (connectionManager != null) + { + connectionManager.initialize(environment); + } } return new ConnectionFactory( - this.environmentConfiguration, - credentialBuilders, - connectionBuilders, - this.storeInstancesIndex + this.environment, + this.storeInstanceProvider, + this.credentialBuilders, + this.connectionBuilders ); } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionManager.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionManager.java index ef2a03f5f86..42848c1e625 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionManager.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionManager.java @@ -16,5 +16,5 @@ public interface ConnectionManager { - void initialize(); + void initialize(LegendEnvironment environment); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialBuilder.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialBuilder.java index 59239f8d311..0f4726abfd7 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialBuilder.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialBuilder.java @@ -22,9 +22,9 @@ import java.lang.reflect.Type; import java.util.Objects; -public abstract class CredentialBuilder +public abstract class CredentialBuilder { - public abstract OUTPUT_CRED makeCredential(Identity identity, SPEC spec, INPUT_CRED cred, EnvironmentConfiguration configuration) throws Exception; + public abstract OUTPUT_CRED makeCredential(Identity identity, CONFIG config, INPUT_CRED cred, LegendEnvironment environment) throws Exception; public Class getAuthenticationConfigurationType() { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialExtractor.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialExtractor.java deleted file mode 100644 index d21f9ea5e5f..00000000000 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialExtractor.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection; - -import org.finos.legend.connection.protocol.AuthenticationConfiguration; -import org.finos.legend.engine.shared.core.identity.Credential; -import org.finos.legend.engine.shared.core.identity.Identity; - -import java.util.Optional; - -public abstract class CredentialExtractor extends CredentialBuilder -{ - - @Override - public Class getOutputCredentialType() - { - return (Class) actualTypeArguments()[1]; - } - - @Override - public CRED makeCredential(Identity identity, SPEC spec, CRED cred, EnvironmentConfiguration configuration) throws Exception - { - Optional credentialOptional = identity.getCredential((Class) this.getOutputCredentialType()); - if (!credentialOptional.isPresent()) - { - throw new RuntimeException(String.format("Can't extract credential of type '%s' from the specified identity", this.getOutputCredentialType().getSimpleName())); - } - return credentialOptional.get(); - } -} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultAuthenticationMechanismLoader.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultAuthenticationMechanismLoader.java deleted file mode 100644 index d099129b9b5..00000000000 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultAuthenticationMechanismLoader.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection; - -import org.eclipse.collections.api.factory.Lists; -import org.finos.legend.connection.protocol.AuthenticationMechanism; -import org.finos.legend.connection.protocol.AuthenticationMechanismType; - -import java.util.List; - -public class DefaultAuthenticationMechanismLoader implements AuthenticationMechanismLoader -{ - @Override - public List getMechanisms() - { - return Lists.mutable.with(AuthenticationMechanismType.values()); - } -} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultAuthenticationMechanismProvider.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultAuthenticationMechanismProvider.java deleted file mode 100644 index cc19a7b9174..00000000000 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultAuthenticationMechanismProvider.java +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection; - -import org.eclipse.collections.api.factory.Lists; - -import java.util.List; -import java.util.ServiceLoader; - -public class DefaultAuthenticationMechanismProvider implements AuthenticationMechanismProvider -{ - @Override - public List getLoaders() - { - return Lists.mutable.withAll(ServiceLoader.load(AuthenticationMechanismLoader.class)); - } -} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/EnvironmentConfiguration.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/EnvironmentConfiguration.java deleted file mode 100644 index c43c983f909..00000000000 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/EnvironmentConfiguration.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection; - -import org.eclipse.collections.api.factory.Lists; -import org.eclipse.collections.impl.factory.Maps; -import org.eclipse.collections.impl.utility.ListIterate; -import org.finos.legend.authentication.vault.CredentialVault; -import org.finos.legend.connection.protocol.AuthenticationConfiguration; -import org.finos.legend.connection.protocol.AuthenticationMechanism; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.authentication.vault.CredentialVaultSecret; -import org.finos.legend.engine.shared.core.identity.Identity; - -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; - -/** - * This is meant to the place we package common configs, such as vaults, - * that can be passed to various parts of engine, authentication, connection factory, etc. - */ -public class EnvironmentConfiguration -{ - private final List vaults; - private final Map, CredentialVault> vaultsIndex; - private final Map storeSupportsIndex; - - private final Map authenticationMechanismsIndex; - - private EnvironmentConfiguration(List vaults, Map storeSupportsIndex, Map authenticationMechanismsIndex) - { - this.vaults = Lists.mutable.withAll(vaults); - this.vaultsIndex = Maps.mutable.empty(); - for (CredentialVault vault : vaults) - { - vaultsIndex.put(vault.getSecretType(), vault); - } - this.storeSupportsIndex = storeSupportsIndex; - this.authenticationMechanismsIndex = authenticationMechanismsIndex; - } - - /** - * This method is meant for testing. - * The recommended usage is to include all the vaults during initialization - */ - public void injectVault(CredentialVault vault) - { - if (this.vaultsIndex.containsKey(vault.getSecretType())) - { - throw new RuntimeException(String.format("Can't register credential vault: found multiple vaults with secret type '%s'", vault.getSecretType().getSimpleName())); - } - this.vaultsIndex.put(vault.getSecretType(), vault); - this.vaults.add(vault); - } - - public String lookupVaultSecret(CredentialVaultSecret credentialVaultSecret, Identity identity) throws Exception - { - Class secretClass = credentialVaultSecret.getClass(); - if (!this.vaultsIndex.containsKey(secretClass)) - { - throw new RuntimeException(String.format("Can't find secret: credential vault for secret of type '%s' has not been registered", secretClass.getSimpleName())); - } - CredentialVault vault = this.vaultsIndex.get(secretClass); - return vault.lookupSecret(credentialVaultSecret, identity); - } - - /** - * This method is meant for testing. - * The recommended usage is to include all the store supports during initialization - */ - public void injectStoreSupport(StoreSupport storeSupport) - { - if (this.storeSupportsIndex.containsKey(storeSupport.getIdentifier())) - { - throw new RuntimeException(String.format("Can't register store support: found multiple store supports with identifier '%s'", storeSupport.getIdentifier())); - } - this.storeSupportsIndex.put(storeSupport.getIdentifier(), storeSupport); - } - - public StoreSupport findStoreSupport(String identifier) - { - return Objects.requireNonNull(this.storeSupportsIndex.get(identifier), String.format("Can't find store support with identifier '%s'", identifier)); - } - - public AuthenticationMechanism findAuthenticationMechanismForConfiguration(AuthenticationConfiguration configuration) - { - return this.authenticationMechanismsIndex.get(configuration.getClass().getSimpleName()); - } - - public static class Builder - { - private final List vaults = Lists.mutable.empty(); - private final Map storeSupportsIndex = new LinkedHashMap<>(); - private AuthenticationMechanismProvider authenticationMechanismProvider; - private final Set authenticationMechanisms = new LinkedHashSet<>(); - - public Builder() - { - - } - - public Builder withVaults(List vaults) - { - this.vaults.addAll(vaults); - return this; - } - - public Builder withVaults(CredentialVault... vaults) - { - this.vaults.addAll(Lists.mutable.with(vaults)); - return this; - } - - public Builder withVault(CredentialVault vault) - { - this.vaults.add(vault); - return this; - } - - public Builder withStoreSupports(List storeSupports) - { - storeSupports.forEach(this::registerStoreSupport); - return this; - } - - public Builder withStoreSupports(StoreSupport... storeSupports) - { - ListIterate.forEach(Lists.mutable.with(storeSupports), this::registerStoreSupport); - return this; - } - - public Builder withStoreSupport(StoreSupport storeSupport) - { - this.registerStoreSupport(storeSupport); - return this; - } - - private void registerStoreSupport(StoreSupport storeSupport) - { - if (this.storeSupportsIndex.containsKey(storeSupport.getIdentifier())) - { - throw new RuntimeException(String.format("Can't register store support: found multiple store supports with identifier '%s'", storeSupport.getIdentifier())); - } - this.storeSupportsIndex.put(storeSupport.getIdentifier(), storeSupport); - } - - public Builder withAuthenticationMechanismProvider(AuthenticationMechanismProvider authenticationMechanismProvider) - { - this.authenticationMechanismProvider = authenticationMechanismProvider; - return this; - } - - public Builder withAuthenticationMechanisms(List authenticationMechanisms) - { - this.authenticationMechanisms.addAll(authenticationMechanisms); - return this; - } - - - public Builder withAuthenticationMechanisms(AuthenticationMechanism... authenticationMechanisms) - { - this.authenticationMechanisms.addAll(Lists.mutable.with(authenticationMechanisms)); - return this; - } - - public Builder withAuthenticationMechanism(AuthenticationMechanism authenticationMechanism) - { - this.authenticationMechanisms.add(authenticationMechanism); - return this; - } - - public EnvironmentConfiguration build() - { - List authenticationMechanisms = this.authenticationMechanismProvider != null ? ListIterate.flatCollect(this.authenticationMechanismProvider.getLoaders(), AuthenticationMechanismLoader::getMechanisms) : Lists.mutable.empty(); - authenticationMechanisms.addAll(this.authenticationMechanisms); - Map authenticationMechanismsIndex = new LinkedHashMap<>(); - authenticationMechanisms.forEach(mechanism -> - { - String key = mechanism.getAuthenticationConfigurationType().getSimpleName(); - if (authenticationMechanismsIndex.containsKey(key)) - { - throw new IllegalStateException(String.format("Can't build environment configuration: found multiple authentication mechanisms (%s, %s) associated with the same configuration type '%s'", - authenticationMechanismsIndex.get(key).getLabel(), - mechanism.getLabel(), - key - )); - } - AuthenticationConfiguration configuration = mechanism.generateConfiguration(); - if (configuration != null && !configuration.getClass().equals(mechanism.getAuthenticationConfigurationType())) - { - throw new IllegalStateException(String.format("Can't build environment configuration: authentication mechanism '%s' is misconfigured, its associated configuration type is '%s' and its generated configuration type is '%s'", - mechanism.getLabel(), - mechanism.getAuthenticationConfigurationType().getSimpleName(), - configuration.getClass().getSimpleName() - )); - } - authenticationMechanismsIndex.put(key, mechanism); - }); - - return new EnvironmentConfiguration(this.vaults, this.storeSupportsIndex, authenticationMechanismsIndex); - } - } -} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentityFactory.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentityFactory.java index 917f3996657..a02600a70a6 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentityFactory.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentityFactory.java @@ -17,21 +17,21 @@ import org.eclipse.collections.api.factory.Lists; import org.finos.legend.engine.shared.core.identity.Credential; import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.engine.shared.core.identity.credential.AnonymousCredential; import org.finos.legend.engine.shared.core.identity.factory.DefaultIdentityFactory; import java.util.List; public class IdentityFactory { - private final EnvironmentConfiguration environmentConfiguration; + private final LegendEnvironment environment; - private IdentityFactory(EnvironmentConfiguration environmentConfiguration) + private IdentityFactory(LegendEnvironment environment) { - this.environmentConfiguration = environmentConfiguration; + this.environment = environment; } - // TODO: @akphi - this clones the logic from IdentityFactoryProvider, we should - // think of when we can unify them + // TODO: @akphi - this clones the logic from IdentityFactoryProvider, we should think about unifying them private static final DefaultIdentityFactory DEFAULT = new DefaultIdentityFactory(); public Identity createIdentity(IdentitySpecification identitySpecification) @@ -39,23 +39,33 @@ public Identity createIdentity(IdentitySpecification identitySpecification) List credentials = Lists.mutable.empty(); credentials.addAll(identitySpecification.getCredentials()); // TODO: @akphi - should we restrict here that we can only either specify the subject/profiles? - credentials.addAll(DEFAULT.makeIdentity(identitySpecification.getSubject()).getCredentials().toList()); - credentials.addAll(DEFAULT.makeIdentity(Lists.mutable.withAll(identitySpecification.getProfiles())).getCredentials().toList()); + if (identitySpecification.getSubject() != null) + { + credentials.addAll(DEFAULT.makeIdentity(identitySpecification.getSubject()).getCredentials().toList()); + } + if (!identitySpecification.getProfiles().isEmpty()) + { + credentials.addAll(DEFAULT.makeIdentity(Lists.mutable.withAll(identitySpecification.getProfiles())).getCredentials().toList()); + } + if (credentials.isEmpty()) + { + return identitySpecification.getName() != null ? new Identity(identitySpecification.getName(), new AnonymousCredential()) : DEFAULT.makeUnknownIdentity(); + } return new Identity(identitySpecification.getName(), credentials); } public static class Builder { - private final EnvironmentConfiguration environmentConfiguration; + private final LegendEnvironment environment; - public Builder(EnvironmentConfiguration environmentConfiguration) + public Builder(LegendEnvironment environment) { - this.environmentConfiguration = environmentConfiguration; + this.environment = environment; } public IdentityFactory build() { - return new IdentityFactory(this.environmentConfiguration); + return new IdentityFactory(this.environment); } } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentitySpecification.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentitySpecification.java index f0f1d0b5eee..7a07c2f6989 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentitySpecification.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/IdentitySpecification.java @@ -103,7 +103,7 @@ public Builder withCredential(Credential credential) public IdentitySpecification build() { return new IdentitySpecification( - Objects.requireNonNull(this.name, "Identity specification name is required"), + this.name, this.profiles, this.subject, this.credentials diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/LegendEnvironment.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/LegendEnvironment.java new file mode 100644 index 00000000000..7751f5bdecc --- /dev/null +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/LegendEnvironment.java @@ -0,0 +1,129 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.ImmutableList; +import org.eclipse.collections.api.map.ImmutableMap; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.impl.factory.Maps; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.authentication.vault.CredentialVault; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.authentication.vault.CredentialVaultSecret; +import org.finos.legend.engine.shared.core.identity.Identity; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * This is the runtime instance of configuration for Legend Engine, the place we package common configs, + * such as vaults, that can be passed to various parts of engine, authentication, connection factory, etc. + */ +public class LegendEnvironment +{ + protected final ImmutableList vaults; + protected final ImmutableMap, CredentialVault> vaultsIndex; + protected final ImmutableMap storeSupportsIndex; + + protected LegendEnvironment(List vaults, Map storeSupportsIndex) + { + this.vaults = Lists.immutable.withAll(vaults); + MutableMap, CredentialVault> vaultsIndex = Maps.mutable.empty(); + for (CredentialVault vault : vaults) + { + vaultsIndex.put(vault.getSecretType(), vault); + } + this.vaultsIndex = vaultsIndex.toImmutable(); + this.storeSupportsIndex = Maps.immutable.withAll(storeSupportsIndex); + } + + public String lookupVaultSecret(CredentialVaultSecret credentialVaultSecret, Identity identity) throws Exception + { + Class secretClass = credentialVaultSecret.getClass(); + if (!this.vaultsIndex.containsKey(secretClass)) + { + throw new RuntimeException(String.format("Can't find secret: credential vault for secret of type '%s' has not been registered", secretClass.getSimpleName())); + } + CredentialVault vault = this.vaultsIndex.get(secretClass); + return vault.lookupSecret(credentialVaultSecret, identity); + } + + public StoreSupport findStoreSupport(String identifier) + { + return Objects.requireNonNull(this.storeSupportsIndex.get(identifier), String.format("Can't find store support with identifier '%s'", identifier)); + } + + public static class Builder + { + private final List vaults = Lists.mutable.empty(); + private final Map storeSupportsIndex = new LinkedHashMap<>(); + + public Builder() + { + } + + public Builder withVaults(List vaults) + { + this.vaults.addAll(vaults); + return this; + } + + public Builder withVaults(CredentialVault... vaults) + { + this.vaults.addAll(Lists.mutable.with(vaults)); + return this; + } + + public Builder withVault(CredentialVault vault) + { + this.vaults.add(vault); + return this; + } + + public Builder withStoreSupports(List storeSupports) + { + storeSupports.forEach(this::registerStoreSupport); + return this; + } + + public Builder withStoreSupports(StoreSupport... storeSupports) + { + ListIterate.forEach(Lists.mutable.with(storeSupports), this::registerStoreSupport); + return this; + } + + public Builder withStoreSupport(StoreSupport storeSupport) + { + this.registerStoreSupport(storeSupport); + return this; + } + + private void registerStoreSupport(StoreSupport storeSupport) + { + if (this.storeSupportsIndex.containsKey(storeSupport.getIdentifier())) + { + throw new RuntimeException(String.format("Can't register store support: found multiple store supports with identifier '%s'", storeSupport.getIdentifier())); + } + this.storeSupportsIndex.put(storeSupport.getIdentifier(), storeSupport); + } + + public LegendEnvironment build() + { + return new LegendEnvironment(this.vaults, this.storeSupportsIndex); + } + } +} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreInstance.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreInstance.java index af25e716a92..2ced76c35a8 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreInstance.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreInstance.java @@ -15,33 +15,104 @@ package org.finos.legend.connection; import org.eclipse.collections.api.factory.Lists; -import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.connection.protocol.AuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationMechanism; import org.finos.legend.connection.protocol.ConnectionSpecification; import java.util.ArrayList; -import java.util.LinkedHashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; -import java.util.Set; -public class StoreInstance +/** + * A StoreInstance represents a named instance of a Store. + */ +public final class StoreInstance { private final String identifier; private final StoreSupport storeSupport; - private final List authenticationMechanisms; - private final List> authenticationConfigurationTypes; private final ConnectionSpecification connectionSpecification; + private final Map authenticationMechanismConfigurationIndex; + private final Map, AuthenticationMechanism> authenticationMechanismIndex; - private StoreInstance(String identifier, StoreSupport storeSupport, List authenticationMechanisms, ConnectionSpecification connectionSpecification) + private StoreInstance(String identifier, StoreSupport storeSupport, List authenticationMechanismConfigurations, ConnectionSpecification connectionSpecification) { - this.identifier = identifier; + this.identifier = Objects.requireNonNull(identifier, "Can't create store instance: identifier is missing"); this.storeSupport = storeSupport; - this.authenticationMechanisms = authenticationMechanisms; - this.authenticationConfigurationTypes = ListIterate.collect(authenticationMechanisms, AuthenticationMechanism::getAuthenticationConfigurationType); - this.connectionSpecification = connectionSpecification; + this.connectionSpecification = Objects.requireNonNull(connectionSpecification, "Connection specification is missing"); + + Map authenticationMechanismConfigurationIndex = new LinkedHashMap<>(); + + if (authenticationMechanismConfigurations.isEmpty()) + { + for (AuthenticationMechanism authenticationMechanism : this.storeSupport.getAuthenticationMechanisms()) + { + authenticationMechanismConfigurationIndex.put(authenticationMechanism, this.storeSupport.getAuthenticationMechanismConfiguration(authenticationMechanism)); + } + } + else + { + for (AuthenticationMechanismConfiguration authenticationMechanismConfiguration : authenticationMechanismConfigurations) + { + AuthenticationMechanism authenticationMechanism = authenticationMechanismConfiguration.getAuthenticationMechanism(); + if (authenticationMechanismConfigurationIndex.containsKey(authenticationMechanism)) + { + throw new RuntimeException(String.format("Found multiple configurations for authentication mechanism '%s'", authenticationMechanism.getLabel())); + } + AuthenticationMechanismConfiguration configFromStoreSupport = this.storeSupport.getAuthenticationMechanismConfiguration(authenticationMechanism); + if (configFromStoreSupport == null) + { + throw new RuntimeException(String.format("Authentication mechanism '%s' is not covered by store support '%s'. Supported mechanism(s):\n%s", + authenticationMechanism.getLabel(), + this.storeSupport.getIdentifier(), + ListIterate.collect(this.storeSupport.getAuthenticationMechanisms(), mechanism -> "- " + mechanism.getLabel()).makeString("\n") + )); + } + ImmutableList> authenticationConfigTypesFromStoreSupport = configFromStoreSupport.getAuthenticationConfigurationTypes(); + List> authenticationConfigurationTypes = Lists.mutable.empty(); + for (Class authenticationConfigurationType : authenticationMechanismConfiguration.getAuthenticationConfigurationTypes()) + { + if (!authenticationConfigTypesFromStoreSupport.contains(authenticationConfigurationType)) + { + throw new RuntimeException(String.format("Authentication configuration type '%s' is not covered by store support '%s' for authentication mechanism '%s'. Supported configuration type(s):\n%s", + authenticationConfigurationType.getSimpleName(), + this.storeSupport.getIdentifier(), + authenticationMechanism.getLabel(), + authenticationConfigTypesFromStoreSupport.collect(type -> "- " + type.getSimpleName()).makeString("\n") + )); + } + else + { + authenticationConfigurationTypes.add(authenticationConfigurationType); + } + } + authenticationMechanismConfigurationIndex.put(authenticationMechanism, new AuthenticationMechanismConfiguration.Builder(authenticationMechanism) + // NOTE: if no configuration type is specified, it means the store instance supports all configuration types configured for that mechanism in the store support + .withAuthenticationConfigurationTypes(!authenticationConfigurationTypes.isEmpty() ? authenticationConfigurationTypes : authenticationConfigTypesFromStoreSupport.toList()) + .withDefaultAuthenticationConfigurationGenerator(authenticationMechanismConfiguration.getDefaultAuthenticationConfigurationGenerator() != null ? authenticationMechanismConfiguration.getDefaultAuthenticationConfigurationGenerator() : configFromStoreSupport.getDefaultAuthenticationConfigurationGenerator()) + .build()); + + } + + } + + this.authenticationMechanismConfigurationIndex = authenticationMechanismConfigurationIndex; + Map, AuthenticationMechanism> authenticationMechanismIndex = new LinkedHashMap<>(); + authenticationMechanismConfigurationIndex.forEach((authenticationMechanism, authenticationMechanismConfiguration) -> + { + if (authenticationMechanismConfiguration.getAuthenticationConfigurationTypes().isEmpty()) + { + throw new RuntimeException(String.format("No authentication configuration type is associated with authentication mechanism '%s'", authenticationMechanism.getLabel())); + } + authenticationMechanismConfiguration.getAuthenticationConfigurationTypes().forEach(configurationType -> + { + authenticationMechanismIndex.put(configurationType, authenticationMechanism); + }); + }); + this.authenticationMechanismIndex = authenticationMechanismIndex; } public String getIdentifier() @@ -56,12 +127,17 @@ public StoreSupport getStoreSupport() public List getAuthenticationMechanisms() { - return authenticationMechanisms; + return new ArrayList<>(this.authenticationMechanismConfigurationIndex.keySet()); } public List> getAuthenticationConfigurationTypes() { - return authenticationConfigurationTypes; + return new ArrayList<>(this.authenticationMechanismIndex.keySet()); + } + + public AuthenticationMechanism getAuthenticationMechanism(Class authenticationConfigurationType) + { + return this.authenticationMechanismIndex.get(authenticationConfigurationType); } public ConnectionSpecification getConnectionSpecification() @@ -69,17 +145,31 @@ public ConnectionSpecification getConnectionSpecification() return connectionSpecification; } + public AuthenticationMechanismConfiguration getAuthenticationMechanismConfiguration(AuthenticationMechanism authenticationMechanism) + { + return authenticationMechanismConfigurationIndex.get(authenticationMechanism); + } + + public T getConnectionSpecification(Class clazz) + { + if (!this.connectionSpecification.getClass().equals(clazz)) + { + throw new RuntimeException(String.format("Can't get connection specification of type '%s' for store '%s'", clazz.getSimpleName(), this.identifier)); + } + return (T) this.connectionSpecification; + } + public static class Builder { - private final EnvironmentConfiguration environmentConfiguration; + private final LegendEnvironment environment; private String identifier; private String storeSupportIdentifier; - private final Set authenticationMechanisms = new LinkedHashSet<>(); + private final List authenticationMechanismConfigurations = Lists.mutable.empty(); private ConnectionSpecification connectionSpecification; - public Builder(EnvironmentConfiguration environmentConfiguration) + public Builder(LegendEnvironment environment) { - this.environmentConfiguration = environmentConfiguration; + this.environment = environment; } public Builder withIdentifier(String identifier) @@ -94,21 +184,21 @@ public Builder withStoreSupportIdentifier(String storeSupportIdentifier) return this; } - public Builder withAuthenticationMechanisms(List authenticationMechanisms) + public Builder withAuthenticationMechanismConfiguration(AuthenticationMechanismConfiguration authenticationMechanismConfiguration) { - this.authenticationMechanisms.addAll(authenticationMechanisms); + this.authenticationMechanismConfigurations.add(authenticationMechanismConfiguration); return this; } - public Builder withAuthenticationMechanism(AuthenticationMechanism authenticationMechanism) + public Builder withAuthenticationMechanismConfigurations(List authenticationMechanismConfigurations) { - this.authenticationMechanisms.add(authenticationMechanism); + this.authenticationMechanismConfigurations.addAll(authenticationMechanismConfigurations); return this; } - public Builder withAuthenticationMechanisms(AuthenticationMechanism... authenticationMechanisms) + public Builder withAuthenticationMechanismConfigurations(AuthenticationMechanismConfiguration... authenticationMechanismConfigurations) { - this.authenticationMechanisms.addAll(Lists.mutable.of(authenticationMechanisms)); + this.authenticationMechanismConfigurations.addAll(Lists.mutable.of(authenticationMechanismConfigurations)); return this; } @@ -120,18 +210,11 @@ public Builder withConnectionSpecification(ConnectionSpecification connectionSpe public StoreInstance build() { - StoreSupport storeSupport = this.environmentConfiguration.findStoreSupport(Objects.requireNonNull(this.storeSupportIdentifier, "Store instance store support identifier is required")); - MutableList unsupportedAuthenticationMechanisms = ListIterate.select(new ArrayList<>(this.authenticationMechanisms), mechanism -> !storeSupport.getAuthenticationMechanisms().contains(mechanism)); - if (!unsupportedAuthenticationMechanisms.isEmpty()) - { - throw new RuntimeException(String.format("Store instance specified with authentication configuration types (%s) which are not covered by its store support '%s'", unsupportedAuthenticationMechanisms.makeString(", "), storeSupport.getIdentifier())); - } return new StoreInstance( - Objects.requireNonNull(this.identifier, "Store instance identifier is required"), - storeSupport, - // NOTE: if no mechanism is specified, it means the store instance supports all mechanisms - this.authenticationMechanisms.isEmpty() ? storeSupport.getAuthenticationMechanisms() : new ArrayList<>(this.authenticationMechanisms), - Objects.requireNonNull(this.connectionSpecification, "Store instance connection specification is required") + this.identifier, + this.environment.findStoreSupport(Objects.requireNonNull(this.storeSupportIdentifier, "Store support identifier is missing")), + this.authenticationMechanismConfigurations, + this.connectionSpecification ); } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionBuilderProvider.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreInstanceProvider.java similarity index 85% rename from legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionBuilderProvider.java rename to legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreInstanceProvider.java index 20bdca4c7d9..8b83db33c91 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/ConnectionBuilderProvider.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreInstanceProvider.java @@ -14,9 +14,7 @@ package org.finos.legend.connection; -import java.util.List; - -public interface ConnectionBuilderProvider +public interface StoreInstanceProvider { - List getBuilders(); + StoreInstance lookup(String identifier); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreSupport.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreSupport.java index 982c6049507..fb36a14dc05 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreSupport.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/StoreSupport.java @@ -15,28 +15,56 @@ package org.finos.legend.connection; import org.eclipse.collections.api.factory.Lists; -import org.eclipse.collections.api.list.ImmutableList; -import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.connection.protocol.AuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationMechanism; import java.util.ArrayList; -import java.util.LinkedHashSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Objects; -import java.util.Set; +/** + * A StoreSupport describes the capabilities supported by a Store. + * For now, it describes the authentication mechanisms. + */ public class StoreSupport { private final String identifier; - private final List authenticationMechanisms; - private final List> authenticationConfigurationTypes; + private final Map authenticationMechanismConfigurationIndex; - protected StoreSupport(String identifier, List authenticationMechanisms) + protected StoreSupport(String identifier, List authenticationMechanismConfigurations) { - this.identifier = identifier; - this.authenticationMechanisms = authenticationMechanisms; - this.authenticationConfigurationTypes = ListIterate.collect(authenticationMechanisms, AuthenticationMechanism::getAuthenticationConfigurationType); + this.identifier = Objects.requireNonNull(identifier, "Identifier is missing"); + + Map authenticationMechanismConfigurationIndex = new LinkedHashMap<>(); + Map, AuthenticationMechanism> authenticationConfigurationTypeIndex = new LinkedHashMap<>(); + for (AuthenticationMechanismConfiguration authenticationMechanismConfiguration : authenticationMechanismConfigurations) + { + AuthenticationMechanism authenticationMechanism = authenticationMechanismConfiguration.getAuthenticationMechanism(); + if (authenticationMechanismConfigurationIndex.containsKey(authenticationMechanism)) + { + throw new RuntimeException(String.format("Found multiple configurations for authentication mechanism '%s'", authenticationMechanism.getLabel())); + } + authenticationMechanismConfigurationIndex.put(authenticationMechanism, authenticationMechanismConfiguration); + authenticationMechanismConfiguration.getAuthenticationConfigurationTypes().forEach(authenticationConfigurationType -> + { + if (authenticationConfigurationTypeIndex.containsKey(authenticationConfigurationType)) + { + throw new RuntimeException(String.format("Authentication configuration type '%s' is associated with multiple authentication mechanisms", authenticationConfigurationType.getSimpleName())); + } + authenticationConfigurationTypeIndex.put(authenticationConfigurationType, authenticationMechanism); + }); + } + + this.authenticationMechanismConfigurationIndex = authenticationMechanismConfigurationIndex; + this.authenticationMechanismConfigurationIndex.forEach((authenticationMechanism, authenticationMechanismConfiguration) -> + { + if (authenticationMechanismConfiguration.getAuthenticationConfigurationTypes().isEmpty()) + { + throw new RuntimeException(String.format("No authentication configuration type is associated with authentication mechanism '%s'", authenticationMechanism.getLabel())); + } + }); } public String getIdentifier() @@ -44,20 +72,20 @@ public String getIdentifier() return identifier; } - public List getAuthenticationMechanisms() + public AuthenticationMechanismConfiguration getAuthenticationMechanismConfiguration(AuthenticationMechanism authenticationMechanism) { - return authenticationMechanisms; + return authenticationMechanismConfigurationIndex.get(authenticationMechanism); } - public ImmutableList> getAuthenticationConfigurationTypes() + public List getAuthenticationMechanisms() { - return Lists.immutable.withAll(authenticationConfigurationTypes); + return new ArrayList<>(this.authenticationMechanismConfigurationIndex.keySet()); } public static class Builder { private String identifier; - private final Set authenticationMechanisms = new LinkedHashSet<>(); + private final List authenticationMechanismConfigurations = Lists.mutable.empty(); public Builder withIdentifier(String identifier) { @@ -65,29 +93,29 @@ public Builder withIdentifier(String identifier) return this; } - public Builder withAuthenticationMechanisms(List authenticationMechanisms) + public Builder withAuthenticationMechanismConfiguration(AuthenticationMechanismConfiguration authenticationMechanismConfiguration) { - this.authenticationMechanisms.addAll(authenticationMechanisms); + this.authenticationMechanismConfigurations.add(authenticationMechanismConfiguration); return this; } - public Builder withAuthenticationMechanisms(AuthenticationMechanism... authenticationMechanisms) + public Builder withAuthenticationMechanismConfigurations(List authenticationMechanismConfigurations) { - this.authenticationMechanisms.addAll(Lists.mutable.of(authenticationMechanisms)); + this.authenticationMechanismConfigurations.addAll(authenticationMechanismConfigurations); return this; } - public Builder withAuthenticationMechanism(AuthenticationMechanism authenticationMechanism) + public Builder withAuthenticationMechanismConfigurations(AuthenticationMechanismConfiguration... authenticationMechanismConfigurations) { - this.authenticationMechanisms.add(authenticationMechanism); + this.authenticationMechanismConfigurations.addAll(Lists.mutable.of(authenticationMechanismConfigurations)); return this; } public StoreSupport build() { return new StoreSupport( - Objects.requireNonNull(this.identifier, "Store support identifier is required"), - new ArrayList<>(this.authenticationMechanisms) + this.identifier, + this.authenticationMechanismConfigurations ); } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/ApiKeyAuthenticationConfiguration.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/ApiKeyAuthenticationConfiguration.java index f7e614c0b49..4e22064c7c7 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/ApiKeyAuthenticationConfiguration.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/ApiKeyAuthenticationConfiguration.java @@ -39,4 +39,13 @@ public enum Location HEADER, COOKIE } + + @Override + public String shortId() + { + return "ApiKey" + + "--location=" + location + + "--keyName=" + keyName + + "--value=" + value.shortId(); + } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/DefaultStoreInstanceProvider.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/DefaultStoreInstanceProvider.java new file mode 100644 index 00000000000..718b6404929 --- /dev/null +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/DefaultStoreInstanceProvider.java @@ -0,0 +1,85 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection.impl; + +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.map.ImmutableMap; +import org.finos.legend.connection.StoreInstance; +import org.finos.legend.connection.StoreInstanceProvider; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class DefaultStoreInstanceProvider implements StoreInstanceProvider +{ + private final ImmutableMap storeInstancesIndex; + + private DefaultStoreInstanceProvider(Map storeInstancesIndex) + { + + this.storeInstancesIndex = Maps.immutable.withAll(storeInstancesIndex); + } + + @Override + public StoreInstance lookup(String identifier) + { + return Objects.requireNonNull(this.storeInstancesIndex.get(identifier), String.format("Can't find store instance with identifier '%s'", identifier)); + } + + public static class Builder + { + private final Map storeInstancesIndex = new HashMap<>(); + + public Builder() + { + + } + + public Builder withStoreInstances(List storeInstances) + { + storeInstances.forEach(this::registerStoreInstance); + return this; + } + + public Builder withStoreInstances(StoreInstance... storeInstances) + { + Lists.mutable.with(storeInstances).forEach(this::registerStoreInstance); + return this; + } + + public Builder withStoreInstance(StoreInstance storeInstance) + { + this.registerStoreInstance(storeInstance); + return this; + } + + private void registerStoreInstance(StoreInstance storeInstance) + { + if (this.storeInstancesIndex.containsKey(storeInstance.getIdentifier())) + { + throw new RuntimeException(String.format("Found multiple store instances with identifier '%s'", storeInstance.getIdentifier())); + } + this.storeInstancesIndex.put(storeInstance.getIdentifier(), storeInstance); + } + + public DefaultStoreInstanceProvider build() + { + return new DefaultStoreInstanceProvider(this.storeInstancesIndex); + } + } +} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/EncryptedPrivateKeyPairAuthenticationConfiguration.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/EncryptedPrivateKeyPairAuthenticationConfiguration.java index b821f70d519..0bd50613949 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/EncryptedPrivateKeyPairAuthenticationConfiguration.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/EncryptedPrivateKeyPairAuthenticationConfiguration.java @@ -33,4 +33,13 @@ public EncryptedPrivateKeyPairAuthenticationConfiguration(String userName, Crede this.privateKey = privateKey; this.passphrase = passphrase; } + + @Override + public String shortId() + { + return "EncryptedPrivateKeyPair" + + "--userName=" + userName + + "--privateKey=" + privateKey.shortId() + + "--passphrase=" + passphrase.shortId(); + } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/InstrumentedStoreInstanceProvider.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/InstrumentedStoreInstanceProvider.java new file mode 100644 index 00000000000..c3438eab9fd --- /dev/null +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/InstrumentedStoreInstanceProvider.java @@ -0,0 +1,45 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection.impl; + +import org.eclipse.collections.api.factory.Maps; +import org.finos.legend.connection.StoreInstance; +import org.finos.legend.connection.StoreInstanceProvider; + +import java.util.Map; +import java.util.Objects; + +/** + * This is the instrumented version of {@link StoreInstanceProvider} which is used for testing. + */ +public class InstrumentedStoreInstanceProvider implements StoreInstanceProvider +{ + private final Map storeInstancesIndex = Maps.mutable.empty(); + + public void injectStoreInstance(StoreInstance storeInstance) + { + if (this.storeInstancesIndex.containsKey(storeInstance.getIdentifier())) + { + throw new RuntimeException(String.format("Found multiple store instances with identifier '%s'", storeInstance.getIdentifier())); + } + this.storeInstancesIndex.put(storeInstance.getIdentifier(), storeInstance); + } + + @Override + public StoreInstance lookup(String identifier) + { + return Objects.requireNonNull(this.storeInstancesIndex.get(identifier), String.format("Can't find store instance with identifier '%s'", identifier)); + } +} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosAuthenticationConfiguration.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosAuthenticationConfiguration.java index 41443983524..61b6ac6594a 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosAuthenticationConfiguration.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosAuthenticationConfiguration.java @@ -20,6 +20,11 @@ public class KerberosAuthenticationConfiguration extends AuthenticationConfigura { public KerberosAuthenticationConfiguration() { + } + @Override + public String shortId() + { + return "Kerberos"; } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosCredentialExtractor.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosCredentialExtractor.java index 22499b65ad8..a36c3add1cc 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosCredentialExtractor.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KerberosCredentialExtractor.java @@ -14,9 +14,23 @@ package org.finos.legend.connection.impl; -import org.finos.legend.connection.CredentialExtractor; +import org.finos.legend.connection.CredentialBuilder; +import org.finos.legend.connection.LegendEnvironment; +import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.engine.shared.core.identity.credential.LegendKerberosCredential; -public class KerberosCredentialExtractor extends CredentialExtractor +import java.util.Optional; + +public class KerberosCredentialExtractor extends CredentialBuilder { + @Override + public LegendKerberosCredential makeCredential(Identity identity, KerberosAuthenticationConfiguration authenticationConfiguration, LegendKerberosCredential credential, LegendEnvironment environment) throws Exception + { + Optional credentialOptional = identity.getCredential(LegendKerberosCredential.class); + if (!credentialOptional.isPresent()) + { + throw new RuntimeException(String.format("Can't extract credential of type '%s' from the specified identity", LegendKerberosCredential.class.getSimpleName())); + } + return credentialOptional.get(); + } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KeyPairCredentialBuilder.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KeyPairCredentialBuilder.java index da47ee7def9..46cc4ddc880 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KeyPairCredentialBuilder.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/KeyPairCredentialBuilder.java @@ -25,7 +25,7 @@ import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.impl.factory.Strings; import org.finos.legend.connection.CredentialBuilder; -import org.finos.legend.connection.EnvironmentConfiguration; +import org.finos.legend.connection.LegendEnvironment; import org.finos.legend.engine.shared.core.identity.Credential; import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.engine.shared.core.identity.credential.PrivateKeyCredential; @@ -42,10 +42,10 @@ public class KeyPairCredentialBuilder extends CredentialBuilder { @Override - public PrivateKeyCredential makeCredential(Identity identity, EncryptedPrivateKeyPairAuthenticationConfiguration authenticationConfiguration, Credential credential, EnvironmentConfiguration environmentConfiguration) throws Exception + public PrivateKeyCredential makeCredential(Identity identity, EncryptedPrivateKeyPairAuthenticationConfiguration authenticationConfiguration, Credential credential, LegendEnvironment environment) throws Exception { - String encryptedPrivateKey = environmentConfiguration.lookupVaultSecret(authenticationConfiguration.privateKey, identity); - String passphrase = environmentConfiguration.lookupVaultSecret(authenticationConfiguration.passphrase, identity); + String encryptedPrivateKey = environment.lookupVaultSecret(authenticationConfiguration.privateKey, identity); + String passphrase = environment.lookupVaultSecret(authenticationConfiguration.passphrase, identity); PrivateKey privateKey = this.getDecryptedPrivateKey(encryptedPrivateKey, passphrase); return new PrivateKeyCredential(authenticationConfiguration.userName, privateKey); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordAuthenticationConfiguration.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordAuthenticationConfiguration.java index 350ccb62005..61e8fb5c096 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordAuthenticationConfiguration.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordAuthenticationConfiguration.java @@ -31,4 +31,12 @@ public UserPasswordAuthenticationConfiguration(String username, CredentialVaultS this.username = username; this.password = password; } + + @Override + public String shortId() + { + return "UserPassword" + + "--username=" + username + + "--password=" + password.shortId(); + } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordCredentialBuilder.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordCredentialBuilder.java index 7739c6ca171..7cec18bd446 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordCredentialBuilder.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/impl/UserPasswordCredentialBuilder.java @@ -15,7 +15,7 @@ package org.finos.legend.connection.impl; import org.finos.legend.connection.CredentialBuilder; -import org.finos.legend.connection.EnvironmentConfiguration; +import org.finos.legend.connection.LegendEnvironment; import org.finos.legend.engine.shared.core.identity.Credential; import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.engine.shared.core.identity.credential.PlaintextUserPasswordCredential; @@ -23,10 +23,10 @@ public class UserPasswordCredentialBuilder extends CredentialBuilder { @Override - public PlaintextUserPasswordCredential makeCredential(Identity identity, UserPasswordAuthenticationConfiguration authenticationConfiguration, Credential credential, EnvironmentConfiguration environmentConfiguration) throws Exception + public PlaintextUserPasswordCredential makeCredential(Identity identity, UserPasswordAuthenticationConfiguration authenticationConfiguration, Credential credential, LegendEnvironment environment) throws Exception { - String password = environmentConfiguration.lookupVaultSecret(authenticationConfiguration.password, identity); + String password = environment.lookupVaultSecret(authenticationConfiguration.password, identity); return new PlaintextUserPasswordCredential(authenticationConfiguration.username, password); } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationConfiguration.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationConfiguration.java index 7718300d6fe..10774902332 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationConfiguration.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationConfiguration.java @@ -16,4 +16,5 @@ public abstract class AuthenticationConfiguration { + public abstract String shortId(); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanism.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanism.java index 284ac36c7f3..7a104453eee 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanism.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanism.java @@ -14,16 +14,7 @@ package org.finos.legend.connection.protocol; -import java.util.List; - public interface AuthenticationMechanism { String getLabel(); - - Class getAuthenticationConfigurationType(); - - default AuthenticationConfiguration generateConfiguration() - { - return null; - } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanismType.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanismType.java index e74db8cc985..b60214e8c54 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanismType.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/AuthenticationMechanismType.java @@ -18,50 +18,20 @@ import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration; import org.finos.legend.connection.impl.KerberosAuthenticationConfiguration; import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; +import org.finos.legend.engine.shared.core.identity.Credential; +import org.finos.legend.engine.shared.core.identity.credential.ApiTokenCredential; +import org.finos.legend.engine.shared.core.identity.credential.LegendKerberosCredential; +import org.finos.legend.engine.shared.core.identity.credential.OAuthCredential; +import org.finos.legend.engine.shared.core.identity.credential.PlaintextUserPasswordCredential; +import org.finos.legend.engine.shared.core.identity.credential.PrivateKeyCredential; public enum AuthenticationMechanismType implements AuthenticationMechanism { - USER_PASSWORD("UsernamePassword") - { - @Override - public Class getAuthenticationConfigurationType() - { - return UserPasswordAuthenticationConfiguration.class; - } - }, - - API_KEY("APIKey") - { - @Override - public Class getAuthenticationConfigurationType() - { - return ApiKeyAuthenticationConfiguration.class; - } - }, - - KEY_PAIR("KeyPair") - { - @Override - public Class getAuthenticationConfigurationType() - { - return EncryptedPrivateKeyPairAuthenticationConfiguration.class; - } - }, - - KERBEROS("Kerberos") - { - @Override - public Class getAuthenticationConfigurationType() - { - return KerberosAuthenticationConfiguration.class; - } - - @Override - public AuthenticationConfiguration generateConfiguration() - { - return new KerberosAuthenticationConfiguration(); - } - }; + USER_PASSWORD("UsernamePassword"), + API_KEY("APIKey"), + KEY_PAIR("KeyPair"), + KERBEROS("Kerberos"), + OAUTH("OAuth"); private final String label; diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/ConnectionSpecification.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/ConnectionSpecification.java index 450c8786ae2..908ab661327 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/ConnectionSpecification.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/protocol/ConnectionSpecification.java @@ -16,4 +16,5 @@ public abstract class ConnectionSpecification { + public abstract String shortId(); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/resources/META-INF/services/org.finos.legend.connection.AuthenticationMechanismLoader b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/resources/META-INF/services/org.finos.legend.connection.AuthenticationMechanismLoader deleted file mode 100644 index f77e7b4a4ab..00000000000 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/resources/META-INF/services/org.finos.legend.connection.AuthenticationMechanismLoader +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.DefaultAuthenticationMechanismLoader diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/resources/META-INF/services/org.finos.legend.connection.CredentialBuilder b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/resources/META-INF/services/org.finos.legend.connection.CredentialBuilder deleted file mode 100644 index 032f2e6497f..00000000000 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/resources/META-INF/services/org.finos.legend.connection.CredentialBuilder +++ /dev/null @@ -1,2 +0,0 @@ -org.finos.legend.connection.impl.UserPasswordCredentialBuilder -org.finos.legend.connection.impl.KerberosCredentialExtractor diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/ConnectionFactoryTest.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/ConnectionFactoryTest.java index d6bc8dbf398..df1398edd7c 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/ConnectionFactoryTest.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/ConnectionFactoryTest.java @@ -15,47 +15,20 @@ package org.finos.legend.connection; import org.eclipse.collections.api.factory.Lists; +import org.finos.legend.connection.impl.InstrumentedStoreInstanceProvider; import org.finos.legend.connection.protocol.AuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationMechanism; import org.finos.legend.connection.protocol.ConnectionSpecification; import org.finos.legend.engine.shared.core.identity.Credential; import org.finos.legend.engine.shared.core.identity.Identity; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; public class ConnectionFactoryTest { - @Test - public void testStoreInstanceManagement() - { - TestEnv env = TestEnv.create(); - StoreInstance storeInstance = new StoreInstance.Builder(env.environmentConfiguration) - .withIdentifier("test-store") - .withStoreSupportIdentifier("test") - .withConnectionSpecification(new TestConnectionSpecification()) - .build(); - env.connectionFactory.injectStoreInstance(storeInstance); - - // failure - Exception exception; - - // error: store already registered - exception = Assert.assertThrows(RuntimeException.class, () -> - { - env.connectionFactory.injectStoreInstance(storeInstance); - }); - Assert.assertEquals("Can't register store instance: found multiple store instances with identifier 'test-store'", exception.getMessage()); - - // error: store not found - exception = Assert.assertThrows(RuntimeException.class, () -> - { - env.connectionFactory.getAuthenticator(new Identity("test"), "unknown"); - }); - Assert.assertEquals("Can't find store instance with identifier 'unknown'", exception.getMessage()); - } - @Test public void testGetConnection_WithFailures() throws Exception { @@ -67,63 +40,61 @@ public void testGetConnection_WithFailures() throws Exception new ConnectionBuilder_A() ), Lists.mutable.with( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build(), + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.Y) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_Y.class) + .build() ) ).newStore("test", Lists.mutable.empty()); Identity identity = new Identity("test"); // success - env.connectionFactory.getConnection(env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X())); + env.connectionFactory.getConnection(identity, env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X())); Exception exception; // error: store not found - exception = Assert.assertThrows(RuntimeException.class, () -> + exception = Assertions.assertThrows(RuntimeException.class, () -> { - env.connectionFactory.getConnection(env.connectionFactory.getAuthenticator(identity, "unknown", new AuthenticationConfiguration_X())); + env.connectionFactory.getConnection(identity, env.connectionFactory.getAuthenticator(identity, "unknown", new AuthenticationConfiguration_X())); }); - Assert.assertEquals("Can't find store instance with identifier 'unknown'", exception.getMessage()); + Assertions.assertEquals("Can't find store instance with identifier 'unknown'", exception.getMessage()); // error: unsupported authentication mechanism - exception = Assert.assertThrows(RuntimeException.class, () -> + exception = Assertions.assertThrows(RuntimeException.class, () -> { - env.connectionFactory.getConnection(env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_Z())); + env.connectionFactory.getConnection(identity, env.connectionFactory.getAuthenticator(identity, "test", TestAuthenticationMechanismType.Z)); }); - Assert.assertEquals("Can't get authenticator: authentication mechanism 'Z' is not supported by store 'test'. Supported mechanism(s):\n- X (config: AuthenticationConfiguration_X)\n- Y (config: AuthenticationConfiguration_Y)", exception.getMessage()); + Assertions.assertEquals("Store 'test' does not support authentication mechanism 'Z'. Supported mechanism(s):\n" + + "- X\n" + + "- Y", exception.getMessage()); - // error: unresolvable authentication flow - exception = Assert.assertThrows(RuntimeException.class, () -> + // error: authentication mechanism does not come with a default config generator + exception = Assertions.assertThrows(RuntimeException.class, () -> { - env.connectionFactory.getConnection(env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_Y())); + env.connectionFactory.getConnection(identity, env.connectionFactory.getAuthenticator(identity, "test", TestAuthenticationMechanismType.X)); }); - Assert.assertEquals("Can't get authenticator: no authentication flow for store 'test' can be resolved for the specified identity using authentication mechanism 'Y' (authentication configuration: AuthenticationConfiguration_Y, connection specification: TestConnectionSpecification)", exception.getMessage()); + Assertions.assertEquals("Can't auto-generate authentication configuration for store 'test' with authentication mechanism 'X'. Please provide a configuration of one of the following type(s):\n" + + "- AuthenticationConfiguration_X", exception.getMessage()); - // alternate error message when authentication mechanisms are not properly registered - TestEnv env2 = TestEnv.create( - Lists.mutable.with(), - Lists.mutable.with(), - Lists.mutable.empty(), - Lists.mutable.with( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y - ) - ).newStore("test", Lists.mutable.empty()); - - // error: unsupported authentication mechanism - exception = Assert.assertThrows(RuntimeException.class, () -> + // error: unsupported authentication configuration + exception = Assertions.assertThrows(RuntimeException.class, () -> { - env2.connectionFactory.getConnection(env2.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_Z())); + env.connectionFactory.getConnection(identity, env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_Z())); }); - Assert.assertEquals("Can't get authenticator: authentication mechanism with configuration 'AuthenticationConfiguration_Z' is not supported by store 'test'. Supported mechanism(s):\n- X (config: AuthenticationConfiguration_X)\n- Y (config: AuthenticationConfiguration_Y)", exception.getMessage()); + Assertions.assertEquals("Store 'test' does not accept authentication configuration type 'AuthenticationConfiguration_Z'. Supported configuration type(s):\n" + + "- AuthenticationConfiguration_X\n" + + "- AuthenticationConfiguration_Y", exception.getMessage()); // error: unresolvable authentication flow - exception = Assert.assertThrows(RuntimeException.class, () -> + exception = Assertions.assertThrows(RuntimeException.class, () -> { - env2.connectionFactory.getConnection(env2.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_Y())); + env.connectionFactory.getConnection(identity, env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_Y())); }); - Assert.assertEquals("Can't get authenticator: no authentication flow for store 'test' can be resolved for the specified identity using authentication mechanism with configuration 'AuthenticationConfiguration_Y' (authentication configuration: AuthenticationConfiguration_Y, connection specification: TestConnectionSpecification)", exception.getMessage()); + Assertions.assertEquals("No authentication flow for store 'test' can be resolved for the specified identity (authentication configuration: AuthenticationConfiguration_Y, connection specification: TestConnectionSpecification)", exception.getMessage()); } /** @@ -142,14 +113,18 @@ public void testGetConnection_WithSimpleFlow() throws Exception new ConnectionBuilder_B() ), Lists.mutable.with( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build(), + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.Y) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_Y.class) + .build() ) ).newStore("test", Lists.mutable.empty()); Identity identity = new Identity("test"); Authenticator authenticator = env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X()); - assertAuthenticator(env.connectionFactory, authenticator, Credential.class, Lists.mutable.with( + assertAuthenticator(identity, env.connectionFactory, authenticator, Credential.class, Lists.mutable.with( "Credential->Credential_A [AuthenticationConfiguration_X]" ), ConnectionBuilder_A.class); } @@ -171,13 +146,15 @@ public void testGetConnection_WithSpecificBuilderOrder() throws Exception new ConnectionBuilder_B() ), Lists.mutable.with( - TestAuthenticationMechanismType.X + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build() ) ).newStore("test", Lists.mutable.empty()); Identity identity = new Identity("test"); Authenticator authenticator = env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X()); - assertAuthenticator(env.connectionFactory, authenticator, Credential.class, Lists.mutable.with( + assertAuthenticator(identity, env.connectionFactory, authenticator, Credential.class, Lists.mutable.with( "Credential->Credential_B [AuthenticationConfiguration_X]" ), ConnectionBuilder_B.class); } @@ -198,13 +175,15 @@ public void testGetConnection_WithChainFlow() throws Exception new ConnectionBuilder_C() ), Lists.mutable.with( - TestAuthenticationMechanismType.X + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build() ) ).newStore("test", Lists.mutable.empty()); Identity identity = new Identity("test"); Authenticator authenticator = env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X()); - assertAuthenticator(env.connectionFactory, authenticator, Credential.class, Lists.mutable.with( + assertAuthenticator(identity, env.connectionFactory, authenticator, Credential.class, Lists.mutable.with( "Credential->Credential_A [AuthenticationConfiguration_X]", "Credential_A->Credential_B [AuthenticationConfiguration_X]", "Credential_B->Credential_C [AuthenticationConfiguration_X]" @@ -227,13 +206,15 @@ public void testGetConnection_WithShortestFlowResolved() throws Exception new ConnectionBuilder_C() ), Lists.mutable.with( - TestAuthenticationMechanismType.X + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build() ) ).newStore("test", Lists.mutable.empty()); Identity identity = new Identity("test", new Credential_B()); Authenticator authenticator = env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X()); - assertAuthenticator(env.connectionFactory, authenticator, Credential_B.class, Lists.mutable.with( + assertAuthenticator(identity, env.connectionFactory, authenticator, Credential_B.class, Lists.mutable.with( "Credential_B->Credential_C [AuthenticationConfiguration_X]" ), ConnectionBuilder_C.class); } @@ -253,9 +234,16 @@ public void testGetConnection_WithNoAuthConfigProvided() throws Exception new ConnectionBuilder_B() ), Lists.mutable.with( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y, - TestAuthenticationMechanismType.Z + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build(), + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.Y) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_Y.class) + .withDefaultAuthenticationConfigurationGenerator(AuthenticationConfiguration_Y::new) + .build(), + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.Z) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_Z.class) + .build() ) ).newStore("test", Lists.mutable.empty()); @@ -263,16 +251,19 @@ public void testGetConnection_WithNoAuthConfigProvided() throws Exception // success Authenticator authenticator = env.connectionFactory.getAuthenticator(identity, "test"); - assertAuthenticator(env.connectionFactory, authenticator, Credential_A.class, Lists.mutable.with( + assertAuthenticator(identity, env.connectionFactory, authenticator, Credential_A.class, Lists.mutable.with( "Credential_A->Credential_B [AuthenticationConfiguration_Y]" ), ConnectionBuilder_B.class); // error: unresolvable authentication flow - Exception exception = Assert.assertThrows(RuntimeException.class, () -> + Exception exception = Assertions.assertThrows(RuntimeException.class, () -> { env.connectionFactory.getAuthenticator(new Identity("test"), "test"); }); - Assert.assertEquals("Can't get authenticator: no authentication flow for store 'test' can be resolved for the specified identity using auto-generated authentication configuration. Try specifying an authentication mechanism by providing a configuration of one of the following types:\n- AuthenticationConfiguration_X (mechanism: X)\n- AuthenticationConfiguration_Z (mechanism: Z)", exception.getMessage()); + Assertions.assertEquals("No authentication flow for store 'test' can be resolved for the specified identity. Try specifying an authentication mechanism or authentication configuration. Supported configuration type(s):\n" + + "- AuthenticationConfiguration_X (X)\n" + + "- AuthenticationConfiguration_Y (Y)\n" + + "- AuthenticationConfiguration_Z (Z)", exception.getMessage()); } /** @@ -289,13 +280,15 @@ public void testGetConnection_WithCredentialExtractor() throws Exception new ConnectionBuilder_A() ), Lists.mutable.with( - TestAuthenticationMechanismType.X + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build() ) ).newStore("test", Lists.mutable.empty()); Identity identity = new Identity("test", new Credential_A()); Authenticator authenticator = env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X()); - assertAuthenticator(env.connectionFactory, authenticator, Credential_A.class, Lists.mutable.with( + assertAuthenticator(identity, env.connectionFactory, authenticator, Credential_A.class, Lists.mutable.with( "Credential_A->Credential_A [AuthenticationConfiguration_X]" ), ConnectionBuilder_A.class); @@ -308,12 +301,14 @@ public void testGetConnection_WithCredentialExtractor() throws Exception new ConnectionBuilder_A() ), Lists.mutable.with( - TestAuthenticationMechanismType.X + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build() ) ).newStore("test", Lists.mutable.empty()); authenticator = env2.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X()); - assertAuthenticator(env2.connectionFactory, authenticator, Credential_A.class, Lists.mutable.with( + assertAuthenticator(identity, env2.connectionFactory, authenticator, Credential_A.class, Lists.mutable.with( "Credential_A->Credential_A [AuthenticationConfiguration_X]" ), ConnectionBuilder_A.class); } @@ -333,53 +328,55 @@ public void testGetConnection_WithoutCredentialExtractor() throws Exception new ConnectionBuilder_A() ), Lists.mutable.with( - TestAuthenticationMechanismType.X + new AuthenticationMechanismConfiguration.Builder(TestAuthenticationMechanismType.X) + .withAuthenticationConfigurationTypes(AuthenticationConfiguration_X.class) + .build() ) ).newStore("test", Lists.mutable.empty()); Identity identity = new Identity("test", new Credential_A()); - Exception exception = Assert.assertThrows(RuntimeException.class, () -> + Exception exception = Assertions.assertThrows(RuntimeException.class, () -> { - env.connectionFactory.getConnection(env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X())); + env.connectionFactory.getConnection(identity, env.connectionFactory.getAuthenticator(identity, "test", new AuthenticationConfiguration_X())); }); - Assert.assertEquals("Can't get authenticator: no authentication flow for store 'test' can be resolved for the specified identity using authentication mechanism 'X' (authentication configuration: AuthenticationConfiguration_X, connection specification: TestConnectionSpecification)", exception.getMessage()); + Assertions.assertEquals("No authentication flow for store 'test' can be resolved for the specified identity (authentication configuration: AuthenticationConfiguration_X, connection specification: TestConnectionSpecification)", exception.getMessage()); } - private void assertAuthenticator(ConnectionFactory connectionFactory, Authenticator authenticator, Class sourceCredentialType, List credentialBuilders, Class connectionBuilderType) throws Exception + private void assertAuthenticator(Identity identity, ConnectionFactory connectionFactory, Authenticator authenticator, Class sourceCredentialType, List credentialBuilders, Class connectionBuilderType) throws Exception { - Assert.assertEquals(sourceCredentialType, authenticator.getSourceCredentialType()); - Assert.assertEquals(connectionBuilderType, authenticator.getConnectionBuilder().getClass()); - Assert.assertArrayEquals(credentialBuilders.toArray(), authenticator.getCredentialBuilders().stream().map(builder -> String.format("%s->%s [%s]", builder.getInputCredentialType().getSimpleName(), builder.getOutputCredentialType().getSimpleName(), builder.getAuthenticationConfigurationType().getSimpleName())).toArray()); - connectionFactory.getConnection(authenticator); + Assertions.assertEquals(sourceCredentialType, authenticator.getSourceCredentialType()); + Assertions.assertEquals(connectionBuilderType, authenticator.getConnectionBuilder().getClass()); + Assertions.assertArrayEquals(credentialBuilders.toArray(), authenticator.getCredentialBuilders().stream().map(builder -> String.format("%s->%s [%s]", builder.getInputCredentialType().getSimpleName(), builder.getOutputCredentialType().getSimpleName(), builder.getAuthenticationConfigurationType().getSimpleName())).toArray()); + connectionFactory.getConnection(identity, authenticator); } private static class TestEnv { - final EnvironmentConfiguration environmentConfiguration; + final LegendEnvironment environment; + final InstrumentedStoreInstanceProvider storeInstanceProvider; final ConnectionFactory connectionFactory; - private TestEnv(List> credentialBuilders, List> connectionBuilders, List authenticationMechanisms, List supportedAuthenticationMechanisms) + private TestEnv(List credentialBuilders, List connectionBuilders, List authenticationMechanismConfigurations) { - this.environmentConfiguration = new EnvironmentConfiguration.Builder() + this.environment = new LegendEnvironment.Builder() .withStoreSupport(new StoreSupport.Builder() .withIdentifier("test") - .withAuthenticationMechanisms(supportedAuthenticationMechanisms) + .withAuthenticationMechanismConfigurations(authenticationMechanismConfigurations) .build()) - .withAuthenticationMechanisms(authenticationMechanisms) .build(); - - this.connectionFactory = new ConnectionFactory.Builder(environmentConfiguration) + this.storeInstanceProvider = new InstrumentedStoreInstanceProvider(); + this.connectionFactory = new ConnectionFactory.Builder(this.environment, this.storeInstanceProvider) .withCredentialBuilders(credentialBuilders) .withConnectionBuilders(connectionBuilders) .build(); } - TestEnv newStore(String identifier, List authenticationMechanisms) + TestEnv newStore(String identifier, List authenticationMechanismConfigurations) { - this.connectionFactory.injectStoreInstance(new StoreInstance.Builder(environmentConfiguration) + this.storeInstanceProvider.injectStoreInstance(new StoreInstance.Builder(this.environment) .withIdentifier(identifier) .withStoreSupportIdentifier("test") - .withAuthenticationMechanisms(authenticationMechanisms) + .withAuthenticationMechanismConfigurations(authenticationMechanismConfigurations) .withConnectionSpecification(new TestConnectionSpecification()) .build()); return this; @@ -387,21 +384,12 @@ TestEnv newStore(String identifier, List authentication static TestEnv create() { - return new TestEnv(Lists.mutable.empty(), Lists.mutable.empty(), Lists.mutable.empty(), Lists.mutable.empty()); + return new TestEnv(Lists.mutable.empty(), Lists.mutable.empty(), Lists.mutable.empty()); } - static TestEnv create(List> credentialBuilders, List> connectionBuilders, List authenticationMechanisms, List supportedAuthenticationMechanisms) + static TestEnv create(List credentialBuilders, List connectionBuilders, List authenticationMechanismConfigurations) { - return new TestEnv(credentialBuilders, connectionBuilders, authenticationMechanisms, supportedAuthenticationMechanisms); - } - - static TestEnv create(List> credentialBuilders, List> connectionBuilders, List supportedAuthenticationMechanisms) - { - return new TestEnv(credentialBuilders, connectionBuilders, Lists.mutable.with( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y, - TestAuthenticationMechanismType.Z - ), supportedAuthenticationMechanisms); + return new TestEnv(credentialBuilders, connectionBuilders, authenticationMechanismConfigurations); } } @@ -423,48 +411,36 @@ private static class Credential_C implements Credential private static class AuthenticationConfiguration_X extends AuthenticationConfiguration { + @Override + public String shortId() + { + return null; + } } private static class AuthenticationConfiguration_Y extends AuthenticationConfiguration { + @Override + public String shortId() + { + return null; + } } private static class AuthenticationConfiguration_Z extends AuthenticationConfiguration { + @Override + public String shortId() + { + return null; + } } private enum TestAuthenticationMechanismType implements AuthenticationMechanism { - X - { - @Override - public Class getAuthenticationConfigurationType() - { - return AuthenticationConfiguration_X.class; - } - }, - Y - { - @Override - public Class getAuthenticationConfigurationType() - { - return AuthenticationConfiguration_Y.class; - } - - @Override - public AuthenticationConfiguration generateConfiguration() - { - return new AuthenticationConfiguration_Y(); - } - }, - Z - { - @Override - public Class getAuthenticationConfigurationType() - { - return AuthenticationConfiguration_Z.class; - } - }; + X, + Y, + Z; @Override public String getLabel() @@ -476,7 +452,7 @@ public String getLabel() private static class CredentialBuilder_A_to_A__withX extends CredentialBuilder { @Override - public Credential_A makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential_A credential, EnvironmentConfiguration configuration) throws Exception + public Credential_A makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential_A credential, LegendEnvironment environment) throws Exception { return new Credential_A(); } @@ -485,7 +461,7 @@ public Credential_A makeCredential(Identity identity, AuthenticationConfiguratio private static class CredentialBuilder_A_to_B__withX extends CredentialBuilder { @Override - public Credential_B makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential_A credential, EnvironmentConfiguration configuration) throws Exception + public Credential_B makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential_A credential, LegendEnvironment environment) throws Exception { return new Credential_B(); } @@ -494,7 +470,7 @@ public Credential_B makeCredential(Identity identity, AuthenticationConfiguratio private static class CredentialBuilder_B_to_C__withX extends CredentialBuilder { @Override - public Credential_C makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential_B credential, EnvironmentConfiguration configuration) throws Exception + public Credential_C makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential_B credential, LegendEnvironment environment) throws Exception { return new Credential_C(); } @@ -503,7 +479,7 @@ public Credential_C makeCredential(Identity identity, AuthenticationConfiguratio private static class CredentialBuilder_Any_to_A__withX extends CredentialBuilder { @Override - public Credential_A makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential credential, EnvironmentConfiguration configuration) throws Exception + public Credential_A makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential credential, LegendEnvironment environment) throws Exception { return new Credential_A(); } @@ -512,7 +488,7 @@ public Credential_A makeCredential(Identity identity, AuthenticationConfiguratio private static class CredentialBuilder_A_to_B__withY extends CredentialBuilder { @Override - public Credential_B makeCredential(Identity identity, AuthenticationConfiguration_Y authenticationConfiguration, Credential_A credential, EnvironmentConfiguration configuration) throws Exception + public Credential_B makeCredential(Identity identity, AuthenticationConfiguration_Y authenticationConfiguration, Credential_A credential, LegendEnvironment environment) throws Exception { return new Credential_B(); } @@ -521,46 +497,88 @@ public Credential_B makeCredential(Identity identity, AuthenticationConfiguratio private static class CredentialBuilder_Any_to_B__withX extends CredentialBuilder { @Override - public Credential_B makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfigurationX, Credential credential, EnvironmentConfiguration configuration) throws Exception + public Credential_B makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfigurationX, Credential credential, LegendEnvironment environment) throws Exception { return new Credential_B(); } } - private static class CredentialExtractor_A__withX extends CredentialExtractor + private static class CredentialExtractor_A__withX extends CredentialBuilder { + @Override + public Credential_A makeCredential(Identity identity, AuthenticationConfiguration_X authenticationConfiguration, Credential_A credential, LegendEnvironment environment) throws Exception + { + + Optional credentialOptional = identity.getCredential(Credential_A.class); + if (!credentialOptional.isPresent()) + { + throw new RuntimeException(""); + } + return credentialOptional.get(); + } } // -------------------------- Connection ------------------------------- + private static class TestConnectionManager implements ConnectionManager + { + @Override + public void initialize(LegendEnvironment environment) + { + } + } + private static class TestConnectionSpecification extends ConnectionSpecification { + @Override + public String shortId() + { + return null; + } } private static class ConnectionBuilder_A extends ConnectionBuilder { @Override - public Object getConnection(Credential_A credential, TestConnectionSpecification connectionSpecification, StoreInstance storeInstance) throws Exception + public Object getConnection(TestConnectionSpecification connectionSpecification, Authenticator authenticator, Identity identity) throws Exception { return null; } + + @Override + public ConnectionManager getConnectionManager() + { + return new TestConnectionManager(); + } } private static class ConnectionBuilder_B extends ConnectionBuilder { @Override - public Object getConnection(Credential_B credential, TestConnectionSpecification connectionSpecification, StoreInstance storeInstance) throws Exception + public Object getConnection(TestConnectionSpecification connectionSpecification, Authenticator authenticator, Identity identity) throws Exception { return null; } + + @Override + public ConnectionManager getConnectionManager() + { + return new TestConnectionManager(); + } } private static class ConnectionBuilder_C extends ConnectionBuilder { @Override - public Object getConnection(Credential_C credential, TestConnectionSpecification connectionSpecification, StoreInstance storeInstance) throws Exception + public Object getConnection(TestConnectionSpecification connectionSpecification, Authenticator authenticator, Identity identity) throws Exception { return null; } + + @Override + public ConnectionManager getConnectionManager() + { + return new TestConnectionManager(); + } } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/EnvironmentConfigurationTest.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/EnvironmentConfigurationTest.java deleted file mode 100644 index b06d773479a..00000000000 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/EnvironmentConfigurationTest.java +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection; - -import org.eclipse.collections.api.factory.Lists; -import org.finos.legend.connection.protocol.AuthenticationConfiguration; -import org.finos.legend.connection.protocol.AuthenticationMechanism; -import org.junit.Assert; -import org.junit.Test; - -public class EnvironmentConfigurationTest -{ - @Test - public void testValidateBuilder() - { - // success - new EnvironmentConfiguration.Builder() - .withAuthenticationMechanisms(Lists.mutable.of( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y - )).build(); - - Exception exception; - - // failure: found invalid mechanism - exception = Assert.assertThrows(RuntimeException.class, () -> - { - new EnvironmentConfiguration.Builder() - .withAuthenticationMechanisms(Lists.mutable.of( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y, - TestAuthenticationMechanismType.Z - )).build(); - }); - Assert.assertEquals("Can't build environment configuration: authentication mechanism 'Z' is misconfigured, its associated configuration type is 'AuthenticationConfiguration_Z' and its generated configuration type is 'AuthenticationConfiguration_X'", exception.getMessage()); - - // failure: found conflicting mechanisms - exception = Assert.assertThrows(RuntimeException.class, () -> - { - new EnvironmentConfiguration.Builder() - .withAuthenticationMechanisms(Lists.mutable.of( - TestAuthenticationMechanismType.X, - TestAuthenticationMechanismType.Y, - TestAuthenticationMechanismType.T - )).build(); - }); - Assert.assertEquals("Can't build environment configuration: found multiple authentication mechanisms (Y, T) associated with the same configuration type 'AuthenticationConfiguration_Y'", exception.getMessage()); - } - - private static class AuthenticationConfiguration_X extends AuthenticationConfiguration - { - } - - private static class AuthenticationConfiguration_Y extends AuthenticationConfiguration - { - } - - private static class AuthenticationConfiguration_Z extends AuthenticationConfiguration - { - } - - private enum TestAuthenticationMechanismType implements AuthenticationMechanism - { - X - { - @Override - public Class getAuthenticationConfigurationType() - { - return AuthenticationConfiguration_X.class; - } - }, - Y - { - @Override - public Class getAuthenticationConfigurationType() - { - return AuthenticationConfiguration_Y.class; - } - - @Override - public AuthenticationConfiguration generateConfiguration() - { - return new AuthenticationConfiguration_Y(); - } - }, - Z - { - @Override - public Class getAuthenticationConfigurationType() - { - return AuthenticationConfiguration_Z.class; - } - - @Override - public AuthenticationConfiguration generateConfiguration() - { - return new AuthenticationConfiguration_X(); - } - }, - T - { - @Override - public Class getAuthenticationConfigurationType() - { - return AuthenticationConfiguration_Y.class; - } - }; - - @Override - public String getLabel() - { - return this.toString(); - } - } -} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/StoreSupportTest.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/StoreSupportTest.java index 18a13ab84a7..3a7f85d05ff 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/StoreSupportTest.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/test/java/org/finos/legend/connection/StoreSupportTest.java @@ -14,11 +14,16 @@ package org.finos.legend.connection; +import org.eclipse.collections.api.factory.Lists; +import org.finos.legend.connection.impl.DefaultStoreInstanceProvider; +import org.finos.legend.connection.impl.EncryptedPrivateKeyPairAuthenticationConfiguration; +import org.finos.legend.connection.impl.KerberosAuthenticationConfiguration; +import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationMechanism; import org.finos.legend.connection.protocol.AuthenticationMechanismType; import org.finos.legend.connection.protocol.ConnectionSpecification; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; public class StoreSupportTest { @@ -33,68 +38,217 @@ public void testValidateStoreSupportBuilder() // failure Exception exception; - exception = Assert.assertThrows(RuntimeException.class, () -> + exception = Assertions.assertThrows(RuntimeException.class, () -> { new StoreSupport.Builder().build(); }); - Assert.assertEquals("Store support identifier is required", exception.getMessage()); + Assertions.assertEquals("Identifier is missing", exception.getMessage()); + + exception = Assertions.assertThrows(RuntimeException.class, () -> + { + new StoreSupport.Builder() + .withIdentifier("test") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .withAuthenticationConfigurationTypes(UserPasswordAuthenticationConfiguration.class) + .build(), + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .withAuthenticationConfigurationTypes(UserPasswordAuthenticationConfiguration.class) + .build() + ).build(); + }); + Assertions.assertEquals("Found multiple configurations for authentication mechanism 'UsernamePassword'", exception.getMessage()); + + exception = Assertions.assertThrows(RuntimeException.class, () -> + { + new StoreSupport.Builder() + .withIdentifier("test") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .withAuthenticationConfigurationTypes(UserPasswordAuthenticationConfiguration.class) + .build(), + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KERBEROS) + .withAuthenticationConfigurationTypes(UserPasswordAuthenticationConfiguration.class) + .build() + ).build(); + }); + Assertions.assertEquals("Authentication configuration type 'UserPasswordAuthenticationConfiguration' is associated with multiple authentication mechanisms", exception.getMessage()); + + exception = Assertions.assertThrows(RuntimeException.class, () -> + { + new StoreSupport.Builder() + .withIdentifier("test") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .build() + ).build(); + }); + Assertions.assertEquals("No authentication configuration type is associated with authentication mechanism 'UsernamePassword'", exception.getMessage()); } @Test public void testValidateStoreInstanceBuilder() { - EnvironmentConfiguration environmentConfiguration = new EnvironmentConfiguration.Builder() + LegendEnvironment environment = new LegendEnvironment.Builder() .withStoreSupport(new StoreSupport.Builder() .withIdentifier("test") - .withAuthenticationMechanisms( - AuthenticationMechanismType.USER_PASSWORD, - AuthenticationMechanismType.KERBEROS + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .withAuthenticationConfigurationTypes( + UserPasswordAuthenticationConfiguration.class, + EncryptedPrivateKeyPairAuthenticationConfiguration.class + ) + .build(), + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KERBEROS) + .withAuthenticationConfigurationTypes(KerberosAuthenticationConfiguration.class) + .build() ) .build()) .build(); // success - StoreInstance testStore = new StoreInstance.Builder(environmentConfiguration) + StoreInstance testStore = new StoreInstance.Builder(environment) .withIdentifier("test-store") .withStoreSupportIdentifier("test") - .withAuthenticationMechanisms(AuthenticationMechanismType.USER_PASSWORD) + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .build() + ) .withConnectionSpecification(new TestConnectionSpecification()) .build(); - Assert.assertArrayEquals(new AuthenticationMechanism[]{AuthenticationMechanismType.USER_PASSWORD}, testStore.getAuthenticationMechanisms().toArray()); + Assertions.assertArrayEquals(new AuthenticationMechanism[]{AuthenticationMechanismType.USER_PASSWORD}, testStore.getAuthenticationMechanisms().toArray()); // make sure if no auth mechanisms is specified, all mechanisms will be supported - StoreInstance testStore2 = new StoreInstance.Builder(environmentConfiguration) + StoreInstance testStore2 = new StoreInstance.Builder(environment) + .withIdentifier("test-store") + .withStoreSupportIdentifier("test") + .withConnectionSpecification(new TestConnectionSpecification()) + .build(); + Assertions.assertArrayEquals(new AuthenticationMechanism[]{AuthenticationMechanismType.USER_PASSWORD, AuthenticationMechanismType.KERBEROS}, testStore2.getAuthenticationMechanisms().toArray()); + + // make sure if no authentication configuration type is specified, all types will be supported + StoreInstance testStore3 = new StoreInstance.Builder(environment) .withIdentifier("test-store") .withStoreSupportIdentifier("test") .withConnectionSpecification(new TestConnectionSpecification()) + .withAuthenticationMechanismConfiguration(new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).build()) .build(); - Assert.assertArrayEquals(new AuthenticationMechanism[]{AuthenticationMechanismType.USER_PASSWORD, AuthenticationMechanismType.KERBEROS}, testStore2.getAuthenticationMechanisms().toArray()); + Assertions.assertArrayEquals(Lists.mutable.of( + UserPasswordAuthenticationConfiguration.class, + EncryptedPrivateKeyPairAuthenticationConfiguration.class + ).toArray(), testStore3.getAuthenticationMechanismConfiguration(AuthenticationMechanismType.USER_PASSWORD).getAuthenticationConfigurationTypes().toArray()); // failure Exception exception; - exception = Assert.assertThrows(RuntimeException.class, () -> + exception = Assertions.assertThrows(RuntimeException.class, () -> { - new StoreInstance.Builder(environmentConfiguration) + new StoreInstance.Builder(environment) .withIdentifier("test-store") .withStoreSupportIdentifier("test") .build(); }); - Assert.assertEquals("Store instance connection specification is required", exception.getMessage()); + Assertions.assertEquals("Connection specification is missing", exception.getMessage()); - exception = Assert.assertThrows(RuntimeException.class, () -> + exception = Assertions.assertThrows(RuntimeException.class, () -> { - new StoreInstance.Builder(environmentConfiguration) + new StoreInstance.Builder(environment) .withIdentifier("test-store") .withStoreSupportIdentifier("test") - .withAuthenticationMechanisms(AuthenticationMechanismType.API_KEY) + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .build(), + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .build() + ) .withConnectionSpecification(new TestConnectionSpecification()) .build(); }); - Assert.assertEquals("Store instance specified with authentication configuration types (API_KEY) which are not covered by its store support 'test'", exception.getMessage()); + Assertions.assertEquals("Found multiple configurations for authentication mechanism 'UsernamePassword'", exception.getMessage()); + + exception = Assertions.assertThrows(RuntimeException.class, () -> + { + new StoreInstance.Builder(environment) + .withIdentifier("test-store") + .withStoreSupportIdentifier("test") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.API_KEY) + .build() + ) + .withConnectionSpecification(new TestConnectionSpecification()) + .build(); + }); + Assertions.assertEquals("Authentication mechanism 'APIKey' is not covered by store support 'test'. Supported mechanism(s):\n" + + "- UsernamePassword\n" + + "- Kerberos", exception.getMessage()); + + exception = Assertions.assertThrows(RuntimeException.class, () -> + { + new StoreInstance.Builder(environment) + .withIdentifier("test-store") + .withStoreSupportIdentifier("test") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .withAuthenticationConfigurationTypes(KerberosAuthenticationConfiguration.class) + .build() + ) + .withConnectionSpecification(new TestConnectionSpecification()) + .build(); + }); + Assertions.assertEquals("Authentication configuration type 'KerberosAuthenticationConfiguration' is not covered by store support 'test' for authentication mechanism 'UsernamePassword'. Supported configuration type(s):\n" + + "- UserPasswordAuthenticationConfiguration\n" + + "- EncryptedPrivateKeyPairAuthenticationConfiguration", exception.getMessage()); } private static class TestConnectionSpecification extends ConnectionSpecification { + @Override + public String shortId() + { + return null; + } + } + + @Test + public void testStoreInstanceManagement() + { + LegendEnvironment environment = new LegendEnvironment.Builder() + .withStoreSupport(new StoreSupport.Builder() + .withIdentifier("test") + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD) + .withAuthenticationConfigurationTypes(UserPasswordAuthenticationConfiguration.class) + .build(), + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.KERBEROS) + .withAuthenticationConfigurationTypes(KerberosAuthenticationConfiguration.class) + .build() + ) + .build()) + .build(); + + StoreInstance storeInstance = new StoreInstance.Builder(environment) + .withIdentifier("test-store") + .withStoreSupportIdentifier("test") + .withConnectionSpecification(new TestConnectionSpecification()) + .build(); + + StoreInstanceProvider storeInstanceProvider = new DefaultStoreInstanceProvider.Builder().withStoreInstance(storeInstance).build(); + + // failure + Exception exception; + + // error: store already registered + exception = Assertions.assertThrows(RuntimeException.class, () -> + { + new DefaultStoreInstanceProvider.Builder().withStoreInstances(storeInstance, storeInstance).build(); + }); + Assertions.assertEquals("Found multiple store instances with identifier 'test-store'", exception.getMessage()); + + // error: store not found + exception = Assertions.assertThrows(RuntimeException.class, () -> + { + storeInstanceProvider.lookup("unknown"); + }); + Assertions.assertEquals("Can't find store instance with identifier 'unknown'", exception.getMessage()); } } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-grammar/pom.xml b/legend-engine-xts-authentication/legend-engine-xt-authentication-grammar/pom.xml index 4652f5ccdd3..862f6eac3f3 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-grammar/pom.xml +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-authentication - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-core/pom.xml b/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-core/pom.xml index aea667763dd..efbfce9fb9d 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-core/pom.xml +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-core/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-authentication - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-gcp-federation/pom.xml b/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-gcp-federation/pom.xml index 111da6e60b6..78e13b4fcd7 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-gcp-federation/pom.xml +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-gcp-federation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-authentication - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-vault-aws/pom.xml b/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-vault-aws/pom.xml index a6b539c795e..76418f9d906 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-vault-aws/pom.xml +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-implementation-vault-aws/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-authentication - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/pom.xml b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/pom.xml index d363d2ba0b1..c922d68497d 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/pom.xml +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-authentication - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/CredentialVaultSecret.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/CredentialVaultSecret.java index a7974149099..e28191a88ca 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/CredentialVaultSecret.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/CredentialVaultSecret.java @@ -29,7 +29,13 @@ public CredentialVaultSecret() // jackson } - public T accept(PackageableElementVisitor visitor) + // TODO: @akphi - turn this into abstract, this default implementation is meant for backward compatibility + public String shortId() + { + return "CredentialVaultSecret"; + } + + public T accept(PackageableElementVisitor visitor) { return visitor.visit(this); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/EnvironmentCredentialVaultSecret.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/EnvironmentCredentialVaultSecret.java index 5558932e881..b217f15c179 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/EnvironmentCredentialVaultSecret.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/EnvironmentCredentialVaultSecret.java @@ -31,6 +31,13 @@ public EnvironmentCredentialVaultSecret(String envVariableName) this.envVariableName = envVariableName; } + @Override + public String shortId() + { + return "EnvironmentCredentialVaultSecret" + + "--envVariableName=" + envVariableName; + } + @Override public T accept(PackageableElementVisitor visitor) { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/PropertiesFileSecret.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/PropertiesFileSecret.java index 1d2bb110620..eccfc23b6cd 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/PropertiesFileSecret.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/PropertiesFileSecret.java @@ -31,6 +31,13 @@ public PropertiesFileSecret(String propertyName) this.propertyName = propertyName; } + @Override + public String shortId() + { + return "PropertiesFileSecret" + + "--propertyName=" + propertyName; + } + @Override public T accept(PackageableElementVisitor visitor) { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/SystemPropertiesSecret.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/SystemPropertiesSecret.java index 571cc0f441b..f506ad5442f 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/SystemPropertiesSecret.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/SystemPropertiesSecret.java @@ -31,6 +31,13 @@ public SystemPropertiesSecret(String systemPropertyName) this.systemPropertyName = systemPropertyName; } + @Override + public String shortId() + { + return "SystemPropertiesSecret" + + "--systemPropertyName=" + systemPropertyName; + } + @Override public T accept(PackageableElementVisitor visitor) { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSCredentials.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSCredentials.java index 52555234afb..778607a8740 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSCredentials.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSCredentials.java @@ -22,5 +22,7 @@ public abstract class AWSCredentials { public SourceInformation sourceInformation; + public abstract String shortId(); + public abstract T accept(AWSCredentialsVisitor visitor); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSDefaultCredentials.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSDefaultCredentials.java index 4781c894843..b1b1b863755 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSDefaultCredentials.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSDefaultCredentials.java @@ -23,6 +23,12 @@ public AWSDefaultCredentials() { } + @Override + public String shortId() + { + return "AWSDefaultCredentials"; + } + @Override public T accept(AWSCredentialsVisitor visitor) { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSTSAssumeRoleCredentials.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSTSAssumeRoleCredentials.java index 29137588b61..ce8634efce5 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSTSAssumeRoleCredentials.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSTSAssumeRoleCredentials.java @@ -36,6 +36,15 @@ public AWSSTSAssumeRoleCredentials(AWSCredentials awsCredentials, String roleArn this.roleSessionName = roleSessionName; } + @Override + public String shortId() + { + return "AWSSTSAssumeRoleCredentials" + + "--roleArn=" + roleArn + + "--roleSessionName=" + roleSessionName + + "--awsCredentials=" + awsCredentials.shortId(); + } + @Override public T accept(AWSCredentialsVisitor visitor) { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSecretsManagerSecret.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSecretsManagerSecret.java index 24f0456bbaa..ac33c9bd39a 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSecretsManagerSecret.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSSecretsManagerSecret.java @@ -40,6 +40,16 @@ public AWSSecretsManagerSecret(String secretId, String versionId, String version this.awsCredentials = awsCredentials; } + @Override + public String shortId() + { + return "AWSSecretsManagerSecret" + + "--secretId=" + secretId + + "--versionId=" + versionId + + "--versionStage=" + versionStage + + "--awsCredentials=" + awsCredentials.shortId(); + } + @Override public T accept(PackageableElementVisitor visitor) { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSStaticCredentials.java b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSStaticCredentials.java index e82640e30d4..e164fd88829 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSStaticCredentials.java +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/authentication/vault/aws/AWSStaticCredentials.java @@ -33,6 +33,14 @@ public AWSStaticCredentials(CredentialVaultSecret accessKeyId, CredentialVaultSe this.secretAccessKey = secretAccessKey; } + @Override + public String shortId() + { + return "AWSStaticCredentials" + + "--accessKeyId=" + accessKeyId.shortId() + + "--secretAccessKey=" + secretAccessKey.shortId(); + } + @Override public T accept(AWSCredentialsVisitor visitor) { diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-pure/pom.xml b/legend-engine-xts-authentication/legend-engine-xt-authentication-pure/pom.xml index ccafbe7a821..a0cef8d5af6 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-pure/pom.xml +++ b/legend-engine-xts-authentication/legend-engine-xt-authentication-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-authentication - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-authentication/pom.xml b/legend-engine-xts-authentication/pom.xml index 629f27f1067..cc85a7a15aa 100644 --- a/legend-engine-xts-authentication/pom.xml +++ b/legend-engine-xts-authentication/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-avro/legend-engine-xt-avro-pure/pom.xml b/legend-engine-xts-avro/legend-engine-xt-avro-pure/pom.xml index a3cd7240d9c..91e4f8b8804 100644 --- a/legend-engine-xts-avro/legend-engine-xt-avro-pure/pom.xml +++ b/legend-engine-xts-avro/legend-engine-xt-avro-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-avro - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-avro/legend-engine-xt-avro/pom.xml b/legend-engine-xts-avro/legend-engine-xt-avro/pom.xml index cd2d036c5d3..d8425a2f80f 100644 --- a/legend-engine-xts-avro/legend-engine-xt-avro/pom.xml +++ b/legend-engine-xts-avro/legend-engine-xt-avro/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-avro - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-avro/pom.xml b/legend-engine-xts-avro/pom.xml index ea9c79d867d..184fe2ba48f 100644 --- a/legend-engine-xts-avro/pom.xml +++ b/legend-engine-xts-avro/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-changetoken/README.md b/legend-engine-xts-changetoken/README.md index 2e11c1abb55..d01d986a285 100644 --- a/legend-engine-xts-changetoken/README.md +++ b/legend-engine-xts-changetoken/README.md @@ -15,6 +15,7 @@ For those kind of detected operations user confirmation should be required, in o ## Change Token Types - **AddedClass** / **RemovedClass** - current no-op - should be emitted if a class of entity was added/removed. +- **RenamedClass** - token describing that a class was renamed or moved to another package. - **AddField** / **RemoveField** - token describing that a field was added or removed to specific entity class. This token should provide **fieldName**, **fieldType** and **class** properties together with **defaultValue**, of e.g. **ConstValue** **@type**. The **value** should be of expected primitive or nested object type (in case of object type **@type** property is mandatory). diff --git a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-compiler/pom.xml b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-compiler/pom.xml index 9c72d7f010a..041ca82feaf 100644 --- a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-compiler/pom.xml +++ b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-compiler/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-changetoken org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-compiler/src/test/java/org/finos/legend/engine/changetoken/generation/GenerateCastRenamedClassTest.java b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-compiler/src/test/java/org/finos/legend/engine/changetoken/generation/GenerateCastRenamedClassTest.java new file mode 100644 index 00000000000..8ee7752ff70 --- /dev/null +++ b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-compiler/src/test/java/org/finos/legend/engine/changetoken/generation/GenerateCastRenamedClassTest.java @@ -0,0 +1,180 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.changetoken.generation; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Map; + +import static org.junit.Assert.assertThrows; + +public class GenerateCastRenamedClassTest extends GenerateCastTestBase +{ + @BeforeClass + public static void setupSuite() throws IOException, ClassNotFoundException + { + setupSuiteFromJson("{\n" + + " \"@type\": \"meta::pure::changetoken::Versions\",\n" + + " \"versions\": [\n" + + " {\n" + + " \"@type\": \"meta::pure::changetoken::Version\",\n" + + " \"version\": \"ftdm:abcdefg123\"\n" + + " },\n" + + " {\n" + + " \"@type\": \"meta::pure::changetoken::Version\",\n" + + " \"version\": \"ftdm:abcdefg456\",\n" + + " \"prevVersion\": \"ftdm:abcdefg123\",\n" + + " \"changeTokens\": [\n" + + " {\n" + + " \"@type\": \"meta::pure::changetoken::RenamedClass\",\n" + + " \"newName\": \"meta::pure::changetoken::tests::nested::NewSampleClass\",\n" + + " \"class\": \"meta::pure::changetoken::tests::SampleClass\"\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}\n"); + } + + @Test + public void testUpcast() throws JsonProcessingException, NoSuchMethodException, InvocationTargetException, IllegalAccessException + { + Map jsonNode = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg123\", \n" + + " \"@type\": \"meta::pure::changetoken::tests::SampleClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::SampleClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"1d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::SampleClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"2d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::SampleClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"3d\"}}]\n" + + " ],\n" + + " \"abc\": {\"@type\":\"Custom\", \"value\":\"4d\"}\n" + + "}", Map.class); + Map jsonNodeOut = (Map) compiledClass.getMethod("upcast", Map.class).invoke(null, jsonNode); + + Map expectedJsonNodeOut = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg456\",\n" + + " \"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"1d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"2d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"3d\"}}]\n" + + " ],\n" + + " \"abc\": {\"@type\":\"Custom\", \"value\":\"4d\"}\n" + + "}", Map.class); // updated version and new default value field added + Assert.assertEquals(expectedJsonNodeOut, jsonNodeOut); + } + + @Test + public void testUpcastType() throws JsonProcessingException, NoSuchMethodException, InvocationTargetException, IllegalAccessException + { + Map jsonNode = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg123\", \n" + + " \"@type\": \"meta::pure::changetoken::tests::OtherClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"1d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"2d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"3d\"}}]\n" + + " ],\n" + + " \"abc\": {\"@type\":\"Custom\", \"value\":\"4d\"}\n" + + "}", Map.class); + Map jsonNodeOut = (Map) compiledClass.getMethod("upcast", Map.class).invoke(null, jsonNode); + + Map expectedJsonNodeOut = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg456\",\n" + + " \"@type\": \"meta::pure::changetoken::tests::OtherClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"1d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"2d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"abc\": {\"@type\":\"Custom\", \"value\":\"3d\"}}]\n" + + " ],\n" + + " \"abc\": {\"@type\":\"Custom\", \"value\":\"4d\"}\n" + + "}", Map.class); // updated version and new default value field added + Assert.assertEquals(expectedJsonNodeOut, jsonNodeOut); + } + + @Test + public void testDowncast() throws JsonProcessingException, NoSuchMethodException, InvocationTargetException, IllegalAccessException + { + ObjectMapper mapper = new ObjectMapper(); + Map jsonNode = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg456\",\n" + + " \"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"4d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"3d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::nested::NewSampleClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"2d\"}}]\n" + + " ],\n" + + " \"xyz\": {\"@type\":\"Custom\", \"value\":\"1d\"}\n" + + "}", Map.class); + Map jsonNodeOut = (Map) compiledClass.getMethod("downcast", Map.class, String.class) + .invoke(null, jsonNode, "ftdm:abcdefg123"); + Map expectedJsonNodeOut = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg123\", \n" + + " \"@type\": \"meta::pure::changetoken::tests::SampleClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::SampleClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"4d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::SampleClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"3d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::SampleClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"2d\"}}]\n" + + " ],\n" + + " \"xyz\": {\"@type\":\"Custom\", \"value\":\"1d\"}\n" + + "}", Map.class); // remove default values + Assert.assertEquals(expectedJsonNodeOut, jsonNodeOut); + } + + @Test + public void testDowncastType() throws JsonProcessingException, NoSuchMethodException, InvocationTargetException, IllegalAccessException + { + ObjectMapper mapper = new ObjectMapper(); + Map jsonNode = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg456\",\n" + + " \"@type\": \"meta::pure::changetoken::tests::OtherClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"4d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"3d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"2d\"}}]\n" + + " ],\n" + + " \"xyz\": {\"@type\":\"Custom\", \"value\":\"1d\"}\n" + + "}", Map.class); + Map jsonNodeOut = (Map) compiledClass.getMethod("downcast", Map.class, String.class) + .invoke(null, jsonNode, "ftdm:abcdefg123"); + Map expectedJsonNodeOut = mapper.readValue( + "{\n" + + " \"version\":\"ftdm:abcdefg123\", \n" + + " \"@type\": \"meta::pure::changetoken::tests::OtherClass\",\n" + + " \"innerObject\": {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"4d\"}},\n" + + " \"innerNestedArray\":[\n" + + " {\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"3d\"}},\n" + + " [{\"@type\": \"meta::pure::changetoken::tests::OtherClass\", \"xyz\": {\"@type\":\"Custom\", \"value\":\"2d\"}}]\n" + + " ],\n" + + " \"xyz\": {\"@type\":\"Custom\", \"value\":\"1d\"}\n" + + "}", Map.class); // remove default values + Assert.assertEquals(expectedJsonNodeOut, jsonNodeOut); + } +} + diff --git a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/pom.xml b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/pom.xml index 94bae3339fd..75678cbcfb2 100644 --- a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/pom.xml +++ b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-changetoken - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/cast_generation.pure b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/cast_generation.pure index e777e1b268b..bf0afebd2b8 100644 --- a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/cast_generation.pure +++ b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/cast_generation.pure @@ -268,6 +268,7 @@ function <> meta::pure::changetoken::cast_generation::_generateS t:RenameField[1]|_handleRenameFieldUpcast($class2, $t, $objVar, $typeVar, $version.version, $pathVar, $rootObjVar, $resVar), t:AddedClass[1]|[], t:RemovedClass[1]|[], + t:RenamedClass[1]|_handleRenamedClassUpcast($class2, $t, $objVar, $typeVar, $version.version, $pathVar, $rootObjVar, $resVar), t:ChangeFieldType[1]|_handleChangeFieldTypeUpcast($class2, $t, $objVar, $typeVar, $version.version, $pathVar, $rootObjVar, $resVar) ])) ->fold({t,res|$res->concatenate($t)}, []); @@ -343,6 +344,7 @@ function <> meta::pure::changetoken::cast_generation::_generateS t:RenameField[1]|_handleRenameFieldDowncast($class2, $t, $objVar, $typeVar, $version.version, $pathVar, $rootObjVar, $resVar), t:AddedClass[1]|[], t:RemovedClass[1]|[], + t:RenamedClass[1]|_handleRenamedClassDowncast($class2, $t, $objVar, $typeVar, $version.version, $pathVar, $rootObjVar, $resVar), t:ChangeFieldType[1]|_handleChangeFieldTypeDowncast($class2, $t, $objVar, $typeVar, $version.version, $pathVar, $rootObjVar, $resVar) ])) ->fold({t,res|$res->concatenate($t)}, []); @@ -569,6 +571,31 @@ function <> meta::pure::changetoken::cast_generation::_handleRen ); } +function <> meta::pure::changetoken::cast_generation::_handleRenamedClassUpcast +( +class:meta::external::language::java::metamodel::Class[1], +token:RenamedClass[1], objVar:Code[1], typeVar:Code[1], version:String[1], pathVar:Code[1], rootObjVar:Code[1], resVar:Code[1] +):Code[1] +{ + // if (type.equals(token.class)) + j_if( + $typeVar->j_invoke('equals', [j_string($token.class)]), + $objVar->j_invoke('put', [j_string('@type'), j_string($token.newName)]) + ); +} + +function <> meta::pure::changetoken::cast_generation::_handleRenamedClassDowncast +( +class:meta::external::language::java::metamodel::Class[1], +token:RenamedClass[1], objVar:Code[1], typeVar:Code[1], version:String[1], pathVar:Code[1], rootObjVar:Code[1], resVar:Code[1] +):Code[1] +{ + // if (type.equals(token.newName)) + j_if( + $typeVar->j_invoke('equals', [j_string($token.newName)]), + $objVar->j_invoke('put', [j_string('@type'), j_string($token.class)]) + ); +} function <> meta::pure::changetoken::cast_generation::_handleAddFieldUpcast ( diff --git a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/changetoken.pure b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/changetoken.pure index bbeecfe3b36..4872e4904a9 100644 --- a/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/changetoken.pure +++ b/legend-engine-xts-changetoken/legend-engine-xt-changetoken-pure/src/main/resources/core_pure_changetoken/changetoken.pure @@ -85,6 +85,18 @@ meta::pure::changetoken::RemovedClass extends ClassChangeToken + ')' }: String[1]; } +Class +{doc.doc='a class was renamed in this version'} +meta::pure::changetoken::RenamedClass extends ClassChangeToken +{ + newName: String[1]; + + toString() { 'RenamedClass(' + + 'class=' + $this.class->toString() + + 'newName=' + $this.newName->toString() + + ')' }: String[1]; +} + Class <> {doc.doc='root of a hierarchy of getters for a default value of a newly-added field'} meta::pure::changetoken::FieldGetter diff --git a/legend-engine-xts-changetoken/pom.xml b/legend-engine-xts-changetoken/pom.xml index aa9e90efc89..7e31aba7d38 100644 --- a/legend-engine-xts-changetoken/pom.xml +++ b/legend-engine-xts-changetoken/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-daml/legend-engine-xt-daml-grammar/pom.xml b/legend-engine-xts-daml/legend-engine-xt-daml-grammar/pom.xml index 336cea660f8..f3fb4c9d12e 100644 --- a/legend-engine-xts-daml/legend-engine-xt-daml-grammar/pom.xml +++ b/legend-engine-xts-daml/legend-engine-xt-daml-grammar/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-daml org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-daml/legend-engine-xt-daml-model/pom.xml b/legend-engine-xts-daml/legend-engine-xt-daml-model/pom.xml index 4b216ab5c14..1c840bf5aac 100644 --- a/legend-engine-xts-daml/legend-engine-xt-daml-model/pom.xml +++ b/legend-engine-xts-daml/legend-engine-xt-daml-model/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-daml org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-daml/legend-engine-xt-daml-pure/pom.xml b/legend-engine-xts-daml/legend-engine-xt-daml-pure/pom.xml index f4563ad65fa..a9e03fa4936 100644 --- a/legend-engine-xts-daml/legend-engine-xt-daml-pure/pom.xml +++ b/legend-engine-xts-daml/legend-engine-xt-daml-pure/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-daml org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-daml/pom.xml b/legend-engine-xts-daml/pom.xml index f691d743bcf..9ff6a3a1df7 100644 --- a/legend-engine-xts-daml/pom.xml +++ b/legend-engine-xts-daml/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-push/legend-engine-xt-data-push-server/pom.xml b/legend-engine-xts-data-push/legend-engine-xt-data-push-server/pom.xml index f7c067d0480..82ac1d32555 100644 --- a/legend-engine-xts-data-push/legend-engine-xt-data-push-server/pom.xml +++ b/legend-engine-xts-data-push/legend-engine-xt-data-push-server/pom.xml @@ -3,7 +3,7 @@ legend-engine-xts-data-push org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-push/legend-engine-xt-data-push-server/src/main/java/org/finos/legend/engine/datapush/server/ConnectionFactoryBundle.java b/legend-engine-xts-data-push/legend-engine-xt-data-push-server/src/main/java/org/finos/legend/engine/datapush/server/ConnectionFactoryBundle.java index 51bbcec8268..2b96dfd3940 100644 --- a/legend-engine-xts-data-push/legend-engine-xt-data-push-server/src/main/java/org/finos/legend/engine/datapush/server/ConnectionFactoryBundle.java +++ b/legend-engine-xts-data-push/legend-engine-xt-data-push-server/src/main/java/org/finos/legend/engine/datapush/server/ConnectionFactoryBundle.java @@ -21,14 +21,15 @@ import org.finos.legend.authentication.vault.CredentialVault; import org.finos.legend.authentication.vault.impl.EnvironmentCredentialVault; import org.finos.legend.authentication.vault.impl.SystemPropertiesCredentialVault; +import org.finos.legend.connection.AuthenticationMechanismConfiguration; import org.finos.legend.connection.ConnectionFactory; import org.finos.legend.connection.DatabaseType; -import org.finos.legend.connection.DefaultAuthenticationMechanismProvider; -import org.finos.legend.connection.DefaultConnectionBuilderProvider; -import org.finos.legend.connection.DefaultCredentialBuilderProvider; -import org.finos.legend.connection.EnvironmentConfiguration; +import org.finos.legend.connection.impl.DefaultStoreInstanceProvider; import org.finos.legend.connection.IdentityFactory; +import org.finos.legend.connection.LegendEnvironment; import org.finos.legend.connection.RelationalDatabaseStoreSupport; +import org.finos.legend.connection.StoreInstanceProvider; +import org.finos.legend.connection.impl.UserPasswordAuthenticationConfiguration; import org.finos.legend.connection.protocol.AuthenticationMechanismType; import java.util.List; @@ -36,8 +37,9 @@ public class ConnectionFactoryBundle implements ConfiguredBundle { - private static EnvironmentConfiguration environmentConfiguration; + private static LegendEnvironment environment; private static IdentityFactory identityFactory; + private static StoreInstanceProvider storeInstanceProvider; private static ConnectionFactory connectionFactory; private final List credentialVaults; private final Function configSupplier; @@ -56,39 +58,39 @@ public void initialize(Bootstrap bootstrap) @Override public void run(C configuration, Environment environment) { - environmentConfiguration = new EnvironmentConfiguration.Builder() + ConnectionFactoryBundle.environment = new LegendEnvironment.Builder() // TODO: @akphi - add a property credential vault and load its content up from the config // .withVault(propertiesFileCredentialVault) .withVault(new SystemPropertiesCredentialVault()) .withVault(new EnvironmentCredentialVault()) .withVaults(this.credentialVaults) - .withStoreSupport(new RelationalDatabaseStoreSupport.Builder() + .withStoreSupport(new RelationalDatabaseStoreSupport.Builder(DatabaseType.POSTGRES) .withIdentifier("Postgres") - .withDatabase(DatabaseType.POSTGRES) - .withAuthenticationMechanisms( - AuthenticationMechanismType.USER_PASSWORD + .withAuthenticationMechanismConfigurations( + new AuthenticationMechanismConfiguration.Builder(AuthenticationMechanismType.USER_PASSWORD).withAuthenticationConfigurationTypes(UserPasswordAuthenticationConfiguration.class).build() ).build()) - .withAuthenticationMechanismProvider(new DefaultAuthenticationMechanismProvider()) // can also use service loader .build(); - identityFactory = new IdentityFactory.Builder(environmentConfiguration) + identityFactory = new IdentityFactory.Builder(ConnectionFactoryBundle.environment) .build(); - connectionFactory = new ConnectionFactory.Builder(environmentConfiguration) - .withCredentialBuilderProvider(new DefaultCredentialBuilderProvider()) // can also use service loader - .withConnectionBuilderProvider(new DefaultConnectionBuilderProvider()) // can also use service loader + storeInstanceProvider = new DefaultStoreInstanceProvider.Builder().build(); + + connectionFactory = new ConnectionFactory.Builder(ConnectionFactoryBundle.environment, storeInstanceProvider) +// .withCredentialBuilderProvider(new DefaultCredentialBuilderProvider()) // can also use service loader +// .withConnectionBuilderProvider(new DefaultConnectionBuilderProvider()) // can also use service loader .build(); // TODO: register store instances } - public static EnvironmentConfiguration getEnvironmentConfiguration() + public static LegendEnvironment getEnvironment() { - if (environmentConfiguration == null) + if (environment == null) { throw new IllegalStateException("Environment configuration has not been set!"); } - return environmentConfiguration; + return environment; } public static IdentityFactory getIdentityFactory() diff --git a/legend-engine-xts-data-push/pom.xml b/legend-engine-xts-data-push/pom.xml index 05ff37fd3d7..5643018d0f0 100644 --- a/legend-engine-xts-data-push/pom.xml +++ b/legend-engine-xts-data-push/pom.xml @@ -3,7 +3,7 @@ legend-engine org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-api/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-api/pom.xml index f7edda0c1e8..51f9c3604ae 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-api/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-data-space - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml index 4a82e4aebcc..c2df0185646 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-compiler/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-data-space org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/pom.xml index afb98755655..02c4c5502f8 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-generation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-data-space - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/pom.xml index 5ff01cc6e41..c7dd83d2cca 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-grammar/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-data-space - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/pom.xml index fab95901935..69d4c4c1b5f 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-data-space - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-pure-metamodel/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-pure-metamodel/pom.xml index 30c23429182..ea17b458e66 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-pure-metamodel/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-pure-metamodel/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-data-space - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/legend-engine-xt-data-space-pure/pom.xml b/legend-engine-xts-data-space/legend-engine-xt-data-space-pure/pom.xml index a323a275c24..59a7613dd4f 100644 --- a/legend-engine-xts-data-space/legend-engine-xt-data-space-pure/pom.xml +++ b/legend-engine-xts-data-space/legend-engine-xt-data-space-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-data-space - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-data-space/pom.xml b/legend-engine-xts-data-space/pom.xml index 55a747dbf5c..8229fb5e8ff 100644 --- a/legend-engine-xts-data-space/pom.xml +++ b/legend-engine-xts-data-space/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-diagram/legend-engine-xt-diagram-api/pom.xml b/legend-engine-xts-diagram/legend-engine-xt-diagram-api/pom.xml index 2dbb985c06a..456d1cc2c40 100644 --- a/legend-engine-xts-diagram/legend-engine-xt-diagram-api/pom.xml +++ b/legend-engine-xts-diagram/legend-engine-xt-diagram-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-diagram - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-diagram/legend-engine-xt-diagram-compiler/pom.xml b/legend-engine-xts-diagram/legend-engine-xt-diagram-compiler/pom.xml index eb046794fe0..2af9714265e 100644 --- a/legend-engine-xts-diagram/legend-engine-xt-diagram-compiler/pom.xml +++ b/legend-engine-xts-diagram/legend-engine-xt-diagram-compiler/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-diagram org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-diagram/legend-engine-xt-diagram-grammar/pom.xml b/legend-engine-xts-diagram/legend-engine-xt-diagram-grammar/pom.xml index 6cfa82371ea..57e083ddecf 100644 --- a/legend-engine-xts-diagram/legend-engine-xt-diagram-grammar/pom.xml +++ b/legend-engine-xts-diagram/legend-engine-xt-diagram-grammar/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-diagram - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-diagram/legend-engine-xt-diagram-protocol/pom.xml b/legend-engine-xts-diagram/legend-engine-xt-diagram-protocol/pom.xml index 515e11603cd..0860e0fd381 100644 --- a/legend-engine-xts-diagram/legend-engine-xt-diagram-protocol/pom.xml +++ b/legend-engine-xts-diagram/legend-engine-xt-diagram-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-diagram - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-diagram/legend-engine-xt-diagram-pure-metamodel/pom.xml b/legend-engine-xts-diagram/legend-engine-xt-diagram-pure-metamodel/pom.xml index 86f77322da7..c47395b7356 100644 --- a/legend-engine-xts-diagram/legend-engine-xt-diagram-pure-metamodel/pom.xml +++ b/legend-engine-xts-diagram/legend-engine-xt-diagram-pure-metamodel/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-diagram - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-diagram/legend-engine-xt-diagram-pure/pom.xml b/legend-engine-xts-diagram/legend-engine-xt-diagram-pure/pom.xml index 9015e6fac3b..1c888511471 100644 --- a/legend-engine-xts-diagram/legend-engine-xt-diagram-pure/pom.xml +++ b/legend-engine-xts-diagram/legend-engine-xt-diagram-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-diagram - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-diagram/pom.xml b/legend-engine-xts-diagram/pom.xml index 77da59cd96a..a07e9f8cc0a 100644 --- a/legend-engine-xts-diagram/pom.xml +++ b/legend-engine-xts-diagram/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/pom.xml b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/pom.xml index 7b81f53bac1..134273b6041 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/pom.xml +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-executionPlan/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-elasticsearch - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-grammar/pom.xml b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-grammar/pom.xml index b8cea53834e..54182e2bf8b 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-grammar/pom.xml +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-elasticsearch - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-protocol/pom.xml b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-protocol/pom.xml index 36fa09fb5b2..f1154c4618e 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-protocol/pom.xml +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-elasticsearch - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/pom.xml b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/pom.xml index b6d1474a1ec..5de2370ded0 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/pom.xml +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-V7-pure-metamodel/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-elasticsearch - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/pom.xml b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/pom.xml index b88e2248122..c53a1621445 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/pom.xml +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-executionPlan-test/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-elasticsearch - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-protocol-utils/pom.xml b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-protocol-utils/pom.xml index bcd8dadaa47..1215486c10d 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-protocol-utils/pom.xml +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-protocol-utils/pom.xml @@ -4,7 +4,7 @@ org.finos.legend.engine legend-engine-xts-elasticsearch - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-elasticsearch-protocol-utils diff --git a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-pure-specification-metamodel/pom.xml b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-pure-specification-metamodel/pom.xml index 4a8d617ad84..c30e19cf8d2 100644 --- a/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-pure-specification-metamodel/pom.xml +++ b/legend-engine-xts-elasticsearch/legend-engine-xt-elasticsearch-pure-specification-metamodel/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-elasticsearch - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-elasticsearch/pom.xml b/legend-engine-xts-elasticsearch/pom.xml index 450d1bb5a24..239aa220e65 100644 --- a/legend-engine-xts-elasticsearch/pom.xml +++ b/legend-engine-xts-elasticsearch/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-driver-bloomberg/pom.xml b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-driver-bloomberg/pom.xml index fa962fa9162..2bb9c91570c 100644 --- a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-driver-bloomberg/pom.xml +++ b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-driver-bloomberg/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-flatdata org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/pom.xml b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/pom.xml index 867cdf67f85..61b63ff0b32 100644 --- a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/pom.xml +++ b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-flatdata - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-test/pom.xml b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-test/pom.xml index d511adc3fb9..242baa0efd3 100644 --- a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-test/pom.xml +++ b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-javaPlatformBinding-test/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-flatdata - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/pom.xml b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/pom.xml index a87df1bcd5c..7b8b4cb0fc5 100644 --- a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/pom.xml +++ b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-model/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-flatdata - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/pom.xml b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/pom.xml index 6101a164fe4..5c764beb36d 100644 --- a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/pom.xml +++ b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-flatdata - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-runtime/pom.xml b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-runtime/pom.xml index 27541d20596..1a1dd5a2e4f 100644 --- a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-runtime/pom.xml +++ b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-runtime/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-flatdata - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/pom.xml b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/pom.xml index 89074abad8b..eae18b2f3af 100644 --- a/legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/pom.xml +++ b/legend-engine-xts-flatdata/legend-engine-xt-flatdata-shared/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-flatdata - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-flatdata/pom.xml b/legend-engine-xts-flatdata/pom.xml index f085be8a74f..1efad88f112 100644 --- a/legend-engine-xts-flatdata/pom.xml +++ b/legend-engine-xts-flatdata/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/pom.xml b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/pom.xml index 5829f1b3066..3c4b9b397cf 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/pom.xml +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-functionActivator - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java index cd27af444ae..7e314bd1645 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/api/FunctionActivatorAPI.java @@ -24,13 +24,14 @@ import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.functionActivator.api.input.FunctionActivatorInput; import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; import org.finos.legend.engine.functionActivator.service.FunctionActivatorLoader; import org.finos.legend.engine.functionActivator.service.FunctionActivatorService; import org.finos.legend.engine.language.pure.compiler.Compiler; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.modelManager.ModelManager; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentResult; +//import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; +import org.finos.legend.engine.functionActivator.deployment.DeploymentResult; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage; import org.finos.legend.engine.protocol.pure.PureClientVersions; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; @@ -63,7 +64,7 @@ public class FunctionActivatorAPI private final ModelManager modelManager; private final PureModel emptyModel; private final Function> routerExtensions; - private List runtimeDeploymentConfig = Lists.mutable.empty(); + private List runtimeDeploymentConfig = Lists.mutable.empty(); public FunctionActivatorAPI(ModelManager modelManager, Function> routerExtensions) { @@ -72,7 +73,7 @@ public FunctionActivatorAPI(ModelManager modelManager, Function activatorConfigurations, Function> routerExtensions) + public FunctionActivatorAPI(ModelManager modelManager, List activatorConfigurations, Function> routerExtensions) { this(modelManager, routerExtensions); this.runtimeDeploymentConfig = activatorConfigurations; @@ -103,7 +104,7 @@ public Response validate(FunctionActivatorInput input, @ApiParam(hidden = true) String clientVersion = input.clientVersion == null ? PureClientVersions.production : input.clientVersion; PureModel pureModel = modelManager.loadModel(input.model, clientVersion, profiles, null); Root_meta_external_function_activator_FunctionActivator activator = (Root_meta_external_function_activator_FunctionActivator) pureModel.getPackageableElement(input.functionActivator); - FunctionActivatorService service = getActivatorService(activator, pureModel); + FunctionActivatorService service = getActivatorService(activator, pureModel); return Response.ok(objectMapper.writeValueAsString(service.validate(profiles, pureModel, activator, input.model, routerExtensions))).type(MediaType.APPLICATION_JSON_TYPE).build(); } catch (Exception ex) @@ -126,8 +127,8 @@ public Response publishToSandbox(FunctionActivatorInput input, @ApiParam(hidden String clientVersion = input.clientVersion == null ? PureClientVersions.production : input.clientVersion; PureModel pureModel = modelManager.loadModel(input.model, clientVersion, profiles, null); Root_meta_external_function_activator_FunctionActivator activator = (Root_meta_external_function_activator_FunctionActivator) pureModel.getPackageableElement(input.functionActivator); - FunctionActivatorService service = getActivatorService(activator,pureModel); - return Response.ok(objectMapper.writeValueAsString(service.publishToSandbox(profiles, pureModel, activator, input.model, service.selectConfig(this.runtimeDeploymentConfig, DeploymentStage.SANDBOX), routerExtensions))).type(MediaType.APPLICATION_JSON_TYPE).build(); + FunctionActivatorService service = getActivatorService(activator,pureModel); + return Response.ok(objectMapper.writeValueAsString(service.publishToSandbox(profiles, pureModel, activator, input.model, service.selectConfig(this.runtimeDeploymentConfig), routerExtensions))).type(MediaType.APPLICATION_JSON_TYPE).build(); } catch (Exception ex) { @@ -149,7 +150,7 @@ public Response renderArtifact(FunctionActivatorInput input, @ApiParam(hidden = String clientVersion = input.clientVersion == null ? PureClientVersions.production : input.clientVersion; PureModel pureModel = modelManager.loadModel(input.model, clientVersion, profiles, null); Root_meta_external_function_activator_FunctionActivator activator = (Root_meta_external_function_activator_FunctionActivator) pureModel.getPackageableElement(input.functionActivator); - FunctionActivatorService service = getActivatorService(activator, pureModel); + FunctionActivatorService service = getActivatorService(activator, pureModel); return Response.ok(objectMapper.writeValueAsString(service.renderArtifact(pureModel, activator, input.model, clientVersion,routerExtensions))).type(MediaType.APPLICATION_JSON_TYPE).build(); } catch (Exception ex) @@ -159,9 +160,9 @@ public Response renderArtifact(FunctionActivatorInput input, @ApiParam(hidden = } } - public FunctionActivatorService getActivatorService(Root_meta_external_function_activator_FunctionActivator activator, PureModel pureModel) + public FunctionActivatorService getActivatorService(Root_meta_external_function_activator_FunctionActivator activator, PureModel pureModel) { - FunctionActivatorService service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst(); + FunctionActivatorService service = FunctionActivatorLoader.extensions().select(c -> c.supports(activator)).getFirst(); if (service == null) { throw new RuntimeException(activator.getClass().getSimpleName() + "is not supported!"); diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/DeploymentManager.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/DeploymentManager.java index c2221880524..302270e5c49 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/DeploymentManager.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/DeploymentManager.java @@ -15,20 +15,17 @@ package org.finos.legend.engine.functionActivator.deployment; import org.eclipse.collections.api.list.MutableList; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentResult; -import org.finos.legend.pure.generated.Root_meta_external_function_activator_FunctionActivator; import org.pac4j.core.profile.CommonProfile; import java.util.List; -public interface DeploymentManager +public interface DeploymentManager { - public V deploy(MutableList profiles, U artifact, T activator); + public V deploy(MutableList profiles, U artifact); - public V deploy(MutableList profiles, U artifact, T activator, List availableRuntimeConfigurations); + public V deploy(MutableList profiles, U artifact, List availableRuntimeConfigurations); - public boolean canDeploy(T activator); + public boolean canDeploy(FunctionActivatorArtifact activatorArtifact); } diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/src/main/java/org/finos/legend/engine/protocol/functionActivator/metamodel/DeploymentResult.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/DeploymentResult.java similarity index 86% rename from legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/src/main/java/org/finos/legend/engine/protocol/functionActivator/metamodel/DeploymentResult.java rename to legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/DeploymentResult.java index 5edb409f35d..4f487838f45 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/src/main/java/org/finos/legend/engine/protocol/functionActivator/metamodel/DeploymentResult.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/DeploymentResult.java @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.engine.protocol.functionActivator.metamodel; +package org.finos.legend.engine.functionActivator.deployment; public class DeploymentResult { + public String activatorIdentifier; public boolean successful; } diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorArtifact.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorArtifact.java index 2b5ce1a61b6..5a29cfb0043 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorArtifact.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorArtifact.java @@ -16,4 +16,6 @@ public class FunctionActivatorArtifact { + public FunctionActivatorDeploymentContent content; + public FunctionActivatorDeploymentConfiguration deploymentConfiguration; } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialBuilderProvider.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorDeploymentConfiguration.java similarity index 80% rename from legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialBuilderProvider.java rename to legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorDeploymentConfiguration.java index 51700704cb6..963ec5e2587 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/CredentialBuilderProvider.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorDeploymentConfiguration.java @@ -12,11 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection; +package org.finos.legend.engine.functionActivator.deployment; -import java.util.List; - -public interface CredentialBuilderProvider +public class FunctionActivatorDeploymentConfiguration { - List getBuilders(); } diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismLoader.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorDeploymentContent.java similarity index 72% rename from legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismLoader.java rename to legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorDeploymentContent.java index 377d2aa4656..47ecf7745ee 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismLoader.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/deployment/FunctionActivatorDeploymentContent.java @@ -12,13 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection; +package org.finos.legend.engine.functionActivator.deployment; -import org.finos.legend.connection.protocol.AuthenticationMechanism; - -import java.util.List; - -public interface AuthenticationMechanismLoader +public class FunctionActivatorDeploymentContent { - List getMechanisms(); } diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/service/FunctionActivatorService.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/service/FunctionActivatorService.java index bec3d361535..7086ba02aa6 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/service/FunctionActivatorService.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-api/src/main/java/org/finos/legend/engine/functionActivator/service/FunctionActivatorService.java @@ -19,10 +19,11 @@ import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo; import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorArtifact; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentResult; +import org.finos.legend.engine.functionActivator.deployment.DeploymentResult; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext; import org.finos.legend.pure.generated.Root_meta_external_function_activator_FunctionActivator; import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; @@ -30,7 +31,7 @@ import java.util.List; -public interface FunctionActivatorService +public interface FunctionActivatorService { FunctionActivatorInfo info(PureModel pureModel, String version); @@ -42,5 +43,5 @@ public interface FunctionActivatorService> routerExtensions); - List selectConfig(List configurations, DeploymentStage state); + List selectConfig(List configurations); } diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/pom.xml b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/pom.xml index dbc3bad3d00..d124e32547e 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/pom.xml +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-functionActivator - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/src/main/java/org/finos/legend/engine/protocol/functionActivator/metamodel/DeploymentConfiguration.java b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/src/main/java/org/finos/legend/engine/protocol/functionActivator/metamodel/DeploymentConfiguration.java index a9e8b0749bb..8997bd54abf 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/src/main/java/org/finos/legend/engine/protocol/functionActivator/metamodel/DeploymentConfiguration.java +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-protocol/src/main/java/org/finos/legend/engine/protocol/functionActivator/metamodel/DeploymentConfiguration.java @@ -20,5 +20,5 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "_type") public class DeploymentConfiguration extends PackageableElement { - public DeploymentStage stage; + //public DeploymentStage stage; } diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/pom.xml b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/pom.xml index f8d762220d5..80638a86ec4 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/pom.xml +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-functionActivator - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/src/main/resources/core_function_activator/metamodel.pure b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/src/main/resources/core_function_activator/metamodel.pure index 9944f30b3c1..c3a89576d7b 100644 --- a/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/src/main/resources/core_function_activator/metamodel.pure +++ b/legend-engine-xts-functionActivator/legend-engine-xt-functionActivator-pure/src/main/resources/core_function_activator/metamodel.pure @@ -9,7 +9,6 @@ Class meta::external::function::activator::FunctionActivator extends Packageable Class meta::external::function::activator::DeploymentConfiguration extends PackageableElement { - stage: DeploymentStage[1]; } Class meta::external::function::activator::DeploymentResult diff --git a/legend-engine-xts-functionActivator/pom.xml b/legend-engine-xts-functionActivator/pom.xml index 40cdee952af..d97c3ce3acd 100644 --- a/legend-engine-xts-functionActivator/pom.xml +++ b/legend-engine-xts-functionActivator/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-generation/legend-engine-external-shared/pom.xml b/legend-engine-xts-generation/legend-engine-external-shared/pom.xml index 1ed8d13f5de..a527e670f5f 100644 --- a/legend-engine-xts-generation/legend-engine-external-shared/pom.xml +++ b/legend-engine-xts-generation/legend-engine-external-shared/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation-pure/pom.xml b/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation-pure/pom.xml index 0336d8acdfe..2c85be6c76d 100644 --- a/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation-pure/pom.xml +++ b/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation/pom.xml b/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation/pom.xml index 774c39a7eea..79517ed8cad 100644 --- a/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation/pom.xml +++ b/legend-engine-xts-generation/legend-engine-language-pure-dsl-generation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-dsl-generation diff --git a/legend-engine-xts-generation/legend-engine-xt-artifact-generation-api/pom.xml b/legend-engine-xts-generation/legend-engine-xt-artifact-generation-api/pom.xml index ff205e703d7..b569e5bdbe5 100644 --- a/legend-engine-xts-generation/legend-engine-xt-artifact-generation-api/pom.xml +++ b/legend-engine-xts-generation/legend-engine-xt-artifact-generation-api/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-generation org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-generation/pom.xml b/legend-engine-xts-generation/pom.xml index ee35eb80ecb..332904fefd8 100644 --- a/legend-engine-xts-generation/pom.xml +++ b/legend-engine-xts-generation/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-compiler/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-compiler/pom.xml index 9f68c834864..65d910abb1b 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-compiler/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-compiler/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -73,7 +73,7 @@ org.finos.legend.engine legend-engine-protocol-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT org.finos.legend.pure diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar-integration/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar-integration/pom.xml index 1402dab035d..3b2b096fa26 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar-integration/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar-integration/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar/pom.xml index 86eac9580b3..16671c61e26 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-protocol/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-protocol/pom.xml index be3cbcee1ff..013104268cb 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-protocol/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure-metamodel/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure-metamodel/pom.xml index cdedf3eb709..0f80e7c9ed8 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure-metamodel/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure-metamodel/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure/pom.xml index 4507e3b736e..af7cbedd4fc 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-query/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-query/pom.xml index 02b39603367..f8a6712504a 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-query/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-query/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-relational-extension/pom.xml b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-relational-extension/pom.xml index 80057cb9c40..b2974b5c830 100644 --- a/legend-engine-xts-graphQL/legend-engine-xt-graphQL-relational-extension/pom.xml +++ b/legend-engine-xts-graphQL/legend-engine-xt-graphQL-relational-extension/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-graphQL - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-graphQL/pom.xml b/legend-engine-xts-graphQL/pom.xml index eed65e274c3..a1529278701 100644 --- a/legend-engine-xts-graphQL/pom.xml +++ b/legend-engine-xts-graphQL/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-haskell/legend-engine-xt-haskell-grammar/pom.xml b/legend-engine-xts-haskell/legend-engine-xt-haskell-grammar/pom.xml index 0001024b5be..79e0e331d24 100644 --- a/legend-engine-xts-haskell/legend-engine-xt-haskell-grammar/pom.xml +++ b/legend-engine-xts-haskell/legend-engine-xt-haskell-grammar/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-haskell org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-haskell/legend-engine-xt-haskell-protocol/pom.xml b/legend-engine-xts-haskell/legend-engine-xt-haskell-protocol/pom.xml index ffe3c0fd8b0..4aa9c9ada1a 100644 --- a/legend-engine-xts-haskell/legend-engine-xt-haskell-protocol/pom.xml +++ b/legend-engine-xts-haskell/legend-engine-xt-haskell-protocol/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-haskell org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-haskell/legend-engine-xt-haskell-pure/pom.xml b/legend-engine-xts-haskell/legend-engine-xt-haskell-pure/pom.xml index 559c846b8b1..78c939989b0 100644 --- a/legend-engine-xts-haskell/legend-engine-xt-haskell-pure/pom.xml +++ b/legend-engine-xts-haskell/legend-engine-xt-haskell-pure/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-haskell org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-haskell/pom.xml b/legend-engine-xts-haskell/pom.xml index 2988d02021d..310338be31b 100644 --- a/legend-engine-xts-haskell/pom.xml +++ b/legend-engine-xts-haskell/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/pom.xml b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/pom.xml index 1e449ac5d30..039d09da9ef 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/pom.xml +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-hostedService - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -65,10 +65,6 @@ org.finos.legend.engine legend-engine-xt-functionActivator-api - - org.finos.legend.engine - legend-engine-xt-functionActivator-protocol - org.finos.legend.engine legend-engine-xt-hostedService-pure @@ -94,20 +90,20 @@ - - org.apache.httpcomponents - httpclient - - - commons-codec - commons-codec - - - - - org.apache.httpcomponents - httpcore - + + + + + + + + + + + + + + diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/api/HostedServiceService.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/api/HostedServiceService.java index 66883f4133c..a35aa36f5a0 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/api/HostedServiceService.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/api/HostedServiceService.java @@ -19,7 +19,9 @@ import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; import org.finos.legend.engine.language.hostedService.deployment.HostedServiceArtifact; +import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentConfiguration; import org.finos.legend.engine.language.hostedService.generation.model.GenerationInfoData; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage; import org.finos.legend.engine.functionActivator.service.FunctionActivatorError; @@ -27,8 +29,8 @@ import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentManager; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; import org.finos.legend.engine.protocol.hostedService.metamodel.HostedService; -import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration; -import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentResult; +//import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration; +import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentResult; import org.finos.legend.engine.language.hostedService.generation.HostedServiceArtifactGenerator; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceProtocolExtension; @@ -95,9 +97,9 @@ public HostedServiceArtifact renderArtifact(PureModel pureModel, Root_meta_exter } @Override - public List selectConfig(List configurations, DeploymentStage stage) + public List selectConfig(List configurations) { - return Lists.mutable.withAll(configurations).select(e -> e instanceof HostedServiceDeploymentConfiguration && e.stage.equals(stage)).collect(e -> (HostedServiceDeploymentConfiguration)e); + return Lists.mutable.withAll(configurations).select(e -> e instanceof HostedServiceDeploymentConfiguration).collect(e -> (HostedServiceDeploymentConfiguration)e); } @@ -106,7 +108,7 @@ public HostedServiceDeploymentResult publishToSandbox(MutableList { GenerationInfoData generation = this.hostedServiceArtifactgenerator.renderArtifact(pureModel, activator, inputModel, "vX_X_X",routerExtensions); HostedServiceArtifact artifact = new HostedServiceArtifact(generation, fetchHostedService(activator, (PureModelContextData)inputModel, pureModel)); - return this.hostedServiceDeploymentManager.deploy(profiles, artifact, activator, runtimeConfigs); + return this.hostedServiceDeploymentManager.deploy(profiles, artifact, runtimeConfigs); // return new HostedServiceDeploymentResult(); } diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceArtifact.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceArtifact.java index 60a380f14c4..1bfe026b012 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceArtifact.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceArtifact.java @@ -20,8 +20,6 @@ public class HostedServiceArtifact extends FunctionActivatorArtifact { - public GenerationInfo info; - public PureModelContextData serviceData; public HostedServiceArtifact() { @@ -30,13 +28,12 @@ public HostedServiceArtifact() public HostedServiceArtifact(GenerationInfo info) { - this.info = info; + this.content = new HostedServiceContent(info); } public HostedServiceArtifact(GenerationInfo info, PureModelContextData serviceData) { - this(info); - this.serviceData = serviceData; + this.content = new HostedServiceContent(info, serviceData); } diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceContent.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceContent.java new file mode 100644 index 00000000000..36de3185647 --- /dev/null +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceContent.java @@ -0,0 +1,37 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.hostedService.deployment; + +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentContent; +import org.finos.legend.engine.language.hostedService.generation.model.GenerationInfo; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; + +public class HostedServiceContent extends FunctionActivatorDeploymentContent +{ + + public GenerationInfo info; + public PureModelContextData serviceData; + + public HostedServiceContent(GenerationInfo info) + { + this.info = info; + } + + public HostedServiceContent(GenerationInfo info, PureModelContextData serviceData) + { + this.info = info; + this.serviceData = serviceData; + } +} diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismProvider.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentConfiguration.java similarity index 69% rename from legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismProvider.java rename to legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentConfiguration.java index d9ad6a7195e..f8627b853e7 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/AuthenticationMechanismProvider.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentConfiguration.java @@ -12,13 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection; +package org.finos.legend.engine.language.hostedService.deployment; -import org.finos.legend.connection.protocol.AuthenticationMechanism; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; -import java.util.List; - -public interface AuthenticationMechanismProvider +public class HostedServiceDeploymentConfiguration extends FunctionActivatorDeploymentConfiguration { - List getLoaders(); + } diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentManager.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentManager.java index 39fd6425da3..8309de32973 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentManager.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentManager.java @@ -15,94 +15,73 @@ package org.finos.legend.engine.language.hostedService.deployment; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.http.HttpResponse; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.Credentials; -import org.apache.http.client.CookieStore; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.DefaultHttpClient; import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.functionActivator.deployment.DeploymentManager; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage; -import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentConfiguration; -import org.finos.legend.engine.protocol.hostedService.metamodel.HostedServiceDeploymentResult; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorArtifact; import org.finos.legend.engine.shared.core.ObjectMapperFactory; -import org.finos.legend.engine.shared.core.kerberos.HttpClientBuilder; -import org.finos.legend.engine.shared.core.kerberos.ProfileManagerHelper; -import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; -import org.finos.legend.pure.generated.Root_meta_external_function_activator_hostedService_HostedService; -import org.finos.legend.pure.generated.Root_meta_external_function_activator_hostedService_HostedServiceDeploymentConfiguration; import org.pac4j.core.profile.CommonProfile; - -import javax.security.auth.Subject; -import java.security.Principal; -import java.security.PrivilegedExceptionAction; +import org.finos.legend.engine.language.hostedService.deployment.HostedServiceDeploymentConfiguration; import java.util.List; -public class HostedServiceDeploymentManager implements DeploymentManager +public class HostedServiceDeploymentManager implements DeploymentManager { public static ObjectMapper mapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); - public boolean canDeploy(Root_meta_external_function_activator_hostedService_HostedService element) + public boolean canDeploy(FunctionActivatorArtifact element) { - return element._activationConfiguration() != null; + return element instanceof HostedServiceArtifact; } - public HostedServiceDeploymentResult deploy(MutableList profiles, HostedServiceArtifact artifact, Root_meta_external_function_activator_hostedService_HostedService activator) + public HostedServiceDeploymentResult deploy(MutableList profiles, HostedServiceArtifact artifact) { return new HostedServiceDeploymentResult(); } - public HostedServiceDeploymentResult deploy(MutableList profiles, HostedServiceArtifact artifact, Root_meta_external_function_activator_hostedService_HostedService activator, List availableRuntimeConfigurations) + public HostedServiceDeploymentResult deploy(MutableList profiles, HostedServiceArtifact artifact, List availableRuntimeConfigurations) { String host; String path; int port; - if (activator._activationConfiguration() == null || activator._activationConfiguration()._stage()._name().equals(DeploymentStage.SANDBOX.name())) - { - if (availableRuntimeConfigurations.size() > 0) - { - host = availableRuntimeConfigurations.get(0).host; - path = availableRuntimeConfigurations.get(0).path; - port = availableRuntimeConfigurations.get(0).port; - } - else - { - throw new EngineException("No available configuration for sandbox deployment"); - } - try - { - HttpPost request = new HttpPost(new URIBuilder() - .setScheme("http") - .setHost(host) - .setPort(port) - .setPath(path) - .build()); - StringEntity stringEntity = new StringEntity(mapper.writeValueAsString(artifact)); - stringEntity.setContentType("application/json"); - request.setEntity(stringEntity); - CloseableHttpClient httpclient = (CloseableHttpClient) HttpClientBuilder.getHttpClient(new BasicCookieStore()); - Subject s = ProfileManagerHelper.extractSubject(profiles); - Subject.doAs(s, (PrivilegedExceptionAction) () -> - { - HttpResponse response = httpclient.execute(request); - return new HostedServiceDeploymentResult(); - }); - } - catch (Exception e) - { - throw new EngineException("No available configuration for sandbox deployment"); - - } - } +// if (artifact.deploymentConfiguration == null || activator._activationConfiguration()._stage()._name().equals(DeploymentStage.SANDBOX.name())) +// { +// if (availableRuntimeConfigurations.size() > 0) +// { +// host = availableRuntimeConfigurations.get(0).host; +// path = availableRuntimeConfigurations.get(0).path; +// port = availableRuntimeConfigurations.get(0).port; +// } +// else +// { +// throw new EngineException("No available configuration for sandbox deployment"); +// } +// try +// { +// HttpPost request = new HttpPost(new URIBuilder() +// .setScheme("http") +// .setHost(host) +// .setPort(port) +// .setPath(path) +// .build()); +// StringEntity stringEntity = new StringEntity(mapper.writeValueAsString(artifact)); +// stringEntity.setContentType("application/json"); +// request.setEntity(stringEntity); +// CloseableHttpClient httpclient = (CloseableHttpClient) HttpClientBuilder.getHttpClient(new BasicCookieStore()); +// Subject s = ProfileManagerHelper.extractSubject(profiles); +// Subject.doAs(s, (PrivilegedExceptionAction) () -> +// { +// HttpResponse response = httpclient.execute(request); +// return new HostedServiceDeploymentResult(); +// }); +// } +// catch (Exception e) +// { +// throw new EngineException("No available configuration for sandbox deployment"); +// +// } +// } // else if (activator._activationConfiguration() != null) // { // host = ((Root_meta_external_function_activator_hostedService_HostedServiceDeploymentConfiguration)activator._activationConfiguration())._; diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/src/main/java/org/finos/legend/engine/protocol/hostedService/metamodel/HostedServiceDeploymentResult.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentResult.java similarity index 81% rename from legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/src/main/java/org/finos/legend/engine/protocol/hostedService/metamodel/HostedServiceDeploymentResult.java rename to legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentResult.java index e8515da15d7..8917f5a219e 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/src/main/java/org/finos/legend/engine/protocol/hostedService/metamodel/HostedServiceDeploymentResult.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-api/src/main/java/org/finos/legend/engine/language/hostedService/deployment/HostedServiceDeploymentResult.java @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.engine.protocol.hostedService.metamodel; +package org.finos.legend.engine.language.hostedService.deployment; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentResult; +import org.finos.legend.engine.functionActivator.deployment.DeploymentResult; public class HostedServiceDeploymentResult extends DeploymentResult { diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/pom.xml b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/pom.xml index dc6dbb3e2d2..255341205a5 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/pom.xml +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-hostedService - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -53,10 +53,10 @@ org.finos.legend.engine legend-engine-xt-hostedService-protocol - - org.finos.legend.engine - legend-engine-xt-functionActivator-protocol - + + + + org.finos.legend.engine legend-engine-language-pure-dsl-service diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/src/main/java/org/finos/legend/engine/language/hostedService/compiler/toPureGraph/HostedServiceCompilerExtension.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/src/main/java/org/finos/legend/engine/language/hostedService/compiler/toPureGraph/HostedServiceCompilerExtension.java index b3993fecf5b..c25ca1a6ec2 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/src/main/java/org/finos/legend/engine/language/hostedService/compiler/toPureGraph/HostedServiceCompilerExtension.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-compiler/src/main/java/org/finos/legend/engine/language/hostedService/compiler/toPureGraph/HostedServiceCompilerExtension.java @@ -80,8 +80,8 @@ public Iterable> getExtraProcessors() public Root_meta_external_function_activator_hostedService_HostedServiceDeploymentConfiguration buildDeploymentConfig(HostedServiceDeploymentConfiguration config, CompileContext context) { - return new Root_meta_external_function_activator_hostedService_HostedServiceDeploymentConfiguration_Impl("", null, context.pureModel.getClass("meta::external::function::activator::hostedService::HostedServiceDeploymentConfiguration")) - ._stage(context.pureModel.getEnumValue("meta::external::function::activator::DeploymentStage", config.stage.name())); + return new Root_meta_external_function_activator_hostedService_HostedServiceDeploymentConfiguration_Impl("", null, context.pureModel.getClass("meta::external::function::activator::hostedService::HostedServiceDeploymentConfiguration")); + // ._stage(context.pureModel.getEnumValue("meta::external::function::activator::DeploymentStage", config.stage.name())); } public Root_meta_external_function_activator_hostedService_HostedService buildHostedService(HostedService app, CompileContext context) diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-generation/pom.xml b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-generation/pom.xml index 37a2717b01b..d71ff976b14 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-generation/pom.xml +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-generation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-hostedService - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/pom.xml b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/pom.xml index ae2c8df4c80..d8b05b1839e 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/pom.xml +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-hostedService - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -92,10 +92,10 @@ org.finos.legend.engine legend-engine-xt-hostedService-protocol - - org.finos.legend.engine - legend-engine-xt-functionActivator-protocol - + + + + org.finos.legend.engine legend-engine-language-pure-grammar diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/HostedServiceParserGrammar.g4 b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/HostedServiceParserGrammar.g4 index 39b4d8ab4d8..a148bf77558 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/HostedServiceParserGrammar.g4 +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/HostedServiceParserGrammar.g4 @@ -198,7 +198,6 @@ embeddedRuntimeContent: ISLAND_START | ISLAND_BRACE_OPEN | ISLAN // ----------------------------------- Deployment ------------------------------------------------------ deploymentConfigs: SERVICE_CONFIGURATION qualifiedName BRACE_OPEN - deploymentStage BRACE_CLOSE ; deploymentStage: SERVICE_DEPLOYMENT_STAGE COLON STRING SEMI_COLON diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/java/org/finos/legend/engine/language/hostedService/grammar/from/HostedServiceTreeWalker.java b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/java/org/finos/legend/engine/language/hostedService/grammar/from/HostedServiceTreeWalker.java index 610e81af12e..6465d2a8c3d 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/java/org/finos/legend/engine/language/hostedService/grammar/from/HostedServiceTreeWalker.java +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-grammar/src/main/java/org/finos/legend/engine/language/hostedService/grammar/from/HostedServiceTreeWalker.java @@ -151,19 +151,6 @@ private List visitStereotypes(HostedServiceParserGrammar.Stereoty private HostedServiceDeploymentConfiguration visitDeploymentConfig(HostedServiceParserGrammar.DeploymentConfigsContext ctx) { HostedServiceDeploymentConfiguration config = new HostedServiceDeploymentConfiguration(); - String stage = ctx.deploymentStage().getText(); - if (stage.equals("PRODUCTION")) - { - config.stage = DeploymentStage.PRODUCTION; - } - else if (stage.equals("SANDBOX")) - { - config.stage = DeploymentStage.SANDBOX; - } - else - { - throw new EngineException("Valid types for deployment stage are: SANDBOX, PRODUCTION"); - } return config; } diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/pom.xml b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/pom.xml index db2f9caed69..a77a0c424cd 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/pom.xml +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-hostedService - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/pom.xml b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/pom.xml index c8c88999f3e..13fe32ebd20 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/pom.xml +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-hostedService - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice/showcase/showcaseServices.pure b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice/showcase/showcaseServices.pure index 6d04770c27e..25d0d8fb18b 100644 --- a/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice/showcase/showcaseServices.pure +++ b/legend-engine-xts-hostedService/legend-engine-xt-hostedService-pure/src/main/resources/core_hostedservice/showcase/showcaseServices.pure @@ -13,7 +13,7 @@ import meta::pure::graphFetch::*; function meta::external::function::activator::hostedService::tests::defaultConfig():HostedServiceDeploymentConfiguration[1] { - ^HostedServiceDeploymentConfiguration(stage = DeploymentStage.PRODUCTION); + ^HostedServiceDeploymentConfiguration(); } // ========================================================================================================= diff --git a/legend-engine-xts-hostedService/pom.xml b/legend-engine-xts-hostedService/pom.xml index 03f3b1dffd7..d09d888e33a 100644 --- a/legend-engine-xts-hostedService/pom.xml +++ b/legend-engine-xts-hostedService/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/pom.xml b/legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/pom.xml index be5cf040888..1c3b0a7a1b4 100644 --- a/legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/pom.xml +++ b/legend-engine-xts-iceberg/legend-engine-xt-iceberg-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-iceberg - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-iceberg/legend-engine-xt-iceberg-test-support/pom.xml b/legend-engine-xts-iceberg/legend-engine-xt-iceberg-test-support/pom.xml index 071163f054a..f4255b4a2a9 100644 --- a/legend-engine-xts-iceberg/legend-engine-xt-iceberg-test-support/pom.xml +++ b/legend-engine-xts-iceberg/legend-engine-xt-iceberg-test-support/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-iceberg - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-iceberg/pom.xml b/legend-engine-xts-iceberg/pom.xml index 04553ce0dae..8bb64007435 100644 --- a/legend-engine-xts-iceberg/pom.xml +++ b/legend-engine-xts-iceberg/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-java/legend-engine-external-language-java/pom.xml b/legend-engine-xts-java/legend-engine-external-language-java/pom.xml index 3b29d25de42..d4bdb261bdd 100644 --- a/legend-engine-xts-java/legend-engine-external-language-java/pom.xml +++ b/legend-engine-xts-java/legend-engine-external-language-java/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-java - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-java/legend-engine-xt-javaGeneration-featureBased-pure/pom.xml b/legend-engine-xts-java/legend-engine-xt-javaGeneration-featureBased-pure/pom.xml index 7eb182df474..71891c3f647 100644 --- a/legend-engine-xts-java/legend-engine-xt-javaGeneration-featureBased-pure/pom.xml +++ b/legend-engine-xts-java/legend-engine-xt-javaGeneration-featureBased-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-java - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-java/legend-engine-xt-javaGeneration-pure/pom.xml b/legend-engine-xts-java/legend-engine-xt-javaGeneration-pure/pom.xml index 7261bbe0d67..68f7ebcc969 100644 --- a/legend-engine-xts-java/legend-engine-xt-javaGeneration-pure/pom.xml +++ b/legend-engine-xts-java/legend-engine-xt-javaGeneration-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-java - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/pom.xml b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/pom.xml index 3a3719cf02f..e969691777d 100644 --- a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/pom.xml +++ b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-java - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/shared.pure b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/shared.pure index 1cc9824474d..62ef87646da 100644 --- a/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/shared.pure +++ b/legend-engine-xts-java/legend-engine-xt-javaPlatformBinding-pure/src/main/resources/core_java_platform_binding/legendJavaPlatformBinding/binding/shared.pure @@ -194,10 +194,7 @@ function meta::external::format::shared::executionPlan::platformBinding::legendJ let itemType = if($javaType->isJavaList(), |$javaType->elementType(), |$javaType)->toUnboxed(); let occurs = occursFromMultiplicity($p.multiplicity); - let getSize = if($javaType->isJavaList(), - | j_this($baseClass)->j_field($fieldName, $javaType)->j_invoke('size', [], javaInt()), - | j_this($baseClass)->j_field($sizeFieldName, javaInt()) - ); + let getSize = j_this($baseClass)->j_field($sizeFieldName, javaInt()); let lowerBoundClause = $getSize->j_lt($occurs.first); let upperBoundClause = $getSize->j_gt($occurs.second); let expectedMultiplicity = $p.multiplicity->printMultiplicity(); @@ -234,37 +231,29 @@ function meta::external::format::shared::executionPlan::platformBinding::legendJ let withFieldsAndGetters = $properties->fold( {p, c| - if($p.javaType->isJavaList(), - {| - $c ->addField(javaField('private', $p.javaType, $p.fieldName)) - ->addMethod({cls| - let field = j_this($cls)->j_field($p.fieldName); - - javaMethod('public', $p.javaType, $p.getterName, [], - j_return(j_conditional($field->j_eq(j_null()), javaCollections()->j_invoke($p.itemType, 'emptyList', [], $p.javaType), $field)) - ); - }) - ->addMethods({cls| - $cls->filter(c| $p.property->in($mappedProperties))->map(c| $c->adderMethods($p, $conventions)); - }); - }, - {| - $c ->addField(javaField('private', $p.javaType->toUnboxed(), $p.fieldName)) - ->addField(javaField('private', javaInt(), $p.sizeFieldName, j_int(0))) - ->addMethod({cls| - let field = j_this($cls)->j_field($p.fieldName); - let fieldSize = j_this($cls)->j_field($p.sizeFieldName); - - javaMethod('public', $p.javaType, $p.getterName, [], - j_return(j_conditional($fieldSize->j_eq(j_int(0)), j_null(), $field)) - ); - }) - ->addMethods({cls| - $cls->filter(c| $p.property->in($mappedProperties))->map(c| $c->adderMethods($p, $conventions)); - }); - - } + let javaType = if($p.javaType->isJavaList(), + |$p.javaType, + |$p.javaType->toUnboxed() ); + + let nullValue = if($p.javaType->isJavaList(), + |javaCollections()->j_invoke('emptyList', [], $p.javaType), + |j_null() + ); + + $c ->addField(javaField('private', $javaType, $p.fieldName)) + ->addField(javaField('private', javaInt(), $p.sizeFieldName, j_int(0))) + ->addMethod({cls| + let field = j_this($cls)->j_field($p.fieldName); + let fieldSize = j_this($cls)->j_field($p.sizeFieldName); + + javaMethod('public', $p.javaType, $p.getterName, [], + j_return(j_conditional($fieldSize->j_eq(j_int(0)), $nullValue, $field)) + ); + }) + ->addMethods({cls| + $cls->filter(c| $p.property->in($mappedProperties))->map(c| $c->adderMethods($p, $conventions)); + }); }, $baseClass ); @@ -396,6 +385,7 @@ function <> meta::external::format::shared::executionPlan::platf )); + let fieldSize = j_this($jc)->j_field($prop.sizeFieldName); let impl = $adderValue.create->concatenate( if($prop.javaType->isJavaList(), {| @@ -403,12 +393,11 @@ function <> meta::external::format::shared::executionPlan::platf j_if($field->j_eq(j_null()), $field->j_assign(javaArrayList($prop.itemType)->j_new([])) ), - $field->j_invoke('add', $adderValue.access) - ] + $field->j_invoke('add', $adderValue.access), + $fieldSize->j_inc() + ]; }, {| - let fieldSize = j_this($jc)->j_field($prop.sizeFieldName); - [ j_if($fieldSize->j_eq(j_int(0)), $field->j_assign($adderValue.access) diff --git a/legend-engine-xts-java/pom.xml b/legend-engine-xts-java/pom.xml index 599ef2e4830..6745dae6ff4 100644 --- a/legend-engine-xts-java/pom.xml +++ b/legend-engine-xts-java/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-json/legend-engine-external-format-jsonSchema/pom.xml b/legend-engine-xts-json/legend-engine-external-format-jsonSchema/pom.xml index 1eb7284b036..d6408f9a55d 100644 --- a/legend-engine-xts-json/legend-engine-external-format-jsonSchema/pom.xml +++ b/legend-engine-xts-json/legend-engine-external-format-jsonSchema/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-json - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-pure/pom.xml b/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-pure/pom.xml index 519a1e0f2d3..b4e1b423c42 100644 --- a/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-pure/pom.xml +++ b/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-json - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-test/pom.xml b/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-test/pom.xml index 841c9d0f58d..b089377922b 100644 --- a/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-test/pom.xml +++ b/legend-engine-xts-json/legend-engine-xt-json-javaPlatformBinding-test/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-json - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-json/legend-engine-xt-json-model/pom.xml b/legend-engine-xts-json/legend-engine-xt-json-model/pom.xml index db08e226556..efe3e7557d4 100644 --- a/legend-engine-xts-json/legend-engine-xt-json-model/pom.xml +++ b/legend-engine-xts-json/legend-engine-xt-json-model/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-json - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-json/legend-engine-xt-json-pure/pom.xml b/legend-engine-xts-json/legend-engine-xt-json-pure/pom.xml index f2742aa7fad..ce7e9084083 100644 --- a/legend-engine-xts-json/legend-engine-xt-json-pure/pom.xml +++ b/legend-engine-xts-json/legend-engine-xt-json-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-json - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-json/legend-engine-xt-json-runtime/pom.xml b/legend-engine-xts-json/legend-engine-xt-json-runtime/pom.xml index d938ad75d6d..1597c96f3d1 100644 --- a/legend-engine-xts-json/legend-engine-xt-json-runtime/pom.xml +++ b/legend-engine-xts-json/legend-engine-xt-json-runtime/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-json - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-json/pom.xml b/legend-engine-xts-json/pom.xml index 90ac7c240ae..f742f78dc5d 100644 --- a/legend-engine-xts-json/pom.xml +++ b/legend-engine-xts-json/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/pom.xml b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/pom.xml index 762f1dbe22d..f821b96137a 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/pom.xml +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-mastery org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 index 306e56dae49..7a15d60935f 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryLexerGrammar.g4 @@ -52,6 +52,7 @@ IDENTITY_RESOLUTION: 'identityResolution'; RESOLUTION_QUERIES: 'resolutionQueries'; RESOLUTION_QUERY_EXPRESSIONS: 'queries'; RESOLUTION_QUERY_KEY_TYPE: 'keyType'; +RESOLUTION_QUERY_OPTIONAL: 'optional'; RESOLUTION_QUERY_KEY_TYPE_GENERATED_PRIMARY_KEY:'GeneratedPrimaryKey'; //Validated against equality key to ensure an acrual PK and fail if don't find match RESOLUTION_QUERY_KEY_TYPE_SUPPLIED_PRIMARY_KEY: 'SuppliedPrimaryKey'; //Validated against equality key to ensure an actuial PK and create if don't find match RESOLUTION_QUERY_KEY_TYPE_ALTERNATE_KEY: 'AlternateKey'; //AlternateKey (In an AlternateKey is specified then at least one required in the input record or fail resolution). AlternateKey && (CurationModel field == Create) then the input source is attempting to create a new record (e.g. from UI) block if existing record found diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 index 98379331ac0..b039d6becae 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/MasteryParserGrammar.g4 @@ -275,6 +275,7 @@ resolutionQueries: RESOLUTION_QUERIES COLON resolutionQuery: BRACE_OPEN (queryExpressions | resolutionQueryKeyType + | resolutionQueryOptional | resolutionQueryPrecedence )* BRACE_CLOSE @@ -295,6 +296,8 @@ resolutionQueryKeyType: RESOLUTION_QUERY_KEY_TYPE COLON ( ) SEMI_COLON ; +resolutionQueryOptional: RESOLUTION_QUERY_OPTIONAL COLON boolean_value SEMI_COLON +; resolutionQueryPrecedence: PRECEDENCE COLON INTEGER SEMI_COLON ; // -------------------------------------- PRECEDENCE RULES-------------------------------------- diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java index bfc9ba0a848..db5a9ebe315 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/toPureGraph/HelperMasterRecordDefinitionBuilder.java @@ -120,7 +120,8 @@ private Iterable visi Root_meta_pure_mastery_metamodel_identity_ResolutionQuery resQuery = new Root_meta_pure_mastery_metamodel_identity_ResolutionQuery_Impl(""); String KEY_TYPE_FULL_PATH = MASTERY_PACKAGE_PREFIX + "::identity::ResolutionKeyType"; - resQuery._keyType(context.resolveEnumValue(KEY_TYPE_FULL_PATH, protocolQuery.keyType.name())); + resQuery._keyType(protocolQuery.keyType == null ? null : context.resolveEnumValue(KEY_TYPE_FULL_PATH, protocolQuery.keyType.name())); + resQuery._optional(protocolQuery.optional); resQuery._precedence(protocolQuery.precedence); ListIterate.forEachWithIndex(protocolQuery.queries, (lambda, i) -> diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java index 3ab85c09c89..8a252c6b062 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/from/MasteryParseTreeWalker.java @@ -719,8 +719,15 @@ private ResolutionQuery visitResolutionQuery(MasteryParserGrammar.ResolutionQuer resolutionQuery.queries = ListIterate.flatCollect(ctx.queryExpressions(), this::visitQueryExpressions); //keyType - MasteryParserGrammar.ResolutionQueryKeyTypeContext resolutionQueryKeyTypeContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.resolutionQueryKeyType(), "keyType", sourceInformation); - resolutionQuery.keyType = visitResolutionKeyType(resolutionQueryKeyTypeContext); + MasteryParserGrammar.ResolutionQueryKeyTypeContext resolutionQueryKeyTypeContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.resolutionQueryKeyType(), "keyType", sourceInformation); + if (resolutionQueryKeyTypeContext != null) + { + resolutionQuery.keyType = visitResolutionKeyType(resolutionQueryKeyTypeContext); + } + + //optional + MasteryParserGrammar.ResolutionQueryOptionalContext resolutionQueryOptionalContext = PureGrammarParserUtility.validateAndExtractOptionalField(ctx.resolutionQueryOptional(), "optional", sourceInformation); + resolutionQuery.optional = evaluateBoolean(resolutionQueryOptionalContext, (resolutionQueryOptionalContext != null ? resolutionQueryOptionalContext.boolean_value() : null), null); //precedence - Field 'precedence' should be specified only once MasteryParserGrammar.ResolutionQueryPrecedenceContext resolutionQueryPrecedenceContext = PureGrammarParserUtility.validateAndExtractRequiredField(ctx.resolutionQueryPrecedence(), "precedence", sourceInformation); diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java index f7a4726e3e0..38aace1908c 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/main/java/org/finos/legend/engine/language/pure/dsl/mastery/grammar/to/HelperMasteryGrammarComposer.java @@ -445,7 +445,14 @@ private static String renderResolutionQueries(IdentityResolution identityResolut builder.append("\n").append(getTabString(indentLevel + 4)).append("queries: [ "); builder.append(renderQueries(resolutionQuery, indentLevel + 5, context)); builder.append(getTabString(indentLevel + 4)).append(" ];\n"); - builder.append(getTabString(indentLevel + 4)).append("keyType: ").append(resolutionQuery.keyType).append(";\n"); + if (resolutionQuery.keyType != null) + { + builder.append(getTabString(indentLevel + 4)).append("keyType: ").append(resolutionQuery.keyType).append(";\n"); + } + if (resolutionQuery.optional != null) + { + builder.append(getTabString(indentLevel + 4)).append("optional: ").append(resolutionQuery.optional).append(";\n"); + } builder.append(getTabString(indentLevel + 4)).append("precedence: ").append(resolutionQuery.precedence).append(";\n"); builder.append(getTabString(indentLevel + 3)).append("}"); }); diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java index 0850ab66f96..b9953070433 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-grammar/src/test/java/org/finos/legend/engine/language/pure/dsl/mastery/compiler/test/TestMasteryCompilationFromGrammar.java @@ -113,18 +113,21 @@ public class TestMasteryCompilationFromGrammar extends TestCompilationFromGramma " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + " ];\n" + " keyType: GeneratedPrimaryKey;\n" + + " optional: true;\n" + " precedence: 1;\n" + " },\n" + " {\n" + " queries: [ {input: org::dataeng::Widget[1],EFFECTIVE_DATE: StrictDate[1]|org::dataeng::Widget.all()->filter(widget|((($widget.identifiers.identifierType == 'ISIN') && ($input.identifiers->filter(idType|$idType.identifierType == 'ISIN').identifier == $widget.identifiers->filter(idType|$idType.identifierType == 'ISIN').identifier)) && ($widget.identifiers.FROM_Z->toOne() <= $EFFECTIVE_DATE)) && ($widget.identifiers.THRU_Z->toOne() > $EFFECTIVE_DATE))}\n" + " ];\n" + " keyType: AlternateKey;\n" + + " optional: true;\n" + " precedence: 2;\n" + " },\n" + " {\n" + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.trigger == $input.trigger)}\n" + " ];\n" + " keyType: Optional;\n" + + " optional: true;\n" + " precedence: 3;\n" + " }\n" + " ]\n" + @@ -393,7 +396,6 @@ public class TestMasteryCompilationFromGrammar extends TestCompilationFromGramma " {\n" + " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + " ];\n" + - " keyType: GeneratedPrimaryKey;\n" + " precedence: 1;\n" + " }\n" + " ]\n" + @@ -477,6 +479,7 @@ public class TestMasteryCompilationFromGrammar extends TestCompilationFromGramma " queries: [ {input: org::dataeng::Widget[1]|org::dataeng::Widget.all()->filter(widget|$widget.widgetId == $input.widgetId)}\n" + " ];\n" + " keyType: GeneratedPrimaryKey;\n" + + " optional: true;\n" + " precedence: 1;\n" + " }" + " ]\n" + @@ -550,10 +553,12 @@ public void testMasteryFullModel() Object[] queriesArray = idRes._resolutionQueries().toArray(); assertEquals(1, ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._precedence()); assertEquals("GeneratedPrimaryKey", ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._keyType()._name()); + assertTrue(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._optional()); assertResolutionQueryLambdas(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._queries().toList()); assertEquals(2, ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[1])._precedence()); assertEquals("AlternateKey", ((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[1])._keyType()._name()); + assertTrue(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._optional()); assertResolutionQueryLambdas(((Root_meta_pure_mastery_metamodel_identity_ResolutionQuery) queriesArray[0])._queries().toList()); //PrecedenceRule @@ -1000,6 +1005,6 @@ protected String getDuplicatedElementTestCode() @Override public String getDuplicatedElementTestExpectedErrorMessage() { - return "COMPILATION error at [8:1-35:1]: Duplicated element 'org::dataeng::Widget'"; + return "COMPILATION error at [8:1-36:1]: Duplicated element 'org::dataeng::Widget'"; } } \ No newline at end of file diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/pom.xml b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/pom.xml index 7453b78289b..181a4305544 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/pom.xml +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-mastery - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java index 19c950d2ef6..3c9500f0522 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/RecordSource.java @@ -26,8 +26,8 @@ public class RecordSource { public String parseService; public String transformService; - public List tags = new ArrayList(); - public List partitions = Collections.emptyList(); + public List tags; + public List partitions; public String id; public String description; public RecordSourceStatus status; diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java index abfc2f1b099..f2e0b634dfc 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-protocol/src/main/java/org/finos/legend/engine/protocol/pure/v1/model/packageableElement/mastery/identity/ResolutionQuery.java @@ -20,8 +20,8 @@ public class ResolutionQuery { - //public List queries; public List queries; public ResolutionKeyType keyType; + public Boolean optional; public Integer precedence; } diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/pom.xml b/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/pom.xml index 7b4e634a713..9807dbd25e2 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/pom.xml +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-mastery - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure b/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure index 318972add4d..f989941b7b4 100644 --- a/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure +++ b/legend-engine-xts-mastery/legend-engine-xt-mastery-pure/src/main/resources/core_mastery/mastery/metamodel.pure @@ -171,7 +171,10 @@ meta::pure::mastery::metamodel::identity::ResolutionQuery queries : meta::pure::metamodel::function::LambdaFunction<{Any[1],StrictDate[0..1]->Any[*]}>[1..*]; //TODO in compiler check that parameter and return type are of same type, via subclass of LambdaFunction? {doc.doc='The key type used to define influences how to generate queries issued against the store and validate the results of query execution.'} - keyType : meta::pure::mastery::metamodel::identity::ResolutionKeyType[1]; + keyType : meta::pure::mastery::metamodel::identity::ResolutionKeyType[0..1]; + + {doc.doc='Field to identify if key is optional.'} + optional : Boolean[0..1]; {doc.doc='The query precedence applied when there are more than one ResoultionQuery definitions on an IdentityResolution, 1 is hiughest.'} precedence : Integer[1]; diff --git a/legend-engine-xts-mastery/pom.xml b/legend-engine-xts-mastery/pom.xml index d09dcd774a7..5ae5aad57ac 100644 --- a/legend-engine-xts-mastery/pom.xml +++ b/legend-engine-xts-mastery/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan-test/pom.xml b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan-test/pom.xml index 0bfb3241b53..01805ce9072 100644 --- a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan-test/pom.xml +++ b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan-test/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-mongodb - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan/pom.xml b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan/pom.xml index 29d4f0c3f39..124c43e1675 100644 --- a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan/pom.xml +++ b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-executionPlan/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-mongodb - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-nonrelationalStore-mongodb-executionPlan diff --git a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar-integration/pom.xml b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar-integration/pom.xml index c534a953b7c..a57e0c592fa 100644 --- a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar-integration/pom.xml +++ b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar-integration/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-mongodb - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-nonrelationalStore-mongodb-grammar-integration diff --git a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar/pom.xml b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar/pom.xml index 584f93c03f0..3dd12a79342 100644 --- a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar/pom.xml +++ b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-grammar/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-mongodb - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-nonrelationalStore-mongodb-grammar diff --git a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure/pom.xml b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure/pom.xml index 8d203b2910b..f49c2c4e3ed 100644 --- a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure/pom.xml +++ b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-mongodb - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-nonrelationalStore-mongodb-javaPlatformBinding-pure diff --git a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-protocol/pom.xml b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-protocol/pom.xml index 615a7bd60dd..ace71a1d405 100644 --- a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-protocol/pom.xml +++ b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-mongodb - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-nonrelationalStore-mongodb-protocol diff --git a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-pure/pom.xml b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-pure/pom.xml index 81e587c14ef..8e9f1be0a31 100644 --- a/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-pure/pom.xml +++ b/legend-engine-xts-mongodb/legend-engine-xt-nonrelationalStore-mongodb-pure/pom.xml @@ -20,7 +20,7 @@ org.finos.legend.engine legend-engine-xts-mongodb - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-nonrelationalStore-mongodb-pure diff --git a/legend-engine-xts-mongodb/pom.xml b/legend-engine-xts-mongodb/pom.xml index 056dbb4842f..3508ca59d65 100644 --- a/legend-engine-xts-mongodb/pom.xml +++ b/legend-engine-xts-mongodb/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-morphir/legend-engine-xt-morphir-pure/pom.xml b/legend-engine-xts-morphir/legend-engine-xt-morphir-pure/pom.xml index 6516912b897..2dddf063d0c 100644 --- a/legend-engine-xts-morphir/legend-engine-xt-morphir-pure/pom.xml +++ b/legend-engine-xts-morphir/legend-engine-xt-morphir-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-morphir - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-morphir/legend-engine-xt-morphir/pom.xml b/legend-engine-xts-morphir/legend-engine-xt-morphir/pom.xml index 3b81dd25016..4819222ccb2 100644 --- a/legend-engine-xts-morphir/legend-engine-xt-morphir/pom.xml +++ b/legend-engine-xts-morphir/legend-engine-xt-morphir/pom.xml @@ -19,7 +19,7 @@ legend-engine-xts-morphir org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-morphir/pom.xml b/legend-engine-xts-morphir/pom.xml index e490e346671..0e81e4950aa 100644 --- a/legend-engine-xts-morphir/pom.xml +++ b/legend-engine-xts-morphir/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/pom.xml b/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/pom.xml index 50cfa164e50..f4dd11e72e3 100644 --- a/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/pom.xml +++ b/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/pom.xml @@ -20,7 +20,7 @@ org.finos.legend.engine legend-engine-xts-openapi - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-openapi-generation @@ -70,6 +70,10 @@ org.eclipse.collections eclipse-collections-api + + org.eclipse.collections + eclipse-collections + diff --git a/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/src/main/java/org/finos/legend/engine/generation/OpenApiArtifactGenerationExtension.java b/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/src/main/java/org/finos/legend/engine/generation/OpenApiArtifactGenerationExtension.java index 765ec5a4cf2..4ff2b8dfc90 100644 --- a/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/src/main/java/org/finos/legend/engine/generation/OpenApiArtifactGenerationExtension.java +++ b/legend-engine-xts-openapi/legend-engine-xt-openapi-generation/src/main/java/org/finos/legend/engine/generation/OpenApiArtifactGenerationExtension.java @@ -14,10 +14,13 @@ package org.finos.legend.engine.generation; +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class; import org.finos.legend.pure.generated.Root_meta_external_function_description_openapi_metamodel_Server_Impl; import org.finos.legend.pure.generated.Root_meta_legend_service_metamodel_Service; import org.finos.legend.pure.generated.core_external_format_openapi_transformation_fromPure_pureToOpenApi; @@ -26,6 +29,7 @@ import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; public class OpenApiArtifactGenerationExtension implements ArtifactGenerationExtension { @@ -60,7 +64,8 @@ public List generate(PackageableElement element, PureModel pureModel, { try { - String result = core_external_format_openapi_transformation_fromPure_pureToOpenApi.Root_meta_external_function_description_openapi_transformation_fromPure_serviceToOpenApi_Service_1__Server_1__String_1_((Root_meta_legend_service_metamodel_Service) element, new Root_meta_external_function_description_openapi_metamodel_Server_Impl("")._url(HOST), pureModel.getExecutionSupport()); + RichIterable packageableElements = Lists.immutable.withAll(data.getElements().stream().filter(e -> e instanceof Class).map(e -> pureModel.getPackageableElement(e._package + "::" + e.name)).collect(Collectors.toList())); + String result = core_external_format_openapi_transformation_fromPure_pureToOpenApi.Root_meta_external_function_description_openapi_transformation_fromPure_serviceToOpenApi_Service_1__PackageableElement_MANY__Server_1__String_1_((Root_meta_legend_service_metamodel_Service) element, packageableElements, new Root_meta_external_function_description_openapi_metamodel_Server_Impl("")._url(HOST), pureModel.getExecutionSupport()); Artifact output = new Artifact(result, element.getName() + "_spec.json", "json"); return Collections.singletonList(output); } diff --git a/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/pom.xml b/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/pom.xml index 019000b3d2b..f5de5af9557 100644 --- a/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/pom.xml +++ b/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-openapi - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-openapi-pure diff --git a/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/pureToOpenApi.pure b/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/pureToOpenApi.pure index 9e5fe861eb4..173940126fd 100644 --- a/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/pureToOpenApi.pure +++ b/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/pureToOpenApi.pure @@ -20,7 +20,6 @@ import meta::legend::service::metamodel::*; import meta::external::function::description::openapi::transformation::fromPure::*; import meta::pure::functions::meta::*; import meta::pure::metamodel::path::*; -import meta::pure::service::*; import meta::external::function::description::openapi::profiles::*; import meta::pure::generation::metamodel::*; import meta::external::query::sql::transformation::queryToPure::tests::*; @@ -73,6 +72,11 @@ function meta::external::function::description::openapi::transformation::fromPur } function <> meta::external::function::description::openapi::transformation::fromPure::serviceToOpenApi(service: Service[1], server: Server[1]):String[1] +{ + $service->serviceToOpenApi([], $server); +} + +function <> meta::external::function::description::openapi::transformation::fromPure::serviceToOpenApi(service: Service[1], elements: PackageableElement[*], server: Server[1]):String[1] { ^FunctionInfo( func = $service.execution->cast(@PureExecution).func, @@ -81,13 +85,12 @@ function <> meta::external::function::description version = '1.0.0', title = 'Legend API', paramsExample = ^Map() - )->meta::external::function::description::openapi::transformation::fromPure::funcInfoToOpenApi($server) + )->meta::external::function::description::openapi::transformation::fromPure::funcInfoToOpenApi($elements, $server) ->asString()->makeString()->replace('\\/','/'); } -function <> meta::external::function::description::openapi::transformation::fromPure::funcInfoToOpenApi(funcInfos:FunctionInfo[*], server:Server[1]):OpenApi[1] +function <> meta::external::function::description::openapi::transformation::fromPure::funcInfoToOpenApi(funcInfos:FunctionInfo[*], elements: PackageableElement[*], server:Server[1]):OpenApi[1] { - let allClasses = $funcInfos->returnTypesFromFunctions(); ^OpenApi( openapi='3.0.0', info=^Info(title='Legend API', version='1.0.0'), @@ -118,28 +121,61 @@ function <> meta::external::function::description::openapi::tran ); ) ), - components= ^Components(schemas=newMap( - $allClasses->map( - c|pair( - $c.rawType.name->toOne(), - ^Schema - ( - type='object', - properties=newMap( - $c.rawType->cast(@Class).properties - ->filter(p|$p.name!='elementOverride' && $p.name!='classifierGenericType') - ->map(p| - pair - ( - $p.name->toOne(), - $p->functionReturnType()->buildComponent($p.multiplicity) - ) + components= generateComponent($funcInfos, $elements) + + ); +} + +function <> meta::external::function::description::openapi::transformation::fromPure::generateComponent(funcInfos: FunctionInfo[*], elements: PackageableElement[*]):Components[1] +{ + let schema = $funcInfos->map(f| if($f->getReturnType()->instanceOf(RootGraphFetchTree), + |let allClasses = $elements->filter(e|$e->instanceOf(Class)); + $allClasses->map( + c|pair( + $c.name->toOne(), + ^Schema + ( + type='object', + properties=newMap( + $c->cast(@Class).properties + ->filter(p|$p.name!='elementOverride' && $p.name!='classifierGenericType') + ->map(p| + pair + ( + $p.name->toOne(), + $p->functionReturnType()->buildComponent($p.multiplicity) + ) + ) + ) + ) + ); + );, + |let allClasses = $funcInfos->returnTypesFromFunctions(); + $allClasses->map( + c|pair( + $c.rawType.name->toOne(), + ^Schema + ( + type='object', + properties=newMap( + $c.rawType->cast(@Class).properties + ->filter(p|$p.name!='elementOverride' && $p.name!='classifierGenericType') + ->map(p| + pair + ( + $p.name->toOne(), + $p->functionReturnType()->buildComponent($p.multiplicity) + ) + ) ) - ) - ) - ) - ) - ))); + ) + ) + ); + ); + ); + + + ^Components(schemas = newMap($schema)); } function <> meta::external::function::description::openapi::transformation::fromPure::getEvaluatedExecutionFunction(funcInfo:FunctionInfo[1]):FunctionDefinition[1] @@ -169,7 +205,7 @@ function <> meta::external::function::description::openapi::tran $funcInfos->map(f|$f->getReturnType()) ->map(r| if($r->instanceOf(RootGraphFetchTree), | $r.classifierGenericType->toOne(), | $r)) ->map(r|$r->cast(@GenericType)->toOne()) - ->map(r|if($r.rawType==JSONResult||$r.rawType==RootGraphFetchTree,|$r.typeArguments,|$r)) + ->map(r|if($r.rawType== meta::external::function::description::openapi::tostring::JSONResult||$r.rawType==RootGraphFetchTree,|$r.typeArguments,|$r)) ->filter(c|!$c.rawType->isEmpty() && $c.rawType->toOne()->instanceOf(Class)) ->map(t|if($t.rawType.specializations->isEmpty(),|$t,|$t.rawType->toOne().classifierGenericType.typeArguments)) ->removeDuplicates(cmpGenericType_GenericType_1__GenericType_1__Boolean_1_) diff --git a/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/tests/tests.pure b/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/tests/tests.pure index c6ba94c2261..febfad58185 100644 --- a/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/tests/tests.pure +++ b/legend-engine-xts-openapi/legend-engine-xt-openapi-pure/src/main/resources/core_external_format_openapi/transformation/fromPure/tests/tests.pure @@ -149,7 +149,7 @@ function <> meta::external::function::description::openapi::transformation::fromPure::tests::testServiceWithExtendAndManyParam():Service[1] +function <> meta::external::function::description::openapi::transformation::fromPure::tests::serviceWithExtendAndManyParam():Service[1] { ^Service( pattern = '/test/service/with/extend', @@ -157,54 +157,55 @@ function <filter(item | $planningAreas->isEmpty() || $planningAreas->contains($item.firstName))->project( [ col(x | $x.lastName, 'kerberos'), col(x | $x.age, 'planningArea') ]) ->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age') ->toString()}));, + func = planningAreas: String[*]| Person.all()->filter(item | $planningAreas->isEmpty() || $planningAreas->contains($item.firstName))->project( [ col(x | $x.lastName, 'kerberos'), col(x | $x.age, 'planningArea') ])->extend(^BasicColumnSpecification(name = 'ageString', func = {row:TDSRow[1]|$row.getInteger('age') ->toString()}));, mapping = personMapping, runtime = [] ) ); } - - - - - - function <> meta::external::function::description::openapi::transformation::fromPure::tests::testServiceWithoutColumnSpecificationOpenapiString():Boolean[1] { - let openapi = meta::external::function::description::openapi::transformation::fromPure::tests::testServiceWithoutColumnSpecification()->serviceToOpenApi(^Server(url='test')); + let openapi = meta::external::function::description::openapi::transformation::fromPure::tests::testServiceWithoutColumnSpecification()->serviceToOpenApi(packageableElem(), ^Server(url='test')); let expected = readFile('/core_external_format_openapi/transformation/fromPure/tests/resources/testOpenApiSpecWithoutColumnSpec.txt')->toOne(); assertJsonStringsEqual($expected, $openapi); } function <> meta::external::function::description::openapi::transformation::fromPure::tests::testServiceShouldReturnCorrectOpenapiString():Boolean[1] { - let openapi = simpleService()->serviceToOpenApi(^Server(url='test')); + let openapi = simpleService()->serviceToOpenApi(packageableElem(), ^Server(url='test')); let expected = readFile('/core_external_format_openapi/transformation/fromPure/tests/resources/testOpenApiSpec.txt')->toOne(); assertJsonStringsEqual($expected, $openapi); } -function <> meta::external::function::description::openapi::transformation::fromPure::tests::testFailedService():Boolean[1] +function <> meta::external::function::description::openapi::transformation::fromPure::tests::testServiceWithExtendAndManyParam():Boolean[1] { - let openapi = meta::external::function::description::openapi::transformation::fromPure::tests::testServiceWithExtendAndManyParam()->serviceToOpenApi(^Server(url='test')); + let openapi = meta::external::function::description::openapi::transformation::fromPure::tests::serviceWithExtendAndManyParam()->serviceToOpenApi(packageableElem(), ^Server(url='test')); let expected = readFile('/core_external_format_openapi/transformation/fromPure/tests/resources/testOpenApiSpecWithExtendAndManyParam.txt')->toOne(); assertJsonStringsEqual($expected, $openapi); } function <> meta::external::function::description::openapi::transformation::fromPure::tests::testGraphFetchServiceShouldReturnCorrectOpenapiString():Boolean[1] { - let openapi = simpleServiceWithGraphFetch()->serviceToOpenApi(^Server(url='test')); + let openapi = simpleServiceWithGraphFetch()->serviceToOpenApi(packageableElem(), ^Server(url='test')); let expected = readFile('/core_external_format_openapi/transformation/fromPure/tests/resources/testGraphFetchOpenApiSpec.txt')->toOne(); assertJsonStringsEqual($expected, $openapi); } function <> meta::external::function::description::openapi::transformation::fromPure::tests::testServiceWithParamShouldReturnCorrectOpenapiString():Boolean[1] { - let openapi = simpleServiceWithParams()->serviceToOpenApi(^Server(url='test')); + let openapi = simpleServiceWithParams()->serviceToOpenApi(packageableElem(), ^Server(url='test')); let expected = readFile('/core_external_format_openapi/transformation/fromPure/tests/resources/testOpenApiSpecWithServiceParams.txt')->toOne(); assertJsonStringsEqual($expected, $openapi); } +function meta::external::function::description::openapi::transformation::fromPure::tests::packageableElem():PackageableElement[*] +{ + [ + pathToElement('meta::external::function::description::openapi::transformation::fromPure::tests::Person') + ] +} + ###Mapping import meta::external::function::description::openapi::transformation::fromPure::tests::*; diff --git a/legend-engine-xts-openapi/pom.xml b/legend-engine-xts-openapi/pom.xml index 2bfe2a458aa..0eb492356f5 100644 --- a/legend-engine-xts-openapi/pom.xml +++ b/legend-engine-xts-openapi/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-api/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-api/pom.xml index 1df22ba3858..89f82f70d74 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-api/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-grammar/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-grammar/pom.xml index a23ce9d269e..167bcbe532b 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-grammar/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-grammar/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-protocol/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-protocol/pom.xml index 9d7fedd352c..5aed6dc6236 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-protocol/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-pure/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-pure/pom.xml index bc15b5280e2..8d0bc85810c 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-pure/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-cloud-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/pom.xml index fd6b57c84a7..38ce5727ce0 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/pom.xml index bc89c4791cb..6f9563ef27c 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/pom.xml index 1d0ac2033b7..24bb9065af5 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/pom.xml @@ -15,7 +15,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml index 5320949adef..6ce4b2fb3bf 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/pom.xml index ca075cde762..1ec08d32c1f 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/pom.xml index c0fd6ad0fcf..bedeca20a3b 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/pom.xml index aa14dc557c1..4cb580c9c24 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/pom.xml index 745e3da9586..a8e51682034 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/pom.xml index 2e2b0b535af..48480c10910 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/pom.xml @@ -15,7 +15,7 @@ org.finos.legend.engine legend-engine-xt-persistence-component - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/pom.xml index 68810141ae6..346eb264ae7 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-grammar/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-grammar/pom.xml index 173307080a7..facdde60178 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-grammar/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-grammar/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-protocol/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-protocol/pom.xml index 99f9c905d46..2e9979db2e6 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-protocol/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-pure/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-pure/pom.xml index 23a591e79cf..57f3c6a0aae 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-pure/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-grammar/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-grammar/pom.xml index aae799618fd..b3752b6992c 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-grammar/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-grammar/pom.xml @@ -3,7 +3,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-protocol/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-protocol/pom.xml index 8142ee1c5e6..14fa9728d66 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-protocol/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-pure/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-pure/pom.xml index 08dfda34a5d..f61f482a32e 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-pure/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-target-relational-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-test-runner/pom.xml b/legend-engine-xts-persistence/legend-engine-xt-persistence-test-runner/pom.xml index b60286c2981..059588ae19f 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-test-runner/pom.xml +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-test-runner/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-persistence - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-persistence/pom.xml b/legend-engine-xts-persistence/pom.xml index 560978be874..e8883a20bf3 100644 --- a/legend-engine-xts-persistence/pom.xml +++ b/legend-engine-xts-persistence/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-protobuf/legend-engine-xt-protobuf-grammar/pom.xml b/legend-engine-xts-protobuf/legend-engine-xt-protobuf-grammar/pom.xml index 95b50c8b79a..c4f6d26f97e 100644 --- a/legend-engine-xts-protobuf/legend-engine-xt-protobuf-grammar/pom.xml +++ b/legend-engine-xts-protobuf/legend-engine-xt-protobuf-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-protobuf - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-protobuf-grammar diff --git a/legend-engine-xts-protobuf/legend-engine-xt-protobuf-protocol/pom.xml b/legend-engine-xts-protobuf/legend-engine-xt-protobuf-protocol/pom.xml index f9d12cdc410..2110a16bdab 100644 --- a/legend-engine-xts-protobuf/legend-engine-xt-protobuf-protocol/pom.xml +++ b/legend-engine-xts-protobuf/legend-engine-xt-protobuf-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-protobuf - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-protobuf-protocol diff --git a/legend-engine-xts-protobuf/legend-engine-xt-protobuf-pure/pom.xml b/legend-engine-xts-protobuf/legend-engine-xt-protobuf-pure/pom.xml index 7d95e761710..39de766d2b2 100644 --- a/legend-engine-xts-protobuf/legend-engine-xt-protobuf-pure/pom.xml +++ b/legend-engine-xts-protobuf/legend-engine-xt-protobuf-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-protobuf - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-protobuf/legend-engine-xt-protobuf/pom.xml b/legend-engine-xts-protobuf/legend-engine-xt-protobuf/pom.xml index 5f213e4664a..ab83b0d422a 100644 --- a/legend-engine-xts-protobuf/legend-engine-xt-protobuf/pom.xml +++ b/legend-engine-xts-protobuf/legend-engine-xt-protobuf/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-protobuf - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -57,7 +57,7 @@ org.finos.legend.engine legend-engine-protocol-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT org.finos.legend.pure diff --git a/legend-engine-xts-protobuf/pom.xml b/legend-engine-xts-protobuf/pom.xml index c479425c12e..5e0d4534792 100644 --- a/legend-engine-xts-protobuf/pom.xml +++ b/legend-engine-xts-protobuf/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-analytics/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-analytics/pom.xml index 36d7c7bc0ee..094dc1f73a7 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-analytics/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-analytics/pom.xml @@ -19,7 +19,7 @@ legend-engine-xt-relationalStore-analytics org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-pure/pom.xml index 27f8c850a38..2157c146b75 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/legend-engine-xt-relationalStore-store-entitlement-pure/pom.xml @@ -19,7 +19,7 @@ legend-engine-xt-relationalStore-analytics org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/pom.xml index ccb3a93a3b3..03771ba0421 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-analytics/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-relationalStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/pom.xml index 06043cb1b1f..6754b87c866 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-relationalStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -33,14 +33,6 @@ org.finos.legend.engine legend-engine-shared-core - - org.finos.legend.engine - legend-engine-xt-authentication-protocol - - - org.finos.legend.engine - legend-engine-xt-authentication-implementation-core - org.finos.legend.engine legend-engine-xt-authentication-connection-factory @@ -62,10 +54,18 @@ + + + com.zaxxer + HikariCP + + + com.h2database h2 + runtime @@ -75,11 +75,6 @@ junit test - - org.finos.legend.engine - legend-engine-xt-authentication-implementation-vault-aws - test - \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/driver/JDBCConnectionDriver.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/DatabaseManager.java similarity index 77% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/driver/JDBCConnectionDriver.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/DatabaseManager.java index 80c80a32b30..d7320f0a503 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/driver/JDBCConnectionDriver.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/DatabaseManager.java @@ -12,17 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection.jdbc.driver; +package org.finos.legend.connection; import java.util.List; import java.util.Properties; -public interface JDBCConnectionDriver +public interface DatabaseManager { List getIds(); - // TODO?: @akphi - should we port over the driver wrapper stuffs from DatabaseManager as well? String getDriver(); - String buildURL(String host, int port, String databaseName, Properties extraUserDataSourceProperties); + String buildURL(String host, int port, String databaseName, Properties properties); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/DatabaseType.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/DatabaseType.java index 159627e6a63..3982c57e35e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/DatabaseType.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/DatabaseType.java @@ -16,7 +16,9 @@ public enum DatabaseType implements Database { + H2("H2"), POSTGRES("Postgres"), + BIG_QUERY("BigQuery"), SNOWFLAKE("Snowflake"); private final String label; diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/HACKY__RelationalDatabaseConnectionAdapter.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/HACKY__RelationalDatabaseConnectionAdapter.java new file mode 100644 index 00000000000..172bdd2a8d2 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/HACKY__RelationalDatabaseConnectionAdapter.java @@ -0,0 +1,41 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection; + +import org.finos.legend.connection.protocol.AuthenticationConfiguration; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.shared.core.identity.Identity; + +/** + * NOTE: this is hacky way of us to realize the relational database connection that we can support + * in the new connection framework (i.e. using {@link ConnectionFactory}), when we have a more + * solid strategy in place for migration to this new framework, we should then remove this mechanism + */ +public interface HACKY__RelationalDatabaseConnectionAdapter +{ + ConnectionFactoryMaterial adapt(RelationalDatabaseConnection relationalDatabaseConnection, Identity identity, LegendEnvironment environment); + + class ConnectionFactoryMaterial + { + public final StoreInstance storeInstance; + public final AuthenticationConfiguration authenticationConfiguration; + + public ConnectionFactoryMaterial(StoreInstance storeInstance, AuthenticationConfiguration authenticationConfiguration) + { + this.storeInstance = storeInstance; + this.authenticationConfiguration = authenticationConfiguration; + } + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/JDBCConnectionBuilder.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/JDBCConnectionBuilder.java new file mode 100644 index 00000000000..0c09de0784e --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/JDBCConnectionBuilder.java @@ -0,0 +1,54 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection; + +import org.finos.legend.connection.impl.JDBCConnectionManager; +import org.finos.legend.connection.protocol.ConnectionSpecification; +import org.finos.legend.engine.shared.core.identity.Credential; + +import java.sql.Connection; + +public abstract class JDBCConnectionBuilder extends ConnectionBuilder +{ + private JDBCConnectionManager.ConnectionPoolConfig connectionPoolConfig; + + @Override + public Class getCredentialType() + { + return (Class) actualTypeArguments()[0]; + } + + @Override + public Class getConnectionSpecificationType() + { + return (Class) actualTypeArguments()[1]; + } + + @Override + public JDBCConnectionManager getConnectionManager() + { + return JDBCConnectionManager.getInstance(); + } + + public void setConnectionPoolConfig(JDBCConnectionManager.ConnectionPoolConfig connectionPoolConfig) + { + this.connectionPoolConfig = connectionPoolConfig; + } + + public JDBCConnectionManager.ConnectionPoolConfig getConnectionPoolConfig() + { + return connectionPoolConfig; + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/RelationalDatabaseStoreSupport.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/RelationalDatabaseStoreSupport.java index 7a80a169137..b81cb654a97 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/RelationalDatabaseStoreSupport.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/RelationalDatabaseStoreSupport.java @@ -15,23 +15,18 @@ package org.finos.legend.connection; import org.eclipse.collections.api.factory.Lists; -import org.finos.legend.connection.protocol.AuthenticationMechanism; -import java.util.ArrayList; -import java.util.LinkedHashSet; import java.util.List; import java.util.Objects; -import java.util.Set; public class RelationalDatabaseStoreSupport extends StoreSupport { private final Database database; - private final Set authenticationMechanisms = new LinkedHashSet<>(); - private RelationalDatabaseStoreSupport(String identifier, Database database, List authenticationMechanisms) + private RelationalDatabaseStoreSupport(String identifier, Database database, List authenticationMechanismConfigurations) { - super(identifier, authenticationMechanisms); - this.database = database; + super(identifier, authenticationMechanismConfigurations); + this.database = Objects.requireNonNull(database, "Relational database store support database type is missing"); } public Database getDatabase() @@ -39,48 +34,67 @@ public Database getDatabase() return database; } + public static RelationalDatabaseStoreSupport cast(StoreSupport storeSupport) + { + return cast(storeSupport, null); + } + + public static RelationalDatabaseStoreSupport cast(StoreSupport storeSupport, Database database) + { + if (!(storeSupport instanceof RelationalDatabaseStoreSupport)) + { + throw new RuntimeException("Expected store support for relational databases"); + } + RelationalDatabaseStoreSupport relationalDatabaseStoreSupport = (RelationalDatabaseStoreSupport) storeSupport; + if (database != null && !database.equals(relationalDatabaseStoreSupport.getDatabase())) + { + + throw new RuntimeException(String.format("Expected relational database store support for '%s'", database.getLabel())); + } + return relationalDatabaseStoreSupport; + } + public static class Builder { + private final Database database; private String identifier; - private Database database; - private final Set authenticationMechanisms = new LinkedHashSet<>(); + private final List authenticationMechanismConfigurations = Lists.mutable.empty(); - public Builder withIdentifier(String identifier) + public Builder(Database database) { - this.identifier = identifier; - return this; + this.database = database; } - public Builder withDatabase(Database database) + public Builder withIdentifier(String identifier) { - this.database = database; + this.identifier = identifier; return this; } - public Builder withAuthenticationMechanisms(List authenticationMechanisms) + public Builder withAuthenticationMechanismConfiguration(AuthenticationMechanismConfiguration authenticationMechanismConfiguration) { - this.authenticationMechanisms.addAll(authenticationMechanisms); + this.authenticationMechanismConfigurations.add(authenticationMechanismConfiguration); return this; } - public Builder withAuthenticationMechanisms(AuthenticationMechanism... authenticationMechanisms) + public Builder withAuthenticationMechanismConfigurations(List authenticationMechanismConfigurations) { - this.authenticationMechanisms.addAll(Lists.mutable.of(authenticationMechanisms)); + this.authenticationMechanismConfigurations.addAll(authenticationMechanismConfigurations); return this; } - public Builder withAuthenticationMechanism(AuthenticationMechanism authenticationMechanism) + public Builder withAuthenticationMechanismConfigurations(AuthenticationMechanismConfiguration... authenticationMechanismConfigurations) { - this.authenticationMechanisms.add(authenticationMechanism); + this.authenticationMechanismConfigurations.addAll(Lists.mutable.of(authenticationMechanismConfigurations)); return this; } public RelationalDatabaseStoreSupport build() { return new RelationalDatabaseStoreSupport( - Objects.requireNonNull(this.identifier, "Store support identifier is required"), - Objects.requireNonNull(this.database, "Store support database type is required"), - new ArrayList<>(this.authenticationMechanisms) + this.identifier, + this.database, + this.authenticationMechanismConfigurations ); } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/driver/H2_JDBCConnectionDriver.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/H2DatabaseManager.java similarity index 79% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/driver/H2_JDBCConnectionDriver.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/H2DatabaseManager.java index 99c86c579ae..44fdf427975 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/driver/H2_JDBCConnectionDriver.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/H2DatabaseManager.java @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection.jdbc.driver; +package org.finos.legend.connection.impl; import org.eclipse.collections.impl.factory.Lists; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; +import org.finos.legend.connection.DatabaseManager; +import org.finos.legend.connection.DatabaseType; import java.util.List; import java.util.Properties; -public class H2_JDBCConnectionDriver implements JDBCConnectionDriver +public class H2DatabaseManager implements DatabaseManager { @Override public List getIds() @@ -35,7 +36,7 @@ public String getDriver() } @Override - public String buildURL(String host, int port, String databaseName, Properties extraUserDataSourceProperties) + public String buildURL(String host, int port, String databaseName, Properties properties) { return String.format("jdbc:h2:tcp://%s:%s/mem:%s", host, port, databaseName); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/JDBCConnectionManager.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/JDBCConnectionManager.java new file mode 100644 index 00000000000..8b96bf0aa7a --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/JDBCConnectionManager.java @@ -0,0 +1,425 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection.impl; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.eclipse.collections.api.block.function.Function0; +import org.eclipse.collections.api.map.ConcurrentMutableMap; +import org.eclipse.collections.impl.map.mutable.ConcurrentHashMap; +import org.finos.legend.connection.Authenticator; +import org.finos.legend.connection.ConnectionManager; +import org.finos.legend.connection.Database; +import org.finos.legend.connection.DatabaseManager; +import org.finos.legend.connection.LegendEnvironment; +import org.finos.legend.connection.StoreInstance; +import org.finos.legend.connection.protocol.AuthenticationConfiguration; +import org.finos.legend.connection.protocol.ConnectionSpecification; +import org.finos.legend.engine.shared.core.identity.Credential; +import org.finos.legend.engine.shared.core.identity.Identity; + +import javax.sql.DataSource; +import java.io.PrintWriter; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.SQLException; +import java.sql.SQLFeatureNotSupportedException; +import java.util.Objects; +import java.util.Properties; +import java.util.ServiceLoader; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.logging.Logger; + +public class JDBCConnectionManager implements ConnectionManager +{ + private static final long HIKARICP_DEFAULT_CONNECTION_TIMEOUT = 30000L; + private static final int HIKARICP_DEFAULT_MAX_POOL_SIZE = 100; + private static final int HIKARICP_DEFAULT_MIN_POOL_SIZE = 0; + + private static final ConcurrentHashMap managerByName = ConcurrentHashMap.newMap(); + private static final AtomicBoolean isInitialized = new AtomicBoolean(); + + private static JDBCConnectionManager INSTANCE; + private final ConcurrentMutableMap poolIndex = ConcurrentHashMap.newMap(); + + protected JDBCConnectionManager() + { + // singleton + } + + public static synchronized JDBCConnectionManager getInstance() + { + if (INSTANCE == null) + { + INSTANCE = new JDBCConnectionManager(); + } + return INSTANCE; + } + + private static void setup() + { + // register database managers + if (!isInitialized.get()) + { + synchronized (isInitialized) + { + if (!isInitialized.get()) + { + for (DatabaseManager manager : ServiceLoader.load(DatabaseManager.class)) + { + manager.getIds().forEach(i -> managerByName.put(i, manager)); + } + isInitialized.set(true); + } + } + } + } + + @Override + public void initialize(LegendEnvironment environment) + { + JDBCConnectionManager.setup(); + } + + public Connection getConnection(Database database, + String host, + int port, + String databaseName, + Properties connectionProperties, + ConnectionPoolConfig connectionPoolConfig, + Function authenticationPropertiesSupplier, + Authenticator authenticator, + Identity identity + ) throws SQLException + { + StoreInstance storeInstance = authenticator.getStoreInstance(); + ConnectionSpecification connectionSpecification = storeInstance.getConnectionSpecification(); + AuthenticationConfiguration authenticationConfiguration = authenticator.getAuthenticationConfiguration(); + String poolName = getPoolName(identity, connectionSpecification, authenticationConfiguration); + + // TODO: @akphi - this is simplistic, we need to handle concurrency and errors + Supplier dataSourceSupplier = () -> this.buildDataSource(database, host, port, databaseName, connectionProperties, connectionPoolConfig, authenticationPropertiesSupplier, authenticator, identity); + Function0 connectionPoolSupplier = () -> new ConnectionPool(dataSourceSupplier.get()); + ConnectionPool connectionPool = this.poolIndex.getIfAbsentPut(poolName, connectionPoolSupplier); + + return connectionPool.dataSource.getConnection(); + +// try (Scope scope = GlobalTracer.get().buildSpan("Get Connection").startActive(true)) +// { +// ConnectionKey connectionKey = this.getConnectionKey(); +// // Logs and traces ----- +// String principal = identityState.getIdentity().getName(); +// scope.span().setTag("Principal", principal); +// scope.span().setTag("DataSourceSpecification", this.toString()); +// LOGGER.info("Get Connection as [{}] for datasource [{}]", principal, connectionKey.shortId()); +// // --------------------- +// try +// { +// DataSourceWithStatistics dataSourceWithStatistics = this.connectionStateManager.getDataSourceForIdentityIfAbsentBuild(identityState, this, dataSourcePoolBuilder); +// // Logs and traces and stats ----- +// String poolName = dataSourceWithStatistics.getPoolName(); +// scope.span().setTag("Pool", poolName); +// int requests = dataSourceWithStatistics.requestConnection(); +// LOGGER.info("Principal [{}] has requested [{}] connections for pool [{}]", principal, requests, poolName); +// return authenticationStrategy.getConnection(dataSourceWithStatistics, identityState.getIdentity()); +// } +// catch (ConnectionException ce) +// { +// LOGGER.error("ConnectionException {{}} : pool stats [{}] ", principal, connectionStateManager.getPoolStatisticsAsJSON(poolNameFor(identityState.getIdentity()))); +// LOGGER.error("ConnectionException ", ce); +// throw ce; +// } +// } + } + + protected HikariDataSource buildDataSource( + Database database, + String host, + int port, + String databaseName, + Properties connectionProperties, + ConnectionPoolConfig connectionPoolConfig, + Function authenticationPropertiesSupplier, + Authenticator authenticator, + Identity identity + ) + { + StoreInstance storeInstance = authenticator.getStoreInstance(); + ConnectionSpecification connectionSpecification = storeInstance.getConnectionSpecification(); + AuthenticationConfiguration authenticationConfiguration = authenticator.getAuthenticationConfiguration(); + DatabaseManager databaseManager = getManagerForDatabase(database); + + String jdbcUrl = databaseManager.buildURL(host, port, databaseName, connectionProperties); + String poolName = getPoolName(identity, connectionSpecification, authenticationConfiguration); + + HikariConfig jdbcConfig = new HikariConfig(); + jdbcConfig.setDriverClassName(databaseManager.getDriver()); + jdbcConfig.setPoolName(poolName); + jdbcConfig.setJdbcUrl(jdbcUrl); + + // NOTE: we could allow more granularity by allow specifying these pooling configurations at specification level + jdbcConfig.setMinimumIdle(connectionPoolConfig != null && connectionPoolConfig.getMinPoolSize() != null ? connectionPoolConfig.getMinPoolSize() : HIKARICP_DEFAULT_MIN_POOL_SIZE); + jdbcConfig.setMaximumPoolSize(connectionPoolConfig != null && connectionPoolConfig.getMaxPoolSize() != null ? connectionPoolConfig.getMaxPoolSize() : HIKARICP_DEFAULT_MAX_POOL_SIZE); + jdbcConfig.setConnectionTimeout(connectionPoolConfig != null && connectionPoolConfig.getConnectionTimeout() != null ? connectionPoolConfig.getConnectionTimeout() : HIKARICP_DEFAULT_CONNECTION_TIMEOUT); + + // specific system configuration to disable statement cache for all databases + // TODO: @akphi - document why we need to do this, check with @pierredebelen, @kevin-m-knight-gs, @epsstan + // See https://github.com/brettwooldridge/HikariCP#statement-cache + jdbcConfig.addDataSourceProperty("cachePrepStmts", false); + jdbcConfig.addDataSourceProperty("prepStmtCacheSize", 0); + jdbcConfig.addDataSourceProperty("prepStmtCacheSqlLimit", 0); + jdbcConfig.addDataSourceProperty("useServerPrepStmts", false); + + jdbcConfig.setDataSource(new DataSourceWrapper(jdbcUrl, connectionProperties, databaseManager, authenticationPropertiesSupplier, authenticator, identity)); + return new HikariDataSource(jdbcConfig); + } + + public ConnectionPool getPool(String poolName) + { + return this.poolIndex.get(poolName); + } + + public int getPoolSize() + { + return this.poolIndex.size(); + } + + public void flushPool() + { + this.poolIndex.forEachKey(this.poolIndex::remove); + } + + public static String getPoolName(Identity identity, ConnectionSpecification connectionSpecification, AuthenticationConfiguration authenticationConfiguration) + { + return String.format("DBPool|%s|%s|%s|%s", + connectionSpecification.shortId(), + authenticationConfiguration.shortId(), + identity.getName(), + identity.getFirstCredential().getClass().getCanonicalName() + ); + } + + private static DatabaseManager getManagerForDatabase(Database database) + { + if (!isInitialized.get()) + { + throw new IllegalStateException("JDBC connection manager has not been configured properly"); + } + DatabaseManager manager = managerByName.get(database.getLabel()); + if (manager == null) + { + throw new RuntimeException(String.format("Can't find any matching managers for database type '%s'", database.getLabel())); + } + return manager; + } + + public static class ConnectionPoolConfig + { + private final Integer minPoolSize; + private final Integer maxPoolSize; + private final Long connectionTimeout; + + private ConnectionPoolConfig(Integer minPoolSize, Integer maxPoolSize, Long connectionTimeout) + { + this.minPoolSize = minPoolSize; + this.maxPoolSize = maxPoolSize; + this.connectionTimeout = connectionTimeout; + } + + public Integer getMinPoolSize() + { + return minPoolSize; + } + + public Integer getMaxPoolSize() + { + return maxPoolSize; + } + + public Long getConnectionTimeout() + { + return connectionTimeout; + } + + public static class Builder + { + private Integer minPoolSize = null; + private Integer maxPoolSize = null; + private Long connectionTimeout = null; + + public Builder withMinPoolSize(Integer minPoolSize) + { + this.minPoolSize = minPoolSize; + return this; + } + + public Builder withMaxPoolSize(Integer maxPoolSize) + { + this.maxPoolSize = maxPoolSize; + return this; + } + + public Builder withConnectionTimeout(Long connectionTimeout) + { + this.connectionTimeout = connectionTimeout; + return this; + } + + public ConnectionPoolConfig build() + { + return new ConnectionPoolConfig(this.minPoolSize, this.maxPoolSize, this.connectionTimeout); + } + } + } + + public static class ConnectionPool + { + private final HikariDataSource dataSource; + + public ConnectionPool(HikariDataSource dataSource) + { + this.dataSource = Objects.requireNonNull(dataSource); + } + + public String getPoolName() + { + return this.dataSource.getPoolName(); + } + + public int getActiveConnections() + { + return this.dataSource.getHikariPoolMXBean().getActiveConnections(); + } + + public int getTotalConnections() + { + return this.dataSource.getHikariPoolMXBean().getTotalConnections(); + } + + public int getIdleConnections() + { + return this.dataSource.getHikariPoolMXBean().getIdleConnections(); + } + + public Properties getProperties() + { + return this.dataSource.getDataSourceProperties(); + } + } + + private static class DataSourceWrapper implements DataSource + { + private final String url; + private final Properties connectionProperties; + private final Function authenticationPropertiesSupplier; + private final Authenticator authenticator; + // TODO: @akphi - how do we get rid of this here? + private final Identity identity; + + private final Driver driver; + + public DataSourceWrapper( + String url, Properties connectionProperties, + DatabaseManager databaseManager, + Function authenticationPropertiesSupplier, + Authenticator authenticator, + Identity identity + ) + { + this.url = url; + this.connectionProperties = connectionProperties; + try + { + this.driver = (Driver) Class.forName(databaseManager.getDriver()).getDeclaredConstructor().newInstance(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + this.authenticationPropertiesSupplier = authenticationPropertiesSupplier; + this.authenticator = authenticator; + this.identity = identity; + } + + @Override + public Connection getConnection() throws SQLException + { + Properties properties = new Properties(); + properties.putAll(this.connectionProperties); + Credential credential; + try + { + credential = authenticator.makeCredential(identity); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + Properties authenticationProperties = authenticationPropertiesSupplier.apply(credential); + properties.putAll(authenticationProperties); + // TODO: @akphi - prune unnecessary properties using DatabaseManager + // TODO: @akphi - add logging and statistics like in execution DriverWrapper + return driver.connect(this.url, properties); + } + + @Override + public Connection getConnection(String username, String password) throws SQLException + { + throw new RuntimeException(); + } + + @Override + public PrintWriter getLogWriter() throws SQLException + { + return null; + } + + @Override + public void setLogWriter(PrintWriter out) throws SQLException + { + } + + @Override + public void setLoginTimeout(int seconds) throws SQLException + { + } + + @Override + public int getLoginTimeout() throws SQLException + { + return 0; + } + + @Override + public T unwrap(Class iface) throws SQLException + { + return null; + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException + { + return false; + } + + @Override + public Logger getParentLogger() throws SQLFeatureNotSupportedException + { + return null; + } + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/StaticJDBCConnectionBuilder.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/StaticJDBCConnectionBuilder.java new file mode 100644 index 00000000000..1cbe12abf4f --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/impl/StaticJDBCConnectionBuilder.java @@ -0,0 +1,52 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection.impl; + +import org.finos.legend.connection.Authenticator; +import org.finos.legend.connection.JDBCConnectionBuilder; +import org.finos.legend.connection.RelationalDatabaseStoreSupport; +import org.finos.legend.connection.StoreInstance; +import org.finos.legend.connection.protocol.StaticJDBCConnectionSpecification; +import org.finos.legend.engine.shared.core.identity.Credential; +import org.finos.legend.engine.shared.core.identity.Identity; +import org.finos.legend.engine.shared.core.identity.credential.PlaintextUserPasswordCredential; + +import java.sql.Connection; +import java.util.Properties; +import java.util.function.Function; + +public class StaticJDBCConnectionBuilder +{ + public static class WithPlaintextUsernamePassword extends JDBCConnectionBuilder + { + public Connection getConnection(StaticJDBCConnectionSpecification connectionSpecification, Authenticator authenticator, Identity identity) throws Exception + { + StoreInstance storeInstance = authenticator.getStoreInstance(); + RelationalDatabaseStoreSupport storeSupport = RelationalDatabaseStoreSupport.cast(storeInstance.getStoreSupport()); + + Properties connectionProperties = new Properties(); + Function authenticationPropertiesSupplier = cred -> + { + PlaintextUserPasswordCredential credential = (PlaintextUserPasswordCredential) cred; + Properties properties = new Properties(); + properties.put("user", credential.getUser()); + properties.put("password", credential.getPassword()); + return properties; + }; + + return this.getConnectionManager().getConnection(storeSupport.getDatabase(), connectionSpecification.host, connectionSpecification.port, connectionSpecification.databaseName, connectionProperties, this.getConnectionPoolConfig(), authenticationPropertiesSupplier, authenticator, identity); + } + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/JDBCConnectionManager.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/JDBCConnectionManager.java deleted file mode 100644 index 12a48d2611f..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/JDBCConnectionManager.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection.jdbc; - -import org.eclipse.collections.impl.map.mutable.ConcurrentHashMap; -import org.finos.legend.connection.ConnectionManager; -import org.finos.legend.connection.Database; -import org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver; - -import java.util.ServiceLoader; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * TODO?: @akphi - This is a temporary hack! - * We probably need to have a mechanism to control the connection pool - * We cloned DatabaseManager from relational executor, we should consider if we can eventually unify these 2 - */ -public class JDBCConnectionManager implements ConnectionManager -{ - private static final ConcurrentHashMap driversByName = ConcurrentHashMap.newMap(); - private static final AtomicBoolean isInitialized = new AtomicBoolean(); - - private static void detectDrivers() - { - if (!isInitialized.get()) - { - synchronized (isInitialized) - { - if (!isInitialized.get()) - { - for (JDBCConnectionDriver driver : ServiceLoader.load(JDBCConnectionDriver.class)) - { - JDBCConnectionManager.register(driver); - } - isInitialized.getAndSet(true); - } - } - } - } - - private static void register(JDBCConnectionDriver driver) - { - driver.getIds().forEach(i -> driversByName.put(i, driver)); - } - - public static JDBCConnectionDriver getDriverForDatabase(Database database) - { - if (!isInitialized.get()) - { - throw new IllegalStateException("JDBC connection manager has not been configured properly"); - } - JDBCConnectionDriver driver = driversByName.get(database.getLabel()); - if (driver == null) - { - throw new RuntimeException(String.format("Can't find matching JDBC connection driver for database type '%s'", database)); - } - return driver; - } - - @Override - public void initialize() - { - JDBCConnectionManager.detectDrivers(); - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/StaticJDBCConnectionBuilder.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/StaticJDBCConnectionBuilder.java deleted file mode 100644 index c5068467bef..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/StaticJDBCConnectionBuilder.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection.jdbc; - -import org.finos.legend.connection.ConnectionBuilder; -import org.finos.legend.connection.RelationalDatabaseStoreSupport; -import org.finos.legend.connection.StoreInstance; -import org.finos.legend.connection.StoreSupport; -import org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver; -import org.finos.legend.engine.shared.core.identity.credential.PlaintextUserPasswordCredential; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.Properties; - -public class StaticJDBCConnectionBuilder -{ - public static class WithPlaintextUsernamePassword extends ConnectionBuilder - { - public Connection getConnection(PlaintextUserPasswordCredential credential, StaticJDBCConnectionSpecification connectionSpecification, StoreInstance storeInstance) throws Exception - { - StoreSupport storeSupport = storeInstance.getStoreSupport(); - if (!(storeSupport instanceof RelationalDatabaseStoreSupport)) - { - throw new RuntimeException("Can't get connection: only support relational databases"); - } - JDBCConnectionDriver driver = JDBCConnectionManager.getDriverForDatabase(((RelationalDatabaseStoreSupport) storeSupport).getDatabase()); - return DriverManager.getConnection( - driver.buildURL(connectionSpecification.host, connectionSpecification.port, connectionSpecification.databaseName, new Properties()), - credential.getUser(), credential.getPassword() - ); - } - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/legacy/JdbcConnectionProvider.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/legacy/JdbcConnectionProvider.java deleted file mode 100644 index 099e2ed1581..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/legacy/JdbcConnectionProvider.java +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection.jdbc.legacy; - -import org.finos.legend.authentication.credentialprovider.CredentialProviderProvider; -import org.finos.legend.connection.legacy.ConnectionProvider; -import org.finos.legend.connection.legacy.ConnectionSpecification; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.authentication.specification.AuthenticationSpecification; -import org.finos.legend.engine.shared.core.identity.Credential; -import org.finos.legend.engine.shared.core.identity.Identity; -import org.finos.legend.engine.shared.core.identity.credential.PlaintextUserPasswordCredential; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.util.Properties; - -public class JdbcConnectionProvider extends ConnectionProvider -{ - public JdbcConnectionProvider(CredentialProviderProvider credentialProviderProvider) - { - super(credentialProviderProvider); - } - - @Override - public Connection makeConnection(ConnectionSpecification connectionSpec, AuthenticationSpecification authenticationSpec, Identity identity) throws Exception - { - assert (connectionSpec instanceof JdbcConnectionSpecification); - JdbcConnectionSpecification jdbcConnectionSpec = (JdbcConnectionSpecification) connectionSpec; - - Credential credential = super.makeCredential(authenticationSpec, identity); - - switch (jdbcConnectionSpec.dbType) - { - case H2: - return connectToH2(jdbcConnectionSpec, credential, identity); - default: - throw new UnsupportedOperationException("Unsupported Db Type " + jdbcConnectionSpec.dbType); - } - } - - private Connection connectToH2(JdbcConnectionSpecification jdbcConnectionSpecification, Credential credential, Identity identity) throws Exception - { - if (!(credential instanceof PlaintextUserPasswordCredential)) - { - String message = String.format("Failed to create connected. Expected credential of type %s but found credential of type %s", PlaintextUserPasswordCredential.class, credential.getClass()); - throw new UnsupportedOperationException(message); - } - PlaintextUserPasswordCredential plaintextUserPasswordCredential = (PlaintextUserPasswordCredential) credential; - Properties properties = new Properties(); - properties.setProperty("user", plaintextUserPasswordCredential.getUser()); - properties.setProperty("password", plaintextUserPasswordCredential.getPassword()); - String url = "jdbc:h2:tcp://" + jdbcConnectionSpecification.dbHostname + ":" + jdbcConnectionSpecification.dbPort + "/mem:" + "db1"; - Connection connection = DriverManager.getConnection(url); - return connection; - } -} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/legacy/JdbcConnectionSpecification.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/legacy/JdbcConnectionSpecification.java deleted file mode 100644 index 7722fa37c45..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/legacy/JdbcConnectionSpecification.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection.jdbc.legacy; - -import org.finos.legend.connection.legacy.ConnectionSpecification; - -public class JdbcConnectionSpecification extends ConnectionSpecification -{ - public enum DbType - { - H2 - } - - public String dbHostname; - public int dbPort; - public DbType dbType; - - public JdbcConnectionSpecification() - { - - } - - public JdbcConnectionSpecification(String dbHostname, int dbPort, DbType dbType) - { - this.dbHostname = dbHostname; - this.dbPort = dbPort; - this.dbType = dbType; - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/StaticJDBCConnectionSpecification.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/protocol/StaticJDBCConnectionSpecification.java similarity index 80% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/StaticJDBCConnectionSpecification.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/protocol/StaticJDBCConnectionSpecification.java index ca531113137..373e76c7c42 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/jdbc/StaticJDBCConnectionSpecification.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/java/org/finos/legend/connection/protocol/StaticJDBCConnectionSpecification.java @@ -12,17 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection.jdbc; +package org.finos.legend.connection.protocol; import org.finos.legend.connection.protocol.ConnectionSpecification; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; public class StaticJDBCConnectionSpecification extends ConnectionSpecification { public String host; public int port; public String databaseName; - public DatabaseType databaseType; public StaticJDBCConnectionSpecification(String host, int port, String databaseName) { @@ -30,4 +28,13 @@ public StaticJDBCConnectionSpecification(String host, int port, String databaseN this.port = port; this.databaseName = databaseName; } + + @Override + public String shortId() + { + return "StaticJDBC" + + "--host=" + host + + "--port=" + port + + "--databaseName=" + databaseName; + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionBuilder b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionBuilder index b0c819c2b3a..a03d3863361 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionBuilder +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionBuilder @@ -1 +1 @@ -org.finos.legend.connection.jdbc.StaticJDBCConnectionBuilder$WithPlaintextUsernamePassword +org.finos.legend.connection.impl.StaticJDBCConnectionBuilder$WithPlaintextUsernamePassword diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionManager b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionManager deleted file mode 100644 index 1344ad1b80f..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionManager +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.jdbc.JDBCConnectionManager diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager new file mode 100644 index 00000000000..069422f1bf1 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager @@ -0,0 +1 @@ +org.finos.legend.connection.impl.H2DatabaseManager diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver deleted file mode 100644 index 098e66f1d92..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.jdbc.driver.H2_JDBCConnectionDriver diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/test/java/org/finos/legend/connection/TestJdbcConnectionProvider.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/test/java/org/finos/legend/connection/TestJdbcConnectionProvider.java deleted file mode 100644 index 9a263f4497f..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-connection/src/test/java/org/finos/legend/connection/TestJdbcConnectionProvider.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.connection; - -import java.sql.Connection; -import java.util.Properties; -import org.finos.legend.authentication.credentialprovider.CredentialProviderProvider; -import org.finos.legend.authentication.credentialprovider.impl.UserPasswordCredentialProvider; -import org.finos.legend.authentication.intermediationrule.IntermediationRuleProvider; -import org.finos.legend.authentication.intermediationrule.impl.UserPasswordFromVaultRule; -import org.finos.legend.authentication.vault.CredentialVaultProvider; -import org.finos.legend.authentication.vault.PlatformCredentialVaultProvider; -import org.finos.legend.authentication.vault.impl.AWSSecretsManagerVault; -import org.finos.legend.authentication.vault.impl.PropertiesFileCredentialVault; -import org.finos.legend.connection.jdbc.legacy.JdbcConnectionProvider; -import org.finos.legend.connection.jdbc.legacy.JdbcConnectionSpecification; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.authentication.specification.UserPasswordAuthenticationSpecification; -import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.authentication.vault.PropertiesFileSecret; -import org.finos.legend.engine.shared.core.identity.Identity; -import org.finos.legend.engine.shared.core.identity.credential.AnonymousCredential; -import org.h2.tools.Server; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -import static org.junit.Assert.assertNotNull; - -public class TestJdbcConnectionProvider -{ - private Server h2Server; - - @Before - public void setup() throws Exception - { - this.h2Server = Server.createTcpServer("-ifNotExists", "-tcpPort", "0").start(); - } - - @After - public void shutdown() - { - if (this.h2Server == null) - { - return; - } - this.h2Server.stop(); - } - - @Test - public void testH2ConnectionCreation() throws Exception - { - Properties properties = new Properties(); - properties.put("passwordRef1", ""); - PropertiesFileCredentialVault propertiesFileCredentialVault = new PropertiesFileCredentialVault(properties); - - PlatformCredentialVaultProvider platformCredentialVaultProvider = PlatformCredentialVaultProvider.builder() - .with(propertiesFileCredentialVault) - .build(); - - AWSSecretsManagerVault awsSecretsManagerVault = AWSSecretsManagerVault.builder() - .with(platformCredentialVaultProvider) - .build(); - - CredentialVaultProvider credentialVaultProvider = CredentialVaultProvider.builder() - .with(platformCredentialVaultProvider) - .with(awsSecretsManagerVault) - .build(); - - IntermediationRuleProvider intermediationRuleProvider = IntermediationRuleProvider.builder() - .with(new UserPasswordFromVaultRule(credentialVaultProvider)) - .build(); - - CredentialProviderProvider credentialProviderProvider = CredentialProviderProvider.builder() - .with(new UserPasswordCredentialProvider()) - .with(intermediationRuleProvider) - .build(); - - JdbcConnectionProvider connectionProvider = new JdbcConnectionProvider(credentialProviderProvider); - JdbcConnectionSpecification connectionSpecification = new JdbcConnectionSpecification("localhost", this.h2Server.getPort(), JdbcConnectionSpecification.DbType.H2); - UserPasswordAuthenticationSpecification authenticationSpecification = new UserPasswordAuthenticationSpecification("sa", new PropertiesFileSecret("passwordRef1")); - - Identity alice = new Identity("alice", new AnonymousCredential()); - Connection connection = connectionProvider.makeConnection(connectionSpecification, authenticationSpecification, alice); - - assertNotNull(connection); - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution-tests/pom.xml index 83e70c1a85a..f54cc552e88 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution-tests/pom.xml @@ -3,7 +3,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-athena - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution/pom.xml index e4c4b03e815..252a324138d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-athena - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-grammar/pom.xml index 8c38c5290f3..7915d4de50b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-athena - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-protocol/pom.xml index 6eb88dec31d..615f857d65d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-athena - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/pom.xml index e5271116c2b..35c20b18636 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-athena - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena.definition.json b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena.definition.json index 7a71e1f48c8..608b107464f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena.definition.json +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena.definition.json @@ -1,6 +1,6 @@ { "name": "core_relational_athena", - "pattern": "(meta::relational::functions::sqlQueryToString::athena|meta::relational::tests::sqlQueryToString::athena|meta::relational::tests::connEquality|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", + "pattern": "(meta::relational::functions::sqlQueryToString::athena|meta::relational::tests::sqlQueryToString::athena|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", "dependencies": [ "platform", "platform_functions", diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena/relational/connection/connectionEqualityTest.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena/relational/connection/connectionEqualityTest.pure deleted file mode 100644 index ad754b2feda..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena/relational/connection/connectionEqualityTest.pure +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::relational::metamodel::execute::tests::*; -import meta::pure::alloy::connections::*; -import meta::pure::runtime::*; -import meta::relational::translation::*; -import meta::pure::extension::*; -import meta::relational::extension::*; -import meta::relational::runtime::*; -import meta::relational::tests::csv::*; -import meta::relational::metamodel::execute::*; -import meta::relational::metamodel::*; -import meta::pure::mapping::*; - -function <> meta::relational::tests::connEquality::testConnectionEqualityAllSameAthena() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.Athena, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::AthenaDatasourceSpecification(awsRegion='awsR', s3OutputLocation='s3OL', databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.Athena, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::AthenaDatasourceSpecification(awsRegion='awsR', s3OutputLocation='s3OL', databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - assert(runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena/relational/connection/metamodel.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena/relational/connection/metamodel.pure index 3d5185442bd..5af0eea4fad 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena/relational/connection/metamodel.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/main/resources/core_relational_athena/relational/connection/metamodel.pure @@ -14,7 +14,7 @@ Class meta::pure::alloy::connections::alloy::specification::AthenaDatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> awsRegion: String[1]; - <> s3OutputLocation: String[1]; - <> databaseName: String[1]; + awsRegion: String[1]; + s3OutputLocation: String[1]; + databaseName: String[1]; } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/pom.xml index 96345450ec3..163b8e2dcc4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution-tests/pom.xml index 4a7c6088d55..439196bb96e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution-tests/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-bigquery org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution/pom.xml index 05bc790ae50..fdd311f8738 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-bigquery - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-grammar/pom.xml index b712e78798e..7c8b03ef5d3 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-bigquery - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-protocol/pom.xml index a2280c9de40..926e3c8454a 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-bigquery - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/pom.xml index a04c9993bd9..90f69ebac50 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-bigquery - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery.definition.json b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery.definition.json index c6169ba4970..a0d5f3d1981 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery.definition.json +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery.definition.json @@ -1,6 +1,6 @@ { "name": "core_relational_bigquery", - "pattern": "(meta::relational::functions::sqlQueryToString::bigQuery|meta::relational::tests::connEquality|meta::relational::tests::sqlQueryToString::bigQuery|meta::relational::bigQuery::tests|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", + "pattern": "(meta::relational::functions::sqlQueryToString::bigQuery|meta::relational::tests::sqlQueryToString::bigQuery|meta::relational::bigQuery::tests|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", "dependencies": [ "platform", "platform_functions", diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery/relational/runtime/connection/bigQuerySpecification.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery/relational/runtime/connection/bigQuerySpecification.pure index b2fd2efd681..2a26ad1a66c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery/relational/runtime/connection/bigQuerySpecification.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery/relational/runtime/connection/bigQuerySpecification.pure @@ -14,8 +14,8 @@ Class meta::pure::alloy::connections::alloy::specification::BigQueryDatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> projectId:String[1]; - <> defaultDataset:String[1]; - <> proxyHost: String[0..1]; - <> proxyPort: String[0..1]; + projectId:String[1]; + defaultDataset:String[1]; + proxyHost: String[0..1]; + proxyPort: String[0..1]; } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery/relational/runtime/connection/connectionEqualityTest.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery/relational/runtime/connection/connectionEqualityTest.pure deleted file mode 100644 index f418fc295bf..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/main/resources/core_relational_bigquery/relational/runtime/connection/connectionEqualityTest.pure +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::relational::metamodel::execute::tests::*; -import meta::pure::alloy::connections::*; -import meta::pure::runtime::*; -import meta::relational::translation::*; -import meta::pure::extension::*; -import meta::relational::extension::*; -import meta::relational::runtime::*; -import meta::relational::tests::csv::*; -import meta::relational::metamodel::execute::*; -import meta::relational::metamodel::*; -import meta::pure::mapping::*; - - -function <> meta::relational::tests::connEquality::testConnectionEqualityAllSameBigQuery() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::BigQueryDatasourceSpecification(projectId='project', defaultDataset='defDs', proxyHost='ph', proxyPort='8080'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::BigQueryDatasourceSpecification(projectId='project', defaultDataset='defDs', proxyHost='ph', proxyPort='8080'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - assert(runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java deleted file mode 100644 index c072278365f..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/legend-engine-xt-relationalStore-bigquery-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2022 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.pure.code.core; - -import junit.framework.TestSuite; -import org.finos.legend.pure.m3.execution.test.PureTestBuilder; -import org.finos.legend.pure.m3.execution.test.TestCollection; -import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; -import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; - -public class Test_Pure_Relational_ConnectionEquality -{ - public static TestSuite suite() - { - String testPackage = "meta::relational::tests::connEquality"; - CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); - return PureTestBuilderCompiled.buildSuite(TestCollection.collectTests(testPackage, executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport); - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/pom.xml index 873ac337f6c..5f52312dd1b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-bigquery/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution-tests/pom.xml index 6575d034125..5847a410ee6 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution-tests/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-databricks - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution/pom.xml index 87cfb490b90..c0e99076ca3 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-databricks - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-grammar/pom.xml index da6a16123a5..61e3e8bf62c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-databricks - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-protocol/pom.xml index 533727ce3a7..725030caf80 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-databricks - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/pom.xml index 30979415dfc..5c4aa68782a 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-databricks - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks.definition.json b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks.definition.json index 9f1b5e9e0ba..878f243e236 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks.definition.json +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks.definition.json @@ -1,5 +1,5 @@ { "name" : "core_relational_databricks", - "pattern" : "(meta::relational::functions::sqlQueryToString::databricks|meta::relational::tests::sqlQueryToString::databricks|meta::relational::tests::connEquality|meta::relational::databricks::tests|meta::relational::tests::functions::sqlstring::databricks|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", + "pattern" : "(meta::relational::functions::sqlQueryToString::databricks|meta::relational::tests::sqlQueryToString::databricks|meta::relational::databricks::tests|meta::relational::tests::functions::sqlstring::databricks|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", "dependencies" : ["platform", "platform_functions", "platform_store_relational", "platform_dsl_mapping", "core_functions", "core", "core_relational"] } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks/relational/connection/connectionEqualityTest.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks/relational/connection/connectionEqualityTest.pure deleted file mode 100644 index 5af95394f2b..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks/relational/connection/connectionEqualityTest.pure +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::relational::metamodel::execute::tests::*; -import meta::pure::alloy::connections::*; -import meta::pure::runtime::*; -import meta::relational::translation::*; -import meta::pure::extension::*; -import meta::relational::extension::*; -import meta::relational::runtime::*; -import meta::relational::tests::csv::*; -import meta::relational::metamodel::execute::*; -import meta::relational::metamodel::*; -import meta::pure::mapping::*; - -function <> meta::relational::tests::connEquality::testConnectionEqualityAllSameDataBricks() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.Databricks, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::DatabricksDatasourceSpecification(hostname='host', port='8080', protocol='http', httpPath='http://path'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.Databricks, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::DatabricksDatasourceSpecification(hostname='host', port='8080', protocol='http', httpPath='http://path'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - assert(runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - - diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks/relational/connection/metamodel.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks/relational/connection/metamodel.pure index fec225e36c7..6c3138786f6 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks/relational/connection/metamodel.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/main/resources/core_relational_databricks/relational/connection/metamodel.pure @@ -14,8 +14,8 @@ Class meta::pure::alloy::connections::alloy::specification::DatabricksDatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> hostname:String[1]; - <> port:String[1]; - <> protocol:String[1]; - <> httpPath:String[1]; + hostname:String[1]; + port:String[1]; + protocol:String[1]; + httpPath:String[1]; } \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java deleted file mode 100644 index c072278365f..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/legend-engine-xt-relationalStore-databricks-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2022 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.pure.code.core; - -import junit.framework.TestSuite; -import org.finos.legend.pure.m3.execution.test.PureTestBuilder; -import org.finos.legend.pure.m3.execution.test.TestCollection; -import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; -import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; - -public class Test_Pure_Relational_ConnectionEquality -{ - public static TestSuite suite() - { - String testPackage = "meta::relational::tests::connEquality"; - CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); - return PureTestBuilderCompiled.buildSuite(TestCollection.collectTests(testPackage, executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport); - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/pom.xml index ea6191c57b7..fa7d5dcc7eb 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-databricks/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/pom.xml index d425c05307e..1faec3ee12d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-dbExtension org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/main/resources/core_relational___dbtype__.definition.json b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/main/resources/core_relational___dbtype__.definition.json index 2c147d30fc1..d199b747ec2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/main/resources/core_relational___dbtype__.definition.json +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/main/resources/core_relational___dbtype__.definition.json @@ -1,5 +1,5 @@ { "name" : "core_relational_${dbtype}", - "pattern" : "(meta::relational::functions::sqlQueryToString::${dbType}|meta::relational::tests::connEquality|meta::relational::tests::sqlQueryToString::${dbType}|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", + "pattern" : "(meta::relational::functions::sqlQueryToString::${dbType}|meta::relational::tests::sqlQueryToString::${dbType}|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", "dependencies" : ["platform", "platform_functions", "platform_store_relational", "core", "core_relational"] } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/main/resources/core_relational___dbtype__/relational/connection/connectionEqualityTest.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/main/resources/core_relational___dbtype__/relational/connection/connectionEqualityTest.pure deleted file mode 100644 index 8467968aaca..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/main/resources/core_relational___dbtype__/relational/connection/connectionEqualityTest.pure +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::relational::metamodel::execute::tests::*; -import meta::pure::alloy::connections::*; -import meta::pure::runtime::*; -import meta::relational::translation::*; -import meta::pure::extension::*; -import meta::relational::extension::*; -import meta::relational::runtime::*; -import meta::relational::tests::csv::*; -import meta::relational::metamodel::execute::*; -import meta::relational::metamodel::*; -import meta::pure::mapping::*; - - -function <> meta::relational::tests::connEquality::testConnectionEqualityAllSame__dbtype__() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.__dbtype__, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::__dbtype__DatasourceSpecification(), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::BigQueryDatasourceSpecification(projectId='project', defaultDataset='defDs', proxyHost='ph', proxyPort='8080'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - assert(runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java deleted file mode 100644 index c072278365f..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-dbExtension-archetype/src/main/resources/archetype-resources/legend-engine-xt-relationalStore-__dbtype__-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2022 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.pure.code.core; - -import junit.framework.TestSuite; -import org.finos.legend.pure.m3.execution.test.PureTestBuilder; -import org.finos.legend.pure.m3.execution.test.TestCollection; -import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; -import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; - -public class Test_Pure_Relational_ConnectionEquality -{ - public static TestSuite suite() - { - String testPackage = "meta::relational::tests::connEquality"; - CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); - return PureTestBuilderCompiled.buildSuite(TestCollection.collectTests(testPackage, executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport); - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/legend-engine-xt-relationalStore-hive-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/legend-engine-xt-relationalStore-hive-pure/pom.xml index c6684af3436..9a48aef5b3c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/legend-engine-xt-relationalStore-hive-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/legend-engine-xt-relationalStore-hive-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-hive - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/pom.xml index 0c654ee63e0..617999ebc71 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-hive/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/pom.xml index e9ba50ef661..3f6402b57aa 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-memsql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/java/org/finos/legend/connection/jdbc/driver/MemSQL_JDBCConnectionDriver.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/java/org/finos/legend/connection/jdbc/driver/MemSQLDatabaseManager.java similarity index 90% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/java/org/finos/legend/connection/jdbc/driver/MemSQL_JDBCConnectionDriver.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/java/org/finos/legend/connection/jdbc/driver/MemSQLDatabaseManager.java index 93bc8c9612d..a9a81b97815 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/java/org/finos/legend/connection/jdbc/driver/MemSQL_JDBCConnectionDriver.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/java/org/finos/legend/connection/jdbc/driver/MemSQLDatabaseManager.java @@ -15,12 +15,13 @@ package org.finos.legend.connection.jdbc.driver; import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.connection.DatabaseManager; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; import java.util.List; import java.util.Properties; -public class MemSQL_JDBCConnectionDriver implements JDBCConnectionDriver +public class MemSQLDatabaseManager implements DatabaseManager { @Override public List getIds() @@ -36,7 +37,7 @@ public String getDriver() } @Override - public String buildURL(String host, int port, String databaseName, Properties extraUserDataSourceProperties) + public String buildURL(String host, int port, String databaseName, Properties properties) { return String.format("jdbc:mysql://%s:%s/%s?permitMysqlScheme", host, port, databaseName); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager new file mode 100644 index 00000000000..29c0a912ba4 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager @@ -0,0 +1 @@ +org.finos.legend.connection.jdbc.driver.MemSQLDatabaseManager diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver deleted file mode 100644 index bf489534a4d..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.jdbc.driver.MemSQL_JDBCConnectionDriver diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution-tests/pom.xml index 4e4a9200f72..936d3fa3d2d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution-tests/pom.xml @@ -19,7 +19,7 @@ legend-engine-xt-relationalStore-memsql org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution/pom.xml index 3273b846b6d..955171c15c9 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-execution/pom.xml @@ -20,7 +20,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-memsql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-relationalStore-memsql-execution diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-pure/pom.xml index f6e5febea57..e3127688fc3 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/legend-engine-xt-relationalStore-memsql-pure/pom.xml @@ -20,7 +20,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-memsql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT legend-engine-xt-relationalStore-memsql-pure diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/pom.xml index ed52c084231..90a45a364dc 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-memsql/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/pom.xml index 9b4aedbaedf..49331903c1c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-postgres - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/java/org/finos/legend/connection/jdbc/driver/PostgreSQL_JDBCConnectionDriver.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/java/org/finos/legend/connection/impl/PostgresDatabaseManager.java similarity index 86% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/java/org/finos/legend/connection/jdbc/driver/PostgreSQL_JDBCConnectionDriver.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/java/org/finos/legend/connection/impl/PostgresDatabaseManager.java index 47f9d6c4e8e..26565642f82 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/java/org/finos/legend/connection/jdbc/driver/PostgreSQL_JDBCConnectionDriver.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/java/org/finos/legend/connection/impl/PostgresDatabaseManager.java @@ -12,15 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection.jdbc.driver; +package org.finos.legend.connection.impl; import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.connection.DatabaseType; +import org.finos.legend.connection.DatabaseManager; import java.util.List; import java.util.Properties; -public class PostgreSQL_JDBCConnectionDriver implements JDBCConnectionDriver +public class PostgresDatabaseManager implements DatabaseManager { @Override public List getIds() @@ -35,7 +36,7 @@ public String getDriver() } @Override - public String buildURL(String host, int port, String databaseName, Properties extraUserDataSourceProperties) + public String buildURL(String host, int port, String databaseName, Properties properties) { return String.format("jdbc:postgresql://%s:%s/%s", host, port, databaseName); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager new file mode 100644 index 00000000000..29f38d7a5c6 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager @@ -0,0 +1 @@ +org.finos.legend.connection.impl.PostgresDatabaseManager diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver deleted file mode 100644 index 4383fc7be57..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.jdbc.driver.PostgreSQL_JDBCConnectionDriver diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution-tests/pom.xml index 41b4aaec071..8ab9b80a9cd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution-tests/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-postgres - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution/pom.xml index 9f6b3ecf819..824e630534e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-postgres - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/pom.xml index 0fab8839dfa..de52baee216 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-postgres - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/pom.xml index fdb9380f8e4..1923479aea9 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-postgres - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/src/main/java/org/finos/legend/connection/PostgresTestContainerWrapper.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/src/main/java/org/finos/legend/connection/PostgresTestContainerWrapper.java index c4b2cea4a04..388576e5836 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/src/main/java/org/finos/legend/connection/PostgresTestContainerWrapper.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/legend-engine-xt-relationalStore-postgres-test-support/src/main/java/org/finos/legend/connection/PostgresTestContainerWrapper.java @@ -21,13 +21,31 @@ public class PostgresTestContainerWrapper { private static final String IMAGE = "postgres"; private static final String TAG = "9.6.12"; - private final PostgreSQLContainer postgreSQLContainer = new PostgreSQLContainer(DockerImageName.parse(IMAGE).withTag(TAG)); + private final PostgreSQLContainer postgreSQLContainer; + + private PostgresTestContainerWrapper() + { + this.postgreSQLContainer = new PostgreSQLContainer(DockerImageName.parse(IMAGE).withTag(TAG)); + } + + private PostgresTestContainerWrapper(String databaseName, String username, String password) + { + this.postgreSQLContainer = new PostgreSQLContainer(DockerImageName.parse(IMAGE).withTag(TAG)) + .withDatabaseName(databaseName) + .withUsername(username) + .withPassword(password); + } public static PostgresTestContainerWrapper build() { return new PostgresTestContainerWrapper(); } + public static PostgresTestContainerWrapper build(String databaseName, String username, String password) + { + return new PostgresTestContainerWrapper(databaseName, username, password); + } + public void start() { this.postgreSQLContainer.start(); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/pom.xml index ea96ab25600..84434a2bf2f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-postgres/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/legend-engine-xt-relationalStore-presto-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/legend-engine-xt-relationalStore-presto-pure/pom.xml index 562c6d03a9f..7e349eee249 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/legend-engine-xt-relationalStore-presto-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/legend-engine-xt-relationalStore-presto-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-presto - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/pom.xml index edc998a5c2b..27a8e313673 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-presto/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution-tests/pom.xml index 0a7afa33582..31692252ee2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution-tests/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-redshift - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution/pom.xml index 03d2e8f1a2e..49584c70168 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-redshift - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-grammar/pom.xml index c65547eac4b..c3ef0746ce7 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-redshift - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-protocol/pom.xml index 3ac849de4fe..1fad81de9c6 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-redshift - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/pom.xml index cdbb3d9ef15..269d1b9c873 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-redshift - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/src/main/resources/core_relational_redshift/relational/connection/metamodel.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/src/main/resources/core_relational_redshift/relational/connection/metamodel.pure index 49c2e1c7f73..e7f45966730 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/src/main/resources/core_relational_redshift/relational/connection/metamodel.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/legend-engine-xt-relationalStore-redshift-pure/src/main/resources/core_relational_redshift/relational/connection/metamodel.pure @@ -14,10 +14,10 @@ Class {doc.doc ='Specification for the AWS redshift database'} meta::pure::legend::connections::legend::specification::RedshiftDatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> {doc.doc ='clusterID'} clusterID:String[1]; - <> {doc.doc ='The aws region'} region:String[1]; - <> {doc.doc ='the full host url'} host:String[1]; - <> {doc.doc ='database name'} databaseName:String[1]; - <> {doc.doc ='port'} port:Integer[1]; - <> {doc.doc ='Optional URL used for redshift service execution'} endpointURL:String[0..1]; + {doc.doc ='clusterID'} clusterID:String[1]; + {doc.doc ='The aws region'} region:String[1]; + {doc.doc ='the full host url'} host:String[1]; + {doc.doc ='database name'} databaseName:String[1]; + {doc.doc ='port'} port:Integer[1]; + {doc.doc ='Optional URL used for redshift service execution'} endpointURL:String[0..1]; } \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/pom.xml index 341ce1371a3..d5ef8686cb1 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-redshift/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/pom.xml index 4a63729d0df..b7aa3c39ef1 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-snowflake - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -37,10 +37,22 @@ org.finos.legend.engine legend-engine-xt-authentication-connection-factory + + org.finos.legend.engine + legend-engine-xt-authentication-protocol + org.finos.legend.engine legend-engine-xt-relationalStore-connection + + org.finos.legend.engine + legend-engine-xt-relationalStore-protocol + + + org.finos.legend.engine + legend-engine-xt-relationalStore-snowflake-protocol + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/HACKY__SnowflakeConnectionAdapter.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/HACKY__SnowflakeConnectionAdapter.java new file mode 100644 index 00000000000..32c89b91622 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/HACKY__SnowflakeConnectionAdapter.java @@ -0,0 +1,75 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.connection.impl; + +import org.finos.legend.connection.HACKY__RelationalDatabaseConnectionAdapter; +import org.finos.legend.connection.LegendEnvironment; +import org.finos.legend.connection.StoreInstance; +import org.finos.legend.connection.protocol.SnowflakeConnectionSpecification; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.authentication.vault.PropertiesFileSecret; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.authentication.SnowflakePublicAuthenticationStrategy; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.specification.SnowflakeDatasourceSpecification; +import org.finos.legend.engine.shared.core.identity.Identity; + +public class HACKY__SnowflakeConnectionAdapter +{ + public static class WithKeyPair implements HACKY__RelationalDatabaseConnectionAdapter + { + @Override + public ConnectionFactoryMaterial adapt(RelationalDatabaseConnection relationalDatabaseConnection, Identity identity, LegendEnvironment environment) + { + if ( + DatabaseType.Snowflake.equals(relationalDatabaseConnection.databaseType) && + relationalDatabaseConnection.datasourceSpecification instanceof SnowflakeDatasourceSpecification && + relationalDatabaseConnection.authenticationStrategy instanceof SnowflakePublicAuthenticationStrategy + ) + { + SnowflakeDatasourceSpecification datasourceSpecification = (SnowflakeDatasourceSpecification) relationalDatabaseConnection.datasourceSpecification; + SnowflakePublicAuthenticationStrategy authenticationStrategy = (SnowflakePublicAuthenticationStrategy) relationalDatabaseConnection.authenticationStrategy; + + SnowflakeConnectionSpecification connectionSpecification = new SnowflakeConnectionSpecification(); + connectionSpecification.accountName = datasourceSpecification.accountName; + connectionSpecification.region = datasourceSpecification.region; + connectionSpecification.warehouseName = datasourceSpecification.warehouseName; + connectionSpecification.databaseName = datasourceSpecification.databaseName; + connectionSpecification.cloudType = datasourceSpecification.cloudType; + connectionSpecification.quotedIdentifiersIgnoreCase = datasourceSpecification.quotedIdentifiersIgnoreCase; + connectionSpecification.enableQueryTags = datasourceSpecification.enableQueryTags; + connectionSpecification.proxyHost = datasourceSpecification.proxyHost; + connectionSpecification.proxyPort = datasourceSpecification.proxyPort; + connectionSpecification.nonProxyHosts = datasourceSpecification.nonProxyHosts; + connectionSpecification.organization = datasourceSpecification.organization; + connectionSpecification.accountType = datasourceSpecification.accountType; + connectionSpecification.role = datasourceSpecification.role; + + StoreInstance storeInstance = new StoreInstance.Builder(environment) + .withIdentifier("adapted-store") + .withStoreSupportIdentifier("Snowflake") + .withConnectionSpecification(connectionSpecification) + .build(); + + EncryptedPrivateKeyPairAuthenticationConfiguration authenticationConfiguration = new EncryptedPrivateKeyPairAuthenticationConfiguration(); + authenticationConfiguration.userName = authenticationStrategy.publicUserName; + authenticationConfiguration.privateKey = new PropertiesFileSecret(authenticationStrategy.privateKeyVaultReference); + authenticationConfiguration.passphrase = new PropertiesFileSecret(authenticationStrategy.passPhraseVaultReference); + + return new ConnectionFactoryMaterial(storeInstance, authenticationConfiguration); + } + return null; + } + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/SnowflakeConnectionBuilder.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/SnowflakeConnectionBuilder.java index 655c9a1f11c..7c6d11bcaec 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/SnowflakeConnectionBuilder.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/SnowflakeConnectionBuilder.java @@ -14,54 +14,57 @@ package org.finos.legend.connection.impl; -import org.finos.legend.connection.ConnectionBuilder; +import org.finos.legend.connection.Authenticator; import org.finos.legend.connection.DatabaseType; +import org.finos.legend.connection.JDBCConnectionBuilder; import org.finos.legend.connection.RelationalDatabaseStoreSupport; import org.finos.legend.connection.StoreInstance; -import org.finos.legend.connection.StoreSupport; -import org.finos.legend.connection.jdbc.JDBCConnectionManager; -import org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver; import org.finos.legend.connection.protocol.SnowflakeConnectionSpecification; +import org.finos.legend.engine.shared.core.identity.Credential; +import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.engine.shared.core.identity.credential.PrivateKeyCredential; import java.sql.Connection; -import java.sql.DriverManager; import java.util.Optional; import java.util.Properties; +import java.util.function.Function; -import static org.finos.legend.connection.jdbc.driver.Snowflake_JDBCConnectionDriver.*; +import static org.finos.legend.connection.impl.SnowflakeDatabaseManager.*; public class SnowflakeConnectionBuilder { - public static class WithKeyPair extends ConnectionBuilder + public static class WithKeyPair extends JDBCConnectionBuilder { @Override - public Connection getConnection(PrivateKeyCredential credential, SnowflakeConnectionSpecification connectionSpecification, StoreInstance storeInstance) throws Exception + public Connection getConnection(SnowflakeConnectionSpecification connectionSpecification, Authenticator authenticator, Identity identity) throws Exception { - StoreSupport storeSupport = storeInstance.getStoreSupport(); - if (!(storeSupport instanceof RelationalDatabaseStoreSupport) || !DatabaseType.SNOWFLAKE.equals(((RelationalDatabaseStoreSupport) storeSupport).getDatabase())) + StoreInstance storeInstance = authenticator.getStoreInstance(); + RelationalDatabaseStoreSupport.cast(storeInstance.getStoreSupport(), DatabaseType.SNOWFLAKE); + + Properties connectionProperties = generateJDBCConnectionProperties(connectionSpecification); + Function authenticationPropertiesSupplier = cred -> { - throw new RuntimeException("Can't get connection: only support Snowflake databases"); - } - JDBCConnectionDriver driver = JDBCConnectionManager.getDriverForDatabase(DatabaseType.SNOWFLAKE); - Properties properties = collectExtraSnowflakeConnectionProperties(connectionSpecification); - properties.put("privateKey", credential.getPrivateKey()); - properties.put("user", credential.getUser()); + PrivateKeyCredential credential = (PrivateKeyCredential) cred; + Properties properties = new Properties(); + properties.put("privateKey", credential.getPrivateKey()); + properties.put("user", credential.getUser()); + return properties; + }; - return DriverManager.getConnection(driver.buildURL(null, 0, connectionSpecification.databaseName, properties), properties); + return this.getConnectionManager().getConnection(DatabaseType.SNOWFLAKE, null, 0, connectionSpecification.databaseName, connectionProperties, this.getConnectionPoolConfig(), authenticationPropertiesSupplier, authenticator, identity); } } - private static Properties collectExtraSnowflakeConnectionProperties(SnowflakeConnectionSpecification connectionSpecification) + public static Properties generateJDBCConnectionProperties(SnowflakeConnectionSpecification connectionSpecification) { Properties properties = new Properties(); // TODO: @akphi - handle quoted identifiers // this is a setting users can control when creating the database connection, we probably don't // want to do this when the database is configured as part of the system boolean quoteIdentifiers = false; - String warehouseName = updateSnowflakeIdentifiers(connectionSpecification.warehouseName, quoteIdentifiers); - String databaseName = updateSnowflakeIdentifiers(connectionSpecification.databaseName, quoteIdentifiers); - properties.put(SNOWFLAKE_ROLE, updateSnowflakeIdentifiers(connectionSpecification.role, quoteIdentifiers)); + String warehouseName = processIdentifier(connectionSpecification.warehouseName, quoteIdentifiers); + String databaseName = processIdentifier(connectionSpecification.databaseName, quoteIdentifiers); + properties.put(SNOWFLAKE_ROLE, processIdentifier(connectionSpecification.role, quoteIdentifiers)); properties.put(SNOWFLAKE_ACCOUNT_NAME, connectionSpecification.accountName); properties.put(SNOWFLAKE_REGION, connectionSpecification.region); @@ -76,22 +79,22 @@ private static Properties collectExtraSnowflakeConnectionProperties(SnowflakeCon properties.put("db", databaseName); properties.put("ocspFailOpen", true); - setProperty(properties, SNOWFLAKE_ACCOUNT_TYPE_NAME, connectionSpecification.accountType); - setProperty(properties, SNOWFLAKE_ORGANIZATION_NAME, connectionSpecification.organization); - setProperty(properties, SNOWFLAKE_PROXY_HOST, connectionSpecification.proxyHost); - setProperty(properties, SNOWFLAKE_PROXY_PORT, connectionSpecification.proxyPort); - setProperty(properties, SNOWFLAKE_NON_PROXY_HOSTS, connectionSpecification.nonProxyHosts); + setNullableProperty(properties, SNOWFLAKE_ACCOUNT_TYPE_NAME, connectionSpecification.accountType); + setNullableProperty(properties, SNOWFLAKE_ORGANIZATION_NAME, connectionSpecification.organization); + setNullableProperty(properties, SNOWFLAKE_PROXY_HOST, connectionSpecification.proxyHost); + setNullableProperty(properties, SNOWFLAKE_PROXY_PORT, connectionSpecification.proxyPort); + setNullableProperty(properties, SNOWFLAKE_NON_PROXY_HOSTS, connectionSpecification.nonProxyHosts); properties.put(SNOWFLAKE_USE_PROXY, properties.get(SNOWFLAKE_PROXY_HOST) != null); return properties; } - private static void setProperty(Properties properties, String key, Object value) + private static void setNullableProperty(Properties properties, String key, Object value) { Optional.ofNullable(value).ifPresent(x -> properties.put(key, value)); } - private static String updateSnowflakeIdentifiers(String identifier, boolean quoteIdentifiers) + public static String processIdentifier(String identifier, boolean quoteIdentifiers) { if (quoteIdentifiers && identifier != null && !(identifier.startsWith("\"") && identifier.endsWith("\""))) { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/jdbc/driver/Snowflake_JDBCConnectionDriver.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/SnowflakeDatabaseManager.java similarity index 77% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/jdbc/driver/Snowflake_JDBCConnectionDriver.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/SnowflakeDatabaseManager.java index aaa27cfcccd..81f78f2a73e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/jdbc/driver/Snowflake_JDBCConnectionDriver.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/impl/SnowflakeDatabaseManager.java @@ -12,17 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection.jdbc.driver; +package org.finos.legend.connection.impl; import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.connection.DatabaseType; import org.finos.legend.connection.SnowflakeAccountType; +import org.finos.legend.connection.DatabaseManager; import org.finos.legend.engine.shared.core.operational.Assert; import java.util.List; import java.util.Properties; -public class Snowflake_JDBCConnectionDriver implements JDBCConnectionDriver +public class SnowflakeDatabaseManager implements DatabaseManager { private static final String PRIVATELINK_SNOWFLAKECOMPUTING_COM = ".privatelink.snowflakecomputing.com"; private static final String SNOWFLAKECOMPUTING_COM = ".snowflakecomputing.com"; @@ -57,18 +58,18 @@ public String getDriver() } @Override - public String buildURL(String host, int port, String databaseName, Properties extraUserDataSourceProperties) + public String buildURL(String host, int port, String databaseName, Properties properties) { - Assert.assertTrue(extraUserDataSourceProperties.getProperty(SNOWFLAKE_ACCOUNT_NAME) != null, () -> SNOWFLAKE_ACCOUNT_NAME + " is not set"); - Assert.assertTrue(extraUserDataSourceProperties.getProperty(SNOWFLAKE_REGION) != null, () -> SNOWFLAKE_REGION + " is not set"); - Assert.assertTrue(extraUserDataSourceProperties.getProperty(SNOWFLAKE_WAREHOUSE_NAME) != null, () -> SNOWFLAKE_WAREHOUSE_NAME + " is not set"); + Assert.assertTrue(properties.getProperty(SNOWFLAKE_ACCOUNT_NAME) != null, () -> SNOWFLAKE_ACCOUNT_NAME + " is not set"); + Assert.assertTrue(properties.getProperty(SNOWFLAKE_REGION) != null, () -> SNOWFLAKE_REGION + " is not set"); + Assert.assertTrue(properties.getProperty(SNOWFLAKE_WAREHOUSE_NAME) != null, () -> SNOWFLAKE_WAREHOUSE_NAME + " is not set"); - String accountName = extraUserDataSourceProperties.getProperty(SNOWFLAKE_ACCOUNT_NAME); - String region = extraUserDataSourceProperties.getProperty(SNOWFLAKE_REGION); - String cloudType = extraUserDataSourceProperties.getProperty(SNOWFLAKE_CLOUD_TYPE); - String organisation = extraUserDataSourceProperties.getProperty(SNOWFLAKE_ORGANIZATION_NAME); + String accountName = properties.getProperty(SNOWFLAKE_ACCOUNT_NAME); + String region = properties.getProperty(SNOWFLAKE_REGION); + String cloudType = properties.getProperty(SNOWFLAKE_CLOUD_TYPE); + String organisation = properties.getProperty(SNOWFLAKE_ORGANIZATION_NAME); - String accountTypeName = extraUserDataSourceProperties.getProperty(SNOWFLAKE_ACCOUNT_TYPE_NAME); + String accountTypeName = properties.getProperty(SNOWFLAKE_ACCOUNT_TYPE_NAME); SnowflakeAccountType accountType = accountTypeName != null ? SnowflakeAccountType.valueOf(accountTypeName) : null; diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/protocol/SnowflakeConnectionSpecification.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/protocol/SnowflakeConnectionSpecification.java index 5a9dbdeb288..555f44c7979 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/protocol/SnowflakeConnectionSpecification.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/java/org/finos/legend/connection/protocol/SnowflakeConnectionSpecification.java @@ -31,4 +31,23 @@ public class SnowflakeConnectionSpecification extends ConnectionSpecification public String accountType; public String role; + + @Override + public String shortId() + { + return "Snowflake" + + "--account=" + accountName + + "--region=" + region + + "--warehouse=" + warehouseName + + "--db=" + databaseName + + "--cloudType=" + cloudType + + "--proxyHost=" + proxyHost + + "--proxyPort=" + proxyPort + + "--nonProxyHosts=" + nonProxyHosts + + "--accountType=" + accountType + + "--organisation=" + organization + + "--quoteIdentifiers=" + quotedIdentifiersIgnoreCase + + "--role=" + role + + "--enableQueryTags=" + enableQueryTags; + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionBuilder b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionBuilder deleted file mode 100644 index 69a94fc1226..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.ConnectionBuilder +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.impl.SnowflakeConnectionBuilder$WithKeyPair diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager new file mode 100644 index 00000000000..fa4a918a8d0 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager @@ -0,0 +1 @@ +org.finos.legend.connection.impl.SnowflakeDatabaseManager diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.HACKY__RelationalDatabaseConnectionAdapter b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.HACKY__RelationalDatabaseConnectionAdapter new file mode 100644 index 00000000000..e92e738a74a --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.HACKY__RelationalDatabaseConnectionAdapter @@ -0,0 +1 @@ +org.finos.legend.connection.impl.HACKY__SnowflakeConnectionAdapter$WithKeyPair \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver deleted file mode 100644 index 54e9611d811..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.jdbc.driver.Snowflake_JDBCConnectionDriver diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution-tests/pom.xml index a4c221688a6..2793ea936d1 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution-tests/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-snowflake - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/pom.xml index 6fd95c707ec..ecab579a53e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-snowflake - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/snowflake/SnowflakeCommands.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/snowflake/SnowflakeCommands.java index b5ccb24ff50..ad04b03715e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/snowflake/SnowflakeCommands.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/driver/vendors/snowflake/SnowflakeCommands.java @@ -14,6 +14,8 @@ package org.finos.legend.engine.plan.execution.stores.relational.connection.driver.vendors.snowflake; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.map.ImmutableMap; import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.Column; import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.IngestionMethod; import org.finos.legend.engine.plan.execution.stores.relational.connection.driver.commands.RelationalDatabaseCommands; @@ -25,6 +27,8 @@ public class SnowflakeCommands extends RelationalDatabaseCommands { + private static final ImmutableMap columnTypeToSqlTextMap = Maps.immutable.of("BIT", "BOOLEAN"); + @Override public String processTempTableName(String tempTableName) { @@ -50,7 +54,7 @@ public List createAndLoadTempTable(String tableName, List column optionalCSVFileLocation = optionalCSVFileLocation.substring(1); } List strings = Arrays.asList( - "CREATE TEMPORARY TABLE " + tableName + " " + columns.stream().map(c -> c.name + " " + c.type).collect(Collectors.joining(",", "(", ")")), + "CREATE TEMPORARY TABLE " + tableName + " " + columns.stream().map(c -> c.name + " " + columnTypeToSqlTextMap.getIfAbsentValue(c.type, c.type)).collect(Collectors.joining(",", "(", ")")), "CREATE OR REPLACE TEMPORARY STAGE " + tempStageName(), "PUT file:///" + optionalCSVFileLocation + " @" + tempStageName() + "/" + optionalCSVFileLocation + " PARALLEL = 16 AUTO_COMPRESS = TRUE", "COPY INTO " + tableName + " FROM @" + tempStageName() + "/" + optionalCSVFileLocation + " file_format = (type = CSV field_optionally_enclosed_by= '\"')", diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/ds/TestSnowflakeCommands.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/ds/TestSnowflakeCommands.java index c47539f68b3..f0ccee880d4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/ds/TestSnowflakeCommands.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-execution/src/test/java/org/finos/legend/engine/plan/execution/stores/relational/connection/ds/TestSnowflakeCommands.java @@ -34,12 +34,13 @@ public void testTempTableCommands() throws IOException ImmutableList columns = Lists.immutable.of( new Column("a", "VARCHAR(100)"), - new Column("b", "VARCHAR(100)") + new Column("b", "VARCHAR(100)"), + new Column("c", "BIT") ); List sqlStatements = snowflakeCommands.createAndLoadTempTable("temp_1", columns.castToList(), "/tmp/temp.csv"); ImmutableList expectedSQLStatements = Lists.immutable.of( - "CREATE TEMPORARY TABLE temp_1 (a VARCHAR(100),b VARCHAR(100))", + "CREATE TEMPORARY TABLE temp_1 (a VARCHAR(100),b VARCHAR(100),c BOOLEAN)", "CREATE OR REPLACE TEMPORARY STAGE LEGEND_TEMP_DB.LEGEND_TEMP_SCHEMA.LEGEND_TEMP_STAGE", "PUT file:///tmp/temp.csv @LEGEND_TEMP_DB.LEGEND_TEMP_SCHEMA.LEGEND_TEMP_STAGE/tmp/temp.csv PARALLEL = 16 AUTO_COMPRESS = TRUE", "COPY INTO temp_1 FROM @LEGEND_TEMP_DB.LEGEND_TEMP_SCHEMA.LEGEND_TEMP_STAGE/tmp/temp.csv file_format = (type = CSV field_optionally_enclosed_by= '\"')", diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-grammar/pom.xml index 5f06d70c5bc..ffca5ba0373 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-snowflake - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-protocol/pom.xml index 7a2324908e4..d91c328de8b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-snowflake - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/pom.xml index 17c03130ab9..1316c73083a 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-snowflake - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake.definition.json b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake.definition.json index c0aa332c785..26e2977f5f2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake.definition.json +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake.definition.json @@ -1,5 +1,5 @@ { "name" : "core_relational_snowflake", - "pattern" : "(meta::relational::functions::sqlQueryToString::snowflake|meta::relational::tests::connEquality|meta::relational::tests::sqlQueryToString::snowflake|meta::relational::tests::sqlToString::snowflake|meta::pure::executionPlan::tests::snowflake|meta::relational::tests::projection::snowflake|meta::relational::tests::query::snowflake|meta::relational::tests::tds::snowflake|meta::relational::tests::mapping::function::snowflake|meta::relational::tests::postProcessor::snowflake|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", + "pattern" : "(meta::relational::functions::sqlQueryToString::snowflake|meta::relational::tests::sqlQueryToString::snowflake|meta::relational::tests::sqlToString::snowflake|meta::pure::executionPlan::tests::snowflake|meta::relational::tests::projection::snowflake|meta::relational::tests::query::snowflake|meta::relational::tests::tds::snowflake|meta::relational::tests::mapping::function::snowflake|meta::relational::tests::postProcessor::snowflake|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", "dependencies" : ["platform", "platform_functions", "platform_store_relational", "platform_dsl_mapping", "core_functions", "core", "core_relational"] } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/connection/connectionEqualityTest.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/connection/connectionEqualityTest.pure deleted file mode 100644 index 4cd7c4e4a8b..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/connection/connectionEqualityTest.pure +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2021 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -import meta::relational::metamodel::execute::tests::*; -import meta::pure::alloy::connections::*; -import meta::pure::runtime::*; -import meta::relational::translation::*; -import meta::pure::extension::*; -import meta::relational::extension::*; -import meta::relational::runtime::*; -import meta::relational::tests::csv::*; -import meta::relational::metamodel::execute::*; -import meta::relational::metamodel::*; -import meta::pure::mapping::*; - - - -function <> meta::relational::tests::connEquality::testConnectionEqualityAllSameSnowflake() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::SnowflakeDatasourceSpecification(accountName='account', region='region', warehouseName='wh', databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::SnowflakePublicAuthenticationStrategy(privateKeyVaultReference='pkvr', publicUserName ='public', passPhraseVaultReference='ppVR') - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::SnowflakeDatasourceSpecification(accountName='account', region='region', warehouseName='wh', databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::SnowflakePublicAuthenticationStrategy(privateKeyVaultReference='pkvr', publicUserName ='public', passPhraseVaultReference='ppVR') - ); - - assert(runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/connection/metamodel.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/connection/metamodel.pure index 19fb053d8f0..5cd3bff5dbb 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/connection/metamodel.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/main/resources/core_relational_snowflake/relational/connection/metamodel.pure @@ -19,28 +19,28 @@ Enum meta::pure::alloy::connections::alloy::specification::SnowflakeAccountType Class meta::pure::alloy::connections::alloy::specification::SnowflakeDatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> accountName:String[1]; - <> region:String[1]; - <> warehouseName:String[1]; - <> databaseName:String[1]; - <> role:String[0..1]; + accountName:String[1]; + region:String[1]; + warehouseName:String[1]; + databaseName:String[1]; + role:String[0..1]; proxyHost:String[0..1]; proxyPort:String[0..1]; nonProxyHosts:String[0..1]; - <> accountType: meta::pure::alloy::connections::alloy::specification::SnowflakeAccountType[0..1]; - <> organization:String[0..1]; - <> cloudType:String[0..1]; + accountType: meta::pure::alloy::connections::alloy::specification::SnowflakeAccountType[0..1]; + organization:String[0..1]; + cloudType:String[0..1]; - <> quotedIdentifiersIgnoreCase:Boolean[0..1]; - <> enableQueryTags: Boolean[0..1]; + quotedIdentifiersIgnoreCase:Boolean[0..1]; + enableQueryTags: Boolean[0..1]; } Class meta::pure::alloy::connections::alloy::authentication::SnowflakePublicAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::AuthenticationStrategy { - <> privateKeyVaultReference:String[1]; - <> passPhraseVaultReference:String[1]; - <> publicUserName:String[1]; + privateKeyVaultReference:String[1]; + passPhraseVaultReference:String[1]; + publicUserName:String[1]; } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java deleted file mode 100644 index c072278365f..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/legend-engine-xt-relationalStore-snowflake-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2022 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package org.finos.legend.pure.code.core; - -import junit.framework.TestSuite; -import org.finos.legend.pure.m3.execution.test.PureTestBuilder; -import org.finos.legend.pure.m3.execution.test.TestCollection; -import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; -import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; - -public class Test_Pure_Relational_ConnectionEquality -{ - public static TestSuite suite() - { - String testPackage = "meta::relational::tests::connEquality"; - CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); - return PureTestBuilderCompiled.buildSuite(TestCollection.collectTests(testPackage, executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport); - } -} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/pom.xml index b1af7dbe6af..219444724ed 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-snowflake/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution-tests/pom.xml index b05ce2bd0f9..c3ab6141c11 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution-tests/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-spanner org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution/pom.xml index 727da722b62..312cb34ace6 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-execution/pom.xml @@ -19,7 +19,7 @@ legend-engine-xt-relationalStore-spanner org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-grammar/pom.xml index 9ae0e6302fc..b983a0e7da4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-grammar/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-spanner org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-jdbc-shaded/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-jdbc-shaded/pom.xml index b4f7def54f3..07e4ce0c61d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-jdbc-shaded/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-jdbc-shaded/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-spanner org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-protocol/pom.xml index 6495480a2cb..b7221efe8da 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-protocol/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-spanner org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/pom.xml index 5fcc25acde0..c7616fd1bcf 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-spanner org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/src/main/resources/core_relational_spanner/relational/runtime/connection/spannerSpecification.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/src/main/resources/core_relational_spanner/relational/runtime/connection/spannerSpecification.pure index 42aec568886..fd243423c9e 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/src/main/resources/core_relational_spanner/relational/runtime/connection/spannerSpecification.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/legend-engine-xt-relationalStore-spanner-pure/src/main/resources/core_relational_spanner/relational/runtime/connection/spannerSpecification.pure @@ -14,9 +14,9 @@ Class meta::pure::alloy::connections::alloy::specification::SpannerDatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> projectId:String[1]; - <> instanceId:String[1]; - <> databaseId:String[1]; - <> proxyHost: String[0..1]; - <> proxyPort: Integer[0..1]; + projectId:String[1]; + instanceId:String[1]; + databaseId:String[1]; + proxyHost: String[0..1]; + proxyPort: Integer[0..1]; } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/pom.xml index ee5e427c085..c8a0de9ca36 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-spanner/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/pom.xml new file mode 100644 index 00000000000..feee5f72129 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/pom.xml @@ -0,0 +1,267 @@ + + + + + + org.finos.legend.engine + legend-engine-xt-relationalStore-sparksql + 4.30.3-SNAPSHOT + + 4.0.0 + + legend-engine-xt-relationalStore-sparksql-pure + jar + Legend Engine - XT - Relational Store - sparksql - Pure + + + + + org.finos.legend.pure + legend-pure-maven-generation-par + + src/main/resources + ${legend.pure.version} + + core_relational_sparksql + + + ${project.basedir}/src/main/resources/core_relational_sparksql.definition.json + + + + + + generate-sources + + build-pure-jar + + + + + + org.finos.legend.pure + legend-pure-m2-functions-pure + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-mapping-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-path-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-graph-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-store-relational-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + org.finos.legend.pure + legend-pure-maven-generation-java + + + compile + + build-pure-compiled-jar + + + true + true + modular + true + + core_relational_sparksql + + + + + + + org.finos.legend.pure + legend-pure-m2-dsl-mapping-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-path-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-graph-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-dsl-diagram-grammar + ${legend.pure.version} + + + org.finos.legend.pure + legend-pure-m2-store-relational-grammar + ${legend.pure.version} + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + ${project.version} + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + ${project.version} + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + dependency-analyze + + + + org.finos.legend.pure:legend-pure-m2-dsl-mapping-pure + org.finos.legend.pure:legend-pure-m2-dsl-path-pure + org.finos.legend.engine:legend-engine-pure-platform-store-relational-java + org.finos.legend.engine:legend-engine-pure-platform-dsl-mapping-java + + + + + + + + + + + org.finos.legend.pure + legend-pure-m4 + + + org.finos.legend.pure + legend-pure-m3-core + + + org.finos.legend.pure + legend-pure-m2-dsl-mapping-pure + + + org.finos.legend.pure + legend-pure-m2-dsl-path-pure + + + org.finos.legend.pure + legend-pure-m2-store-relational-pure + + + org.finos.legend.pure + legend-pure-runtime-java-engine-compiled + + + + org.finos.legend.engine + legend-engine-pure-code-compiled-core + + + org.finos.legend.engine + legend-engine-pure-code-compiled-functions + + + org.finos.legend.engine + legend-engine-pure-platform-dsl-mapping-java + + + org.finos.legend.engine + legend-engine-pure-platform-java + + + org.finos.legend.engine + legend-engine-pure-platform-functions-java + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure + + + org.finos.legend.engine + legend-engine-pure-platform-store-relational-java + + + + org.eclipse.collections + eclipse-collections + + + org.eclipse.collections + eclipse-collections-api + + + + + org.finos.legend.pure + legend-pure-m2-functions-json-pure + test + + + com.fasterxml.jackson.core + jackson-annotations + test + + + com.fasterxml.jackson.core + jackson-databind + test + + + com.fasterxml.jackson.core + jackson-core + test + + + junit + junit + + + + diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultConnectionBuilderProvider.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/java/org/finos/legend/pure/code/core/CoreRelationalSparkSQLCodeRepositoryProvider.java similarity index 54% rename from legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultConnectionBuilderProvider.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/java/org/finos/legend/pure/code/core/CoreRelationalSparkSQLCodeRepositoryProvider.java index e52a422658e..a652f3bddc0 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultConnectionBuilderProvider.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/java/org/finos/legend/pure/code/core/CoreRelationalSparkSQLCodeRepositoryProvider.java @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection; +package org.finos.legend.pure.code.core; -import org.eclipse.collections.api.factory.Lists; +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepository; +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider; +import org.finos.legend.pure.m3.serialization.filesystem.repository.GenericCodeRepository; -import java.util.List; -import java.util.ServiceLoader; - -public class DefaultConnectionBuilderProvider implements ConnectionBuilderProvider +public class CoreRelationalSparkSQLCodeRepositoryProvider implements CodeRepositoryProvider { @Override - public List getBuilders() + public CodeRepository repository() { - return Lists.mutable.withAll(ServiceLoader.load(ConnectionBuilder.class)); + return GenericCodeRepository.build("core_relational_sparksql.definition.json"); } } + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider new file mode 100644 index 00000000000..96a2fb109ac --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/META-INF/services/org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider @@ -0,0 +1 @@ +org.finos.legend.pure.code.core.CoreRelationalSparkSQLCodeRepositoryProvider \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql.definition.json b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql.definition.json new file mode 100644 index 00000000000..ed5893f3ad1 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql.definition.json @@ -0,0 +1,5 @@ +{ + "name" : "core_relational_sparksql", + "pattern" : "(meta::relational::functions::sqlQueryToString::sparkSQL|meta::relational::tests::sqlQueryToString::sparkSQL|meta::relational::tests::sqlToString::sparkSQL|meta::pure::executionPlan::tests::sparkSQL|meta::relational::tests::mapping::sqlFunction::sparkSQL|meta::relational::tests::postProcessor::sparkSQL|meta::relational::tests::query::function::sparkSQL|meta::relational::tests::functions::sqlstring::sparkSQL|meta::relational::tests::tds::sparkSQL|meta::relational::tests::projection::sparkSQL|meta::pure::alloy::connections|meta::protocols::pure)(::.*)?", + "dependencies" : ["platform", "platform_functions", "platform_store_relational", "platform_dsl_mapping", "core_functions", "core", "core_relational"] +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/executionPlan/tests/executionPlanTestSparkSQL.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/executionPlan/tests/executionPlanTestSparkSQL.pure new file mode 100644 index 00000000000..5a3121306e1 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/executionPlan/tests/executionPlanTestSparkSQL.pure @@ -0,0 +1,93 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::pure::alloy::connections::alloy::authentication::*; +import meta::pure::alloy::connections::alloy::specification::*; +import meta::pure::alloy::connections::*; +import meta::pure::mapping::modelToModel::test::createInstances::*; +import meta::relational::postProcessor::*; +import meta::pure::extension::*; +import meta::relational::extension::*; +import meta::pure::mapping::modelToModel::test::shared::*; +import meta::pure::mapping::modelToModel::test::enumerationMapping::enumToEnum::mapping::*; +import meta::pure::mapping::modelToModel::test::enumerationMapping::enumToEnum::model::*; +import meta::pure::mapping::modelToModel::test::enumeration::*; +import meta::pure::graphFetch::execution::*; +import meta::pure::executionPlan::tests::datetime::*; +import meta::relational::tests::tds::tabletds::*; +import meta::pure::mapping::*; +import meta::relational::mapping::*; +import meta::relational::runtime::*; +import meta::relational::tests::mapping::inheritance::relational::*; +import meta::relational::metamodel::join::*; +import meta::relational::tests::tds::tdsJoin::*; +import meta::pure::executionPlan::toString::*; +import meta::pure::executionPlan::tests::*; +import meta::relational::tests::groupBy::datePeriods::mapping::*; +import meta::relational::tests::groupBy::datePeriods::*; +import meta::relational::tests::groupBy::datePeriods::domain::*; +import meta::pure::executionPlan::*; +import meta::relational::tests::*; +import meta::relational::tests::model::simple::*; +import meta::pure::runtime::*; +import meta::pure::mapping::modelToModel::test::shared::src::*; +import meta::pure::graphFetch::executionPlan::*; +import meta::pure::graphFetch::routing::*; +import meta::pure::functions::collection::*; +import meta::pure::executionPlan::tests::SparkSQL::*; + +function <> meta::pure::executionPlan::tests::sparkSQL::testFilterEqualsWithOptionalParameter_SparkSQL():Boolean[1] +{ + let expectedPlan ='Sequence\n'+ + '(\n'+ + ' type = TDS[(Time, Integer, INT, "")]\n'+ + ' (\n'+ + ' FunctionParametersValidationNode\n'+ + ' (\n'+ + ' functionParameters = [optionalID:String[0..1], optionalActive:Boolean[0..1]]\n'+ + ' )\n'+ + ' Relational\n'+ + ' (\n'+ + ' type = TDS[(Time, Integer, INT, "")]\n'+ + ' resultColumns = [("Time", INT)]\n'+ + ' sql = select "root"."time" as "Time" from interactionTable as "root" where ((${optionalVarPlaceHolderOperationSelector(optionalID![], \'"root".ID = ${varPlaceHolderToString(optionalID![] "\\\'" "\\\'" {"\\\'" : "\\\'\\\'"} "null")}\', \'"root".ID is null\')}) and (${optionalVarPlaceHolderOperationSelector(optionalActive![], \'case when "root"."active" = \\\'Y\\\' then \\\'true\\\' else \\\'false\\\' end = ${varPlaceHolderToString(optionalActive![] "\\\'" "\\\'" {} "null")}\', \'case when "root"."active" = \\\'Y\\\' then \\\'true\\\' else \\\'false\\\' end is null\')}))\n'+ + ' connection = DatabaseConnection(type = "SparkSQL")\n'+ + ' )\n'+ + ' )\n'+ + ')\n'; + assertPlanGenerationForOptionalParameter(DatabaseType.SparkSQL, $expectedPlan); +} + +function <> meta::pure::executionPlan::tests::sparkSQL::testGreaterThanLessThanEqualsWithOptionalParameter_SparkSQL():Boolean[1] +{ + let func = {optionalAgeLowerLimit: Integer[0..1], optionalAgeHigherLimit: Integer[0..1]|Person.all()->filter(p|$p.age>$optionalAgeLowerLimit && ($p.age<=$optionalAgeHigherLimit))->project(col(a|$a.firstName, 'firstName'))}; + let expectedPlan ='Sequence\n'+ + '(\n'+ + ' type = TDS[(firstName, String, VARCHAR(200), "")]\n'+ + ' (\n'+ + ' FunctionParametersValidationNode\n'+ + ' (\n'+ + ' functionParameters = [optionalAgeLowerLimit:Integer[0..1], optionalAgeHigherLimit:Integer[0..1]]\n'+ + ' )\n'+ + ' Relational\n'+ + ' (\n'+ + ' type = TDS[(firstName, String, VARCHAR(200), "")]\n'+ + ' resultColumns = [("firstName", VARCHAR(200))]\n'+ + ' sql = select "root".FIRSTNAME as "firstName" from personTable as "root" where ((("root".AGE is not null and ${varPlaceHolderToString(optionalAgeLowerLimit![] "" "" {} "null")} is not null) and "root".AGE > ${varPlaceHolderToString(optionalAgeLowerLimit![] "" "" {} "null")}) and (("root".AGE is not null and ${varPlaceHolderToString(optionalAgeHigherLimit![] "" "" {} "null")} is not null) and "root".AGE <= ${varPlaceHolderToString(optionalAgeHigherLimit![] "" "" {} "null")}))\n'+ + ' connection = DatabaseConnection(type = "SparkSQL")\n'+ + ' )\n'+ + ' )\n'+ + ')\n'; + assertPlanGenerationForOptionalParameterWithGreaterThanLessThan($func, DatabaseType.SparkSQL, $expectedPlan); +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/sparkSQLExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/sparkSQLExtension.pure new file mode 100644 index 00000000000..a14c2f8aad0 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/sparkSQLExtension.pure @@ -0,0 +1,654 @@ +import meta::relational::metamodel::*; +import meta::relational::functions::sqlQueryToString::sparkSQL::*; +import meta::relational::functions::sqlQueryToString::default::*; +import meta::relational::functions::sqlQueryToString::*; +import meta::relational::metamodel::operation::*; +import meta::relational::metamodel::relation::*; +import meta::relational::runtime::*; +import meta::pure::extension::*; +import meta::relational::extension::*; + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::dbExtensionLoaderForSparkSQL():DbExtensionLoader[1] +{ + ^DbExtensionLoader(dbType = DatabaseType.SparkSQL, loader = createDbExtensionForSparkSQL__DbExtension_1_); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::createDbExtensionForSparkSQL():DbExtension[1] +{ + let reservedWords = sybaseReservedWords(); + let literalProcessors = getDefaultLiteralProcessors()->putAll(getLiteralProcessorsForSparkSQL()); + let literalProcessor = {type:Type[1]| $literalProcessors->get(if($type->instanceOf(Enumeration), | Enum, | $type))->toOne()}; + let dynaFuncDispatch = getDynaFunctionToSqlDefault($literalProcessor)->groupBy(d| $d.funcName)->putAll( + getDynaFunctionToSqlForSparkSQL()->groupBy(d| $d.funcName))->getDynaFunctionDispatcher(); + + ^DbExtension( + isBooleanLiteralSupported = false, + collectionThresholdLimit = 250000, + aliasLimit = 255, + isDbReservedIdentifier = {str:String[1]| $str->in($reservedWords)}, + literalProcessor = $literalProcessor, + windowColumnProcessor = processWindowColumn_WindowColumn_1__SqlGenerationContext_1__String_1_, + joinStringsProcessor = processJoinStringsOperationForSparkSQL_JoinStrings_1__SqlGenerationContext_1__String_1_, + selectSQLQueryProcessor = processSelectSQLQueryForSparkSQL_SelectSQLQuery_1__SqlGenerationContext_1__Boolean_1__String_1_, + columnNameToIdentifier = columnNameToIdentifierForSparkSQL_String_1__DbConfig_1__String_1_, + identifierProcessor = processIdentifierWithDoubleQuotes_String_1__DbConfig_1__String_1_, + dynaFuncDispatch = $dynaFuncDispatch, + ddlCommandsTranslator = getDDLCommandsTranslator(), + processTempTableName = processTempTableNameDefault_String_1__String_1_ + ); +} + +function meta::relational::functions::sqlQueryToString::sparkSQL::getDDLCommandsTranslator(): RelationalDDLCommandsTranslator[1] +{ + ^RelationalDDLCommandsTranslator( + createSchema = translateCreateSchemaStatementDefault_CreateSchemaSQL_1__DbConfig_1__String_1_, + dropSchema = translateDropSchemaStatementDefault_DropSchemaSQL_1__DbConfig_1__String_1_, + createTable = translateCreateTableStatementForSparkSQL_CreateTableSQL_1__DbConfig_1__String_1_, + dropTable = translateDropTableStatementDefault_DropTableSQL_1__DbConfig_1__String_1_, + loadTable = loadValuesToDbTableForSparkSQL_LoadTableSQL_1__DbConfig_1__String_MANY_ + ); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::getLiteralProcessorsForSparkSQL():Map[1] +{ + newMap([ + pair(StrictDate, ^LiteralProcessor(format = 'convert(DATE, \'%s\', 121)', transform = {d:StrictDate[1], dbTimeZone:String[0..1] | $d->convertDateToSqlString($dbTimeZone)})), + pair(DateTime, ^LiteralProcessor(format = 'convert(DATETIME, \'%s\', 121)', transform = {d:DateTime[1], dbTimeZone:String[0..1] | $d->convertDateToSqlString($dbTimeZone)})), + pair(Date, ^LiteralProcessor(format = 'convert(DATETIME, \'%s\', 121)', transform = {d:Date[1], dbTimeZone:String[0..1] | $d->convertDateToSqlString($dbTimeZone)})) + ]) +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::getDynaFunctionToSqlForSparkSQL(): DynaFunctionToSql[*] +{ + let allStates = allGenerationStates(); + + [ + dynaFnToSql('contains', $allStates, ^ToSql(format=likePattern('%%%s%%'), transform={p:String[2]|$p->transformLikeParamsForSparkSQL()})), + dynaFnToSql('dateDiff', $allStates, ^ToSql(format='datediff(%s,%s,%s)', transform={p:String[*]|[$p->at(2)->replace('\'', '')->processDateDiffDurationUnitForSparkSQL(),$p->at(0),$p->at(1)]})), + dynaFnToSql('datePart', $allStates, ^ToSql(format='date(%s)')), + dynaFnToSql('endsWith', $allStates, ^ToSql(format=likePattern('%%%s'), transform={p:String[2]|$p->transformLikeParamsForSparkSQL()})), + dynaFnToSql('isAlphaNumeric', $allStates, ^ToSql(format=likePatternWithoutEscape('%%%s%%'), transform={p:String[1]|$p->transformAlphaNumericParamsForSparkSQL()})), + dynaFnToSql('startsWith', $allStates, ^ToSql(format=likePattern('%s%%'), transform={p:String[2]|$p->transformLikeParamsForSparkSQL()})) + ]->concatenate(getDynaFunctionToSqlCommonToBothSybases()); +} + +function meta::relational::functions::sqlQueryToString::sparkSQL::getDynaFunctionToSqlCommonToBothSybases(): DynaFunctionToSql[*] +{ + let allStates = allGenerationStates(); + let selectOutsideWhen = selectOutsideWhenGenerationState(); + let notSelectOutsideWhen = notSelectOutsideWhenGenerationStates(); + + [ + dynaFnToSql('adjust', $allStates, ^ToSql(format='dateadd(%s)', transform={p:String[3] | $p->at(2)->mapToDBUnitType() + ', ' + $p->at(1) + ', ' + $p->at(0)})), + dynaFnToSql('atan2', $allStates, ^ToSql(format='atan2(%s,%s)')), + dynaFnToSql('char', $allStates, ^ToSql(format='char(%s)')), + dynaFnToSql('concat', $allStates, ^ToSql(format='%s', transform={p:String[*]|$p->joinStrings(' + ')})), + dynaFnToSql('convertDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | $p->convertToDateIQ()})), + dynaFnToSql('convertDateTime', $allStates, ^ToSql(format='%s' , transform={p:String[*] | $p->convertToDateTimeIQ()})), + dynaFnToSql('convertVarchar128', $allStates, ^ToSql(format='convert(VARCHAR(128), %s)')), + dynaFnToSql('dayOfMonth', $allStates, ^ToSql(format='datepart(DAY,%s)')), + dynaFnToSql('dayOfWeek', $allStates, ^ToSql(format='datename(WEEKDAY,%s)')), + dynaFnToSql('dayOfWeekNumber', $allStates, ^ToSql(format='%s',transform={p:String[1..2]| if($p->size()==1,| 'datepart(Weekday,'+ $p->at(0)+')',|$p->dayOfWeekNumberSparkSQL());})), + dynaFnToSql('dayOfYear', $allStates, ^ToSql(format='datepart(DAYOFYEAR,%s)')), + dynaFnToSql('firstDayOfMonth', $allStates, ^ToSql(format='dateadd(DAY, -(day(%s) - 1), %s)', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstDayOfQuarter', $allStates, ^ToSql(format='dateadd(QUARTER, quarter(%s) - 1, dateadd(DAY, -(datepart(dayofyear, %s) - 1), %s))', transform={p:String[1] | $p->repeat(3)})), + dynaFnToSql('firstDayOfThisMonth', $allStates, ^ToSql(format='dateadd(DAY, -(day(today()) - 1), today())%s', transform={p:String[*] | ''})), + dynaFnToSql('firstDayOfThisQuarter', $allStates, ^ToSql(format='dateadd(QUARTER, quarter(today()) - 1, dateadd(DAY, -(datepart(dayofyear, today()) - 1), today()))%s', transform={p:String[*] | ''})), + dynaFnToSql('firstDayOfThisYear', $allStates, ^ToSql(format='dateadd(DAY, -(datepart(dayofyear, today()) - 1), today())%s', transform={p:String[*] | ''})), + dynaFnToSql('firstDayOfWeek', $allStates, ^ToSql(format='dateadd(DAY, -(mod(datepart(weekday, %s)+5, 7)), %s)', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstDayOfYear', $allStates, ^ToSql(format='dateadd(DAY, -(datepart(dayofyear, %s) - 1), %s)', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstHourOfDay', $allStates, ^ToSql(format='datetime(date(%s))')), + dynaFnToSql('firstMillisecondOfSecond', $allStates, ^ToSql(format='dateadd(microsecond, -(datepart(microsecond, %s)), %s)', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstMinuteOfHour', $allStates, ^ToSql(format='dateadd(hour, datepart(hour, %s), date(%s))', transform={p:String[1] | $p->repeat(2)})), + dynaFnToSql('firstSecondOfMinute', $allStates, ^ToSql(format='dateadd(minute, datepart(minute, %s), dateadd(hour, datepart(hour, %s), date(%s)))', transform={p:String[1] | $p->repeat(3)})), + dynaFnToSql('hour', $allStates, ^ToSql(format='hour(%s)')), + dynaFnToSql('indexOf', $allStates, ^ToSql(format='LOCATE(%s)', transform={p:String[2] | $p->at(0) + ', ' + $p->at(1)})), + dynaFnToSql('isEmpty', $selectOutsideWhen, ^ToSql(format='case when (%s is null) then \'true\' else \'false\' end', parametersWithinWhenClause=true)), + dynaFnToSql('isEmpty', $notSelectOutsideWhen, ^ToSql(format='%s is null')), + dynaFnToSql('isNotEmpty', $selectOutsideWhen, ^ToSql(format='case when (%s is not null) then \'true\' else \'false\' end', parametersWithinWhenClause=true)), + dynaFnToSql('isNotEmpty', $notSelectOutsideWhen, ^ToSql(format='%s is not null')), + dynaFnToSql('isNotNull', $selectOutsideWhen, ^ToSql(format='case when (%s is not null) then \'true\' else \'false\' end', parametersWithinWhenClause=true)), + dynaFnToSql('isNotNull', $notSelectOutsideWhen, ^ToSql(format='%s is not null')), + dynaFnToSql('isNull', $selectOutsideWhen, ^ToSql(format='case when (%s is null) then \'true\' else \'false\' end', parametersWithinWhenClause=true)), + dynaFnToSql('isNull', $notSelectOutsideWhen, ^ToSql(format='%s is null')), + dynaFnToSql('isNumeric', $allStates, ^ToSql(format='isnumeric(%s)')), + dynaFnToSql('joinStrings', $allStates, ^ToSql(format='list(%s,%s)')), + dynaFnToSql('left', $allStates, ^ToSql(format='left(%s,%s)')), + dynaFnToSql('length', $allStates, ^ToSql(format='char_length(%s)')), + dynaFnToSql('matches', $allStates, ^ToSql(format=regexpPattern('%s'), transform={p:String[2]|$p->transformRegexpParams()})), + dynaFnToSql('md5', $allStates, ^ToSql(format='hash(%s, \'MD5\')')), + dynaFnToSql('minute', $allStates, ^ToSql(format='minute(%s)')), + dynaFnToSql('mod', $allStates, ^ToSql(format='mod(%s,%s)')), + dynaFnToSql('month', $allStates, ^ToSql(format='month(%s)')), + dynaFnToSql('monthNumber', $allStates, ^ToSql(format='month(%s)')), + dynaFnToSql('mostRecentDayOfWeek', $allStates, ^ToSql(format='dateadd(Day, case when %s - dow(%s) > 0 then %s - dow(%s) - 7 else %s - dow(%s) end, %s)', transform={p:String[1..2] | $p->formatMostRecentSybase('today()')}, parametersWithinWhenClause = [false, false])), + dynaFnToSql('now', $allStates, ^ToSql(format='now(%s)', transform={p:String[*] | ''})), + dynaFnToSql('parseDate', $allStates, ^ToSql(format='%s', transform={p:String[*] | if( $p->size()==1,|'cast('+$p->at(0)+' as timestamp)' ,|'convert( datetime,'+ $p->at(0)+','+$p->at(1)+')' )})), + dynaFnToSql('parseDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')), + dynaFnToSql('parseFloat', $allStates, ^ToSql(format='cast(%s as float)')), + dynaFnToSql('parseInteger', $allStates, ^ToSql(format='cast(%s as integer)')), + dynaFnToSql('position', $allStates, ^ToSql(format='charindex(%s, %s)')), + dynaFnToSql('previousDayOfWeek', $allStates, ^ToSql(format='dateadd(DAY, case when %s - dow(%s) >= 0 then %s - dow(%s) - 7 else %s - dow(%s) end, %s)', transform={p:String[1..2] | $p->formatMostRecentSybase('today()')}, parametersWithinWhenClause = [false, false])), + dynaFnToSql('quarter', $allStates, ^ToSql(format='quarter(%s)')), + dynaFnToSql('quarterNumber', $allStates, ^ToSql(format='quarter(%s)')), + dynaFnToSql('rem', $allStates, ^ToSql(format='mod(%s,%s)')), + dynaFnToSql('right', $allStates, ^ToSql(format='right(%s,%s)')), + dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)), + dynaFnToSql('second', $allStates, ^ToSql(format='second(%s)')), + dynaFnToSql('sha1', $allStates, ^ToSql(format='hash(%s, \'SHA1\')')), + dynaFnToSql('sha256', $allStates, ^ToSql(format='hash(%s, \'SHA256\')')), + dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), + dynaFnToSql('stdDevPopulation', $allStates, ^ToSql(format='stddev_pop(%s)')), + dynaFnToSql('stdDevSample', $allStates, ^ToSql(format='stddev_samp(%s)')), + dynaFnToSql('today', $allStates, ^ToSql(format='today(%s)', transform={p:String[*] | ''})), + dynaFnToSql('toDecimal', $allStates, ^ToSql(format='cast(%s as decimal)')), + dynaFnToSql('toFloat', $allStates, ^ToSql(format='cast(%s as double)')), + dynaFnToSql('toString', $allStates, ^ToSql(format='cast(%s as varchar)')), + dynaFnToSql('toTimestamp', $allStates, ^ToSql(format='%s', transform={p:String[2] | $p->transformToTimestampSparkSQL()})), + dynaFnToSql('weekOfYear', $allStates, ^ToSql(format='datepart(WEEK,%s)')), + dynaFnToSql('year', $allStates, ^ToSql(format='year(%s)')) + ]; +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::convertToDateIQ(params:String[*]):String[1] +{ + $params->convertDateFunctionHasCorrectParams(); + let dateFormat = if( $params->size() == 1,| 120,| dateFormats()->get($params->at(1)->replace('\'', ''))->toOne();); + if ($dateFormat == 106, + |'convert ( date,(\'01 \' + ' + 'substring(' + $params->at(0) + ',1,3)' + ' + \' \' + ' + 'substring(' + $params->at(0) + ',4,4))' + ',' + $dateFormat->toString() + ')', + |'convert ( date,'+$params->at(0)+','+$dateFormat->toString() +')';); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::transformLikeParamsForSparkSQL(params: String[2]):String[*] +{ + let likeExpression = $params->at(1)->removeQuotes()->escapeLikeExprForSparkSQL(); + $params->at(0) + ->concatenate($likeExpression) + ->concatenate(likeEscapeClauseForSparkSQL($likeExpression)); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::likeEscapeClauseForSparkSQL(expr: String[1]):String[*] +{ + if($expr->contains('\\'), + |' escape \'\\\'', + |''); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::escapeLikeExprForSparkSQL(query: String[1]):String[1] +{ + // Escaping references... + // Sybase IQ: http://infocenter.sybase.com/help/index.jsp?topic=/com.sybase.infocenter.dc38151.1520/html/iqrefbb/CACGCGGC.htm + + $query + ->replace('_', '\\_') + ->replace('%', '\\%') + ->replace('[', '\\['); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::convertToDateTimeIQ(params:String[*]):String[1] +{ + $params->convertDateTimeFunctionHasCorrectParams(); + let dateTimeFormat = if( $params->size() == 1,| 120 ,| dateTimeFormats()->get($params->at(1)->replace('\'', ''))->toOne();); + //http://infocenter.sybase.com/help/index.jsp?topic=/com.sybase.infocenter.dc38151.1520/html/iqrefbb/Dateformat.htm + 'convert( timestamp,'+$params->at(0)+','+$dateTimeFormat->toString() +')'; +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::transformToTimestampSparkSQL(params:String[2]):String[1] +{ + // Temporarily revert functionality to handle scenarios that have date string of the format yyyyMMdd + 'cast('+$params->at(0)+' as timestamp)'; + + //Standardizing the format as per Postgres specification, will include mappings for the formats in future. + // assert($params->at(1)->replace('\'', '') == 'YYYY-MM-DD HH24:MI:SS', | $params->at(1) +' not supported '); + // let timestampFormat = 121; + // 'convert(datetime,'+$params->at(0)+','+$timestampFormat->toString() +')'; +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::formatMostRecentSybase(p:String[1..2], defaultDay:String[1]):String[*] +{ + let day = $p->last()->toOne()->mapToDBDayOfWeekNumber()->toString(); + let current = if ($p->size() == 2, | $p->first()->toOne(), | $defaultDay); + [$day, $current, $day, $current, $day, $current, $current]; +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::processDateDiffDurationUnitForSparkSQL(durationUnit:String[1]):String[1] +{ + let durationEnumNames = [DurationUnit.YEARS,DurationUnit.MONTHS,DurationUnit.WEEKS,DurationUnit.DAYS,DurationUnit.HOURS,DurationUnit.MINUTES,DurationUnit.SECONDS,DurationUnit.MILLISECONDS]->map(e|$e->toString()); + let durationDbNames = ['yy', 'mm', 'wk', 'dd', 'hh', 'mi', 'ss', 'ms']; + $durationEnumNames->zip($durationDbNames)->filter(h | $h.first == $durationUnit).second->toOne(); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::transformAlphaNumericParamsForSparkSQL(params: String[1]):String[*] +{ + let param = '\'[^a-zA-Z0-9]\''; + let expression = $param->removeQuotes(); + $params->at(0)->concatenate($expression); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::processJoinStringsOperationForSparkSQL(js:JoinStrings[1], sgc:SqlGenerationContext[1]): String[1] +{ + processJoinStringsOperation($js, $sgc, {col, sep| 'list(' + $col + ',' + $sep + ' )'}, {strs, sep| $strs->joinStrings(if('\'\'' == $sep, |'+', |'+' + $sep + '+'))}); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::processSelectSQLQueryForSparkSQL(s:SelectSQLQuery[1], sgc:SqlGenerationContext[1], isSubSelect:Boolean[1]):String[1] +{ + $s->processSelectSQLQueryForSparkSQL($sgc.dbConfig, $sgc.format, $sgc.config, $isSubSelect, $sgc.extensions); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::processSelectSQLQueryForSparkSQL(sq:SelectSQLQuery[1], dbConfig : DbConfig[1], format:Format[1], config:Config[1], isSubSelect : Boolean[1], extensions:Extension[*]):String[1] +{ + // Sybase IQ does not support limit / offset in a subselect, so we need to adjust it, e.g. + // select * from (select top 10 "root".id as "ID" from trades as "root" ) as x + // gives + // SQL Anywhere Error -1001030: Feature, TOP/FIRST/LIMIT in a view, is not supported. + // but instead needs to be + // select * from (select "limitoffset_via_window_subquery"."ID" as "ID" from (select "root".id as "ID", row_number() OVER (Order By "root".id) as "row_number" from trades as "root") + // as "limitoffset_via_window_subquery" where "limitoffset_via_window_subquery".row_number < 10) as x + // Also see https://www.jooq.org/doc/3.1/manual/sql-building/sql-statements/select-statement/limit-clause/#N467C4 + + let s = if($isSubSelect && ($sq.fromRow->isNotEmpty() || $sq.toRow->isNotEmpty()), |$sq->rewriteSliceAsWindowFunction(), |$sq); + let opStr = if($s.filteringOperation->isEmpty(), |'', |$s.filteringOperation->map(s|$s->wrapAsBooleanOperation($extensions)->processOperation($dbConfig, $format->indent(), ^$config(callingFromFilter = true), $extensions))->filter(s|$s != '')->joinStrings(' <||> ')); + let havingStr = if($s.havingOperation->isEmpty(), |'', |$s.havingOperation->map(s|$s->processOperation($dbConfig, $format->indent(), $config, $extensions))->filter(s|$s != '')->joinStrings(' <||> ')); + + $format.separator + 'select ' + if($s.distinct == true,|'distinct ',|'') + processTop($s, $format, $dbConfig, $extensions) + + processSelectColumns($s.columns, $dbConfig, $format->indent(), false, $extensions) + + if ($s.data == [],|'',| ' ' + $format.separator + 'from ' + $s.data->toOne()->processJoinTreeNode([], $dbConfig, $format->indent(), [], $extensions)) + + if (eq($opStr, ''), |'', | ' ' + $format.separator + 'where ' + $opStr) + + if ($s.groupBy->isEmpty(),|'',| ' ' + $format.separator + 'group by '+$s.groupBy->processGroupByColumns($dbConfig, $format->indent(), true, $extensions)->makeString(','))+ + if (eq($havingStr, ''), |'', | ' ' + $format.separator + 'having ' + $havingStr) + + if ($s.orderBy->isEmpty(),|'',| ' ' + $format.separator + 'order by '+ $s.orderBy->processOrderBy($dbConfig, $format->indent(), $config, $extensions)->makeString(','))+ + + processLimit($s, $dbConfig, $format, $extensions, [], processSliceOrDropForSparkSQL_SelectSQLQuery_1__Format_1__DbConfig_1__Extension_MANY__Any_1__String_1_); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::processSliceOrDropForSparkSQL(s:SelectSQLQuery[1], format:Format[1], dbConfig : DbConfig[1], extensions:Extension[*], size:Any[1]):String[1] +{ + // queries where specifying query with "limit 100,1000" would return 1100 rows. + // However when a order by column is specified, the expected number of rows is returned (i.e. 900 rows). + // Given that a slice without an order is largely meaningless, we'll simply report an error (rather than trying to simulate something) + // + // http://infocenter.sybase.com/help/index.jsp?topic=/com.sybase.infocenter.dc00801.1601/doc/html/san1281564978024.html + + assert($s.orderBy->isNotEmpty(), | 'SparkSQL requires an order by column for meaningful limit/offset'); + '%s limit %s,%s'->format([$format.separator, $s.fromRow->toOne()->getValueForTake($format, $dbConfig, $extensions), $size]); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::columnNameToIdentifierForSparkSQL(columnName: String[1], dbConfig: DbConfig[1]): String[1] +{ + if($dbConfig.isDbReservedIdentifier($columnName->toLower()), |'"' + $columnName->toLower() + '"', |$columnName->quoteIdentifierDefault()); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::dayOfWeekNumberSparkSQL(dayOfWeek: String[*]):String[1] +{ + let day = if(startsWith($dayOfWeek->at(1),'\''),|$dayOfWeek->at(1)->removeQuotes(),|$dayOfWeek->at(1)); + assert(or($day == 'Sunday',$day == 'Monday'),'DayOfWeekNumber Function requires either Sunday or Monday as First Day of Week'); + if($day =='Sunday',|'datepart(Weekday,'+$dayOfWeek->at(0)+')',|'mod (datepart(weekday,'+$dayOfWeek->at(0)+')+5,7)+1'); +} + +function meta::relational::functions::sqlQueryToString::sparkSQL::translateCreateTableStatementForSparkSQL(c:CreateTableSQL[1], dbConfig: DbConfig[1]): String[1] +{ + let columns = $c.table.columns->map(r|$r->match([c:Column[1]| '[' + $c.name->processColumnName($dbConfig) + '] ' + $c.type->getColumnTypeSqlTextDefault(), + r:RelationalOperationElement[1]| fail('Only \'Column\' types are supported when creating temporary tables, found: '+$r->type()->toOne()->elementToPath());'';])); + + if($c.isTempTable->isTrue(),| 'DECLARE LOCAL TEMPORARY TABLE ' + $c.table->tableToString($dbConfig) + '('+ $columns->joinStrings(',') + ') ON COMMIT PRESERVE ROWS' + ,| $c->meta::relational::functions::sqlQueryToString::default::translateCreateTableStatementDefault($dbConfig)); + +} + +function meta::relational::functions::sqlQueryToString::sparkSQL::loadValuesToDbTableForSparkSQL(l:LoadTableSQL[1], dbConfig: DbConfig[1]): String[*] +{ + let columns = $l.table.columns->map(r|$r->match([c:Column[1]| $c.name->toOne(), + r:RelationalOperationElement[1]| fail('Only \'Column\' types are supported when creating temporary tables, found: '+$r->type()->toOne()->elementToPath());'';])); + + if($l.absolutePathToFile->isNotEmpty(),| 'load table ' + $l.table->tableToString($dbConfig) +'( '+ $columns->map(c|'['+$c + '] null(blanks)')->joinStrings(',') + ' ) USING CLIENT FILE \'' + $l.absolutePathToFile->toOne()->processOperation($dbConfig.dbType, []) + '\' FORMAT BCP \nwith checkpoint on \nquotes on \nescapes off \ndelimited by \',\' \nROW DELIMITED BY \'\r\n\''; + + ,| $l->meta::relational::functions::sqlQueryToString::default::loadValuesToDbTableDefault($dbConfig)); +} + +function <> meta::relational::functions::sqlQueryToString::sparkSQL::sybaseReservedWords():String[*] +{ + //Based on + // http://infocenter.sybase.com/help/index.jsp?topic=/com.sybase.infocenter.dc38151.1510/html/iqrefbb/Alhakeywords.htm + // and + // http://infocenter.sybase.com/help/index.jsp?topic=/com.sybase.infocenter.dc38151.1601/doc/html/san1278452828146.html + + [ + 'active', + 'add', + 'algorithm', + 'all', + 'alter', + 'and', + 'any', + 'append', + 'array', + 'as', + 'asc', + 'attach', + 'auto', + 'backup', + 'begin', + 'between', + 'bigint', + 'binary', + 'bit', + 'bottom', + 'break', + 'by', + 'calibrate', + 'calibration', + 'call', + 'cancel', + 'capability', + 'cascade', + 'case', + 'cast', + 'certificate', + 'char', + 'char_convert', + 'character', + 'check', + 'checkpoint', + 'checksum', + 'clientport', + 'close', + 'columns', + 'comment', + 'commit', + 'committed', + 'comparisons', + 'compressed', + 'computes', + 'conflict', + 'connect', + 'constraint', + 'contains', + 'continue', + 'convert', + 'create', + 'cross', + 'cube', + 'current', + 'current_timestamp', + 'current_user', + 'cursor', + 'date', + 'datetimeoffset', + 'dbspace', + 'dbspacename', + 'deallocate', + 'debug', + 'dec', + 'decimal', + 'declare', + 'decoupled', + 'decrypted', + 'default', + 'delay', + 'delete', + 'deleting', + 'density', + 'desc', + 'detach', + 'deterministic', + 'disable', + 'distinct', + 'do', + 'double', + 'drop', + 'dynamic', + 'elements', + 'else', + 'elseif', + 'enable', + 'encapsulated', + 'encrypted', + 'end', + 'endif', + 'escape', + 'except', + 'exception', + 'exclude', + 'exec', + 'execute', + 'existing', + 'exists', + 'explicit', + 'express', + 'externlogin', + 'fastfirstrow', + 'fetch', + 'first', + 'float', + 'following', + 'for', + 'force', + 'foreign', + 'forward', + 'from', + 'full', + 'gb', + 'goto', + 'grant', + 'group', + 'grouping', + 'having', + 'hidden', + 'history', + 'holdlock', + 'identified', + 'if', + 'in', + 'inactive', + 'index', + 'index_lparen', + 'inner', + 'inout', + 'input', + 'insensitive', + 'insert', + 'inserting', + 'install', + 'instead', + 'int', + 'integer', + 'integrated', + 'intersect', + 'into', + 'iq', + 'is', + 'isolation', + 'jdk', + 'join', + 'json', + 'kb', + 'kerberos', + 'key', + 'lateral', + 'left', + 'like', + 'limit', + 'lock', + 'logging', + 'login', + 'long', + 'match', + 'mb', + 'membership', + 'merge', + 'message', + 'mode', + 'modify', + 'namespace', + 'natural', + 'nchar', + 'new', + 'no', + 'noholdlock', + 'nolock', + 'not', + 'notify', + 'null', + 'numeric', + 'nvarchar', + 'of', + 'off', + 'on', + 'open', + 'openstring', + 'openxml', + 'optimization', + 'option', + 'options', + 'or', + 'order', + 'others', + 'out', + 'outer', + 'over', + 'pages', + 'paglock', + 'partial', + 'partition', + 'passthrough', + 'password', + 'plan', + 'preceding', + 'precision', + 'prepare', + 'primary', + 'print', + 'privileges', + 'proc', + 'procedure', + 'proxy', + 'publication', + 'raiserror', + 'range', + 'raw', + 'readcommitted', + 'readonly', + 'readpast', + 'readtext', + 'readuncommitted', + 'readwrite', + 'real', + 'recursive', + 'reference', + 'references', + 'refresh', + 'release', + 'relocate', + 'remote', + 'remove', + 'rename', + 'reorganize', + 'repeatable', + 'repeatableread', + 'reserve', + 'resizing', + 'resource', + 'restore', + 'restrict', + 'return', + 'revoke', + 'right', + 'rollback', + 'rollup', + 'root', + 'row', + 'rowlock', + 'rows', + 'rowtype', + 'save', + 'savepoint', + 'schedule', + 'scroll', + 'secure', + 'select', + 'sensitive', + 'serializable', + 'service', + 'session', + 'set', + 'setuser', + 'share', + 'smallint', + 'soapaction', + 'some', + 'space', + 'spatial', + 'sqlcode', + 'sqlstate', + 'start', + 'stop', + 'subtrans', + 'subtransaction', + 'synchronize', + 'syntax_error', + 'table', + 'tablock', + 'tablockx', + 'tb', + 'temporary', + 'then', + 'ties', + 'time', + 'timestamp', + 'tinyint', + 'to', + 'top', + 'tran', + 'transaction', + 'transactional', + 'transfer', + 'treat', + 'tries', + 'trigger', + 'truncate', + 'tsequal', + 'unbounded', + 'uncommitted', + 'union', + 'unique', + 'uniqueidentifier', + 'unknown', + 'unnest', + 'unsigned', + 'update', + 'updating', + 'updlock', + 'url', + 'user', + 'using', + 'utc', + 'validate', + 'values', + 'varbinary', + 'varbit', + 'varchar', + 'variable', + 'varray', + 'varying', + 'view', + 'virtual', + 'wait', + 'waitfor', + 'web', + 'when', + 'where', + 'while', + 'window', + 'with', + 'with_cube', + 'with_lparen', + 'with_rollup', + 'withauto', + 'within', + 'word', + 'work', + 'writeserver', + 'writetext', + 'xlock', + 'xml' + ] +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/tests/testSparkSQLToSQLString.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/tests/testSparkSQLToSQLString.pure new file mode 100644 index 00000000000..53baa3649a6 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/main/resources/core_relational_sparksql/relational/sqlQueryToString/tests/testSparkSQLToSQLString.pure @@ -0,0 +1,45 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::relational::tests::functions::sqlstring::*; +import meta::pure::mapping::*; +import meta::relational::functions::asserts::*; +import meta::relational::mapping::*; +import meta::relational::tests::*; +import meta::relational::tests::model::simple::*; +import meta::pure::profiles::*; +import meta::relational::functions::sqlstring::*; +import meta::relational::runtime::*; + +function <> meta::relational::tests::functions::sqlstring::sparkSQL::testProcessLiteralForSpark():Boolean[1] +{ + let result = toSQLString(|Person.all()->project([ + a | 'String', + b | %2016-03-01, + c | %2016-03-01T12:18:18.976+0200, + d | 1, + e | 1.1 + ], + ['a','b','c','d', 'e'])->take(0), + simpleRelationalMapping, DatabaseType.SparkSQL, meta::relational::extension::relationalExtensions()); + assertEquals('select top 0 \'String\' as "a", convert(DATE, \'2016-03-01\', 121) as "b", convert(DATETIME, \'2016-03-01 10:18:18.976\', 121) as "c", 1 as "d", 1.1 as "e" from personTable as "root"', $result); +} + +function <> meta::relational::tests::functions::sqlstring::sparkSQL::testToSQLStringWithLength():Boolean[1] +{ + [DatabaseType.SparkSQL]->map(db| + let s = toSQLString(|Person.all()->project(p|length($p.firstName), 'nameLength'), simpleRelationalMapping, $db, meta::relational::extension::relationalExtensions()); + assertEquals('select char_length("root".FIRSTNAME) as "nameLength" from personTable as "root"', $s); + ); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_SparkSQL.java similarity index 54% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_SparkSQL.java index c072278365f..c729562435f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-athena/legend-engine-xt-relationalStore-athena-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_ConnectionEquality.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/test/java/org/finos/legend/pure/code/core/Test_Pure_Relational_SparkSQL.java @@ -1,4 +1,4 @@ -// Copyright 2022 Goldman Sachs +// Copyright 2023 Goldman Sachs // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,16 +16,20 @@ import junit.framework.TestSuite; import org.finos.legend.pure.m3.execution.test.PureTestBuilder; +import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; import org.finos.legend.pure.m3.execution.test.TestCollection; import org.finos.legend.pure.runtime.java.compiled.execution.CompiledExecutionSupport; -import org.finos.legend.pure.runtime.java.compiled.testHelper.PureTestBuilderCompiled; -public class Test_Pure_Relational_ConnectionEquality +public class Test_Pure_Relational_SparkSQL { public static TestSuite suite() { - String testPackage = "meta::relational::tests::connEquality"; CompiledExecutionSupport executionSupport = PureTestBuilderCompiled.getClassLoaderExecutionSupport(); - return PureTestBuilderCompiled.buildSuite(TestCollection.collectTests(testPackage, executionSupport.getProcessorSupport(), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport); + executionSupport.getConsole().disable(); + TestSuite suite = new TestSuite(); + + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::pure::executionPlan::tests::sparkSQL", executionSupport.getProcessorSupport(), fn -> PureTestBuilderCompiled.generatePureTestCollection(fn, executionSupport), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); + suite.addTest(PureTestBuilderCompiled.buildSuite(TestCollection.collectTests("meta::relational::tests::functions::sqlstring::sparkSQL", executionSupport.getProcessorSupport(), fn -> PureTestBuilderCompiled.generatePureTestCollection(fn, executionSupport), ci -> PureTestBuilder.satisfiesConditions(ci, executionSupport.getProcessorSupport())), executionSupport)); + return suite; } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/test/java/org/finos/legend/pure/code/core/test/TestSparkSQLCodeRepositoryProviderAvailable.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/test/java/org/finos/legend/pure/code/core/test/TestSparkSQLCodeRepositoryProviderAvailable.java new file mode 100644 index 00000000000..a127f31c56e --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/legend-engine-xt-relationalStore-sparksql-pure/src/test/java/org/finos/legend/pure/code/core/test/TestSparkSQLCodeRepositoryProviderAvailable.java @@ -0,0 +1,36 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.pure.code.core.test; + +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.pure.code.core.CoreRelationalSparkSQLCodeRepositoryProvider; +import org.finos.legend.pure.m3.serialization.filesystem.repository.CodeRepositoryProvider; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ServiceLoader; + +public class TestSparkSQLCodeRepositoryProviderAvailable +{ + @Test + public void testCodeRepositoryProviderAvailable() + { + MutableList> codeRepositoryProviders = + Lists.mutable.withAll(ServiceLoader.load(CodeRepositoryProvider.class)) + .collect(Object::getClass); + Assert.assertTrue(codeRepositoryProviders.contains(CoreRelationalSparkSQLCodeRepositoryProvider.class)); + } +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/pom.xml new file mode 100644 index 00000000000..70d964849ee --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sparksql/pom.xml @@ -0,0 +1,34 @@ + + + + + org.finos.legend.engine + legend-engine-xt-relationalStore-dbExtension + 4.30.3-SNAPSHOT + + 4.0.0 + + legend-engine-xt-relationalStore-sparksql + pom + Legend Engine - XT - Relational Store - DB Extension - sparksql + + + + legend-engine-xt-relationalStore-sparksql-pure + + + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/pom.xml index 7bebfeac1c0..81ef368c6bf 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-sqlserver - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/java/org/finos/legend/connection/jdbc/driver/SQLServer_JDBCConnectionDriver.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/java/org/finos/legend/connection/jdbc/driver/SQLServerDatabaseManager.java similarity index 89% rename from legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/java/org/finos/legend/connection/jdbc/driver/SQLServer_JDBCConnectionDriver.java rename to legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/java/org/finos/legend/connection/jdbc/driver/SQLServerDatabaseManager.java index 50a75513461..f22628f7a8b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/java/org/finos/legend/connection/jdbc/driver/SQLServer_JDBCConnectionDriver.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/java/org/finos/legend/connection/jdbc/driver/SQLServerDatabaseManager.java @@ -15,12 +15,13 @@ package org.finos.legend.connection.jdbc.driver; import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.connection.DatabaseManager; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseType; import java.util.List; import java.util.Properties; -public class SQLServer_JDBCConnectionDriver implements JDBCConnectionDriver +public class SQLServerDatabaseManager implements DatabaseManager { @Override public List getIds() @@ -35,7 +36,7 @@ public String getDriver() } @Override - public String buildURL(String host, int port, String databaseName, Properties extraUserDataSourceProperties) + public String buildURL(String host, int port, String databaseName, Properties properties) { return String.format("jdbc:sqlserver://%s:%s;databaseName=%s", host, port, databaseName); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager new file mode 100644 index 00000000000..c4d426cd4b6 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/resources/META-INF/services/org.finos.legend.connection.DatabaseManager @@ -0,0 +1 @@ +org.finos.legend.connection.jdbc.driver.SQLServerDatabaseManager diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver deleted file mode 100644 index 9ec8d4417bd..00000000000 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-connection/src/main/resources/META-INF/services/org.finos.legend.connection.jdbc.driver.JDBCConnectionDriver +++ /dev/null @@ -1 +0,0 @@ -org.finos.legend.connection.jdbc.driver.SQLServer_JDBCConnectionDriver diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution-tests/pom.xml index e057e6343a2..9b44b9c3aa6 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution-tests/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-sqlserver - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-sqlserver-execution-tests diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution/pom.xml index f88f3c1f00c..8acc343995a 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-sqlserver - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-pure/pom.xml index 63ad1c86a3b..1bf927ec074 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/legend-engine-xt-relationalStore-sqlserver-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-sqlserver - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/pom.xml index e6a19b8bb52..3e4435fab94 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sqlserver/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/legend-engine-xt-relationalStore-sybase-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/legend-engine-xt-relationalStore-sybase-pure/pom.xml index a03c439b22a..4e77024584d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/legend-engine-xt-relationalStore-sybase-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/legend-engine-xt-relationalStore-sybase-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-sybase - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/pom.xml index fc6be7b388b..796630c7e2b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybase/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/legend-engine-xt-relationalStore-sybaseiq-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/legend-engine-xt-relationalStore-sybaseiq-pure/pom.xml index 8caf1531038..f24e6422ef2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/legend-engine-xt-relationalStore-sybaseiq-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/legend-engine-xt-relationalStore-sybaseiq-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-sybaseiq - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/pom.xml index fb4c07d6890..87414bc2e42 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-sybaseiq/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-reports/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-reports/pom.xml index a3f502285e6..9360ce5bb39 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-reports/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-reports/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-test-reports diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-server/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-server/pom.xml index 77e87ca1319..6d25c757b49 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-server/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-test-server/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-test-server diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution-tests/pom.xml index 748d08e2e97..5eb93cb81c1 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution-tests/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-trino - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution/pom.xml index a04571bdb89..f689bec926f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-trino - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-grammar/pom.xml index 96fa6080752..3e779e2db94 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-trino - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-protocol/pom.xml index 82b2bdf41d0..42a593a4b56 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-trino - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-pure/pom.xml index 9ec5c1a8289..ed855c1d657 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/legend-engine-xt-relationalStore-trino-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-trino - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/pom.xml index 4a16c6277ce..e2232e36f68 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/legend-engine-xt-relationalStore-trino/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-dbExtension - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/pom.xml index 3e8f7643639..f87fe778684 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-dbExtension/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-relationalStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -41,6 +41,7 @@ legend-engine-xt-relationalStore-hive legend-engine-xt-relationalStore-presto legend-engine-xt-relationalStore-databricks + legend-engine-xt-relationalStore-sparksql legend-engine-xt-relationalStore-test-reports diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-authorizer/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-authorizer/pom.xml index 9215c4df534..e7bdb518fba 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-authorizer/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-authorizer/pom.xml @@ -3,7 +3,7 @@ legend-engine-xt-relationalStore-execution org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-api/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-api/pom.xml index 86554528935..ced84a76d07 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-api/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-api/pom.xml @@ -19,13 +19,13 @@ legend-engine-xt-relationalStore-execution org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-executionPlan-connection-api jar - Legend Engine - XT - Relational Store - Connection API + Legend Engine - XT - Relational Store - Execution Plan - Connection API diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication-default/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication-default/pom.xml index 796cbd73ac0..ac70578e248 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication-default/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication-default/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-executionPlan-connection-authentication-default diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication/pom.xml index 0fd568b8f34..cc0a9324f3c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-authentication/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-executionPlan-connection-authentication diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-tests/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-tests/pom.xml index 3e0334e600e..007f8d7f8f9 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-tests/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection-tests/pom.xml @@ -18,13 +18,13 @@ org.finos.legend.engine legend-engine-xt-relationalStore-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-executionPlan-connection-tests jar - Legend Engine - XT - Relational Store - Connection - Tests + Legend Engine - XT - Relational Store - Execution Plan - Connection - Tests diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/pom.xml index ae5d5afa4f5..45dd9ec99d3 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/pom.xml @@ -19,13 +19,13 @@ org.finos.legend.engine legend-engine-xt-relationalStore-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-xt-relationalStore-executionPlan-connection jar - Legend Engine - XT - Relational Store - Connection + Legend Engine - XT - Relational Store - Execution Plan - Connection diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java index a8cfff372ac..ec98e7b74b5 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.h2.value.Value; import org.h2.value.ValueBigint; @@ -128,4 +129,22 @@ public static String legend_h2_extension_reverse_string(String string) { return string == null ? null : new StringBuilder(string).reverse().toString(); } + + public static String legend_h2_extension_split_part(String string, String token, Integer part) + { + if (part < 1) + { + throw new IllegalArgumentException("Split part must be greater than zero"); + } + + if (string == null) + { + return null; + } + + String[] parts = StringUtils.split(string, token); + int readjustedPart = part - 1; + + return parts.length > readjustedPart ? parts[readjustedPart] : null; + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java index 8264185e1c5..f63987ef7bd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan-connection/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/authentication/strategy/DefaultH2AuthenticationStrategy.java @@ -106,7 +106,8 @@ private static List getLegendH2ExtensionSQLs() "CREATE ALIAS IF NOT EXISTS legend_h2_extension_json_parse FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_json_parse\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_decode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_base64_decode\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_encode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_base64_encode\";", - "CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_reverse_string\";" + "CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_reverse_string\";", + "CREATE ALIAS IF NOT EXISTS legend_h2_extension_split_part FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions.legend_h2_extension_split_part\";" ); } @@ -119,7 +120,8 @@ private static List getLegendH2_1_4_200_ExtensionSQLs() "CREATE ALIAS IF NOT EXISTS legend_h2_extension_base64_encode FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_base64_encode\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_reverse_string FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_reverse_string\";", "CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_md5 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_md5\";", - "CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_sha1 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_sha1\";" + "CREATE ALIAS IF NOT EXISTS legend_h2_extension_hash_sha1 FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_hash_sha1\";", + "CREATE ALIAS IF NOT EXISTS legend_h2_extension_split_part FOR \"org.finos.legend.engine.plan.execution.stores.relational.LegendH2Extensions_1_4_200.legend_h2_extension_split_part\";" ); } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/pom.xml index 08991114717..2c7005f303f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -127,6 +127,26 @@ + + org.finos.legend.engine + legend-engine-xt-authentication-connection-factory + + + org.bouncycastle + * + + + + + org.finos.legend.engine + legend-engine-xt-relationalStore-connection + + + org.bouncycastle + * + + + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java index 1267e4bdcc2..f4573c59dfd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/RelationalExecutor.java @@ -107,7 +107,7 @@ public RelationalExecutor(TemporaryTestDbConfiguration temporarytestdb, Relation public RelationalExecutor(TemporaryTestDbConfiguration temporarytestdb, RelationalExecutionConfiguration relationalExecutionConfiguration, Optional flowProviderHolder) { this.flowProviderHolder = flowProviderHolder; - this.connectionManager = new ConnectionManagerSelector(temporarytestdb, relationalExecutionConfiguration.oauthProfiles, flowProviderHolder); + this.connectionManager = new ConnectionManagerSelector(temporarytestdb, relationalExecutionConfiguration.oauthProfiles, flowProviderHolder, relationalExecutionConfiguration.getConnectionFactory(), relationalExecutionConfiguration.getRelationalDatabaseConnectionAdapters(), false); this.relationalExecutionConfiguration = relationalExecutionConfiguration; this.resultInterpreterExtensions = Iterate.addAllTo(ResultInterpreterExtensionLoader.extensions(), Lists.mutable.empty()).collect(ResultInterpreterExtension::additionalResultBuilder); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/config/RelationalExecutionConfiguration.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/config/RelationalExecutionConfiguration.java index e344e4a234d..b22846a8447 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/config/RelationalExecutionConfiguration.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/config/RelationalExecutionConfiguration.java @@ -17,6 +17,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.eclipse.collections.api.factory.Lists; import org.finos.legend.authentication.credentialprovider.CredentialProviderProvider; +import org.finos.legend.connection.ConnectionFactory; +import org.finos.legend.connection.HACKY__RelationalDatabaseConnectionAdapter; import org.finos.legend.engine.authentication.provider.DatabaseAuthenticationFlowProvider; import org.finos.legend.engine.authentication.provider.DatabaseAuthenticationFlowProviderConfiguration; import org.finos.legend.engine.plan.execution.stores.StoreExecutorConfiguration; @@ -35,6 +37,8 @@ public class RelationalExecutionConfiguration implements StoreExecutorConfigurat private CredentialProviderProvider credentialProviderProvider; @JsonProperty private RelationalGraphFetchExecutionConfig relationalGraphFetchExecutionConfig; + private ConnectionFactory connectionFactory; + private List relationalDatabaseConnectionAdapters = Lists.mutable.empty(); @Override public StoreType getStoreType() @@ -71,6 +75,16 @@ public CredentialProviderProvider getCredentialProviderProvider() return credentialProviderProvider; } + public ConnectionFactory getConnectionFactory() + { + return connectionFactory; + } + + public List getRelationalDatabaseConnectionAdapters() + { + return relationalDatabaseConnectionAdapters; + } + public void setCredentialProviderProvider(CredentialProviderProvider credentialProviderProvider) { this.credentialProviderProvider = credentialProviderProvider; @@ -86,6 +100,16 @@ public void setFlowProviderConfiguration(DatabaseAuthenticationFlowProviderConfi this.flowProviderConfiguration = flowProviderConfiguration; } + public void setConnectionFactory(ConnectionFactory connectionFactory) + { + this.connectionFactory = connectionFactory; + } + + public void setRelationalDatabaseConnectionAdapters(List adapters) + { + this.relationalDatabaseConnectionAdapters = adapters; + } + public static Builder newInstance() { return new Builder(); @@ -105,6 +129,8 @@ public static class Builder private TemporaryTestDbConfiguration temporaryTestDbConfiguration; private CredentialProviderProvider credentialProviderProvider; private RelationalGraphFetchExecutionConfig relationalGraphFetchExecutionConfig; + private ConnectionFactory connectionFactory; + private final List relationalDatabaseConnectionAdapters = Lists.mutable.empty(); public Builder withTempPath(String tempPath) { @@ -151,6 +177,18 @@ public Builder withRelationalGraphFetchExecutionConfig(RelationalGraphFetchExecu return this; } + public Builder withConnectionFactory(ConnectionFactory connectionFactory) + { + this.connectionFactory = connectionFactory; + return this; + } + + public Builder withRelationalDatabaseConnectionAdapters(List adapters) + { + this.relationalDatabaseConnectionAdapters.addAll(adapters); + return this; + } + public RelationalExecutionConfiguration build() { RelationalExecutionConfiguration relationalExecutionConfiguration = new RelationalExecutionConfiguration(); @@ -161,6 +199,8 @@ public RelationalExecutionConfiguration build() relationalExecutionConfiguration.temporarytestdb = this.temporaryTestDbConfiguration; relationalExecutionConfiguration.credentialProviderProvider = credentialProviderProvider; relationalExecutionConfiguration.relationalGraphFetchExecutionConfig = relationalGraphFetchExecutionConfig; + relationalExecutionConfiguration.connectionFactory = connectionFactory; + relationalExecutionConfiguration.relationalDatabaseConnectionAdapters = relationalDatabaseConnectionAdapters; return relationalExecutionConfiguration; } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/manager/ConnectionManagerSelector.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/manager/ConnectionManagerSelector.java index 358f96fc143..c8a4be55180 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/manager/ConnectionManagerSelector.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/connection/manager/ConnectionManagerSelector.java @@ -17,6 +17,8 @@ import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.utility.Iterate; +import org.finos.legend.connection.ConnectionFactory; +import org.finos.legend.connection.HACKY__RelationalDatabaseConnectionAdapter; import org.finos.legend.engine.authentication.credential.CredentialSupplier; import org.finos.legend.engine.authentication.provider.DatabaseAuthenticationFlowProvider; import org.finos.legend.engine.plan.execution.stores.StoreExecutionState; @@ -44,6 +46,12 @@ public class ConnectionManagerSelector { private final Optional flowProviderHolder; private MutableList connectionManagers; + private final ConnectionFactory connectionFactory; + + // TODO: @akphi - these are temporary hacks to bootstrap the new connection framework + private final List relationalDatabaseConnectionAdapters = Lists.mutable.empty(); + public static final String TEMPORARY__USE_NEW_CONNECTION_FRAMEWORK = "org.finos.legend.engine.execution.enableNewConnectionFramework"; + private boolean enableNewConnectionFramework = false; public ConnectionManagerSelector(TemporaryTestDbConfiguration temporaryTestDb, List oauthProfiles) { @@ -57,6 +65,20 @@ public ConnectionManagerSelector(TemporaryTestDbConfiguration temporaryTestDb, L new RelationalConnectionManager(temporaryTestDb.port, oauthProfiles, flowProviderHolder) ).withAll(extensions.collect(e -> e.getExtensionManager(temporaryTestDb.port, oauthProfiles))); this.flowProviderHolder = flowProviderHolder; + this.connectionFactory = null; + } + + public ConnectionManagerSelector(TemporaryTestDbConfiguration temporaryTestDb, List oauthProfiles, Optional flowProviderHolder, ConnectionFactory connectionFactory, List relationalDatabaseConnectionAdapters, boolean enableNewConnectionFrameworkByDefault) + { + MutableList extensions = Iterate.addAllTo(ServiceLoader.load(ConnectionManagerExtension.class), Lists.mutable.empty()); + this.connectionManagers = Lists.mutable.with( + new RelationalConnectionManager(temporaryTestDb.port, oauthProfiles, flowProviderHolder) + ).withAll(extensions.collect(e -> e.getExtensionManager(temporaryTestDb.port, oauthProfiles))); + this.flowProviderHolder = flowProviderHolder; + + this.connectionFactory = connectionFactory; + this.relationalDatabaseConnectionAdapters.addAll(relationalDatabaseConnectionAdapters); + this.enableNewConnectionFramework = enableNewConnectionFrameworkByDefault; } public Optional getFlowProviderHolder() @@ -114,6 +136,36 @@ public Connection getDatabaseConnectionImpl(Identity identity, DatabaseConnectio if (databaseConnection instanceof RelationalDatabaseConnection) { RelationalDatabaseConnection relationalDatabaseConnection = (RelationalDatabaseConnection) databaseConnection; + + if ("true".equals(System.getenv(TEMPORARY__USE_NEW_CONNECTION_FRAMEWORK)) || this.enableNewConnectionFramework) + { + if (this.connectionFactory != null && !this.relationalDatabaseConnectionAdapters.isEmpty()) + { + HACKY__RelationalDatabaseConnectionAdapter.ConnectionFactoryMaterial connectionFactoryMaterial = null; + for (HACKY__RelationalDatabaseConnectionAdapter adapter : this.relationalDatabaseConnectionAdapters) + { + connectionFactoryMaterial = adapter.adapt(relationalDatabaseConnection, identity, this.connectionFactory.getEnvironment()); + if (connectionFactoryMaterial != null) + { + break; + } + } + + if (connectionFactoryMaterial != null) + { + try + { + return this.connectionFactory.getConnection(identity, connectionFactoryMaterial.storeInstance, connectionFactoryMaterial.authenticationConfiguration); + } + catch (Exception exception) + { + // TODO: @akphi @epsstan - should we throw here? + throw new RuntimeException((exception)); + } + } + } + } + Optional databaseCredentialHolder = RelationalConnectionManager.getCredential(flowProviderHolder, relationalDatabaseConnection, identity, runtimeContext); return datasource.getConnectionUsingIdentity(identity, databaseCredentialHolder); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java index 6c6a3bc35e1..228c41da0a6 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-executionPlan/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/plugin/RelationalExecutionNodeExecutor.java @@ -878,25 +878,26 @@ private static void createTempTableFromRealizedRelationalResultInBlockConnection private static void createTempTableFromRealizedRelationalResultUsingTempTableStrategyInBlockConnection(RelationalTempTableGraphFetchExecutionNode node, RealizedRelationalResult realizedRelationalResult, String tempTableName, DatabaseConnection databaseConnection, String databaseType, String databaseTimeZone, ExecutionState threadExecutionState, MutableList profiles) { + RelationalStoreExecutionState relationalStoreExecutionState = null; + boolean tempTableCreated = false; try (Scope ignored = GlobalTracer.get().buildSpan("create temp table").withTag("tempTableName", tempTableName).withTag("databaseType", databaseType).startActive(true)) { - RelationalStoreExecutionState relationalStoreExecutionState = (RelationalStoreExecutionState) threadExecutionState.getStoreExecutionState(StoreType.Relational); + relationalStoreExecutionState = (RelationalStoreExecutionState) threadExecutionState.getStoreExecutionState(StoreType.Relational); relationalStoreExecutionState.setRetainConnection(true); + node.tempTableStrategy.createTempTableNode.accept(new ExecutionNodeExecutor(profiles, threadExecutionState)); + tempTableCreated = true; + if (node.tempTableStrategy instanceof LoadFromSubQueryTempTableStrategy) { - node.tempTableStrategy.createTempTableNode.accept(new ExecutionNodeExecutor(profiles, threadExecutionState)); node.tempTableStrategy.loadTempTableNode.accept(new ExecutionNodeExecutor(profiles, threadExecutionState)); } else if (node.tempTableStrategy instanceof LoadFromResultSetAsValueTuplesTempTableStrategy) { - node.tempTableStrategy.createTempTableNode.accept(new ExecutionNodeExecutor(profiles, threadExecutionState)); loadValuesIntoTempTablesFromRelationalResult(node.tempTableStrategy.loadTempTableNode, realizedRelationalResult, ((LoadFromResultSetAsValueTuplesTempTableStrategy) node.tempTableStrategy).tupleBatchSize, databaseTimeZone, threadExecutionState, profiles); } else if (node.tempTableStrategy instanceof LoadFromTempFileTempTableStrategy) { - node.tempTableStrategy.createTempTableNode.accept(new ExecutionNodeExecutor(profiles, threadExecutionState)); - String requestId = new RequestIdGenerator().generateId(); String fileName = tempTableName + requestId; try (TemporaryFile tempFile = new TemporaryFile(((RelationalStoreExecutionState) threadExecutionState.getStoreExecutionState(StoreType.Relational)).getRelationalExecutor().getRelationalExecutionConfiguration().tempPath, fileName)) @@ -915,12 +916,17 @@ else if (node.tempTableStrategy instanceof LoadFromTempFileTempTableStrategy) { throw new RuntimeException("Unsupported temp table strategy"); } - - String dropTempTableSqlQuery = ((SQLExecutionNode) node.tempTableStrategy.dropTempTableNode.executionNodes.get(0)).sqlQuery; - BlockConnection blockConnection = relationalStoreExecutionState.getBlockConnectionContext().getBlockConnection(relationalStoreExecutionState, databaseConnection, profiles); - blockConnection.addCommitQuery(dropTempTableSqlQuery); - blockConnection.addRollbackQuery(dropTempTableSqlQuery); - blockConnection.close(); + } + finally + { + if (relationalStoreExecutionState != null && tempTableCreated) + { + String dropTempTableSqlQuery = ((SQLExecutionNode) node.tempTableStrategy.dropTempTableNode.executionNodes.get(0)).sqlQuery; + BlockConnection blockConnection = relationalStoreExecutionState.getBlockConnectionContext().getBlockConnection(relationalStoreExecutionState, databaseConnection, profiles); + blockConnection.addCommitQuery(dropTempTableSqlQuery); + blockConnection.addRollbackQuery(dropTempTableSqlQuery); + blockConnection.close(); + } } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/pom.xml index e765d9a1e1b..520ede2a524 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/pom.xml @@ -20,7 +20,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -60,5 +60,13 @@ legend-engine-shared-core + + + + org.apache.commons + commons-lang3 + + + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java index 2dfa3742af0..17038e00a4d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-h2-1.4.200-execution/src/main/java/org/finos/legend/engine/plan/execution/stores/relational/LegendH2Extensions_1_4_200.java @@ -17,6 +17,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.codec.digest.DigestUtils; import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.h2.value.Value; @@ -139,4 +140,22 @@ public static String legend_h2_extension_hash_sha1(String string) { return string == null ? null : DigestUtils.sha1Hex(string); } + + public static String legend_h2_extension_split_part(String string, String token, Integer part) + { + if (part < 1) + { + throw new IllegalArgumentException("Split part must be greater than zero"); + } + + if (string == null) + { + return null; + } + + String[] parts = StringUtils.split(string, token); + int readjustedPart = part - 1; + + return parts.length > readjustedPart ? parts[readjustedPart] : null; + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-mutation-executionPlan-test/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-mutation-executionPlan-test/pom.xml index 78af676449c..4d9d402b510 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-mutation-executionPlan-test/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/legend-engine-xt-relationalStore-mutation-executionPlan-test/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-execution - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/pom.xml index 712af4f6b51..fa85c60aae4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-execution/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-relationalStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/pom.xml index fc9e02a0ff9..b69a1e35d3c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/java/org/finos/legend/engine/language/pure/relational/api/relationalElement/test/TestRelationalElementApi.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/java/org/finos/legend/engine/language/pure/relational/api/relationalElement/test/TestRelationalElementApi.java index 595c970b544..fa822be6a74 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/java/org/finos/legend/engine/language/pure/relational/api/relationalElement/test/TestRelationalElementApi.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/java/org/finos/legend/engine/language/pure/relational/api/relationalElement/test/TestRelationalElementApi.java @@ -57,23 +57,36 @@ private static String loadFromFile(String filename) throws IOException ); } - @Test - public void shouldGenerateModelsFromDatabaseSpecification() throws IOException + + private void test(String inputGrammarPath, String expectedJsonPath, String dbPath) throws IOException { - String expectedJson = loadFromFile("expectedJson.json"); - String inputGrammar = loadFromFile("inputGrammar.pure"); + String expectedJson = loadFromFile(expectedJsonPath); + String inputGrammar = loadFromFile(inputGrammarPath); Assert.assertNotNull(expectedJson); Assert.assertNotNull(inputGrammar); PureModelContextData inputPmcd = buildPMCDFromString(inputGrammar); - String databasePath = "meta::relational::transform::autogen::tests::testDB"; - DatabaseToModelGenerationInput inputJson = new DatabaseToModelGenerationInput(databasePath, inputPmcd, null); + DatabaseToModelGenerationInput inputJson = new DatabaseToModelGenerationInput(dbPath, inputPmcd, null); RelationalElementAPI relationalElementAPI = new RelationalElementAPI(DeploymentMode.PROD, null); Response response = relationalElementAPI.generateModelsFromDatabaseSpecification(inputJson, null); Assert.assertNotNull(response); String actualJson = response.getEntity().toString(); JsonAssert.assertJsonEquals(expectedJson, actualJson); + PureModelContextData generatedModel = objectMapper.readValue(actualJson, PureModelContextData.class); + PureModelContextData fullModel = generatedModel.combine(inputPmcd); // compile generated model and input database - PureModelContextData pureModelContextData = objectMapper.readValue(actualJson, PureModelContextData.class).combine(inputPmcd); - testManager.loadModelAndData(pureModelContextData, pureModelContextData.serializer.version, null, null); + testManager.loadModelAndData(fullModel, fullModel.serializer.version, null, null); + } + + @Test + public void shouldGenerateModelsFromDatabaseSpecification() throws IOException + { + test("inputGrammar.pure", "expectedJson.json", "meta::relational::transform::autogen::tests::testDB"); + } + + @Test + public void generateModelsWithDbWithIncludes() throws IOException + { + test("inputGrammarWithInclude.pure", "expectedJsonWithInclude.json", "model::MyDB"); + } } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/resources/expectedJsonWithInclude.json b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/resources/expectedJsonWithInclude.json new file mode 100644 index 00000000000..005da823e05 --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/resources/expectedJsonWithInclude.json @@ -0,0 +1,629 @@ +{ + "elements": [ + { + "package": "model", + "_type": "mapping", + "name": "MyDBMapping", + "associationMappings": [ + { + "stores": [ + "model::MyDB" + ], + "_type": "relational", + "propertyMappings": [ + { + "relationalOperation": { + "joins": [ + { + "name": "CompanyEmployee", + "db": "model::MyDB" + } + ], + "_type": "elemtWithJoins" + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "companyEmployeeTestSchema1Company", + "class": "model::testSchema1::Employee" + }, + "source": "model_testSchema1_Employee", + "target": "model_testSchema1_Company" + }, + { + "relationalOperation": { + "joins": [ + { + "name": "CompanyEmployee", + "db": "model::MyDB" + } + ], + "_type": "elemtWithJoins" + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "companyEmployeeTestSchema1Employee", + "class": "model::testSchema1::Company" + }, + "source": "model_testSchema1_Company", + "target": "model_testSchema1_Employee" + } + ], + "association": "model::CompanyEmployee", + "id": "model_CompanyEmployee" + } + ], + "classMappings": [ + { + "mainTable": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Company" + }, + "root": true, + "propertyMappings": [ + { + "relationalOperation": { + "_type": "column", + "column": "name", + "tableAlias": "Company", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Company" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "name", + "class": "model::testSchema1::Company" + }, + "target": "" + }, + { + "relationalOperation": { + "_type": "column", + "column": "location", + "tableAlias": "Company", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Company" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "location", + "class": "model::testSchema1::Company" + }, + "target": "" + } + ], + "_type": "relational", + "distinct": false, + "id": "model_testSchema1_Company", + "class": "model::testSchema1::Company", + "primaryKey": [ + { + "_type": "column", + "column": "name", + "tableAlias": "Company", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Company" + } + } + ] + }, + { + "mainTable": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Employee" + }, + "root": true, + "propertyMappings": [ + { + "relationalOperation": { + "_type": "column", + "column": "fullname", + "tableAlias": "Employee", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Employee" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "fullname", + "class": "model::testSchema1::Employee" + }, + "target": "" + }, + { + "relationalOperation": { + "_type": "column", + "column": "passportId", + "tableAlias": "Employee", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Employee" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "passportId", + "class": "model::testSchema1::Employee" + }, + "target": "" + }, + { + "relationalOperation": { + "_type": "column", + "column": "firmname", + "tableAlias": "Employee", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Employee" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "firmname", + "class": "model::testSchema1::Employee" + }, + "target": "" + }, + { + "relationalOperation": { + "_type": "column", + "column": "location", + "tableAlias": "Employee", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Employee" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "location", + "class": "model::testSchema1::Employee" + }, + "target": "" + } + ], + "_type": "relational", + "distinct": false, + "id": "model_testSchema1_Employee", + "class": "model::testSchema1::Employee", + "primaryKey": [ + { + "_type": "column", + "column": "fullname", + "tableAlias": "Employee", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Employee" + } + }, + { + "_type": "column", + "column": "passportId", + "tableAlias": "Employee", + "table": { + "schema": "testSchema1", + "database": "model::MyDB", + "_type": "table", + "mainTableDb": "model::MyDB", + "table": "Employee" + } + } + ] + } + ] + }, + { + "taggedValues": [ + { + "tag": { + "profile": "meta::pure::profiles::doc", + "value": "doc" + }, + "value": "Generated Element" + } + ], + "package": "model::testSchema1", + "_type": "class", + "name": "Company", + "properties": [ + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "name", + "type": "String" + }, + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "location", + "type": "String" + } + ] + }, + { + "taggedValues": [ + { + "tag": { + "profile": "meta::pure::profiles::doc", + "value": "doc" + }, + "value": "Generated Element" + } + ], + "package": "model::testSchema1", + "_type": "class", + "name": "Employee", + "properties": [ + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "fullname", + "type": "String" + }, + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "passportId", + "type": "Integer" + }, + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 0 + }, + "name": "firmname", + "type": "String" + }, + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 0 + }, + "name": "location", + "type": "String" + } + ] + }, + { + "taggedValues": [ + { + "tag": { + "profile": "meta::pure::profiles::doc", + "value": "doc" + }, + "value": "Generated Element" + } + ], + "package": "model", + "_type": "association", + "name": "CompanyEmployee", + "properties": [ + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "companyEmployeeTestSchema1Company", + "type": "model::testSchema1::Company" + }, + { + "multiplicity": { + "lowerBound": 1 + }, + "name": "companyEmployeeTestSchema1Employee", + "type": "model::testSchema1::Employee" + } + ] + }, + { + "package": "model", + "_type": "mapping", + "name": "MyIncludeDBMapping", + "associationMappings": [ + { + "stores": [ + "model::MyIncludeDB" + ], + "_type": "relational", + "propertyMappings": [ + { + "relationalOperation": { + "joins": [ + { + "name": "PersonAddress", + "db": "model::MyIncludeDB" + } + ], + "_type": "elemtWithJoins" + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "personAddressHrPerson", + "class": "model::HR::Address" + }, + "source": "model_HR_Address", + "target": "model_HR_Person" + }, + { + "relationalOperation": { + "joins": [ + { + "name": "PersonAddress", + "db": "model::MyIncludeDB" + } + ], + "_type": "elemtWithJoins" + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "personAddressHrAddress", + "class": "model::HR::Person" + }, + "source": "model_HR_Person", + "target": "model_HR_Address" + } + ], + "association": "model::PersonAddress", + "id": "model_PersonAddress" + } + ], + "classMappings": [ + { + "mainTable": { + "schema": "HR", + "database": "model::MyIncludeDB", + "_type": "table", + "mainTableDb": "model::MyIncludeDB", + "table": "Person" + }, + "root": true, + "propertyMappings": [ + { + "relationalOperation": { + "_type": "column", + "column": "name", + "tableAlias": "Person", + "table": { + "schema": "HR", + "database": "model::MyIncludeDB", + "_type": "table", + "mainTableDb": "model::MyIncludeDB", + "table": "Person" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "name", + "class": "model::HR::Person" + }, + "target": "" + }, + { + "relationalOperation": { + "_type": "column", + "column": "addressId", + "tableAlias": "Person", + "table": { + "schema": "HR", + "database": "model::MyIncludeDB", + "_type": "table", + "mainTableDb": "model::MyIncludeDB", + "table": "Person" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "addressId", + "class": "model::HR::Person" + }, + "target": "" + } + ], + "_type": "relational", + "distinct": false, + "id": "model_HR_Person", + "class": "model::HR::Person", + "primaryKey": [ + { + "_type": "column", + "column": "name", + "tableAlias": "Person", + "table": { + "schema": "HR", + "database": "model::MyIncludeDB", + "_type": "table", + "mainTableDb": "model::MyIncludeDB", + "table": "Person" + } + } + ] + }, + { + "mainTable": { + "schema": "HR", + "database": "model::MyIncludeDB", + "_type": "table", + "mainTableDb": "model::MyIncludeDB", + "table": "Address" + }, + "root": true, + "propertyMappings": [ + { + "relationalOperation": { + "_type": "column", + "column": "id", + "tableAlias": "Address", + "table": { + "schema": "HR", + "database": "model::MyIncludeDB", + "_type": "table", + "mainTableDb": "model::MyIncludeDB", + "table": "Address" + } + }, + "_type": "relationalPropertyMapping", + "property": { + "property": "id", + "class": "model::HR::Address" + }, + "target": "" + } + ], + "_type": "relational", + "distinct": false, + "id": "model_HR_Address", + "class": "model::HR::Address", + "primaryKey": [ + { + "_type": "column", + "column": "id", + "tableAlias": "Address", + "table": { + "schema": "HR", + "database": "model::MyIncludeDB", + "_type": "table", + "mainTableDb": "model::MyIncludeDB", + "table": "Address" + } + } + ] + } + ] + }, + { + "taggedValues": [ + { + "tag": { + "profile": "meta::pure::profiles::doc", + "value": "doc" + }, + "value": "Generated Element" + } + ], + "package": "model::HR", + "_type": "class", + "name": "Person", + "properties": [ + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "name", + "type": "String" + }, + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 0 + }, + "name": "addressId", + "type": "Integer" + } + ] + }, + { + "taggedValues": [ + { + "tag": { + "profile": "meta::pure::profiles::doc", + "value": "doc" + }, + "value": "Generated Element" + } + ], + "package": "model::HR", + "_type": "class", + "name": "Address", + "properties": [ + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "id", + "type": "String" + } + ] + }, + { + "taggedValues": [ + { + "tag": { + "profile": "meta::pure::profiles::doc", + "value": "doc" + }, + "value": "Generated Element" + } + ], + "package": "model", + "_type": "association", + "name": "PersonAddress", + "properties": [ + { + "multiplicity": { + "lowerBound": 1 + }, + "name": "personAddressHrPerson", + "type": "model::HR::Person" + }, + { + "multiplicity": { + "upperBound": 1, + "lowerBound": 1 + }, + "name": "personAddressHrAddress", + "type": "model::HR::Address" + } + ] + } + ], + "_type": "data", + "serializer": { + "name": "pure", + "version": "vX_X_X" + } +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/resources/inputGrammarWithInclude.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/resources/inputGrammarWithInclude.pure new file mode 100644 index 00000000000..1a1180e431e --- /dev/null +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-api/src/test/resources/inputGrammarWithInclude.pure @@ -0,0 +1,43 @@ +###Relational +Database model::MyIncludeDB +( + Schema HR + ( + Table Person + ( + name VARCHAR(200) PRIMARY KEY, + addressId INTEGER + ) + Table Address + ( + id VARCHAR(200) PRIMARY KEY + ) + ) + + Join PersonAddress(HR.Person.addressId = HR.Address.id) +) + +Database model::MyDB +( + include model::MyIncludeDB + + Schema testSchema1 + ( + Table Company + ( + name VARCHAR(200) PRIMARY KEY, + location VARCHAR(200) NOT NULL + ) + Table Employee + ( + fullname VARCHAR(1000) PRIMARY KEY, + passportId INTEGER PRIMARY KEY, + firmname VARCHAR(200), + location VARCHAR(200) + ) + ) + + Join CompanyEmployee(testSchema1.Company.location != 'Toronto' + and testSchema1.Employee.fullname = 'John Doe' + and testSchema1.Company.name = testSchema1.Employee.firmname) +) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/pom.xml index c7fc665d9c8..d4e55a6aae1 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java index 17087670de7..039ba72730c 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-grammar/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/RelationalCompilerExtension.java @@ -517,6 +517,10 @@ public List>> handlers.m(handlers.m(handlers.h("meta::pure::tds::extensions::columnValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 5)), handlers.m(handlers.h("meta::pure::tds::extensions::columnValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> true))) ), + new FunctionExpressionBuilderRegistrationInfo(null, + handlers.m(handlers.m(handlers.h("meta::pure::tds::extensions::extendWithDigestOnColumns_TabularDataSet_1__String_1__HashType_1__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 4)), + handlers.m(handlers.h("meta::pure::tds::extensions::extendWithDigestOnColumns_TabularDataSet_1__String_1__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 2))) + ), new FunctionExpressionBuilderRegistrationInfo(null, handlers.m(handlers.m(handlers.h("meta::pure::tds::extensions::rowValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 5)), handlers.m(handlers.h("meta::pure::tds::extensions::rowValueDifference_TabularDataSet_1__TabularDataSet_1__String_$1_MANY$__String_$1_MANY$__TabularDataSet_1_", false, ps -> handlers.res("meta::pure::tds::TabularDataSet", "one"), ps -> ps.size() == 4))) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-javaPlatformBinding-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-javaPlatformBinding-pure/pom.xml index 7d33a8aad5a..9c37bad0792 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-javaPlatformBinding-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-javaPlatformBinding-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/pom.xml index ff08f001d04..671cdb0e5e9 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/pom.xml index d973369ed11..0d6ad1b5b1d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xt-relationalStore-generation - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/autogeneration/relationalToPure.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/autogeneration/relationalToPure.pure index 3950ecd0ed1..d8fb0155f5d 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/autogeneration/relationalToPure.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/autogeneration/relationalToPure.pure @@ -309,10 +309,17 @@ function meta::relational::transform::autogen::buildPureModelFromClassesAssociat function meta::relational::transform::autogen::classesAssociationsAndMappingFromDatabase(database:Database[1], targetPackage:String[1]):String[1] { - meta::relational::transform::autogen::classesAssociationsAndMappingFromDatabase($database, $targetPackage, $targetPackage, $targetPackage); + let allDatabases = meta::relational::transform::autogen::getAllDatabases($database, []); + let elements = $allDatabases->map(db | meta::relational::transform::autogen::generateModelsFromDatabase($db, $targetPackage, $targetPackage, $targetPackage)); + meta::relational::transform::autogen::buildPMCD($elements); } -function meta::relational::transform::autogen::classesAssociationsAndMappingFromDatabase(database:Database[1], classesPackage:String[1], associationsPackage:String[1], mappingPackage:String[1]):String[1] +function meta::relational::transform::autogen::getAllDatabases(database : Database[1], visited: Database[*]): Database[*] +{ + if($database->in($visited), | [], | $database->concatenate($database.includes->map(db|$db->cast(@Database)->meta::relational::transform::autogen::getAllDatabases($visited->concatenate($database))))); +} + +function meta::relational::transform::autogen::generateModelsFromDatabase(database:Database[1], classesPackage:String[1], associationsPackage:String[1], mappingPackage:String[1]):meta::protocols::pure::vX_X_X::metamodel::PackageableElement[*] { let allClasses = $database.schemas->map(schema | $schema->meta::relational::transform::autogen::schemaToModel($classesPackage)); let allAssociationJoinPairs = meta::relational::transform::autogen::databaseToAssociationJoinPair($database, $associationsPackage); @@ -320,6 +327,16 @@ function meta::relational::transform::autogen::classesAssociationsAndMappingFrom let mappingWithClassMappings = meta::relational::transform::autogen::databaseToMappingWithClassMappings($database, $mappingPackage, $classesPackage); let associationMappings = meta::relational::transform::autogen::associationsAndJoinsToAssociationMappings($database, $mappingWithClassMappings, $allAssociationJoinPairs); let mapping = ^$mappingWithClassMappings(associationMappings = $associationMappings); - let pmcd = meta::relational::transform::autogen::buildPureModelFromClassesAssociationsAndMapping($mapping, $allClasses, $allAssociations); + $mapping->concatenate($allClasses)->concatenate($allAssociations); +} + +function meta::relational::transform::autogen::buildPMCD(elements: meta::protocols::pure::vX_X_X::metamodel::PackageableElement[*]):String[1] +{ + let pmcd = ^meta::protocols::pure::vX_X_X::metamodel::PureModelContextData + ( + _type = 'data', + serializer = ^meta::protocols::Protocol(name='pure', version='vX_X_X'), + elements = $elements + ); $pmcd->meta::alloy::metadataServer::alloyToJSON(); } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/contract/storeContract.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/contract/storeContract.pure index 946e2cbabc2..161a866cfab 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/contract/storeContract.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/contract/storeContract.pure @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import meta::relational::contract::*; import meta::pure::router::metamodel::*; import meta::pure::router::systemMapping::tests::*; import meta::relational::mapping::*; @@ -30,7 +29,6 @@ import meta::pure::executionPlan::toString::*; import meta::pure::router::clustering::*; import meta::pure::graphFetch::executionPlan::*; import meta::pure::graphFetch::routing::*; -import meta::pure::alloy::connections::*; function meta::relational::contract::relationalStoreContract():StoreContract[1] { @@ -42,20 +40,6 @@ function meta::relational::contract::relationalStoreContract():StoreContract[1] planGraphFetchExecution = meta::relational::contract::planGraphFetchExecution_StoreMappingLocalGraphFetchExecutionNodeGenerationInput_1__LocalGraphFetchExecutionNode_1_, planCrossGraphFetchExecution = meta::relational::contract::planCrossGraphFetchExecution_StoreMappingCrossLocalGraphFetchExecutionNodeGenerationInput_1__LocalGraphFetchExecutionNode_1_, - connectionEquality = { b : Connection [1] | - [ - d: RelationalDatabaseConnection[1]| - let bAsRDB = $b->cast(@RelationalDatabaseConnection); - // connection element is the store name and we dont compare those - let comparison = $d.type == $bAsRDB.type && - $d.timeZone == $bAsRDB.timeZone && - $d.quoteIdentifiers == $bAsRDB.quoteIdentifiers && - $d.datasourceSpecification == $bAsRDB.datasourceSpecification && - compareObjectsWithPossiblyNoProperties($d.authenticationStrategy,$bAsRDB.authenticationStrategy) && - (($d.postProcessors->isEmpty() && $bAsRDB.postProcessors->isEmpty()) || ($d.postProcessors->isNotEmpty() && $bAsRDB.postProcessors->isNotEmpty() && $d.postProcessors == $bAsRDB.postProcessors)); - ] - }, - supports = meta::relational::contract::supports_FunctionExpression_1__Boolean_1_, supportsStreamFunction = meta::relational::contract::supportsStream_FunctionExpression_1__Boolean_1_, shouldStopRouting = [ @@ -238,17 +222,6 @@ function meta::relational::contract::planExecution(sq:meta::pure::mapping::Store ); } -function meta::relational::contract::compareObjectsWithPossiblyNoProperties(obj1: Any[1], obj2: Any[1]): Boolean[1] -{ - let propertyCountForObj1 = $obj1->type()->cast(@Class)->hierarchicalProperties()->size(); - let propertyCountForObj2 = $obj2->type()->cast(@Class)->hierarchicalProperties()->size(); - - if($propertyCountForObj1 == 0 && $propertyCountForObj2 == 0 - ,| true - ,| $obj1 == $obj2 - ); -} - function meta::relational::contract::planGraphFetchExecution(input: StoreMappingLocalGraphFetchExecutionNodeGenerationInput[1]): LocalGraphFetchExecutionNode[1] { meta::relational::graphFetch::executionPlan::planRootGraphFetchExecutionRelational($input.storeQuery, $input.ext, $input.clusteredTree, $input.orderedPaths, $input.mapping, $input.runtime, $input.exeCtx, $input.enableConstraints, $input.checked, $input.extensions, $input.debug) diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/extensions/extension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/extensions/extension.pure index 87262c48cc9..ead7599a286 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/extensions/extension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/extensions/extension.pure @@ -15,8 +15,6 @@ import meta::external::language::java::factory::project::*; import meta::external::language::java::metamodel::project::*; import meta::pure::alloy::connections::*; -import meta::pure::alloy::connections::alloy::specification::*; -import meta::pure::alloy::connections::alloy::authentication::*; import meta::pure::executionPlan::*; import meta::pure::executionPlan::engine::java::*; import meta::pure::executionPlan::toString::*; @@ -203,6 +201,20 @@ function meta::relational::extension::relationalExtension() : meta::pure::extens ) ) )->concatenate( + [ + meta::pure::tds::extensions::extendWithDigestOnColumns_TabularDataSet_1__String_1__HashType_1__String_$1_MANY$__TabularDataSet_1_, + meta::pure::tds::extensions::extendWithDigestOnColumns_TabularDataSet_1__String_1__TabularDataSet_1_ + ]->map(f| + pair($f->cast(@Function), {| + let tdsSchema = resolveSchemaImpl($fe.parametersValues->at(0), $openVars, $extensions); + let digestCol = $fe.parametersValues->last()->toOne()->reactivate($openVars)->cast(@String)->toOne(); + + $tdsSchema.extend(^TDSColumn(name = $digestCol, offset= 0, type = String)); + } + ) + ) + ) + ->concatenate( pair(tableToTDS_Table_1__TableTDS_1_->cast(@Function), {| createSchemaState($fe->reactivate()->cast(@TabularDataSet).columns); }) @@ -210,11 +222,12 @@ function meta::relational::extension::relationalExtension() : meta::pure::extens }, testExtension_testedBy = {allReferenceUses:ReferenceUsage[*], extensions:Extension[*] | {soFarr:TestedByResult[1]| $allReferenceUses.owner->filter(o|$o->instanceOf(Database))->cast(@Database)->fold({db,tbr|$db->testedBy($tbr, $extensions)}, $soFarr)}}, validTestPackages = 'meta::relational::tests' - - ) + ) } + + //Helper Functions function meta::pure::router::clustering::getResolvedStore(rr: meta::relational::mapping::RootRelationalInstanceSetImplementation[*], mapping: Mapping[1]):Store[*] diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tableToTDS.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tableToTDS.pure index 42b8f3778a3..2dc3f05ac89 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tableToTDS.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/functions/tableToTDS.pure @@ -40,7 +40,7 @@ function meta::pure::tds::viewToTDS(view:meta::relational::metamodel::relation:: } function - {doc.doc = 'Project the specified calculated columns from the provided TDS. This is similar to extend, but rather than adding the columns it replaces all of the existing ones '} + {doc.doc = 'Project the specified columns from the provided TableTDS'} meta::pure::tds::project(tds:meta::relational::mapping::TableTDS[1], columnFunctions:ColumnSpecification[*]):TabularDataSet[1] { $tds->project($columnFunctions); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure index 984af1381b7..b4466f24885 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/pureToSQLQuery/pureToSQLQuery.pure @@ -2226,22 +2226,43 @@ function meta::relational::functions::pureToSqlQuery::processNoOp(f:FunctionExpr processValueSpecificationReturnPropertyMapping($f.parametersValues->at(0), $currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions)->toOne() } +function meta::relational::functions::pureToSqlQuery::processSplitPart(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] +{ + //sql is 1 index based, pure is 0 index based + let position = $f.parametersValues->at(2)->zeroToOneBasedIndex(); + + let functionExpression = ^$f(parametersValues = [$f.parametersValues->at(0), $f.parametersValues->at(1), $position]); + $functionExpression->processDynaFunction($currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions); +} + +function meta::relational::functions::pureToSqlQuery::zeroToOneBasedIndex(v:ValueSpecification[1]):ValueSpecification[1] +{ + $v->match([ + i:InstanceValue[1] | ^$i(values = $i.values->toOne()->cast(@Integer) + 1) + ]); +} + function meta::relational::functions::pureToSqlQuery::processHash(f:FunctionExpression[1], currentPropertyMapping:PropertyMapping[*], operation:SelectWithCursor[1], vars:Map[1], state:State[1], joinType:JoinType[1], nodeId:String[1], aggFromMap:List[1], context:DebugContext[1], extensions:Extension[*]):RelationalOperationElement[1] { - let type = $f.parametersValues->at(1)->reactivate()->cast(@meta::pure::functions::hash::HashType)->toOne(); + let name = meta::relational::functions::pureToSqlQuery::hashTypeToHashDynaFuncName($type); + + let oldFunc = $f.func; + let functionExpression = ^$f(func = ^$oldFunc(functionName = $name->toOne()), parametersValues = $f.parametersValues->at(0)); + $functionExpression->processDynaFunction($currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions); +} +function <> meta::relational::functions::pureToSqlQuery::hashTypeToHashDynaFuncName(hashType : meta::pure::functions::hash::HashType[1]): String[1] +{ let name = newMap([ pair(meta::pure::functions::hash::HashType.MD5, 'md5'), pair(meta::pure::functions::hash::HashType.SHA1, 'sha1'), pair(meta::pure::functions::hash::HashType.SHA256, 'sha256') - ])->get($type); + ])->get($hashType); - assert($name->isNotEmpty(), | 'hash type ' + $type.name + ' is not yet supported'); + assert($name->isNotEmpty(), | 'hash type ' + $hashType.name + ' is not yet supported'); - let oldFunc = $f.func; - let functionExpression = ^$f(func = ^$oldFunc(functionName = $name->toOne()), parametersValues = $f.parametersValues->at(0)); - $functionExpression->processDynaFunction($currentPropertyMapping, $operation, $vars, $state, $joinType, $nodeId, $aggFromMap, $context, $extensions); + $name->toOne(); } function <> meta::relational::functions::pureToSqlQuery::canProcessAt(f:FunctionExpression[1]):Boolean[1] @@ -4350,7 +4371,9 @@ function meta::relational::functions::pureToSqlQuery::processAggFn(v:ValueSpecif b:Boolean[1] | ^Literal(value=$b), f:ValueSpecification[1]|$f->processAggFn($params), f:LambdaFunction[1]|$f.expressionSequence->toOne()->cast(@ValueSpecification)->processAggFn($params), - a:Any[1] | fail('Unsupported aggregation operation'); []; + e:Enum[1] | ^Literal(value=$e), + e:Enumeration[1] | ^Literal(value = $v), + a:Any[1] | println($a);fail('Unsupported aggregation operation'); []; ])); ); ] @@ -5026,14 +5049,42 @@ function meta::relational::functions::pureToSqlQuery::processTdsLambda(mapFn:Val ^PureFunctionTDSToRelationalFunctionPair(first = meta::pure::tds::extensions::firstNotNull_T_MANY__T_$0_1$_, second = {| newDynaFunction('coalesce', $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)) }), + ^PureFunctionTDSToRelationalFunctionPair(first = splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_, second = {| + newDynaFunction('splitPart', [ + $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)->toOne(), + $f.parametersValues->at(1)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)->toOne(), + $f.parametersValues->at(2)->zeroToOneBasedIndex()->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context) ->toOne() + ]) + }), ^PureFunctionTDSToRelationalFunctionPair(first = parseDate_String_1__Date_1_, second = {| let format = ^InstanceValue(multiplicity = PureOne, genericType = ^GenericType(rawType=String), values = 'YYYY-MM-DD HH24:MI:SS'); newDynaFunction('toTimestamp', [ $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)->toOne(), ^Literal(value = 'YYYY-MM-DD HH24:MI:SS') ]); - }) - ]; + }), + ^PureFunctionTDSToRelationalFunctionPair(first = meta::pure::functions::hash::hash_String_1__HashType_1__String_1_, second = {| + let type = $f.parametersValues->at(1)->reactivate()->cast(@meta::pure::functions::hash::HashType)->toOne(); + let name = meta::relational::functions::pureToSqlQuery::hashTypeToHashDynaFuncName($type); + + let value = $f.parametersValues->at(0)->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context); + + newDynaFunction($name, $value); + }) + ]->concatenate( + [ + meta::pure::functions::string::joinStrings_String_MANY__String_1_, + meta::pure::functions::string::joinStrings_String_MANY__String_1__String_1_, + meta::pure::functions::string::joinStrings_String_MANY__String_1__String_1__String_1__String_1_ + ]->map(func| + ^PureFunctionTDSToRelationalFunctionPair(first = $func, second = {| + ^DynaFunction( + name = 'concat', + parameters =$f.parametersValues->map(p|$p->processTdsLambda($a, $returnColumnName, $vars, $state, $currentPropertyMapping, $context)) + ) + }) + ) + ); let override = $overrides->filter(o | $o.first == $f.func)->first(); if ($override->isNotEmpty(), @@ -7558,6 +7609,7 @@ function meta::relational::functions::pureToSqlQuery::getSupportedFunctions():Ma ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::replace_String_1__String_1__String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::repeatString_String_$0_1$__Integer_1__String_$0_1$_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::reverseString_String_1__String_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), + ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::string::splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_,second=meta::relational::functions::pureToSqlQuery::processSplitPart_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::math::abs_Integer_1__Integer_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::math::abs_Number_1__Number_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::math::abs_Float_1__Float_1_,second=meta::relational::functions::pureToSqlQuery::processDynaFunction_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), @@ -7673,4 +7725,4 @@ function meta::relational::functions::pureToSqlQuery::getSupportedFunctions():Ma ^PureFunctionToRelationalFunctionPair(first=meta::pure::mutation::save_T_MANY__RootGraphFetchTree_1__Mapping_1__Runtime_1__T_MANY_, second=meta::relational::functions::pureToSqlQuery::processNoOp_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_), ^PureFunctionToRelationalFunctionPair(first=meta::pure::functions::hash::hash_String_1__HashType_1__String_1_, second=meta::relational::functions::pureToSqlQuery::processHash_FunctionExpression_1__PropertyMapping_MANY__SelectWithCursor_1__Map_1__State_1__JoinType_1__String_1__List_1__DebugContext_1__Extension_MANY__RelationalOperationElement_1_) ]) -} +} \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/router/tests/testPreeval.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/router/tests/testPreeval.pure index 53db20dcb43..97c41e40d88 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/router/tests/testPreeval.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/router/tests/testPreeval.pure @@ -1,134 +1,132 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Press F9 to execute the 'go' function... -// Press F10 to run the full test suite - -import meta::pure::extension::*; -import meta::pure::router::preeval::*; -import meta::pure::router::preeval::tests::*; -import meta::external::format::shared::binding::*; -import meta::external::format::shared::functions::*; -import meta::json::*; -import meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::*; -import meta::protocols::pure::vX_X_X::transformation::toPureGrammar::*; -import apps::protocols::tds::preeval::router::tests::prerouting::*; -import meta::pure::graphFetch::execution::*; -import meta::pure::mapping::*; -import meta::pure::mapping::modelToModel::test::simple::*; -import meta::pure::mapping::modelToModel::test::shared::dest::*; -import meta::pure::mapping::modelToModel::test::shared::src::*; -import meta::pure::runtime::*; - -function <> meta::pure::router::preeval::tests::testPrerouting41():Boolean[1] -{ - let runtime = ^Runtime(); - let context = ^ExecutionContext(); - - let input = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') - ])->from(meta::relational::tests::simpleRelationalMapping, $runtime, $context)->join( - meta::pure::router::preeval::tests::Person.all() - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') - ])->from(meta::relational::tests::simpleRelationalMapping, $runtime), - meta::relational::metamodel::join::JoinType.LEFT_OUTER, 'firstName') - }; - - let expected = {| - meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) - ])->from(meta::relational::tests::simpleRelationalMapping, $runtime, $context)->join( - meta::pure::router::preeval::tests::Person.all() - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) - ])->from(meta::relational::tests::simpleRelationalMapping, $runtime), - meta::relational::metamodel::join::JoinType.LEFT_OUTER, 'firstName', 'firstName') - }; - - assertRoundTrip($input, $expected); -} - -function <> meta::pure::router::preeval::tests::testPrerouting42():Boolean[1] -{ - let runtime = ^Runtime(); - - let input = {name:String[1]| - let person = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( - #{ - meta::pure::router::preeval::tests::Person{ - firstName - } - }# - ); - - let person2 = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( - #{ - meta::pure::router::preeval::tests::Person{ - firstName - } - }# - )->from(meta::relational::tests::simpleRelationalMapping, $runtime); - - meta::pure::router::preeval::tests::Person.all() - ->filter(x | ($x.firstName == $person.firstName) || ($x.firstName == $name) || $x.firstName == $person2.firstName) - ->project([ - col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}, 'lastName'), - col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age'), - col({p:meta::pure::router::preeval::tests::Person[1]|today()}, 'today') - ]); - }; - - let expected = {name:String[1]| - - let person = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( - #{ - meta::pure::router::preeval::tests::Person{ - firstName - } - }# - ); - - let person2 = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( - #{ - meta::pure::router::preeval::tests::Person{ - firstName - } - }# - )->from(meta::relational::tests::simpleRelationalMapping, $runtime); - - - meta::pure::router::preeval::tests::Person.all() - ->filter(x | ($x.firstName == $person.firstName) || ($x.firstName == $name) || $x.firstName == $person2.firstName) - ->project([ - ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}), - ^BasicColumnSpecification(name = 'lastName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}), - ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}), - ^BasicColumnSpecification(name = 'today', func = {p:meta::pure::router::preeval::tests::Person[1]|today()}) - ]); - }; - - assertRoundTrip($input, $expected, noDebug()); -} - -function <> meta::pure::router::preeval::tests::testPrerouting_Store():Boolean[1] -{ - let input = {| meta::relational::tests::db}; - let expected = {| meta::relational::tests::db}; - assertRoundTrip($input, $expected); +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Press F9 to execute the 'go' function... +// Press F10 to run the full test suite + +import meta::pure::extension::*; +import meta::pure::router::preeval::*; +import meta::pure::router::preeval::tests::*; +import meta::json::*; +import meta::protocols::pure::vX_X_X::transformation::fromPureGraph::valueSpecification::*; +import meta::protocols::pure::vX_X_X::transformation::toPureGrammar::*; +import apps::protocols::tds::preeval::router::tests::prerouting::*; +import meta::pure::graphFetch::execution::*; +import meta::pure::mapping::*; +import meta::pure::mapping::modelToModel::test::simple::*; +import meta::pure::mapping::modelToModel::test::shared::dest::*; +import meta::pure::mapping::modelToModel::test::shared::src::*; +import meta::pure::runtime::*; + +function <> meta::pure::router::preeval::tests::testPrerouting41():Boolean[1] +{ + let runtime = ^Runtime(); + let context = ^ExecutionContext(); + + let input = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') + ])->from(meta::relational::tests::simpleRelationalMapping, $runtime, $context)->join( + meta::pure::router::preeval::tests::Person.all() + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName') + ])->from(meta::relational::tests::simpleRelationalMapping, $runtime), + meta::relational::metamodel::join::JoinType.LEFT_OUTER, 'firstName') + }; + + let expected = {| + meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) + ])->from(meta::relational::tests::simpleRelationalMapping, $runtime, $context)->join( + meta::pure::router::preeval::tests::Person.all() + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}) + ])->from(meta::relational::tests::simpleRelationalMapping, $runtime), + meta::relational::metamodel::join::JoinType.LEFT_OUTER, 'firstName', 'firstName') + }; + + assertRoundTrip($input, $expected); +} + +function <> meta::pure::router::preeval::tests::testPrerouting42():Boolean[1] +{ + let runtime = ^Runtime(); + + let input = {name:String[1]| + let person = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( + #{ + meta::pure::router::preeval::tests::Person{ + firstName + } + }# + ); + + let person2 = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( + #{ + meta::pure::router::preeval::tests::Person{ + firstName + } + }# + )->from(meta::relational::tests::simpleRelationalMapping, $runtime); + + meta::pure::router::preeval::tests::Person.all() + ->filter(x | ($x.firstName == $person.firstName) || ($x.firstName == $name) || $x.firstName == $person2.firstName) + ->project([ + col({p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}, 'firstName'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}, 'lastName'), + col({p:meta::pure::router::preeval::tests::Person[1]|$p.age}, 'age'), + col({p:meta::pure::router::preeval::tests::Person[1]|today()}, 'today') + ]); + }; + + let expected = {name:String[1]| + + let person = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( + #{ + meta::pure::router::preeval::tests::Person{ + firstName + } + }# + ); + + let person2 = meta::pure::router::preeval::tests::Person.all()->toOne()->graphFetch( + #{ + meta::pure::router::preeval::tests::Person{ + firstName + } + }# + )->from(meta::relational::tests::simpleRelationalMapping, $runtime); + + + meta::pure::router::preeval::tests::Person.all() + ->filter(x | ($x.firstName == $person.firstName) || ($x.firstName == $name) || $x.firstName == $person2.firstName) + ->project([ + ^BasicColumnSpecification(name = 'firstName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.firstName}), + ^BasicColumnSpecification(name = 'lastName', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.lastName}), + ^BasicColumnSpecification(name = 'age', func = {p:meta::pure::router::preeval::tests::Person[1]|$p.age}), + ^BasicColumnSpecification(name = 'today', func = {p:meta::pure::router::preeval::tests::Person[1]|today()}) + ]); + }; + + assertRoundTrip($input, $expected, noDebug()); +} + +function <> meta::pure::router::preeval::tests::testPrerouting_Store():Boolean[1] +{ + let input = {| meta::relational::tests::db}; + let expected = {| meta::relational::tests::db}; + assertRoundTrip($input, $expected); } \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/authenticationStrategy.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/authenticationStrategy.pure index 46f7b8f2500..cea7442a70b 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/authenticationStrategy.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/authenticationStrategy.pure @@ -23,22 +23,23 @@ Class <> meta::pure::alloy::connections::alloy::authent Class meta::pure::alloy::connections::alloy::authentication::DelegatedKerberosAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::AuthenticationStrategy { - <> serverPrincipal: String[0..1]; + serverPrincipal: String[0..1]; } Class {doc.doc = 'Authentication using a middle tier user/password'} meta::pure::alloy::connections::alloy::authentication::MiddleTierUserNamePasswordAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::AuthenticationStrategy { - <> {doc.doc = 'Username/pasword vault reference'} vaultReference: String[1]; + {doc.doc = 'Username/pasword vault reference'} + vaultReference: String[1]; } Class meta::pure::alloy::connections::alloy::authentication::UserNamePasswordAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::AuthenticationStrategy { - <> baseVaultReference: String[0..1]; - <> userNameVaultReference: String[1]; - <> passwordVaultReference: String[1]; + baseVaultReference: String[0..1]; + userNameVaultReference: String[1]; + passwordVaultReference: String[1]; } Class meta::pure::alloy::connections::alloy::authentication::GCPApplicationDefaultCredentialsAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::AuthenticationStrategy @@ -51,7 +52,7 @@ Class meta::pure::alloy::connections::alloy::authentication::DefaultH2Authentica Class meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::AuthenticationStrategy { - <> apiToken:String[1]; + apiToken:String[1]; } Class meta::pure::alloy::connections::alloy::authentication::TestDatabaseAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy @@ -60,6 +61,6 @@ Class meta::pure::alloy::connections::alloy::authentication::TestDatabaseAuthent Class meta::pure::alloy::connections::alloy::authentication::GCPWorkloadIdentityFederationAuthenticationStrategy extends meta::pure::alloy::connections::alloy::authentication::AuthenticationStrategy { - <> serviceAccountEmail : String[1]; - <> additionalGcpScopes: String[*]; + serviceAccountEmail : String[1]; + additionalGcpScopes: String[*]; } \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/datasourceSpecification.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/datasourceSpecification.pure index 0eec4ed5d67..cc96e0c03b0 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/datasourceSpecification.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/runtime/connection/datasourceSpecification.pure @@ -18,21 +18,20 @@ Class <> meta::pure::alloy::connections::alloy::specific Class meta::pure::alloy::connections::alloy::specification::StaticDatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> host: String[1]; - <> port: Integer[1]; - <> databaseName: String[1]; + host: String[1]; + port: Integer[1]; + databaseName: String[1]; } Class meta::pure::alloy::connections::alloy::specification::EmbeddedH2DatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> databaseName:String[1]; - <> directory:String[1]; - <> autoServerMode:Boolean[1]; - + databaseName:String[1]; + directory:String[1]; + autoServerMode:Boolean[1]; } Class meta::pure::alloy::connections::alloy::specification::LocalH2DatasourceSpecification extends meta::pure::alloy::connections::alloy::specification::DatasourceSpecification { - <> testDataSetupCsv:String[0..1]; - <> testDataSetupSqls:String[*]; + testDataSetupCsv:String[0..1]; + testDataSetupSqls:String[*]; } diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure index 7f655b411de..ae23a8953bf 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbExtension.pure @@ -1001,6 +1001,7 @@ Enum meta::relational::functions::sqlQueryToString::DynaFunctionRegistry sign, sin, size, + splitPart, sqlFalse, sqlNull, sqlTrue, diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure index 98b3bc31095..2a64e92ae84 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension1_4_200.pure @@ -122,6 +122,7 @@ function <> meta::relational::functions::sqlQueryToString::h2::v dynaFnToSql('second', $allStates, ^ToSql(format='second(%s)')), dynaFnToSql('sha1', $allStates, ^ToSql(format='legend_h2_extension_hash_sha1(%s)')), dynaFnToSql('sha256', $allStates, ^ToSql(format='rawtohex(hash(\'SHA256\', %s))')), + dynaFnToSql('splitPart', $allStates, ^ToSql(format='legend_h2_extension_split_part(%s, %s, %s)')), dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), dynaFnToSql('stdDevPopulation', $allStates, ^ToSql(format='stddev_pop(%s)')), dynaFnToSql('stdDevSample', $allStates, ^ToSql(format='stddev_samp(%s)')), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure index ab466dbeb1c..bcef6849dfd 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/dbSpecific/h2/h2Extension2_1_214.pure @@ -143,7 +143,8 @@ function <> meta::relational::functions::sqlQueryToString::h2::v dynaFnToSql('round', $allStates, ^ToSql(format='round(%s, %s)', transform=transformRound_String_MANY__String_MANY_)), dynaFnToSql('second', $allStates, ^ToSql(format='second(%s)')), dynaFnToSql('sha1', $allStates, ^ToSql(format='rawtohex(hash(\'SHA-1\', %s))')), - dynaFnToSql('sha256', $allStates, ^ToSql(format='rawtohex(hash(\'SHA-256\', %s))')), + dynaFnToSql('sha256', $allStates, ^ToSql(format='rawtohex(hash(\'SHA-256\', %s))')), + dynaFnToSql('splitPart', $allStates, ^ToSql(format='legend_h2_extension_split_part(%s, %s, %s)')), dynaFnToSql('substring', $allStates, ^ToSql(format='substring%s', transform={p:String[*]|$p->joinStrings('(', ', ', ')')})), dynaFnToSql('stdDevPopulation', $allStates, ^ToSql(format='stddev_pop(%s)')), dynaFnToSql('stdDevSample', $allStates, ^ToSql(format='stddev_samp(%s)')), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure index 8c4ec03d1c2..a338f8194e2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/extensionDefaults.pure @@ -220,6 +220,7 @@ function meta::relational::functions::sqlQueryToString::default::getDynaFunction dynaFnToSql('sha256', $allStates, ^ToSql(format='sha256(%s)')), dynaFnToSql('sign', $allStates, ^ToSql(format='sign(%s)')), dynaFnToSql('sin', $allStates, ^ToSql(format='sin(%s)')), + dynaFnToSql('splitPart', $allStates, ^ToSql(format='split_part(%s, %s, %s)')), dynaFnToSql('size', $allStates, ^ToSql(format='count(%s)', transform={p:String[*]|if($p->isEmpty(),|'*',|$p)})), dynaFnToSql('sqlFalse', $allStates, ^ToSql(format='%s', transform={p:String[*]|processLiteralValue(false, [], $literalProcessor)})), dynaFnToSql('sqlNull', $allStates, ^ToSql(format='null')), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure index 0e7a9db18d5..a4b108a63c2 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/sqlQueryToString/testSuite/dynaFunctions/string.pure @@ -65,6 +65,13 @@ function <> meta::relational::tests::dbSpecificTests::sqlQueryTe runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); } +function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::splitPart::testSplitPart(config:DbTestConfig[1]):Boolean[1] +{ + let dynaFunc = ^DynaFunction(name='splitPart', parameters=[^Literal(value='abc|def|ghi'), ^Literal(value='|'), ^Literal(value=2)]); + let expected = ^Literal(value='def'); + runDynaFunctionDatabaseTest($dynaFunc, $expected, $config); +} + function <> meta::relational::tests::dbSpecificTests::sqlQueryTests::dynaFunctions::length::testWithString(config:DbTestConfig[1]):Boolean[1] { let dynaFunc = ^DynaFunction(name='length', parameters=[^Literal(value='String Random')]); diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tdsExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tdsExtension.pure index 306d74a4ebd..5542d6f3344 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tdsExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tdsExtension.pure @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::relational::mapping::*; +import meta::pure::functions::hash::*; import meta::pure::tds::extensions::*; import meta::relational::metamodel::join::*; @@ -202,4 +204,47 @@ function <> meta::pure::tds::extensions::joinWithOptionalColumns | $q1->join($q2, $joinType, {x,y|true}), | $q1->join($q2, $joinType, $cols->toOneMany()) ); -} \ No newline at end of file +} + +function <> meta::pure::tds::extensions::extendWithDigestOnColumns(input : TabularDataSet[1], digestColumnName : String[1]) : TabularDataSet[1] +{ + $input->extendWithDigestOnColumns($digestColumnName, HashType.MD5, $input.columns.name->toOneMany()); +} + +function <> meta::pure::tds::extensions::extendWithDigestOnColumns(input : TabularDataSet[1], digestValueColumnName : String[1], digestHashType : HashType[1], digestColumns : String[1..*]) : TabularDataSet[1] +{ + $input->extend( + col({row:TDSRow[1]| + $input.columns + ->filter(c|$c.name->in($digestColumns)) + ->map(col|toStringForColAccessor($col)->eval($row)) + ->joinStrings('|') + ->hash($digestHashType) + },$digestValueColumnName) + ) +} + +function <> + meta::pure::tds::extensions::toStringForColAccessor(col: TDSColumn[1]) : Function<{TDSRow[1]->String[1]}>[1] +{ + [ +// Pending release of https://github.com/finos/legend-pure/pull/736 +// if(!$col.type->toOne()->instanceOf(Enumeration), +// | [], +// | {r:TDSRow[1]|$r.getEnum($col.name)->toString()} +// ) + ] + ->concatenate( + [ + pair(Integer, {r:TDSRow[1]|$r.getInteger($col.name)->toString()}), + pair(Float, {r:TDSRow[1]|$r.getFloat($col.name)->toString()}), + pair(String, {r:TDSRow[1]|$r.getString($col.name)->toString()}), + pair(Boolean, {r:TDSRow[1]|$r.getBoolean($col.name)->toString()}), + pair(Date, {r:TDSRow[1]|$r.getDate($col.name)->toString()}), + pair(DateTime, {r:TDSRow[1]|$r.getDateTime($col.name)->toString()}), + pair(StrictDate, {r:TDSRow[1]|$r.getStrictDate($col.name)->toString()}) + ]->filter(p|$p.first == $col.type).second + ) + ->toOne('Unsupported column type: ' + $col.type->toOne()->toString() + '(' + $col.name + ')') +} + diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testGroupBy.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testGroupBy.pure index 7fd08737df4..2708c52fda4 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testGroupBy.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testGroupBy.pure @@ -467,3 +467,16 @@ function <> meta::relational::tests::tds::groupBy::testTDSGroupByEmpt assertEquals([], $result.values.rows->map(r|$r.values->makeString('|'))); meta::relational::functions::asserts::assertSameSQL('select "Firm Count" as "Firm Count" from (select count(*) as "Firm Count" from firmTable as "root") as "subselect" where "Firm Count" > 10', $result); } + +function <> meta::relational::tests::tds::groupBy::testTDSGroupByWithEnumArgumentFunctionCall():Boolean[1] +{ + let result = execute( + meta::pure::router::preeval::preval(|Trade.all() + ->project([col(x | $x.date, 'Date')]) + ->groupBy([], [agg('Date Adjust', x|$x.getStrictDate('Date'), y|$y->max()->toOne()->adjust(1, DurationUnit.WEEKS))]), meta::relational::extension::relationalExtensions()), + simpleRelationalMapping, + meta::relational::tests::testRuntime(), + meta::relational::extension::relationalExtensions()); + + meta::relational::functions::asserts::assertSameSQL('select dateadd(WEEK, 1, max("root".tradeDate)) as "Date Adjust" from tradeTable as "root"', $result); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsExtension.pure index db75161dfb4..9068a1d7d01 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsExtension.pure @@ -12,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::relational::mapping::*; +import meta::pure::router::preeval::*; +import meta::pure::functions::hash::*; import meta::pure::tds::extensions::*; import meta::relational::tests::csv::*; import meta::protocols::tds::preeval::router::prerouting::*; @@ -487,4 +490,41 @@ function <> meta::pure::tds::tests::extensions::testFirstNotNull():Bo { assertEquals(1, [TDSNull, 1, 2]->meta::pure::tds::extensions::firstNotNull()); assertEquals([], [TDSNull, TDSNull]->meta::pure::tds::extensions::firstNotNull()); +} + +function <> + meta::pure::tds::tests::extensions::testExtendDigest_InMemory() : Boolean[1] +{ + let scores = [1,2,3,4,5]; + let data = range($scores->size())->map(i|'student_' +toString($i))->zip($scores); + + let tds = $data->project([col(p|$p.first, 'name'), col(p|$p.second, 'score')]); + + let result = $tds->extendWithDigestOnColumns('_digest'); + + println($result->toCSV()); + + assertEquals(['0da8968758f3a315890e2f9ac3eb2fb3', 'eab9489b8f6517e84642ed09d819fee6', '387c444764879ef47844199bc693effc', '72d92b984f7ba90fe9b1ac4962c74012', 'e9f5f78a0597a8804196a8f9ced00000'], $result->columnValues('_digest')); +} + +function <> + meta::pure::tds::tests::extensions::testExtendDigest_Relational() : Boolean[1] +{ + let mapping = meta::relational::tests::simpleRelationalMapping; + let runtime = meta::relational::tests::testRuntime(); + + let query = {| + Trade.all() + ->project([ + col(x | $x.id, 'Trade ID'), + col(x | $x.quantity, 'Quantity') + ]) + ->extendWithDigestOnColumns('_digest') + ->sort('Trade ID') + ->limit(2); + }; + + let result = execute($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + assertEquals(['b7bbee4d9b6a2736c25b00dded9344c7', '9e103ea06a6999b4c5a86cf25d68b083'], $result.values->columnValues('_digest')); } \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsSchema.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsSchema.pure index 7b5f5a9f51e..eb13bca875a 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsSchema.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tds/tests/testTdsSchema.pure @@ -120,6 +120,16 @@ function <> meta::relational::tds::schema::tests::resolveSchemaTest() ['tradeDate'], ['quantityA']) ->sort(asc('tradeDate')) }); + + assertSchemaRoundTripEquality({| + Trade.all() + ->groupBy([x|$x.date->adjust(0, DurationUnit.DAYS)], + [ agg(x | $x.quantity, y | $y->sum()), agg(x | $x.quantity, y | $y->sum())], + ['tradeDate', 'quantityA', 'quantityB'] + ) + ->extendWithDigestOnColumns('_digest') + ->sort(asc('tradeDate')) + }); } function meta::relational::tds::schema::tests::assertSchemaRoundTripEquality(query : FunctionDefinition<{->TabularDataSet[1]}>[1]) : Boolean[1] diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tests/testRelationalExtension.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tests/testRelationalExtension.pure index ec9c4db84d7..6244fa06434 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tests/testRelationalExtension.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/tests/testRelationalExtension.pure @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import meta::relational::metamodel::execute::tests::*; -import meta::pure::alloy::connections::*; import meta::pure::runtime::*; import meta::relational::translation::*; import meta::pure::extension::*; @@ -24,119 +22,6 @@ import meta::relational::metamodel::execute::*; import meta::relational::metamodel::*; import meta::pure::mapping::*; -function meta::relational::metamodel::execute::tests::runRelationalRouterExtensionConnectionEquality(c1: RelationalDatabaseConnection[1], - c2: RelationalDatabaseConnection[1]) : Boolean[1] -{ - let extensions = meta::relational::extension::relationalExtensions().routerExtensions(); - $c1->match( - $extensions.connectionEquality->map(e | $e->eval($c2))->concatenate([ - a:Connection[1] | true - ])->toOneMany()); -} - -function <> meta::relational::metamodel::execute::tests::testConnectionEqualityAllSameStatic() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.SybaseIQ, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::StaticDatasourceSpecification(host='host', port=8080, databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy() - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.SybaseIQ, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::StaticDatasourceSpecification(host='host', port=8080, databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy() - ); - - assert(runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - -function <> meta::relational::metamodel::execute::tests::testConnectionEqualityAllButOnePropertySame() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::StaticDatasourceSpecification(host='host', port=8090, databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy() - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::StaticDatasourceSpecification(host='host', port=8080, databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - assert(!runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - -function <> meta::relational::metamodel::execute::tests::testConnectionEqualityTypeDiff() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.H2, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::LocalH2DatasourceSpecification(), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy() - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.Snowflake, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::StaticDatasourceSpecification(host='host', port=8080, databaseName='db'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - - assert(!runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - - -function <> meta::relational::metamodel::execute::tests::testConnectionEqualityTypeSameSpecDiff() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.H2, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::LocalH2DatasourceSpecification(), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy() - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.H2, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::LocalH2DatasourceSpecification(testDataSetupCsv='something'), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy() - ); - - - assert(!runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} - -function <> meta::relational::metamodel::execute::tests::testConnectionEqualityTypeSpecSameAuthDiff() : Boolean[1] -{ - let c1 = ^RelationalDatabaseConnection( - element = 'Store1', - type = DatabaseType.H2, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::LocalH2DatasourceSpecification(), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::DefaultH2AuthenticationStrategy() - ); - - let c2 = ^RelationalDatabaseConnection( - element = '', - type = DatabaseType.H2, - datasourceSpecification = ^meta::pure::alloy::connections::alloy::specification::LocalH2DatasourceSpecification(), - authenticationStrategy = ^meta::pure::alloy::connections::alloy::authentication::ApiTokenAuthenticationStrategy(apiToken='token') - ); - - - assert(!runRelationalRouterExtensionConnectionEquality($c1, $c2)); - -} function <> meta::relational::metamodel::execute::tests::testExecuteInDbToTDS() : Boolean[1] { diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure index d74a62e5ce4..9561c6f25bb 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/src/main/resources/core_relational/relational/transform/fromPure/tests/testToSQLString.pure @@ -203,13 +203,24 @@ function <> meta::relational::tests::functions::sqlstring::testToSQLS assertEquals('select reverse("root".FIRSTNAME) as "reverse" from personTable as "root"', $db2Sql); } +function <> meta::relational::tests::functions::sqlstring::testToSQLStringSplitPart():Boolean[1] +{ + let sql = toSQLString({|Person.all() + ->project([ + col(p|splitPart($p.firstName, '|', 0), 'splitBar') + ])->project([ + col(row:TDSRow[1] | splitPart($row.getString('splitBar'), ',', 1), 'splitComma') + ])}, simpleRelationalMapping, DatabaseType.H2, meta::relational::extension::relationalExtensions()); + + assertEquals('select legend_h2_extension_split_part(legend_h2_extension_split_part("root".FIRSTNAME, \'|\', 1), \',\', 2) as "splitComma" from personTable as "root"', $sql); +} + function <> meta::relational::tests::functions::sqlstring::testToSQLStringAscii():Boolean[1] { let sql = toSQLString(|Person.all()->project(p|ascii($p.firstName), 'ascii'), simpleRelationalMapping, DatabaseType.H2, meta::relational::extension::relationalExtensions()); assertEquals('select ascii("root".FIRSTNAME) as "ascii" from personTable as "root"', $sql); } - function <> meta::relational::tests::functions::sqlstring::testToSQLStringChar():Boolean[1] { let sql = toSQLString(|Person.all()->project(p|char($p.age->toOne()), 'char'), simpleRelationalMapping, DatabaseType.H2, meta::relational::extension::relationalExtensions()); @@ -599,15 +610,48 @@ function <> meta::relational::tests::functions::sqlstring::testToSQLS assertEquals('select cast("root".quantity as decimal) as "decimal", cast("root".quantity as double precision) as "float" from tradeTable as "root"', $result); } -function <> meta::relational::tests::functions::sqlstring::testHashFunctions():Boolean[1] +function <> meta::relational::tests::functions::sqlstring::testToSQLStringForTDSStringJoin():Boolean[1] { let result = toSQLString( |Person.all() ->project([ + col(p|$p.firstName, 'firstName'), + col(p|$p.lastName, 'lastName') + ]) + ->extend([ + col(row:TDSRow[1]|joinStrings([$row.getString('firstName'), ' ', $row.getString('lastName')]), 'name1'), + col(row:TDSRow[1]|joinStrings([$row.getString('firstName'), ' ', $row.getString('lastName')], '|'), 'name2'), + col(row:TDSRow[1]|joinStrings([$row.getString('firstName'), ' ', $row.getString('lastName')], '[', ',', ']'), 'name3'), + col(row:TDSRow[1]|joinStrings(['myValue', $row.getString('firstName'), ' ', $row.getString('lastName')], '[', ',', ']'), 'name4') + ]), simpleRelationalMapping, DatabaseType.H2, meta::relational::extension::relationalExtensions()); + + assertEquals('select "root".FIRSTNAME as "firstName", "root".LASTNAME as "lastName", concat("root".FIRSTNAME, \' \', "root".LASTNAME, \'\', \'\', \'\') as "name1", concat("root".FIRSTNAME, \' \', "root".LASTNAME, \'|\') as "name2", ' + +'concat("root".FIRSTNAME, \' \', "root".LASTNAME, \'[\', \',\', \']\') as "name3", concat(\'myValue\', "root".FIRSTNAME, \' \', "root".LASTNAME, \'[\', \',\', \']\') as "name4" from personTable as "root"', $result); +} + +function <> meta::relational::tests::functions::sqlstring::testHashFunctions():Boolean[1] +{ + let result = toSQLString( + |Person.all() + ->project([ + col(p|$p.firstName, 'firstName'), + col(p|$p.lastName, 'lastName'), col(p|$p.firstName->hash(HashType.MD5), 'md5'), col(p|$p.firstName->hash(HashType.SHA1), 'sha1'), col(p|$p.firstName->hash(HashType.SHA256), 'sha256') - ]), simpleRelationalMapping, DatabaseType.H2, meta::relational::extension::relationalExtensions()); + ]) + ->project([ + col({row:TDSRow[1]|$row.getString('firstName')->hash(HashType.MD5)}, 'tds_md5'), + col({row:TDSRow[1]|$row.getString('firstName')->hash(HashType.SHA1)}, 'tds_sha1'), + col({row:TDSRow[1]|$row.getString('firstName')->hash(HashType.SHA256)}, 'tds_sha256'), + col({row:TDSRow[1]|hash(($row.getString('firstName') + $row.getString('lastName')), HashType.MD5)}, 'tds_concat_md5'), + col({row:TDSRow[1]|hash(joinStrings([$row.getString('firstName'), $row.getString('lastName')]), HashType.MD5)}, 'tds_joinstrings_md5'), + col({row:TDSRow[1]|[$row.getString('firstName'), $row.getString('lastName')]->joinStrings('|')->hash(HashType.MD5)}, 'tds_digest') + ]) + , simpleRelationalMapping, DatabaseType.H2, meta::relational::extension::relationalExtensions()); + + assertEquals('select rawtohex(hash(\'MD5\', "root".FIRSTNAME)) as "tds_md5", rawtohex(hash(\'SHA-1\', "root".FIRSTNAME)) as "tds_sha1", rawtohex(hash(\'SHA-256\', "root".FIRSTNAME)) as "tds_sha256", ' + + 'rawtohex(hash(\'MD5\', concat("root".FIRSTNAME, "root".LASTNAME))) as "tds_concat_md5", rawtohex(hash(\'MD5\', concat("root".FIRSTNAME, "root".LASTNAME, \'\', \'\', \'\'))) as "tds_joinstrings_md5", ' + + 'rawtohex(hash(\'MD5\', concat("root".FIRSTNAME, "root".LASTNAME, \'|\'))) as "tds_digest" from personTable as "root"', $result); - assertNotEmpty($result); } \ No newline at end of file diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/pom.xml b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/pom.xml index 85b77598489..39ea484d72f 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/pom.xml +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-relationalStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-relationalStore/pom.xml b/legend-engine-xts-relationalStore/pom.xml index 3fb042bc0d2..1d34bdeef56 100644 --- a/legend-engine-xts-relationalStore/pom.xml +++ b/legend-engine-xts-relationalStore/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-rosetta/legend-engine-xt-rosetta-pure/pom.xml b/legend-engine-xts-rosetta/legend-engine-xt-rosetta-pure/pom.xml index 48d1299b3a1..c5a33b2c66d 100644 --- a/legend-engine-xts-rosetta/legend-engine-xt-rosetta-pure/pom.xml +++ b/legend-engine-xts-rosetta/legend-engine-xt-rosetta-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-rosetta - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-rosetta/legend-engine-xt-rosetta/pom.xml b/legend-engine-xts-rosetta/legend-engine-xt-rosetta/pom.xml index 17009461af1..5f609af7e15 100644 --- a/legend-engine-xts-rosetta/legend-engine-xt-rosetta/pom.xml +++ b/legend-engine-xts-rosetta/legend-engine-xt-rosetta/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-rosetta - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-rosetta/pom.xml b/legend-engine-xts-rosetta/pom.xml index a4aab98a520..59e69ab3eb2 100644 --- a/legend-engine-xts-rosetta/pom.xml +++ b/legend-engine-xts-rosetta/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-service/legend-engine-language-pure-dsl-service-execution/pom.xml b/legend-engine-xts-service/legend-engine-language-pure-dsl-service-execution/pom.xml index 69967de3c2c..515a4633c0a 100644 --- a/legend-engine-xts-service/legend-engine-language-pure-dsl-service-execution/pom.xml +++ b/legend-engine-xts-service/legend-engine-language-pure-dsl-service-execution/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-dsl-service-execution diff --git a/legend-engine-xts-service/legend-engine-language-pure-dsl-service-generation/pom.xml b/legend-engine-xts-service/legend-engine-language-pure-dsl-service-generation/pom.xml index da99039d65a..ce8ff9fab38 100644 --- a/legend-engine-xts-service/legend-engine-language-pure-dsl-service-generation/pom.xml +++ b/legend-engine-xts-service/legend-engine-language-pure-dsl-service-generation/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure/pom.xml b/legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure/pom.xml index 76611f81a9e..cb7ac8f717d 100644 --- a/legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure/pom.xml +++ b/legend-engine-xts-service/legend-engine-language-pure-dsl-service-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-service/legend-engine-language-pure-dsl-service/pom.xml b/legend-engine-xts-service/legend-engine-language-pure-dsl-service/pom.xml index 4a6e28be685..a268f0dc547 100644 --- a/legend-engine-xts-service/legend-engine-language-pure-dsl-service/pom.xml +++ b/legend-engine-xts-service/legend-engine-language-pure-dsl-service/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-language-pure-dsl-service diff --git a/legend-engine-xts-service/legend-engine-service-post-validation-runner/pom.xml b/legend-engine-xts-service/legend-engine-service-post-validation-runner/pom.xml index 06c656af07f..686ebb172d4 100644 --- a/legend-engine-xts-service/legend-engine-service-post-validation-runner/pom.xml +++ b/legend-engine-xts-service/legend-engine-service-post-validation-runner/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-service/legend-engine-services-model-api/pom.xml b/legend-engine-xts-service/legend-engine-services-model-api/pom.xml index 3587482c409..4e607ec1909 100644 --- a/legend-engine-xts-service/legend-engine-services-model-api/pom.xml +++ b/legend-engine-xts-service/legend-engine-services-model-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-services-model-api diff --git a/legend-engine-xts-service/legend-engine-services-model/pom.xml b/legend-engine-xts-service/legend-engine-services-model/pom.xml index 3420a067a70..c86604f6bae 100644 --- a/legend-engine-xts-service/legend-engine-services-model/pom.xml +++ b/legend-engine-xts-service/legend-engine-services-model/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 legend-engine-services-model diff --git a/legend-engine-xts-service/legend-engine-test-runner-service/pom.xml b/legend-engine-xts-service/legend-engine-test-runner-service/pom.xml index a97987ce261..c699c0a17d2 100644 --- a/legend-engine-xts-service/legend-engine-test-runner-service/pom.xml +++ b/legend-engine-xts-service/legend-engine-test-runner-service/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-service - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-service/pom.xml b/legend-engine-xts-service/pom.xml index eeb0fcfd0f1..db6e78f2be3 100644 --- a/legend-engine-xts-service/pom.xml +++ b/legend-engine-xts-service/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-executionPlan/pom.xml b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-executionPlan/pom.xml index cd313afbcdf..3276fb94862 100644 --- a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-executionPlan/pom.xml +++ b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-executionPlan/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-serviceStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-grammar/pom.xml b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-grammar/pom.xml index 8363ae36f78..dccf0583568 100644 --- a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-grammar/pom.xml +++ b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-serviceStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-javaPlatformBinding-pure/pom.xml b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-javaPlatformBinding-pure/pom.xml index 3bcf9f17846..f1fbb73c74e 100644 --- a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-javaPlatformBinding-pure/pom.xml +++ b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-javaPlatformBinding-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-serviceStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-protocol/pom.xml b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-protocol/pom.xml index 62c2ae0445b..6111fe9b625 100644 --- a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-protocol/pom.xml +++ b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-serviceStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-pure/pom.xml b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-pure/pom.xml index 4a451318222..b9d5291ba2b 100644 --- a/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-pure/pom.xml +++ b/legend-engine-xts-serviceStore/legend-engine-xt-serviceStore-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-serviceStore - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-serviceStore/pom.xml b/legend-engine-xts-serviceStore/pom.xml index 8f2641bd947..34499847381 100644 --- a/legend-engine-xts-serviceStore/pom.xml +++ b/legend-engine-xts-serviceStore/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml index b902a7d5443..6a8e81612c3 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-snowflakeApp - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -82,15 +82,15 @@ org.finos.legend.engine - legend-engine-xt-functionActivator-protocol + legend-engine-xt-relationalStore-pure org.finos.legend.engine - legend-engine-xt-relationalStore-pure + legend-engine-xt-relationalStore-protocol org.finos.legend.engine - legend-engine-xt-relationalStore-protocol + legend-engine-executionPlan-execution diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java index a6a4d7c7edd..3b9d06989ac 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/api/SnowflakeAppService.java @@ -20,15 +20,19 @@ import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.list.mutable.FastList; import org.finos.legend.engine.functionActivator.api.output.FunctionActivatorInfo; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; import org.finos.legend.engine.functionActivator.service.FunctionActivatorError; import org.finos.legend.engine.functionActivator.service.FunctionActivatorService; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppArtifact; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppContent; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppDeploymentConfiguration; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppGenerator; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentConfiguration; import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentStage; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentManager; -import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentResult; +import org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeDeploymentResult; import org.finos.legend.engine.plan.execution.stores.relational.config.TemporaryTestDbConfiguration; import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; import org.finos.legend.engine.plan.generation.PlanGenerator; @@ -37,6 +41,7 @@ import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeAppProtocolExtension; import org.finos.legend.pure.generated.Root_meta_external_function_activator_FunctionActivator; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; +import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration; import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_RelationalDatabaseConnection; import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy; import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification; @@ -50,7 +55,7 @@ import java.util.List; -public class SnowflakeAppService implements FunctionActivatorService +public class SnowflakeAppService implements FunctionActivatorService { private ConnectionManagerSelector connectionManager; private SnowflakeDeploymentManager snowflakeDeploymentManager; @@ -89,75 +94,31 @@ public boolean supports(Root_meta_external_function_activator_FunctionActivator @Override public MutableList validate(MutableList profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, Function> routerExtensions) { - RichIterable sqlExpressions = extractSQLExpressions(pureModel, activator, routerExtensions); - return sqlExpressions.size() != 1 ? - Lists.mutable.with(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + sqlExpressions.size() + "' SQL expressions", sqlExpressions.toList())) : + SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions); + int size = ((SnowflakeAppContent)artifact.content).sqlExpressions.size(); + return size != 1 ? + Lists.mutable.with(new SnowflakeAppError("SnowflakeApp can't be used with a plan containing '" + size + "' SQL expressions", ((SnowflakeAppContent)artifact.content).sqlExpressions)) : Lists.mutable.empty(); } @Override - public SnowflakeDeploymentResult publishToSandbox(MutableList profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, List runtimeConfigurations, Function> routerExtensions) + public SnowflakeDeploymentResult publishToSandbox(MutableList profiles, PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, List runtimeConfigurations, Function> routerExtensions) { - Object[] objects = this.extractSQLExpressionsAndConnectionMetadata(pureModel, activator, routerExtensions); - RichIterable sqlExpressions = (RichIterable) objects[0]; - - Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification ds = (Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification) objects[1]; - Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy as = (Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy) objects[2]; - - String applicationName = activator._applicationName(); - SnowflakeDeploymentConfiguration config = new SnowflakeDeploymentConfiguration(applicationName); - - return this.snowflakeDeploymentManager.fakeDeploy(ds, as, applicationName); + SnowflakeAppArtifact artifact = SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions); + return this.snowflakeDeploymentManager.deploy(profiles, artifact, runtimeConfigurations); } @Override public SnowflakeAppArtifact renderArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, PureModelContext inputModel, String clientVersion, Function> routerExtensions) { - RichIterable sqlExpressions = extractSQLExpressions(pureModel, activator, routerExtensions); - return new SnowflakeAppArtifact(sqlExpressions); - } - - private RichIterable extractSQLExpressions(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function> routerExtensions) - { - PackageableFunction function = activator._function(); - Root_meta_pure_executionPlan_ExecutionPlan executionPlan = PlanGenerator.generateExecutionPlanAsPure((FunctionDefinition) function, null, null, null, pureModel, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel)); - Root_meta_pure_executionPlan_ExecutionNode node = executionPlan._rootExecutionNode(); - return collectAllNodes(node) - .selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class) - .collect(Root_meta_relational_mapping_SQLExecutionNode::_sqlQuery) - .select(x -> !x.toLowerCase().startsWith("alter")); - } - - private Object[] extractSQLExpressionsAndConnectionMetadata(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function> routerExtensions) - { - PackageableFunction function = activator._function(); - Root_meta_pure_executionPlan_ExecutionPlan executionPlan = PlanGenerator.generateExecutionPlanAsPure((FunctionDefinition) function, null, null, null, pureModel, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel)); - Root_meta_pure_executionPlan_ExecutionNode node = executionPlan._rootExecutionNode(); - - RichIterable expressions = collectAllNodes(node) - .selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class) - .collect(Root_meta_relational_mapping_SQLExecutionNode::_sqlQuery) - .select(x -> !x.toLowerCase().startsWith("alter")); - - Root_meta_pure_alloy_connections_RelationalDatabaseConnection relCOnn = (Root_meta_pure_alloy_connections_RelationalDatabaseConnection)collectAllNodes(node).selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class) - .getAny() - ._connection(); - Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification ds = (Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification) relCOnn._datasourceSpecification(); - Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy as = (Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy) relCOnn._authenticationStrategy(); - - return new Object[]{expressions, ds, as}; - } - - private RichIterable collectAllNodes(Root_meta_pure_executionPlan_ExecutionNode node) - { - return Lists.mutable.with(node).withAll(node._executionNodes().flatCollect(this::collectAllNodes)); + return SnowflakeAppGenerator.generateArtifact(pureModel, activator, routerExtensions); } @Override - public List selectConfig(List configurations, DeploymentStage stage) + public List selectConfig(List configurations) { - return Lists.mutable.withAll(configurations).select(e -> e instanceof SnowflakeDeploymentConfiguration && e.stage.equals(stage)).collect(e -> (SnowflakeDeploymentConfiguration)e); + return Lists.mutable.withAll(configurations).select(e -> e instanceof SnowflakeAppDeploymentConfiguration).collect(e -> (SnowflakeAppDeploymentConfiguration) e); } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java index 5208fe1fcf1..9ab608335d6 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifact.java @@ -16,19 +16,25 @@ import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorArtifact; public class SnowflakeAppArtifact extends FunctionActivatorArtifact { - public RichIterable sqlExpressions = Lists.mutable.empty(); public SnowflakeAppArtifact() { //empty artifact } - public SnowflakeAppArtifact(RichIterable sqlExpressions) + public SnowflakeAppArtifact(String name, MutableList sqlExpressions) { - this.sqlExpressions = sqlExpressions; + this.content = new SnowflakeAppContent(name, sqlExpressions); + } + + public SnowflakeAppArtifact(String name, MutableList sqlExpressions, SnowflakeAppDeploymentConfiguration config) + { + this.content = new SnowflakeAppContent(name, sqlExpressions); + this.deploymentConfiguration = config; } } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java index 3f71520de0f..1d916df4b67 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppArtifactGenerationExtension.java @@ -26,7 +26,8 @@ public class SnowflakeAppArtifactGenerationExtension implements ArtifactGenerationExtension { - private static final String ROOT_PATH = "Snowflake App "; + private static final String ROOT_PATH = "snowflakeApp"; + private static final String FILENAME = "snowflakeArtifact.json"; @Override public String getKey() diff --git a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultCredentialBuilderProvider.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppContent.java similarity index 52% rename from legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultCredentialBuilderProvider.java rename to legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppContent.java index 1d2bc6f769f..388b537959b 100644 --- a/legend-engine-xts-authentication/legend-engine-xt-authentication-connection-factory/src/main/java/org/finos/legend/connection/DefaultCredentialBuilderProvider.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppContent.java @@ -12,18 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.connection; +package org.finos.legend.engine.language.snowflakeApp.deployment; +import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.list.MutableList; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentContent; -import java.util.List; -import java.util.ServiceLoader; - -public class DefaultCredentialBuilderProvider implements CredentialBuilderProvider +public class SnowflakeAppContent extends FunctionActivatorDeploymentContent { - @Override - public List getBuilders() + public MutableList sqlExpressions = Lists.mutable.empty(); + + public String applicationName; + + public SnowflakeAppContent(String name, MutableList sqlExpressions) { - return Lists.mutable.withAll(ServiceLoader.load(CredentialBuilder.class)); + this.applicationName = name; + this.sqlExpressions = sqlExpressions; } } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentConfiguration.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentConfiguration.java new file mode 100644 index 00000000000..6308e215f8b --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppDeploymentConfiguration.java @@ -0,0 +1,23 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.snowflakeApp.deployment; + +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorDeploymentConfiguration; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; + +public class SnowflakeAppDeploymentConfiguration extends FunctionActivatorDeploymentConfiguration +{ + RelationalDatabaseConnection connection; +} diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppGenerator.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppGenerator.java new file mode 100644 index 00000000000..9a0616a1d61 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeAppGenerator.java @@ -0,0 +1,79 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +package org.finos.legend.engine.language.snowflakeApp.deployment; + +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.impl.factory.Lists; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.plan.generation.PlanGenerator; +import org.finos.legend.engine.plan.platform.PlanPlatform; +import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; +import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_RelationalDatabaseConnection; +import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy; +import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification; +import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionNode; +import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.generated.Root_meta_relational_mapping_SQLExecutionNode; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; + +public class SnowflakeAppGenerator +{ + + public static SnowflakeAppArtifact generateArtifact(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function> routerExtensions) + { + RichIterable sqlExpressions = extractSQLExpressions(pureModel, activator, routerExtensions); + return new SnowflakeAppArtifact(activator._applicationName(), Lists.mutable.withAll(sqlExpressions)); + } + + private static RichIterable extractSQLExpressions(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function> routerExtensions) + { + PackageableFunction function = activator._function(); + Root_meta_pure_executionPlan_ExecutionPlan executionPlan = PlanGenerator.generateExecutionPlanAsPure((FunctionDefinition) function, null, null, null, pureModel, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel)); + Root_meta_pure_executionPlan_ExecutionNode node = executionPlan._rootExecutionNode(); + return collectAllNodes(node) + .selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class) + .collect(Root_meta_relational_mapping_SQLExecutionNode::_sqlQuery) + .select(x -> !x.toLowerCase().startsWith("alter")); + } + + private static Object[] extractSQLExpressionsAndConnectionMetadata(PureModel pureModel, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, Function> routerExtensions) + { + PackageableFunction function = activator._function(); + Root_meta_pure_executionPlan_ExecutionPlan executionPlan = PlanGenerator.generateExecutionPlanAsPure((FunctionDefinition) function, null, null, null, pureModel, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel)); + Root_meta_pure_executionPlan_ExecutionNode node = executionPlan._rootExecutionNode(); + + RichIterable expressions = collectAllNodes(node) + .selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class) + .collect(Root_meta_relational_mapping_SQLExecutionNode::_sqlQuery) + .select(x -> !x.toLowerCase().startsWith("alter")); + + Root_meta_pure_alloy_connections_RelationalDatabaseConnection relCOnn = (Root_meta_pure_alloy_connections_RelationalDatabaseConnection)collectAllNodes(node).selectInstancesOf(Root_meta_relational_mapping_SQLExecutionNode.class) + .getAny() + ._connection(); + Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification ds = (Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification) relCOnn._datasourceSpecification(); + Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy as = (Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy) relCOnn._authenticationStrategy(); + + return new Object[]{expressions, ds, as}; + } + + private static RichIterable collectAllNodes(Root_meta_pure_executionPlan_ExecutionNode node) + { + return Lists.mutable.with(node).withAll(node._executionNodes().flatCollect(SnowflakeAppGenerator::collectAllNodes)); + } +} diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java index de9d2bc4240..5cd7369a2f4 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentManager.java @@ -14,49 +14,89 @@ package org.finos.legend.engine.language.snowflakeApp.deployment; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.ImmutableList; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.factory.Lists; import org.finos.legend.engine.functionActivator.deployment.DeploymentManager; -import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; -import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension; +import org.finos.legend.engine.functionActivator.deployment.FunctionActivatorArtifact; import org.finos.legend.engine.language.snowflakeApp.api.SnowflakeAppDeploymentTool; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.plan.execution.stores.relational.RelationalExecutor; +import org.finos.legend.engine.plan.execution.stores.relational.connection.manager.ConnectionManagerSelector; +import org.finos.legend.engine.plan.execution.stores.relational.plugin.RelationalStoreState; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.DatabaseConnection; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection; import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentConfiguration; -import org.finos.legend.engine.protocol.snowflakeApp.metamodel.SnowflakeDeploymentResult; -import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp; +import org.finos.legend.engine.shared.core.identity.Identity; import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_authentication_SnowflakePublicAuthenticationStrategy; import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_alloy_specification_SnowflakeDatasourceSpecification; import org.pac4j.core.profile.CommonProfile; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; import java.util.List; +import java.util.Map; -public class SnowflakeDeploymentManager implements DeploymentManager +public class SnowflakeDeploymentManager implements DeploymentManager { + private static final Logger LOGGER = LoggerFactory.getLogger(SnowflakeDeploymentManager.class); + private SnowflakeAppDeploymentTool snowflakeAppDeploymentTool; + private PlanExecutor planExecutor; + private ConnectionManagerSelector connectionManager; public SnowflakeDeploymentManager(SnowflakeAppDeploymentTool deploymentTool) { this.snowflakeAppDeploymentTool = deploymentTool; } + public SnowflakeDeploymentManager(PlanExecutor planExecutor) + { + this.planExecutor = planExecutor; + connectionManager = ((RelationalStoreState)planExecutor.getExtraExecutors().select(c -> c instanceof RelationalExecutor).getFirst().getStoreState()).getRelationalExecutor().getConnectionManager(); + } + @Override - public SnowflakeDeploymentResult deploy(MutableList profiles, SnowflakeAppArtifact artifact, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator) + public boolean canDeploy(FunctionActivatorArtifact artifact) { - return new SnowflakeDeploymentResult(true); + return artifact instanceof SnowflakeAppArtifact; } @Override - public SnowflakeDeploymentResult deploy(MutableList profiles, SnowflakeAppArtifact artifact, Root_meta_external_function_activator_snowflakeApp_SnowflakeApp activator, List availableRuntimeConfigurations) + public SnowflakeDeploymentResult deploy(MutableList profiles, SnowflakeAppArtifact artifact) { - return null; + return new SnowflakeDeploymentResult("",true); } @Override - public boolean canDeploy(Root_meta_external_function_activator_snowflakeApp_SnowflakeApp artifact) + public SnowflakeDeploymentResult deploy(MutableList profiles, SnowflakeAppArtifact artifact, List availableRuntimeConfigurations) { - return true; + LOGGER.info("Starting deployment"); + SnowflakeDeploymentResult result; + try (Connection jdbcConnection = this.getDeploymentConnection(profiles, artifact)) + { + String appName = ((SnowflakeAppContent)artifact.content).applicationName; + jdbcConnection.setAutoCommit(false); + this.deployImpl(jdbcConnection, appName); + jdbcConnection.commit(); + LOGGER.info("Completed deployment successfully"); + result = new SnowflakeDeploymentResult(appName, true); + } + catch (Exception e) + { + LOGGER.info("Completed deployment with error"); + result = new SnowflakeDeploymentResult(Lists.mutable.with(e.getMessage())); + } + return result; } + public SnowflakeAppDeploymentTool getSnowflakeAppDeploymentTool() { return snowflakeAppDeploymentTool; @@ -68,7 +108,7 @@ public SnowflakeDeploymentResult fakeDeploy(Root_meta_pure_alloy_connections_all try { this.snowflakeAppDeploymentTool.deploy(datasourceSpecification, authenticationStrategy, applicationName); - return new SnowflakeDeploymentResult(true); + return new SnowflakeDeploymentResult("",true); } catch (Exception e) { @@ -76,4 +116,77 @@ public SnowflakeDeploymentResult fakeDeploy(Root_meta_pure_alloy_connections_all } } + public java.sql.Connection getDeploymentConnection(MutableList profiles, RelationalDatabaseConnection connection) + { + return this.connectionManager.getDatabaseConnection(profiles, (DatabaseConnection) connection); + } + + public void deployImpl(Connection jdbcConnection, String context) throws Exception + { + Statement statement = jdbcConnection.createStatement(); + String deploymentTableName = this.getDeploymentTableName(jdbcConnection); + String createTableSQL = String.format("create table %s (id INTEGER, message VARCHAR(1000)) if not exists", deploymentTableName); + boolean createTableStatus = statement.execute(createTableSQL); + String insertSQL = String.format("insert into %s(id, message) values(%d, '%s')", deploymentTableName, System.currentTimeMillis(), context); + boolean insertStatus = statement.execute(insertSQL); + } + + public String getDeploymentTableName(Connection jdbcConnection) throws SQLException + { + String catalogName = jdbcConnection.getCatalog(); + String schema = "NATIVE_APP"; + return String.format("%s.%s.LEGEND_SNOWFLAKE_APP_DEPLOYMENT", catalogName, schema); + } + + public java.sql.Connection getDeploymentConnection(MutableList profiles, SnowflakeAppArtifact artifact) + { + RelationalDatabaseConnection connection = extractConnectionFromArtifact(artifact); + return this.connectionManager.getDatabaseConnection(profiles, connection); + } + + public RelationalDatabaseConnection extractConnectionFromArtifact(SnowflakeAppArtifact artifact) + { + return ((SnowflakeAppDeploymentConfiguration)artifact.deploymentConfiguration).connection; + } + + public ImmutableList getDeployed(MutableList profiles, RelationalDatabaseConnection connection) throws Exception + { + ImmutableList deployments = null; + + LOGGER.info("Querying deployment"); + try (Connection jdbcConnection = this.getDeploymentConnection(profiles, connection)) + { + deployments = this.getDeployedImpl(jdbcConnection); + LOGGER.info("Completed querying deployments successfully"); + } + catch (Exception e) + { + LOGGER.info("Completed querying deployments with error"); + throw e; + } + return deployments; + } + + public ImmutableList getDeployedImpl(Connection jdbcConnection) throws Exception + { + MutableList deployments = Lists.mutable.empty(); + String deploymentTableName = this.getDeploymentTableName(jdbcConnection); + String querySql = String.format("select * from %s order by id", deploymentTableName); + Statement statement = jdbcConnection.createStatement(); + ResultSet resultSet = statement.executeQuery(querySql); + while (resultSet.next()) + { + DeploymentInfo deploymentInfo = new DeploymentInfo(); + deploymentInfo.attributes.put("id", resultSet.getLong(1)); + deploymentInfo.attributes.put("message", resultSet.getString(2)); + deployments.add(deploymentInfo); + } + return deployments.toImmutable(); + } + + public static class DeploymentInfo + { + public Map attributes = Maps.mutable.empty(); + } + } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentResult.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentResult.java similarity index 73% rename from legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentResult.java rename to legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentResult.java index 6fb26b26684..cfc0a154fee 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentResult.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/java/org/finos/legend/engine/language/snowflakeApp/deployment/SnowflakeDeploymentResult.java @@ -12,19 +12,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.engine.protocol.snowflakeApp.metamodel; +package org.finos.legend.engine.language.snowflakeApp.deployment; import org.eclipse.collections.api.list.MutableList; -import org.finos.legend.engine.protocol.functionActivator.metamodel.DeploymentResult; +import org.finos.legend.engine.functionActivator.deployment.DeploymentResult; public class SnowflakeDeploymentResult extends DeploymentResult { public MutableList errors; - public SnowflakeDeploymentResult(boolean result) + public SnowflakeDeploymentResult(String activatorIdentifier, boolean result) { - this.successful = false; + this.successful = result; + this.activatorIdentifier = activatorIdentifier; } public SnowflakeDeploymentResult(MutableList errors) diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension new file mode 100644 index 00000000000..cc58504ac24 --- /dev/null +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-api/src/main/resources/META-INF/services/org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension @@ -0,0 +1 @@ +org.finos.legend.engine.language.snowflakeApp.deployment.SnowflakeAppArtifactGenerationExtension diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/pom.xml index 13e457fe3f6..b8d03d2d61e 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-snowflakeApp org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -50,11 +50,15 @@ org.finos.legend.engine - legend-engine-xt-functionActivator-protocol + legend-engine-language-pure-compiler org.finos.legend.engine - legend-engine-language-pure-compiler + legend-engine-protocol-pure + + + org.finos.legend.engine + legend-engine-xt-relationalStore-pure diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java index abd8ab77ef6..c96373bb55f 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-compiler/src/main/java/org/finos/legend/engine/language/snowflakeApp/compiler/toPureGraph/SnowflakeAppCompilerExtension.java @@ -25,6 +25,7 @@ import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeApp_Impl; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration; import org.finos.legend.pure.generated.Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration_Impl; +import org.finos.legend.pure.generated.Root_meta_pure_alloy_connections_RelationalDatabaseConnection; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.PackageableFunction; import org.finos.legend.pure.m3.navigation.function.FunctionDescriptor; @@ -82,6 +83,7 @@ public Root_meta_external_function_activator_snowflakeApp_SnowflakeApp buildSnow public Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration buildDeploymentConfig(SnowflakeDeploymentConfiguration configuration, CompileContext context) { return new Root_meta_external_function_activator_snowflakeApp_SnowflakeDeploymentConfiguration_Impl("") - ._stage(context.pureModel.getEnumValue("meta::external::function::activator::DeploymentStage", configuration.stage.name())); + ._target((Root_meta_pure_alloy_connections_RelationalDatabaseConnection) context.resolveConnection(configuration.activationConnection.connection, configuration.sourceInformation)); + // ._stage(context.pureModel.getEnumValue("meta::external::function::activator::DeploymentStage", configuration.stage.name())); } } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml index aec1ee63b31..541a62bf835 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-snowflakeApp - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -108,10 +108,6 @@ - - org.finos.legend.engine - legend-engine-shared-core - org.finos.legend.engine legend-engine-xt-snowflakeApp-protocol @@ -120,10 +116,6 @@ org.finos.legend.engine legend-engine-language-pure-grammar - - org.finos.legend.engine - legend-engine-xt-functionActivator-protocol - org.finos.legend.engine legend-engine-protocol-pure diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 index f6600e0f521..be29564551c 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/SnowflakeAppParserGrammar.g4 @@ -56,7 +56,7 @@ activation: SNOWFLAKE_APP__ACTIVATION COLON qualifiedName SE // ----------------------------------- Deployment ------------------------------------------------------ deploymentConfig: CONFIGURATION qualifiedName BRACE_OPEN - (activationConnection | stage) + activationConnection BRACE_CLOSE ; diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java index 21c043b1c85..31444f92ce3 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-grammar/src/main/java/org/finos/legend/engine/language/snowflakeApp/grammar/from/SnowflakeAppTreeWalker.java @@ -70,19 +70,6 @@ private SnowflakeDeploymentConfiguration visitDeploymentConfig(SnowflakeAppParse ? Collections.emptyList() : ctx.activationConnection().qualifiedName().packagePath().identifier(), ctx.activationConnection().qualifiedName().identifier()); pointer.sourceInformation = walkerSourceInformation.getSourceInformation(ctx.activationConnection().qualifiedName()); config.activationConnection = pointer; - String stage = ctx.stage().getText(); - if (stage.equals("PRODUCTION")) - { - config.stage = DeploymentStage.PRODUCTION; - } - else if (stage.equals("SANDBOX")) - { - config.stage = DeploymentStage.SANDBOX; - } - else - { - throw new EngineException("Valid types for deployment stage are: SANDBOX, PRODUCTION"); - } return config; } diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/pom.xml index 4259f418927..a6f0fa7bdbb 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-snowflakeApp - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentConfiguration.java b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentConfiguration.java index 1db41b25cd6..0aff2c51431 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentConfiguration.java +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-protocol/src/main/java/org/finos/legend/engine/protocol/snowflakeApp/metamodel/SnowflakeDeploymentConfiguration.java @@ -21,12 +21,7 @@ public class SnowflakeDeploymentConfiguration extends DeploymentConfiguration { public ConnectionPointer activationConnection; - public String applicationName; - - public SnowflakeDeploymentConfiguration(String applicationName) - { - this.applicationName = applicationName; - } +// public String applicationName; public SnowflakeDeploymentConfiguration() { diff --git a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml index ea89181fc3b..f99e4583b8b 100644 --- a/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml +++ b/legend-engine-xts-snowflakeApp/legend-engine-xt-snowflakeApp-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-snowflakeApp - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-snowflakeApp/pom.xml b/legend-engine-xts-snowflakeApp/pom.xml index 2eb7f9dfa24..0c085498dcf 100644 --- a/legend-engine-xts-snowflakeApp/pom.xml +++ b/legend-engine-xts-snowflakeApp/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-compiler/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-compiler/pom.xml index cc6153b6d9f..5beacacb8f2 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-compiler/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-compiler/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-sql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -52,6 +52,22 @@ test + + Generate translator for protocol + generate-sources + + java + + + true + org.finos.legend.engine.protocol.generation.GenerateMetamodelToProtocolTranslator + + core_external_query_sql_schema_metamodel.protocol.json + ${project.build.directory}/generated-sources/ + + test + + diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-grammar-integration/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-grammar-integration/pom.xml index bbd89ed83fa..8abde6bb126 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-grammar-integration/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-grammar-integration/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-sql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/pom.xml index cbd98d0d779..4c758a1408c 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-sql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java index 1e6b49eb802..57aaa1af936 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/main/java/org/finos/legend/engine/language/sql/grammar/to/SQLGrammarComposer.java @@ -527,7 +527,7 @@ public String visit(TableFunction val) @Override public String visit(TableSubquery val) { - return null; + return "(" + visit(val.query) + ")"; } @Override diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java index 4c9901b5bb0..befce651ac1 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-grammar/src/test/java/org/finos/legend/engine/language/sql/grammar/test/roundtrip/TestSQLRoundTrip.java @@ -288,6 +288,12 @@ public void testWithinGroup() check("SELECT percentile_cont(0.1) WITHIN GROUP (ORDER BY a ASC) FROM myTable"); } + @Test + public void testNested() + { + check("SELECT * from (select col from myTable)"); + } + private void fail(String sql, int start, int end, String message) { try diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/pom.xml index c152db7d53e..968d8d1253d 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/pom.xml @@ -3,7 +3,7 @@ legend-engine-xts-sql org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/handler/legend/LegendTdsResultParser.java b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/handler/legend/LegendTdsResultParser.java index 03b0e8ba6f9..d134a3bc3c6 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/handler/legend/LegendTdsResultParser.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/main/java/org/finos/legend/engine/postgres/handler/legend/LegendTdsResultParser.java @@ -27,6 +27,8 @@ import java.util.Objects; import static java.util.Objects.requireNonNull; import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class LegendTdsResultParser { @@ -34,12 +36,14 @@ public class LegendTdsResultParser public static final String TYPE = "type"; public static final String COLUMNS = "columns"; public static final String NAME = "name"; - public static final String RELATIONAL_TYPE = "relationalType"; public static final String RESULT = "result"; public static final String ROWS = "rows"; public static final String VALUES = "values"; private static final List SKIP_CHILDREN_TOKEN = Collections.unmodifiableList(Arrays.asList(JsonToken.START_ARRAY, JsonToken.START_OBJECT)); + + private static final Logger LOGGER = LoggerFactory.getLogger(LegendTdsResultParser.class); + public static final String _TYPE = "_type"; private final JsonParser parser; private List legendColumns; @@ -96,14 +100,11 @@ private void parserHeader() throws IOException acceptNextToken(JsonToken.START_ARRAY); while (parser.nextToken() != JsonToken.END_ARRAY) { - - //parse column acceptCurrent(JsonToken.START_OBJECT); String columnName = parseNextTextField(NAME); String type = parseNextTextField(TYPE); - String relationType = parseNextTextField(RELATIONAL_TYPE); - acceptNextToken(JsonToken.END_OBJECT); + skipUntilToken(JsonToken.END_OBJECT); legendColumns.add(new LegendColumn(columnName, type)); } acceptNextToken(JsonToken.END_OBJECT); @@ -190,6 +191,19 @@ private void skipChildren() throws IOException parser.skipChildren(); } + private void skipUntilToken(JsonToken expectedToken) throws IOException + { + parser.nextToken(); + while (!expectedToken.equals(parser.currentToken())) + { + if (LOGGER.isDebugEnabled()) + { + LOGGER.debug("Skipping token :" + parser.currentToken()); + } + parser.nextToken(); + } + } + private void acceptNextToken(JsonToken expectedToken) throws IOException { JsonToken jsonToken = parser.nextToken(); diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTest.java b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTest.java index 48c46ab089e..5f39cca3175 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTest.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/PostgresServerTest.java @@ -75,13 +75,17 @@ public void testMetadata() throws SQLException ) { ResultSetMetaData resultSetMetaData = statement.getMetaData(); - Assert.assertEquals(3, resultSetMetaData.getColumnCount()); + Assert.assertEquals(5, resultSetMetaData.getColumnCount()); Assert.assertEquals("Id", resultSetMetaData.getColumnName(1)); Assert.assertEquals("Name", resultSetMetaData.getColumnName(2)); Assert.assertEquals("Employee Type", resultSetMetaData.getColumnName(3)); + Assert.assertEquals("Full Name", resultSetMetaData.getColumnName(4)); + Assert.assertEquals("Derived Name", resultSetMetaData.getColumnName(5)); Assert.assertEquals("int4", resultSetMetaData.getColumnTypeName(1)); Assert.assertEquals("varchar", resultSetMetaData.getColumnTypeName(2)); Assert.assertEquals("varchar", resultSetMetaData.getColumnTypeName(3)); + Assert.assertEquals("varchar", resultSetMetaData.getColumnTypeName(4)); + Assert.assertEquals("varchar", resultSetMetaData.getColumnTypeName(5)); } } diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/handler/legend/LegendTDSResultParserTest.java b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/handler/legend/LegendTDSResultParserTest.java index dacac73ae7f..8c79ebce61f 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/handler/legend/LegendTDSResultParserTest.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/java/org/finos/legend/engine/postgres/handler/legend/LegendTDSResultParserTest.java @@ -47,7 +47,33 @@ public void testParseDataValidateResults() throws IOException LOGGER.info("Row: {}, Column: {}: Value: {}", row, column, value); column++; } - Assert.assertEquals("Verify number of column", 8, column); + Assert.assertEquals("Verify number of columns", 8, column); + } + Assert.assertEquals("Verify number of rows", 2, row); + } + } + + @Test + public void testParseDataNoRelationalTypeValidateResults() throws IOException + { + try (InputStream pureProjectInputStream = ClassLoader.getSystemClassLoader().getResourceAsStream("legendTdsResultNoRelationalType.json");) + { + LegendTdsResultParser parser = new LegendTdsResultParser(pureProjectInputStream); + List legendColumns = parser.getLegendColumns(); + int row = 0; + int column = 0; + while (parser.hasNext()) + { + row++; + column = 0; + List nextRow = parser.next(); + for (int i = 0; i < legendColumns.size(); i++) + { + Object value = nextRow.get(i); + LOGGER.info("Row: {}, Column: {}: Value: {}", row, column, value); + column++; + } + Assert.assertEquals("Verify number of columns", 2, column); } Assert.assertEquals("Verify number of rows", 2, row); } diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/resources/legendTdsResultNoRelationalType.json b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/resources/legendTdsResultNoRelationalType.json new file mode 100644 index 00000000000..f50797ad373 --- /dev/null +++ b/legend-engine-xts-sql/legend-engine-xt-sql-postgres-server/src/test/resources/legendTdsResultNoRelationalType.json @@ -0,0 +1,42 @@ +{ + "builder": { + "_type": "tdsBuilder", + "columns": [ + { + "name": "stringColumn1", + "type": "String" + }, + { + "name": "stringColumn2", + "type": "String", + "relationalType": "sqlType" + } + ] + }, + "activities": [ + { + "_type": "relational", + "sql": "select ..." + } + ], + "result": { + "columns": [ + "stringColumn1", + "stringColumn2" + ], + "rows": [ + { + "values": [ + "foo", + "bar" + ] + }, + { + "values": [ + "foo1", + "bar2" + ] + } + ] + } +} \ No newline at end of file diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-protocol/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-protocol/pom.xml index 652f7656073..e9bebe9d248 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-protocol/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-protocol/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-sql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -52,6 +52,22 @@ test + + generate schema metamodel + generate-sources + + java + + + true + org.finos.legend.engine.protocol.generation.GenerateMetaClasses + + core_external_query_sql_schema_metamodel.protocol.json + ${project.build.directory}/generated-sources/ + + test + + diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-protocol/src/main/resources/core_external_query_sql_schema_metamodel.protocol.json b/legend-engine-xts-sql/legend-engine-xt-sql-protocol/src/main/resources/core_external_query_sql_schema_metamodel.protocol.json new file mode 100644 index 00000000000..e973ac4b716 --- /dev/null +++ b/legend-engine-xts-sql/legend-engine-xt-sql-protocol/src/main/resources/core_external_query_sql_schema_metamodel.protocol.json @@ -0,0 +1,6 @@ +{ + "purePackage": "meta::external::query::sql::schema::metamodel", + "javaPackage": "org.finos.legend.engine.protocol.sql.schema.metamodel", + "elementsToBeExcluded": [ + ] +} \ No newline at end of file diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/pom.xml index 1c3269faae3..4849c69df26 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-sql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure index 477a5487c44..f412a054db9 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/metamodel.pure @@ -24,10 +24,10 @@ Class meta::external::query::sql::metamodel::Statement extends meta::external::q Class meta::external::query::sql::metamodel::Query extends meta::external::query::sql::metamodel::Statement { - queryBody: meta::external::query::sql::metamodel::QueryBody[1]; - limit: meta::external::query::sql::metamodel::Expression[0..1]; - orderBy: meta::external::query::sql::metamodel::SortItem[*]; - offset: meta::external::query::sql::metamodel::Expression[0..1]; + <> queryBody: meta::external::query::sql::metamodel::QueryBody[1]; + <> limit: meta::external::query::sql::metamodel::Expression[0..1]; + <> orderBy: meta::external::query::sql::metamodel::SortItem[*]; + <> offset: meta::external::query::sql::metamodel::Expression[0..1]; } Class meta::external::query::sql::metamodel::Relation extends meta::external::query::sql::metamodel::Node @@ -40,24 +40,24 @@ Class meta::external::query::sql::metamodel::QueryBody extends meta::external::q Class meta::external::query::sql::metamodel::TableSubquery extends meta::external::query::sql::metamodel::QueryBody { - query: meta::external::query::sql::metamodel::Query[1]; + <> query: meta::external::query::sql::metamodel::Query[1]; } Class meta::external::query::sql::metamodel::SubqueryExpression extends meta::external::query::sql::metamodel::Expression { - query: meta::external::query::sql::metamodel::Query[1]; + <> query: meta::external::query::sql::metamodel::Query[1]; } Class meta::external::query::sql::metamodel::QuerySpecification extends meta::external::query::sql::metamodel::QueryBody { - select: meta::external::query::sql::metamodel::Select[1]; - from: meta::external::query::sql::metamodel::Relation[*]; - where: meta::external::query::sql::metamodel::Expression[0..1]; - groupBy: meta::external::query::sql::metamodel::Expression[*]; - having: meta::external::query::sql::metamodel::Expression[0..1]; - orderBy: meta::external::query::sql::metamodel::SortItem[*]; - limit: meta::external::query::sql::metamodel::Expression[0..1]; - offset: meta::external::query::sql::metamodel::Expression[0..1]; + <> select: meta::external::query::sql::metamodel::Select[1]; + <> from: meta::external::query::sql::metamodel::Relation[*]; + <> where: meta::external::query::sql::metamodel::Expression[0..1]; + <> groupBy: meta::external::query::sql::metamodel::Expression[*]; + <> having: meta::external::query::sql::metamodel::Expression[0..1]; + <> orderBy: meta::external::query::sql::metamodel::SortItem[*]; + <> limit: meta::external::query::sql::metamodel::Expression[0..1]; + <> offset: meta::external::query::sql::metamodel::Expression[0..1]; } Class meta::external::query::sql::metamodel::SetOperation extends meta::external::query::sql::metamodel::QueryBody @@ -66,22 +66,22 @@ Class meta::external::query::sql::metamodel::SetOperation extends meta::external Class meta::external::query::sql::metamodel::AliasedRelation extends meta::external::query::sql::metamodel::Relation { - relation:meta::external::query::sql::metamodel::Relation[1]; - alias: String[1]; - columnNames: String[*]; + <> relation:meta::external::query::sql::metamodel::Relation[1]; + <> alias: String[1]; + <> columnNames: String[*]; } Class meta::external::query::sql::metamodel::Union extends meta::external::query::sql::metamodel::SetOperation { - left: meta::external::query::sql::metamodel::Relation[1]; - right: meta::external::query::sql::metamodel::Relation[1]; - distinct: Boolean[1]; + <> left: meta::external::query::sql::metamodel::Relation[1]; + <> right: meta::external::query::sql::metamodel::Relation[1]; + <> distinct: Boolean[1]; } Class meta::external::query::sql::metamodel::Select extends meta::external::query::sql::metamodel::Node { - distinct: Boolean[1]; - selectItems: meta::external::query::sql::metamodel::SelectItem[*]; + <> distinct: Boolean[1]; + <> selectItems: meta::external::query::sql::metamodel::SelectItem[*]; } Class meta::external::query::sql::metamodel::SelectItem extends meta::external::query::sql::metamodel::Node @@ -90,23 +90,23 @@ Class meta::external::query::sql::metamodel::SelectItem extends meta::external:: Class meta::external::query::sql::metamodel::AllColumns extends meta::external::query::sql::metamodel::SelectItem { - prefix: String[0..1]; + <> prefix: String[0..1]; } Class meta::external::query::sql::metamodel::SingleColumn extends meta::external::query::sql::metamodel::SelectItem { - alias: String[0..1]; - expression: meta::external::query::sql::metamodel::Expression[1]; + <> alias: String[0..1]; + <> expression: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::Table extends meta::external::query::sql::metamodel::QueryBody { - name: meta::external::query::sql::metamodel::QualifiedName[1]; + <> name: meta::external::query::sql::metamodel::QualifiedName[1]; } Class meta::external::query::sql::metamodel::TableFunction extends meta::external::query::sql::metamodel::QueryBody { - functionCall: meta::external::query::sql::metamodel::FunctionCall[1]; + <> functionCall: meta::external::query::sql::metamodel::FunctionCall[1]; } Class meta::external::query::sql::metamodel::Expression extends meta::external::query::sql::metamodel::Node @@ -119,33 +119,33 @@ Class meta::external::query::sql::metamodel::Literal extends meta::external::que Class meta::external::query::sql::metamodel::LongLiteral extends meta::external::query::sql::metamodel::Literal { - value: Integer[1]; + <> value: Integer[1]; } Class meta::external::query::sql::metamodel::BooleanLiteral extends meta::external::query::sql::metamodel::Literal { - value: Boolean[1]; + <> value: Boolean[1]; } Class meta::external::query::sql::metamodel::DoubleLiteral extends meta::external::query::sql::metamodel::Literal { - value: Float[1]; + <> value: Float[1]; } Class meta::external::query::sql::metamodel::IntegerLiteral extends meta::external::query::sql::metamodel::Literal { - value: Integer[1]; + <> value: Integer[1]; } Class meta::external::query::sql::metamodel::StringLiteral extends meta::external::query::sql::metamodel::Literal { - value: String[1]; - quoted: Boolean[0..1]; + <> value: String[1]; + <> quoted: Boolean[0..1]; } Class meta::external::query::sql::metamodel::ArrayLiteral extends meta::external::query::sql::metamodel::Literal { - values: meta::external::query::sql::metamodel::Expression[*]; + <> values: meta::external::query::sql::metamodel::Expression[*]; } Class meta::external::query::sql::metamodel::NullLiteral extends meta::external::query::sql::metamodel::Literal @@ -154,125 +154,125 @@ Class meta::external::query::sql::metamodel::NullLiteral extends meta::external: Class meta::external::query::sql::metamodel::IntervalLiteral extends meta::external::query::sql::metamodel::Literal { - ago: Boolean[0..1]; - years: Integer[0..1]; - months: Integer[0..1]; - weeks: Integer[0..1]; - days: Integer[0..1]; - hours: Integer[0..1]; - minutes: Integer[0..1]; - seconds: Integer[0..1]; + <> ago: Boolean[0..1]; + <> years: Integer[0..1]; + <> months: Integer[0..1]; + <> weeks: Integer[0..1]; + <> days: Integer[0..1]; + <> hours: Integer[0..1]; + <> minutes: Integer[0..1]; + <> seconds: Integer[0..1]; } Class meta::external::query::sql::metamodel::NamedArgumentExpression extends meta::external::query::sql::metamodel::Expression { - name: String[1]; - expression: meta::external::query::sql::metamodel::Expression[1]; + <> name: String[1]; + <> expression: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::LikePredicate extends meta::external::query::sql::metamodel::Expression { - value: meta::external::query::sql::metamodel::Expression[1]; - pattern: meta::external::query::sql::metamodel::Expression[1]; - escape: meta::external::query::sql::metamodel::Expression[0..1]; - ignoreCase: Boolean[1]; + <> value: meta::external::query::sql::metamodel::Expression[1]; + <> pattern: meta::external::query::sql::metamodel::Expression[1]; + <> escape: meta::external::query::sql::metamodel::Expression[0..1]; + <> ignoreCase: Boolean[1]; } Class meta::external::query::sql::metamodel::SortItem extends meta::external::query::sql::metamodel::Node { - sortKey: meta::external::query::sql::metamodel::Expression[1]; - ordering: meta::external::query::sql::metamodel::SortItemOrdering[1]; - nullOrdering: meta::external::query::sql::metamodel::SortItemNullOrdering[1]; + <> sortKey: meta::external::query::sql::metamodel::Expression[1]; + <> ordering: meta::external::query::sql::metamodel::SortItemOrdering[1]; + <> nullOrdering: meta::external::query::sql::metamodel::SortItemNullOrdering[1]; } Class meta::external::query::sql::metamodel::ComparisonExpression extends meta::external::query::sql::metamodel::Expression { - left: meta::external::query::sql::metamodel::Expression[1]; - right: meta::external::query::sql::metamodel::Expression[1]; - operator: meta::external::query::sql::metamodel::ComparisonOperator[1]; + <> left: meta::external::query::sql::metamodel::Expression[1]; + <> right: meta::external::query::sql::metamodel::Expression[1]; + <> operator: meta::external::query::sql::metamodel::ComparisonOperator[1]; } Class meta::external::query::sql::metamodel::LogicalBinaryExpression extends meta::external::query::sql::metamodel::Expression { - type: meta::external::query::sql::metamodel::LogicalBinaryType[1]; - left: meta::external::query::sql::metamodel::Expression[1]; - right: meta::external::query::sql::metamodel::Expression[1]; + <> type: meta::external::query::sql::metamodel::LogicalBinaryType[1]; + <> left: meta::external::query::sql::metamodel::Expression[1]; + <> right: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::NotExpression extends meta::external::query::sql::metamodel::Expression { - value: meta::external::query::sql::metamodel::Expression[1]; + <> value: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::ArithmeticExpression extends meta::external::query::sql::metamodel::Expression { - type: meta::external::query::sql::metamodel::ArithmeticType[1]; - left: meta::external::query::sql::metamodel::Expression[1]; - right: meta::external::query::sql::metamodel::Expression[1]; + <> type: meta::external::query::sql::metamodel::ArithmeticType[1]; + <> left: meta::external::query::sql::metamodel::Expression[1]; + <> right: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::NegativeExpression extends meta::external::query::sql::metamodel::Expression { - value: meta::external::query::sql::metamodel::Expression[1]; + <> value: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::IsNullPredicate extends meta::external::query::sql::metamodel::Expression { - value: meta::external::query::sql::metamodel::Expression[1]; + <> value: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::IsNotNullPredicate extends meta::external::query::sql::metamodel::Expression { - value: meta::external::query::sql::metamodel::Expression[1]; + <> value: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::CurrentTime extends meta::external::query::sql::metamodel::Expression { - type: meta::external::query::sql::metamodel::CurrentTimeType[1]; - precision: Integer[0..1]; + <> type: meta::external::query::sql::metamodel::CurrentTimeType[1]; + <> precision: Integer[0..1]; } Class meta::external::query::sql::metamodel::FunctionCall extends meta::external::query::sql::metamodel::Expression { - name: meta::external::query::sql::metamodel::QualifiedName[1]; - distinct: Boolean[1]; - arguments: meta::external::query::sql::metamodel::Expression[*]; - filter: meta::external::query::sql::metamodel::Expression[0..1]; - window: meta::external::query::sql::metamodel::Window[0..1]; - group: meta::external::query::sql::metamodel::Group[0..1]; + <> name: meta::external::query::sql::metamodel::QualifiedName[1]; + <> distinct: Boolean[1]; + <> arguments: meta::external::query::sql::metamodel::Expression[*]; + <> filter: meta::external::query::sql::metamodel::Expression[0..1]; + <> window: meta::external::query::sql::metamodel::Window[0..1]; + <> group: meta::external::query::sql::metamodel::Group[0..1]; } Class meta::external::query::sql::metamodel::SimpleCaseExpression extends meta::external::query::sql::metamodel::Expression { - operand: meta::external::query::sql::metamodel::Expression[1]; - whenClauses: meta::external::query::sql::metamodel::WhenClause[*]; - defaultValue: meta::external::query::sql::metamodel::Expression[0..1]; + <> operand: meta::external::query::sql::metamodel::Expression[1]; + <> whenClauses: meta::external::query::sql::metamodel::WhenClause[*]; + <> defaultValue: meta::external::query::sql::metamodel::Expression[0..1]; } Class meta::external::query::sql::metamodel::SearchedCaseExpression extends meta::external::query::sql::metamodel::Expression { - whenClauses: meta::external::query::sql::metamodel::WhenClause[*]; - defaultValue: meta::external::query::sql::metamodel::Expression[0..1]; + <> whenClauses: meta::external::query::sql::metamodel::WhenClause[*]; + <> defaultValue: meta::external::query::sql::metamodel::Expression[0..1]; } Class meta::external::query::sql::metamodel::WhenClause extends meta::external::query::sql::metamodel::Expression { - operand: meta::external::query::sql::metamodel::Expression[1]; - result: meta::external::query::sql::metamodel::Expression[1]; + <> operand: meta::external::query::sql::metamodel::Expression[1]; + <> result: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::Extract extends meta::external::query::sql::metamodel::Expression { - expression: meta::external::query::sql::metamodel::Expression[1]; - field: meta::external::query::sql::metamodel::ExtractField[1]; + <> expression: meta::external::query::sql::metamodel::Expression[1]; + <> field: meta::external::query::sql::metamodel::ExtractField[1]; } Class meta::external::query::sql::metamodel::Join extends meta::external::query::sql::metamodel::Relation { - type: meta::external::query::sql::metamodel::JoinType[1]; - left: meta::external::query::sql::metamodel::Relation[1]; - right: meta::external::query::sql::metamodel::Relation[1]; - criteria: meta::external::query::sql::metamodel::JoinCriteria[0..1]; + <> type: meta::external::query::sql::metamodel::JoinType[1]; + <> left: meta::external::query::sql::metamodel::Relation[1]; + <> right: meta::external::query::sql::metamodel::Relation[1]; + <> criteria: meta::external::query::sql::metamodel::JoinCriteria[0..1]; } Class meta::external::query::sql::metamodel::JoinCriteria @@ -285,72 +285,72 @@ Class meta::external::query::sql::metamodel::NaturalJoin extends meta::external: Class meta::external::query::sql::metamodel::JoinOn extends meta::external::query::sql::metamodel::JoinCriteria { - expression: meta::external::query::sql::metamodel::Expression[1]; + <> expression: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::JoinUsing extends meta::external::query::sql::metamodel::JoinCriteria { - columns: String[*]; + <> columns: String[*]; } Class meta::external::query::sql::metamodel::Cast extends meta::external::query::sql::metamodel::Expression { - expression: meta::external::query::sql::metamodel::Expression[1]; - type: meta::external::query::sql::metamodel::ColumnType[1]; + <> expression: meta::external::query::sql::metamodel::Expression[1]; + <> type: meta::external::query::sql::metamodel::ColumnType[1]; } Class meta::external::query::sql::metamodel::ColumnType extends meta::external::query::sql::metamodel::Expression { - name: String[1]; - parameters: Integer[*]; + <> name: String[1]; + <> parameters: Integer[*]; } Class meta::external::query::sql::metamodel::QualifiedName { - parts: String[*]; + <> parts: String[*]; } Class meta::external::query::sql::metamodel::QualifiedNameReference extends meta::external::query::sql::metamodel::Expression { - name: meta::external::query::sql::metamodel::QualifiedName[1]; + <> name: meta::external::query::sql::metamodel::QualifiedName[1]; } Class meta::external::query::sql::metamodel::InListExpression extends meta::external::query::sql::metamodel::Expression { - values: meta::external::query::sql::metamodel::Expression[*]; + <> values: meta::external::query::sql::metamodel::Expression[*]; } Class meta::external::query::sql::metamodel::InPredicate extends meta::external::query::sql::metamodel::Expression { - value: meta::external::query::sql::metamodel::Expression[1]; - valueList: meta::external::query::sql::metamodel::Expression[1]; + <> value: meta::external::query::sql::metamodel::Expression[1]; + <> valueList: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::BetweenPredicate extends meta::external::query::sql::metamodel::Expression { - min: meta::external::query::sql::metamodel::Expression[1]; - max: meta::external::query::sql::metamodel::Expression[1]; - value: meta::external::query::sql::metamodel::Expression[1]; + <> min: meta::external::query::sql::metamodel::Expression[1]; + <> max: meta::external::query::sql::metamodel::Expression[1]; + <> value: meta::external::query::sql::metamodel::Expression[1]; } Class meta::external::query::sql::metamodel::Group extends meta::external::query::sql::metamodel::Statement { - orderBy: meta::external::query::sql::metamodel::SortItem[1]; + <> orderBy: meta::external::query::sql::metamodel::SortItem[1]; } Class meta::external::query::sql::metamodel::Window extends meta::external::query::sql::metamodel::Statement { - windowRef: String[0..1]; - partitions: meta::external::query::sql::metamodel::Expression[*]; - orderBy: meta::external::query::sql::metamodel::SortItem[*]; - windowFrame: meta::external::query::sql::metamodel::WindowFrame[0..1]; + <> windowRef: String[0..1]; + <> partitions: meta::external::query::sql::metamodel::Expression[*]; + <> orderBy: meta::external::query::sql::metamodel::SortItem[*]; + <> windowFrame: meta::external::query::sql::metamodel::WindowFrame[0..1]; } Class meta::external::query::sql::metamodel::WindowFrame extends meta::external::query::sql::metamodel::Node { - mode: meta::external::query::sql::metamodel::WindowFrameMode[1]; - start: meta::external::query::sql::metamodel::FrameBound[1]; - end: meta::external::query::sql::metamodel::FrameBound[0..1]; + <> mode: meta::external::query::sql::metamodel::WindowFrameMode[1]; + <> start: meta::external::query::sql::metamodel::FrameBound[1]; + <> end: meta::external::query::sql::metamodel::FrameBound[0..1]; } Enum meta::external::query::sql::metamodel::WindowFrameMode { @@ -362,8 +362,8 @@ Enum meta::external::query::sql::metamodel::FrameBoundType { Class meta::external::query::sql::metamodel::FrameBound extends meta::external::query::sql::metamodel::Node { - type: meta::external::query::sql::metamodel::FrameBoundType[1]; - value: meta::external::query::sql::metamodel::Expression[0..1]; + <> type: meta::external::query::sql::metamodel::FrameBoundType[1]; + <> value: meta::external::query::sql::metamodel::Expression[0..1]; } Enum meta::external::query::sql::metamodel::TrimMode diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/schema_metamodel.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/schema_metamodel.pure new file mode 100644 index 00000000000..495a29cf632 --- /dev/null +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure-metamodel/src/main/resources/core_external_query_sql_metamodel/schema_metamodel.pure @@ -0,0 +1,41 @@ +Class meta::external::query::sql::schema::metamodel::SchemaColumn +{ + <> name: String[1]; +} + +Class meta::external::query::sql::schema::metamodel::PrimitiveSchemaColumn extends meta::external::query::sql::schema::metamodel::SchemaColumn +{ + <> type: meta::external::query::sql::schema::metamodel::PrimitiveType[1]; +} + +Class meta::external::query::sql::schema::metamodel::EnumSchemaColumn extends meta::external::query::sql::schema::metamodel::SchemaColumn +{ + <> type: String[1]; +} + + +Class meta::external::query::sql::schema::metamodel::Schema +{ + <> columns: meta::external::query::sql::schema::metamodel::SchemaColumn[*]; + <> enums: meta::external::query::sql::schema::metamodel::Enum[*]; +} + +Enum meta::external::query::sql::schema::metamodel::PrimitiveType +{ + Boolean, + StrictDate, + Number, + String, + LatestDate, + Float, + DateTime, + Date, + Integer, + Decimal +} + +Class meta::external::query::sql::schema::metamodel::Enum +{ + <> type: String[1]; + <> values: String[*]; +} \ No newline at end of file diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-pure/pom.xml index 17d3611b561..af4fbb211de 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-sql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure index 811882e12af..f451545280f 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/fromPure.pure @@ -155,13 +155,13 @@ function meta::external::query::sql::transformation::queryToPure::getSchemaFromS sources: SQLSource[*], query: meta::external::query::sql::metamodel::Node[1], extensions: meta::pure::extension::Extension[*] - ): meta::external::query::sql::Schema[1] + ): meta::external::query::sql::schema::metamodel::Schema[1] { let sqlTransformContext = processRootQuery($sources, $query, $extensions); getSchema($sqlTransformContext); } -function meta::external::query::sql::transformation::queryToPure::getSchema(context:SqlTransformContext[1]): meta::external::query::sql::Schema[1] +function meta::external::query::sql::transformation::queryToPure::getSchema(context:SqlTransformContext[1]): meta::external::query::sql::schema::metamodel::Schema[1] { $context.columns()->meta::external::query::sql::tdsColsToSchema(); } @@ -291,6 +291,9 @@ function <> meta::external::query::sql::transformation::queryToP l:LogicalBinaryExpression[1] | $l.left->extractAggregatesFromExpression()->concatenate($l.right->extractAggregatesFromExpression()), n:NegativeExpression[1] | $n.value->extractAggregatesFromExpression(), n:NotExpression[1] | $n.value->extractAggregatesFromExpression(), + s:SimpleCaseExpression[1] | $s->convertToSearchedCaseExpression()->extractAggregatesFromExpression(), + s:SearchedCaseExpression[1] | + $s.whenClauses->map(w | $w.operand->extractAggregatesFromExpression()->concatenate($w.result->extractAggregatesFromExpression()))->concatenate($s.defaultValue->extractAggregatesFromExpression());, e:meta::external::query::sql::metamodel::Expression[0..1] | [] ]); } @@ -407,18 +410,24 @@ function <> meta::external::query::sql::transformation::queryToP let aggregateExpression = extractAggregatesFromExpression($column.expression); - let functionCall = $aggregateExpression->match([ - f:FunctionCall[1] | $f, + let functionCalls = $aggregateExpression->match([ + f:FunctionCall[1] | ensureAggregateFunction($f), + f:FunctionCall[1..*] | + // we can only support multiple aggs being used within a single expression if they use the case arguments + //e.g. case when sum(col) > 10 then max(col) else min(col) end + //to support more complex cases we need to extract the aggregates into separate columns + assertEquals(1, $f.arguments->removeDuplicates()->size(), 'unsupported aggregation'); + + $f->map(fc | ensureAggregateFunction($fc));, e:meta::external::query::sql::metamodel::Expression[*] | fail('invalid aggregate expression'); - ])->toOne()->cast(@FunctionCall); + ])->cast(@FunctionCall); - let mapExpression = processMapFunction($functionCall, $context); + // we take the first as the map only cares about the argument and we ensure they are the same above + let mapExpression = processMapFunction($functionCalls->at(0), $context); let mapReturnType = $mapExpression.genericType.rawType->toOne(); let mapFunctionType = functionType('row', TDSRow, PureOne, $mapReturnType, ZeroOne); let mapLambda = lambda($mapFunctionType, $mapExpression); - ensureAggregateFunction($functionCall); - let aggLambda = processAggregationLambda($column.expression, $mapReturnType, $context); let aggReturnType = $aggLambda->functionReturnType(); @@ -524,10 +533,11 @@ function <> meta::external::query::sql::transformation::queryToP } //We currently need to ensure that all the functions used in group by are in fact aggregate functions. e.g. select count(*), pi() would not currently translate well to Pure. -function <> meta::external::query::sql::transformation::queryToPure::ensureAggregateFunction(functionCall:FunctionCall[1]):Boolean[1] +function <> meta::external::query::sql::transformation::queryToPure::ensureAggregateFunction(functionCall:FunctionCall[1]):FunctionCall[1] { let processor = functionProcessor($functionCall.name); assert($processor.isAggregate, | 'function ' + extractNameFromQualifiedName($functionCall.name, []) + ' is not currently supported as aggregate function'); + $functionCall; } function <> meta::external::query::sql::transformation::queryToPure::processExtend(select: Select[1], context: SqlTransformContext[1]):FunctionExpression[1] @@ -551,7 +561,7 @@ function <> meta::external::query::sql::transformation::queryToP if (isSelectStar($select), | $context.expression->toOne(), | if (allColumnsSimpleSelect($select), - | processSelectToRenameAndRestrict($select, $restrict, $context), + | processSelectToRestrictAndRename($select, $restrict, $context), | processSelectToProject($select, $context))); } @@ -567,7 +577,7 @@ function <> meta::external::query::sql::transformation::queryToP $si->instanceOf(SingleColumn) && isExpressionAggregate($si->cast(@SingleColumn).expression, true, false) } -function <> meta::external::query::sql::transformation::queryToPure::isExpressionAggregate(e:meta::external::query::sql::metamodel::Expression[1], includeParameters:Boolean[1], includeWindow:Boolean[1]):Boolean[1] +function <> meta::external::query::sql::transformation::queryToPure::isExpressionAggregate(e:meta::external::query::sql::metamodel::Expression[0..1], includeParameters:Boolean[1], includeWindow:Boolean[1]):Boolean[1] { $e->match([ a:ArithmeticExpression[1] | $a.left->isExpressionAggregate($includeParameters, $includeWindow) || $a.right->isExpressionAggregate($includeParameters, $includeWindow), @@ -582,6 +592,10 @@ function <> meta::external::query::sql::transformation::queryToP l:LogicalBinaryExpression[1] | $l.left->isExpressionAggregate($includeParameters, $includeWindow) || $l.right->isExpressionAggregate($includeParameters, $includeWindow), n:NegativeExpression[1] | $n.value->isExpressionAggregate($includeParameters, $includeWindow), n:NotExpression[1] | $n.value->isExpressionAggregate($includeParameters, $includeWindow), + s:SimpleCaseExpression[1] | $s->convertToSearchedCaseExpression()->isExpressionAggregate($includeParameters, $includeWindow), + s:SearchedCaseExpression[1] | + $s.whenClauses->exists(w | $w.operand->isExpressionAggregate($includeParameters, $includeWindow) || $w.result->isExpressionAggregate($includeParameters, $includeWindow)) + || $s.defaultValue->isExpressionAggregate($includeParameters, $includeWindow), e:meta::external::query::sql::metamodel::Expression[0..1] | false ]) } @@ -613,12 +627,18 @@ function <> meta::external::query::sql::transformation::queryToP appendTdsFunc($context.expression->toOne(), project_TabularDataSet_1__ColumnSpecification_MANY__TabularDataSet_1_, list($iv)); } -function <> meta::external::query::sql::transformation::queryToPure::processSelectToRenameAndRestrict(select: Select[1], restrict:Boolean[1], context: SqlTransformContext[1]):FunctionExpression[1] +function <> meta::external::query::sql::transformation::queryToPure::processSelectToRestrictAndRename(select: Select[1], restrict:Boolean[1], context: SqlTransformContext[1]):FunctionExpression[1] { debug('processSelectToRestrict', $context.debug); - let rename = $select.selectItems->cast(@SingleColumn)->processRename($context); - if ($restrict, | processRestrict($select.selectItems->processSelectItems($context, true).second, ^$context(expression = $rename)), | $rename); + let restricted = if ($restrict, + | $select.selectItems->map(si | $si->match([ + s:SingleColumn[1] | ^$s(alias = []), + s:SelectItem[1] | $s + ]))->processSelectItems($context, true).second->processRestrict($context), + | $context.expression); + + $select.selectItems->cast(@SingleColumn)->processRename(^$context(expression = $restricted)); } function <> meta::external::query::sql::transformation::queryToPure::processRestrict(columns:String[*], context: SqlTransformContext[1]):FunctionExpression[1] @@ -638,7 +658,7 @@ function <> meta::external::query::sql::transformation::queryToP if ($si.alias->isNotEmpty() && $si.alias != $defaultName, - | + | sfe(pair_U_1__V_1__Pair_1_, ^GenericType(rawType = Pair, typeArguments = $typeArguments), $typeArguments, [iv($defaultName), $si.alias->toOne()->iv()]), | []); ); @@ -923,7 +943,9 @@ function <> meta::external::query::sql::transformation::queryToP function meta::external::query::sql::transformation::queryToPure::extractSourceArguments(expressions:meta::external::query::sql::metamodel::Expression[*]):SQLSourceArgument[*] { - $expressions->map(e | $e->extractSourceArgument($expressions->indexOf($e), [])); + range(0, $expressions->size())->map(i | + extractSourceArgument($expressions->at($i), $i, []) + ); } function meta::external::query::sql::transformation::queryToPure::extractSourceArgument(expression:meta::external::query::sql::metamodel::Expression[1], index:Integer[1], name:String[0..1]):SQLSourceArgument[0..1] @@ -1263,11 +1285,7 @@ function meta::external::query::sql::transformation::queryToPure::extractNameFro n:NegativeExpression[1] | '-' + $n.value->extractNameFromExpression($context), n:NotExpression[1] | 'NOT ' + $n.value->extractNameFromExpression($context), q:QualifiedNameReference[1] | $q.name->extractNameFromQualifiedName($context), - s:SimpleCaseExpression[1] | - ^SearchedCaseExpression( - whenClauses = $s.whenClauses->map(wc | ^$wc(operand = ^ComparisonExpression(left = $s.operand, right = $wc.operand, operator = ComparisonOperator.EQUAL))), - defaultValue = $s.defaultValue - )->extractNameFromExpression($context), + s:SimpleCaseExpression[1] | $s->convertToSearchedCaseExpression()->extractNameFromExpression($context), s:SearchedCaseExpression[1] | let when = $s.whenClauses->map(w | $w->extractNameFromExpression($context))->joinStrings(' '); let default = if ($s.defaultValue->isEmpty(), | '', | ' ELSE ' + $s.defaultValue->toOne()->extractNameFromExpression($context)); @@ -1628,12 +1646,17 @@ function <> meta::external::query::sql::transformation::queryToP function <> meta::external::query::sql::transformation::queryToPure::processSimpleCaseExpression(s:SimpleCaseExpression[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):SimpleFunctionExpression[1] { debug('processSimpleCaseExpression', $context.debug); - let sce = ^SearchedCaseExpression( + let sce = $s->convertToSearchedCaseExpression(); + + processSearchedCaseExpression($sce, $expContext, $context); +} + +function <> meta::external::query::sql::transformation::queryToPure::convertToSearchedCaseExpression(s:SimpleCaseExpression[1]):SearchedCaseExpression[1] +{ + ^SearchedCaseExpression( whenClauses = $s.whenClauses->map(wc | ^$wc(operand = ^ComparisonExpression(left = $s.operand, right = $wc.operand, operator = ComparisonOperator.EQUAL))), defaultValue = $s.defaultValue ); - - processSearchedCaseExpression($sce, $expContext, $context); } function <> meta::external::query::sql::transformation::queryToPure::processWhenClause(s:WhenClause[1], else:ValueSpecification[1], expContext:SqlTransformExpressionContext[1], context:SqlTransformContext[1]):ValueSpecification[1] @@ -1918,7 +1941,9 @@ function meta::external::query::sql::transformation::queryToPure::functionProces processor('btrim', String, {args, fc, ctx | processTrim(trim_String_1__String_1_, $args)}), processor('char_length', length_String_1__Integer_1_), processor('chr', char_Integer_1__String_1_), - processor('concat', plus_String_MANY__String_1_), + processor('concat', String, {args, fc, ctx | + sfe(plus_String_MANY__String_1_, iv($args)) + }), processor('length', length_String_1__Integer_1_), processor('lower', toLower_String_1__String_1_), processor('ltrim', String, {args, fc, ctx | processTrim(ltrim_String_1__String_1_, $args)}), @@ -1929,6 +1954,21 @@ function meta::external::query::sql::transformation::queryToPure::functionProces processor('reverse', reverseString_String_1__String_1_), processor('rtrim', String, {args, fc, ctx | processTrim(rtrim_String_1__String_1_, $args)}), processor('sha256', String, {args, fc, ctx | processHash($args, meta::pure::functions::hash::HashType.SHA256)}), + processor('split_part', String, {args, fc, ctx | + assertEquals(3, $args->size(), 'split_part must specify three arguments'); + + let position = $args->at(2)->match([ + i:InstanceValue[1] | $i.values->match([ + i:Integer[1] | iv($i + 1), + a:Any[*] | fail('invalid split part position'); iv(1); + ]), + v:ValueSpecification[1] | sfe(plus_Integer_MANY__Integer_1_, iv([$args->at(2), iv(1)])) + ]); + + let arguments = [$args->at(0), $args->at(1), $position]; + + sfe(splitPart_String_$0_1$__String_1__Integer_1__String_$0_1$_, $arguments); + }), processor('starts_with', startsWith_String_1__String_1__Boolean_1_), processor('string_agg', true, false, String, {args, fc, ctx | @@ -2883,4 +2923,4 @@ function meta::external::query::sql::transformation::queryToPure::getParameters( function meta::external::query::sql::transformation::queryToPure::debug(a:String[1], debug:DebugContext[1]):Any[0] { if ($debug.debug, | println($debug.space + $a), | []); -} \ No newline at end of file +} diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure index ef202aa134f..1d745b94da8 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/fromPure/tests/testTranspile.pure @@ -19,6 +19,7 @@ import meta::external::query::sql::transformation::queryToPure::*; import meta::external::query::sql::metamodel::*; import meta::external::query::sql::transformation::queryToPure::tests::*; import meta::external::query::sql::*; +import meta::external::query::sql::schema::metamodel::*; import meta::legend::service::metamodel::*; import meta::pure::functions::meta::*; import meta::pure::mapping::*; @@ -27,34 +28,36 @@ import meta::pure::metamodel::serialization::grammar::*; //SELECT function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectStar():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + } + ) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectNull():Boolean[1] { - let sqlString = 'SELECT NULL FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT NULL FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->project([ col(row:TDSRow[1] | []->cast(@String), 'NULL') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + } + ); } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectStarFromJoinTable():Boolean[1] { - let sqlString = 'SELECT table1.* FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT * FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."Integer"'; + test( + 'SELECT table1.* FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT * FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."Integer"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String']) ->renameColumns([ @@ -83,16 +86,16 @@ function <> meta::external::query::sql::transformation::queryToPure:: pair('DateTime_table1', 'DateTime'), pair('String_table1', 'String') ])->restrict(['Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + } + ) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectStarAndColumnsFromJoinTable():Boolean[1] { - let sqlString = 'SELECT table1.*, table2.id, sin(table2.int) FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT ID as "id", Integer AS "int" FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."int"'; + test( + 'SELECT table1.*, table2.id, sin(table2.int) FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT ID as "id", Integer AS "int" FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."int"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String'])->renameColumns([ pair('Boolean', 'Boolean_table1'), @@ -106,10 +109,10 @@ function <> meta::external::query::sql::transformation::queryToPure:: ->join(FlatInput.all()->project( [x | $x.idIn, x | $x.integerIn, x | $x.stringIn], ['ID', 'Integer', 'String'] - )->renameColumns([ + )->restrict(['ID', 'Integer'])->renameColumns([ pair('ID', 'id'), pair('Integer', 'int') - ])->restrict(['id', 'int'])->renameColumns([ + ])->renameColumns([ pair('id', 'id_table2'), pair('int', 'int_table2') ]), meta::relational::metamodel::join::JoinType.LEFT_OUTER, {row1:TDSRow[1], row2:TDSRow[1] | $row1.getInteger('Integer_table1') == $row2.getInteger('int_table2')} @@ -124,347 +127,358 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | $row.getInteger('id_table2'), 'id'), col(row:TDSRow[1] | sin($row.getInteger('int_table2')), 'sin(int)') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + } + ) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectColumns():Boolean[1] { - let sqlString = 'SELECT Boolean, Integer FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Boolean, Integer FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Boolean', 'Integer']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectWithAliases():Boolean[1] { - let sqlString = 'SELECT Boolean, Integer AS int FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Boolean, Integer AS int FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) + ->restrict(['Boolean', 'Integer']) ->renameColumns(pair('Integer', 'int')) - ->restrict(['Boolean', 'int']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectColumnMultiTimes():Boolean[1] { - let sqlString = 'SELECT String, String as "str" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String, String as "str" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->project([ col(row:TDSRow[1] | $row.getString('String'), 'String'), col(row:TDSRow[1] | $row.getString('String'), 'str') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectColumnMultiTimesRealiasToExisting():Boolean[1] { - let sqlString = 'SELECT String as "str", String as "String" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String as "str", String as "String" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->project([ col(row:TDSRow[1] | $row.getString('String'), 'str'), col(row:TDSRow[1] | $row.getString('String'), 'String') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) +} + +function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectColumnAliasedAsUnusedTableColumnName():Boolean[1] +{ + test( + 'SELECT Integer as "String" FROM service."/service/service1"', + + {| FlatInput.all()->project( + [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], + [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) + ->restrict('Integer')->renameColumns(pair('Integer', 'String')) + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectColumnMultiTimesGroupBy():Boolean[1] { - let sqlString = 'SELECT String, String as "str" FROM service."/service/service1" group by 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String, String as "str" FROM service."/service/service1" group by 1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->project([ col(row:TDSRow[1] | $row.getString('String'), 'String'), col(row:TDSRow[1] | $row.getString('String'), 'str') ])->restrict(['String', 'str'])->distinct() - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectQualified():Boolean[1] { - let sqlString = 'SELECT Boolean, service."/service/service1".Integer FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Boolean, service."/service/service1".Integer FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Boolean', 'Integer']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectQualifiedWithAlias():Boolean[1] { - let sqlString = 'SELECT Boolean, table1.Integer FROM service."/service/service1" AS table1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Boolean, table1.Integer FROM service."/service/service1" AS table1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Boolean', 'Integer']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testTableFunc():Boolean[1] { - let sqlString = 'SELECT Boolean, table1.Integer FROM service(\'/service/service1\') AS table1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Boolean, table1.Integer FROM service(\'/service/service1\') AS table1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Boolean', 'Integer']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectQuotedSingleColumn():Boolean[1] { - let sqlString = 'SELECT "Boolean", "Integer" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT "Boolean", "Integer" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Boolean', 'Integer']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSelectUnquotedSingleColumn():Boolean[1] { - let sqlString = 'SELECT Boolean, Integer FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Boolean, Integer FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Boolean', 'Integer']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testSimplificationOfNullablesComplex():Boolean[1] { - let sqlString = 'SELECT (date_trunc(\'day\', CAST(\'1900-01-01\' AS DATE) + NULL * INTERVAL \'1 DAY\') + 1 * INTERVAL \'1 YEAR\') AS "NULL1", ' + - 'CAST( FLOOR(EXTRACT(EPOCH FROM CAST((CAST(\'1900-01-01 00:00:00\' AS TIMESTAMP) + NULL * INTERVAL \'1 DAY\') AS TIMESTAMP)) / 86400) - FLOOR(EXTRACT(EPOCH FROM CAST((CAST(\'1900-01-01 00:00:00\' AS TIMESTAMP) + NULL * INTERVAL \'1 DAY\') AS TIMESTAMP)) / 86400) AS BIGINT) AS "NULL2" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); + test( + 'SELECT (date_trunc(\'day\', CAST(\'1900-01-01\' AS DATE) + NULL * INTERVAL \'1 DAY\') + 1 * INTERVAL \'1 YEAR\') AS "NULL1", ' + + 'CAST( FLOOR(EXTRACT(EPOCH FROM CAST((CAST(\'1900-01-01 00:00:00\' AS TIMESTAMP) + NULL * INTERVAL \'1 DAY\') AS TIMESTAMP)) / 86400) - ' + + 'FLOOR(EXTRACT(EPOCH FROM CAST((CAST(\'1900-01-01 00:00:00\' AS TIMESTAMP) + NULL * INTERVAL \'1 DAY\') AS TIMESTAMP)) / 86400) AS BIGINT) AS "NULL2" FROM service."/service/service1"', - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->project([ col(row:TDSRow[1] | []->cast(@Date), 'NULL1'), col(row:TDSRow[1] | []->cast(@Integer), 'NULL2') ]) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //DISTINCT function <> meta::external::query::sql::transformation::queryToPure::tests::testDistinctAllColumns():Boolean[1] { - let sqlString = 'SELECT DISTINCT * FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT DISTINCT * FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->distinct() - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testDistinctSingleColumn():Boolean[1] { - let sqlString = 'SELECT DISTINCT Boolean FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT DISTINCT Boolean FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Boolean']) ->distinct() - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //LIMIT-OFFSET function <> meta::external::query::sql::transformation::queryToPure::tests::testLimit():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" LIMIT 2'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" LIMIT 2', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->limit(2) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testLimitAll():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" LIMIT ALL'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" LIMIT ALL', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testOffset():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" OFFSET 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" OFFSET 1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->drop(1) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testLimitOffset():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" OFFSET 2 LIMIT 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" OFFSET 2 LIMIT 1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->slice(2, 2 + 1) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //ORDER BY function <> meta::external::query::sql::transformation::queryToPure::tests::testOrderBy():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" ORDER BY Integer DESC, Boolean ASC'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" ORDER BY Integer DESC, Boolean ASC', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->sort([desc('Integer'), asc('Boolean')]) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testOrderByAlias():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" AS table1 ORDER BY table1.Integer DESC, Boolean'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" AS table1 ORDER BY table1.Integer DESC, Boolean', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->sort([desc('Integer'), asc('Boolean')]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testOrderByQualified():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" ORDER BY service."/service/service1".Integer DESC, Boolean ASC'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" ORDER BY service."/service/service1".Integer DESC, Boolean ASC', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->sort([desc('Integer'), asc('Boolean')]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testOrderByIndex():Boolean[1] { - let sqlString = 'SELECT Integer, String FROM service."/service/service1" ORDER BY 1 DESC, 2 ASC'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Integer, String FROM service."/service/service1" ORDER BY 1 DESC, 2 ASC', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->restrict(['Integer', 'String']) ->sort([desc('Integer'), asc('String')]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //WHERE function <> meta::external::query::sql::transformation::queryToPure::tests::testWhere():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE Integer != 2 AND Integer <> 3'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE Integer != 2 AND Integer <> 3', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | ($row.getInteger('Integer') != 2) && ($row.getInteger('Integer') != 3)) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testCompositeWhere():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE (Integer = 2 AND String = \'abc\') OR (String = \'def\' AND Integer = 1)'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE (Integer = 2 AND String = \'abc\') OR (String = \'def\' AND Integer = 1)', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | ($row.getInteger('Integer') == 2 && $row.getString('String') == 'abc') || ($row.getString('String') == 'def' && $row.getInteger('Integer') == 1)) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testWhereAliases():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" AS table1 WHERE Integer = 2 OR table1.Integer = 3'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" AS table1 WHERE Integer = 2 OR table1.Integer = 3', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | $row.getInteger('Integer') == 2 || $row.getInteger('Integer') == 3) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testWhereQualified():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE service."/service/service1".Integer = 2 OR Integer = 3'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE service."/service/service1".Integer = 2 OR Integer = 3', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | $row.getInteger('Integer') == 2 || $row.getInteger('Integer') == 3) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testInAndNullOperators():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE Integer IN (1,2,3) OR Integer IS NULL OR Integer IS NOT NULL'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE Integer IN (1,2,3) OR Integer IS NULL OR Integer IS NOT NULL', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | $row.getInteger('Integer')->in([1,2,3]) || $row.getInteger('Integer')->isEmpty() || $row.getInteger('Integer')->isNotEmpty()) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testComparisonOperatorsNumber():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE Integer > 1 OR Integer < 1 OR Integer >= 1 OR Integer <= 1 OR Integer BETWEEN 0 AND 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE Integer > 1 OR Integer < 1 OR Integer >= 1 OR Integer <= 1 OR Integer BETWEEN 0 AND 1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | ($row.getInteger('Integer') > 1) @@ -472,52 +486,52 @@ function <> meta::external::query::sql::transformation::queryToPure:: || ($row.getInteger('Integer') >= 1) || ($row.getInteger('Integer') <= 1) || (($row.getInteger('Integer') >= 0) && ($row.getInteger('Integer') <= 1))) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testComparisonOperatorsString():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE String > \'one\' OR String < \'one\' OR String >= \'one\' OR String <= \'one\''; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE String > \'one\' OR String < \'one\' OR String >= \'one\' OR String <= \'one\'', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | ($row.getString('String') > 'one') || ($row.getString('String') < 'one') || ($row.getString('String') >= 'one') || ($row.getString('String') <= 'one')) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testComparisonOperatorsBoolean():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE Boolean > true OR Boolean < true OR Boolean >= false OR Boolean <= false'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE Boolean > true OR Boolean < true OR Boolean >= false OR Boolean <= false', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | ($row.getBoolean('Boolean') > true) || ($row.getBoolean('Boolean') < true) || ($row.getBoolean('Boolean') >= false) || ($row.getBoolean('Boolean') <= false)) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testComparisonOperatorsDate():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE StrictDate > TIMESTAMP \'2023-01-01\' OR StrictDate < CURRENT_DATE OR StrictDate >= TIMESTAMP \'2023-01-02\' OR StrictDate <= CURRENT_DATE OR StrictDate > \'2023-01-03\''; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE StrictDate > TIMESTAMP \'2023-01-01\' OR StrictDate < CURRENT_DATE OR StrictDate >= TIMESTAMP \'2023-01-02\' OR StrictDate <= CURRENT_DATE OR StrictDate > \'2023-01-03\'', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | ($row.getStrictDate('StrictDate') > parseDate('2023-01-01')) || ($row.getStrictDate('StrictDate') < today()) || ($row.getStrictDate('StrictDate') >= parseDate('2023-01-02')) || ($row.getStrictDate('StrictDate') <= today()) || ($row.getStrictDate('StrictDate') > parseDate('2023-01-03'))) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testComparisonOperatorsEnum():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service3" WHERE "The Enum Value" > \'Value2\' or "The Enum Value" < \'Value_3\' or "The Enum Value" >= \'Value2\' or "The Enum Value" <= \'Value_3\' or "The Enum Value" = \'Value1\' or "The Enum Value" != \'Value1\''; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + test( + 'SELECT * FROM service."/service/service3" WHERE "The Enum Value" > \'Value2\' or "The Enum Value" < \'Value_3\' or "The Enum Value" >= \'Value2\' or "The Enum Value" <= \'Value_3\' or "The Enum Value" = \'Value1\' or "The Enum Value" != \'Value1\'', + + {| let const = 123; FlatInput.all()->project( [ @@ -533,15 +547,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: || $row.getEnum('The Enum Value') == MyEnum.Value1 || $row.getEnum('The Enum Value') != MyEnum.Value1 ) - ;}; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + ;}, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testDistinctFromOperator():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1" WHERE String IS DISTINCT FROM \'ABC\' OR String IS NOT DISTINCT FROM \'DEF\''; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1" WHERE String IS DISTINCT FROM \'ABC\' OR String IS NOT DISTINCT FROM \'DEF\'', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | @@ -549,100 +563,99 @@ function <> meta::external::query::sql::transformation::queryToPure:: || (($row.getString('String')->isEmpty() && 'DEF'->isEmpty()) || ($row.getString('String') == 'DEF')) ) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testWhereCombined():Boolean[1] { - let sqlString = 'SELECT String, sum(Float) AS "float" FROM service."/service/service1" WHERE Integer = 1 GROUP BY 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String, sum(Float) AS "float" FROM service."/service/service1" WHERE Integer = 1 GROUP BY 1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->filter(row | ($row.getInteger('Integer') == 1)) ->groupBy('String', agg('float', row | $row.getFloat('Float'), y | $y->sum())) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } //GROUP BY function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupBy():Boolean[1] { - let sqlString = 'SELECT count(Integer) AS "count" FROM service."/service/service1" GROUP BY String'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT count(Integer) AS "count" FROM service."/service/service1" GROUP BY String', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->groupBy('String', agg('count', row | $row.getInteger('Integer'), y | $y->count())) ->restrict('count') - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByNoAggregates():Boolean[1] { - let sqlString = 'SELECT String FROM service."/service/service1" GROUP BY 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String FROM service."/service/service1" GROUP BY 1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->restrict('String')->distinct() - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByIndex():Boolean[1] { - let sqlString = 'SELECT String, sum(Integer) AS "sum" FROM service."/service/service1" GROUP BY 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String, sum(Integer) AS "sum" FROM service."/service/service1" GROUP BY 1', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->groupBy(['String'], agg('sum', row | $row.getInteger('Integer'), y | $y->sum())) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByIndexMultipleWithAlias():Boolean[1] { - let sqlString = 'SELECT String, sum(Integer) AS "sum", Boolean AS "bool" FROM service."/service/service1" GROUP BY 1, 3'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String, sum(Integer) AS "sum", Boolean AS "bool" FROM service."/service/service1" GROUP BY 1, 3', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->renameColumns(pair('Boolean', 'bool')) ->groupBy(['String', 'bool'], agg('sum', row | $row.getInteger('Integer'), y | $y->sum())) ->restrict(['String', 'sum', 'bool']) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByWithAlias():Boolean[1] { - let sqlString = 'SELECT String, sum(Integer) AS "sum", Boolean AS "bool" FROM service."/service/service1" GROUP BY String, "bool"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT String, sum(Integer) AS "sum", Boolean AS "bool" FROM service."/service/service1" GROUP BY String, "bool"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->renameColumns(pair('Boolean', 'bool')) ->groupBy(['String', 'bool'], agg('sum', row | $row.getInteger('Integer'), y | $y->sum())) ->restrict(['String', 'sum', 'bool']) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByFunctions():Boolean[1] { - let sqlString = 'SELECT count(Integer) AS "count", count(DISTINCT Integer) AS "distinctCount", SUM(Integer) AS "sum", avg(Integer) AS "avg", ' + + test('SELECT count(Integer) AS "count", count(DISTINCT Integer) AS "distinctCount", SUM(Integer) AS "sum", avg(Integer) AS "avg", ' + 'stddev_pop(Integer) AS "stdDevPop", stddev_samp(Integer) AS "stdDevSamp", stddev(Integer) AS "stdDev", ' + 'var_pop(Integer) AS "variancePop", var_samp(Integer) AS "varianceSamp", variance(Integer) AS "variance", ' + 'min(Integer) AS "minInt", min(StrictDate) AS "minDate", min(String) AS "minString", ' + 'max(Integer) AS "maxInt", max(StrictDate) AS "maxDate", max(String) AS "maxString", ' + - 'string_agg(String) AS "stringAgg", string_agg(cast(Integer AS VARCHAR), \' \') AS "stringAgg2" FROM service."/service/service1" GROUP BY String'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + 'string_agg(String) AS "stringAgg", string_agg(cast(Integer AS VARCHAR), \' \') AS "stringAgg2" FROM service."/service/service1" GROUP BY String', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->groupBy('String', [ @@ -665,15 +678,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: agg('stringAgg', row | $row.getString('String'), y | $y->joinStrings()), agg('stringAgg2', row | $row.getInteger('Integer')->toString(), y | $y->joinStrings(' ')) ])->restrict(['count', 'distinctCount', 'sum', 'avg', 'stdDevPop', 'stdDevSamp', 'stdDev', 'variancePop', 'varianceSamp', 'variance', 'minInt', 'minDate', 'minString', 'maxInt', 'maxDate', 'maxString', 'stringAgg', 'stringAgg2']) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByAggregateExpressions():Boolean[1] { - let sqlString = 'SELECT Integer, sum(1) AS "sum", count(*) AS "count", sum(Integer + Float) AS "exp", EXTRACT(YEAR FROM MAX("StrictDate")) AS "aggFunctionCall", MAX("StrictDate") + INTERVAL \'1 WEEK\' AS "dateAggMath" FROM service."/service/service1" GROUP BY Integer'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT Integer, sum(1) AS "sum", count(*) AS "count", sum(Integer + Float) AS "exp", EXTRACT(YEAR FROM MAX("StrictDate")) AS "aggFunctionCall", MAX("StrictDate") + INTERVAL \'1 WEEK\' AS "dateAggMath" FROM service."/service/service1" GROUP BY Integer', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->groupBy(['Integer'], [ @@ -683,15 +696,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: agg('aggFunctionCall', row | $row.getStrictDate('StrictDate'), y | $y->max()->toOne()->year()), agg('dateAggMath', row | $row.getStrictDate('StrictDate'), y | $y->max()->toOne()->adjust(1, DurationUnit.WEEKS)) ]) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByMixed():Boolean[1] { - let sqlString = 'SELECT sum(Integer) AS "SUM", CASE WHEN String = \'abc\' THEN 1 WHEN String = \'def\' THEN 2 ELSE 3 END AS "Number" FROM service."/service/service1" GROUP BY 2'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT sum(Integer) AS "SUM", CASE WHEN String = \'abc\' THEN 1 WHEN String = \'def\' THEN 2 ELSE 3 END AS "Number" FROM service."/service/service1" GROUP BY 2', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->extend( @@ -699,16 +712,31 @@ function <> meta::external::query::sql::transformation::queryToPure:: )->groupBy(['Number'], [ agg('SUM', row | $row.getInteger('Integer'), y | $y->sum()) ])->restrict(['SUM', 'Number']) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) +} + +function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByAggWithinCase():Boolean[1] +{ + test('SELECT CASE WHEN(SUM("Integer") <10) THEN max("Integer") ELSE min("Integer") END AS "MIN/MAX", ' + + 'CASE WHEN(SUM("Integer") < 10) THEN \'LOW\' ELSE \'HIGH\' END AS "HIGH/LOW" FROM service."/service/service1"', + + {| FlatInput.all()->project( + [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], + [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] + )->groupBy([], [ + agg('MIN/MAX', row | $row.getInteger('Integer'), y | if ($y->sum() < 10, | max($y), | min($y))), + agg('HIGH/LOW', row | $row.getInteger('Integer'), y | if ($y->sum() < 10, | 'LOW', | 'HIGH')) + ]) + }, false) } //HAVING function <> meta::external::query::sql::transformation::queryToPure::tests::testHaving():Boolean[1] { - let sqlString = 'SELECT count(Integer) AS "count", max(Integer) FROM service."/service/service1" GROUP BY String HAVING count(Integer) > 0 AND count(1) > 0 AND floor(max(Integer)) > 0'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT count(Integer) AS "count", max(Integer) FROM service."/service/service1" GROUP BY String HAVING count(Integer) > 0 AND count(1) > 0 AND floor(max(Integer)) > 0', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->groupBy('String', [ @@ -718,15 +746,14 @@ function <> meta::external::query::sql::transformation::queryToPure:: ])->filter(row | ($row.getInteger('count') > 0) && ($row.getInteger('count(1)') > 0) && (floor($row.getInteger('max(Integer)')) > 0)) ->restrict(['count', 'max(Integer)']) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testHavingNoGroupby():Boolean[1] { - let sqlString = 'SELECT \'abc\', pi() AS "pi" FROM service."/service/service1" HAVING count(1) > 0 AND count(1) < 1000'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test('SELECT \'abc\', pi() AS "pi" FROM service."/service/service1" HAVING count(1) > 0 AND count(1) < 1000', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->extend([ @@ -738,15 +765,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: ])->filter(row | ($row.getInteger('count(1)') > 0) && ($row.getInteger('count(1)') < 1000)) ->restrict(['abc', 'pi']) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testHavingWithJoinAndColumnAlias():Boolean[1] { - let sqlString = 'SELECT table1.*, table2.String as "str" FROM service(\'/service/service1\') "table1" LEFT JOIN service(\'/service/service1\') "table2" USING (String) HAVING (COUNT(1) > 0)'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT table1.*, table2.String as "str" FROM service(\'/service/service1\') "table1" LEFT JOIN service(\'/service/service1\') "table2" USING (String) HAVING (COUNT(1) > 0)', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->renameColumns([ @@ -787,36 +814,35 @@ function <> meta::external::query::sql::transformation::queryToPure:: pair('String_table1', 'String') ]) ->restrict(['Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String', 'str']) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testHavingSelectStar():Boolean[1] { - let sqlString = 'SELECT * FROM service(\'/service/service1\') HAVING (COUNT(1) > 0)'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service(\'/service/service1\') HAVING (COUNT(1) > 0)', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->groupBy(['Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String'], agg('COUNT(1)', row | 1, y | $y->count()))->filter(row | $row.getInteger('COUNT(1)') > 0) ->restrict([ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) - }; - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } //CAST function <> meta::external::query::sql::transformation::queryToPure::tests::testCasts():Boolean[1] { - let sqlString = 'SELECT' - + ' CAST(\'2023-01-01\' AS DATE) AS "constant", CAST(String AS VARCHAR) AS "string", CAST(String AS TEXT) AS "text", CAST(String AS DATE) AS "date",' - + ' CAST(String AS INTEGER) AS "integer", CAST(String AS BIGINT) AS "bigint", CAST(String AS SMALLINT) AS "smallint", CAST(String AS BOOLEAN) AS "boolean", ' - + ' CAST(String AS DOUBLE PRECISION) AS "double", CAST(String AS NUMERIC) AS "numeric",' - + ' CAST(String AS TIMESTAMP) AS "timestamp", CAST(Integer AS TEXT) AS "integerText", CAST(Integer AS VARCHAR) AS "integerString", CAST(Integer AS Integer) AS "expression",' - + ' CAST(String AS VARCHAR(2)) AS "stringChars", CAST(Integer AS VARCHAR(2)) AS "integerStringChars",' - + ' CAST(Float AS NUMERIC) AS "floatNumeric", CAST(Decimal AS NUMERIC) AS "decimalNumeric", CAST(Decimal AS DOUBLE PRECISION) AS "decimalDoublePrecision",' - + ' CAST(Float AS DOUBLE PRECISION) AS "floatDoublePrecision", CAST(String AS NUMERIC(4, 2)) AS "numericParams" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test('SELECT' + + ' CAST(\'2023-01-01\' AS DATE) AS "constant", CAST(String AS VARCHAR) AS "string", CAST(String AS TEXT) AS "text", CAST(String AS DATE) AS "date",' + + ' CAST(String AS INTEGER) AS "integer", CAST(String AS BIGINT) AS "bigint", CAST(String AS SMALLINT) AS "smallint", CAST(String AS BOOLEAN) AS "boolean", ' + + ' CAST(String AS DOUBLE PRECISION) AS "double", CAST(String AS NUMERIC) AS "numeric",' + + ' CAST(String AS TIMESTAMP) AS "timestamp", CAST(Integer AS TEXT) AS "integerText", CAST(Integer AS VARCHAR) AS "integerString", CAST(Integer AS Integer) AS "expression",' + + ' CAST(String AS VARCHAR(2)) AS "stringChars", CAST(Integer AS VARCHAR(2)) AS "integerStringChars",' + + ' CAST(Float AS NUMERIC) AS "floatNumeric", CAST(Decimal AS NUMERIC) AS "decimalNumeric", CAST(Decimal AS DOUBLE PRECISION) AS "decimalDoublePrecision",' + + ' CAST(Float AS DOUBLE PRECISION) AS "floatDoublePrecision", CAST(String AS NUMERIC(4, 2)) AS "numericParams" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->project([ @@ -842,17 +868,17 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | $row.getFloat('Float'), 'floatDoublePrecision'), col(row:TDSRow[1] | round(parseDecimal($row.getString('String')), 2), 'numericParams') ]) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //CAST function <> meta::external::query::sql::transformation::queryToPure::tests::testDateCasts():Boolean[1] { - let sqlString = 'SELECT CAST(\'2023-01-01\' AS DATE) AS "constantDate", CAST(String AS DATE) AS "date", CAST(\'2023-01-01 10:01:01\' AS TIMESTAMP) AS "constantTimestamp", CAST(\'2023-01-01 10:01:01.12\' AS TIMESTAMP) AS "constantTimestampMillis", CAST(String AS TIMESTAMP) AS "timestamp" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT CAST(\'2023-01-01\' AS DATE) AS "constantDate", CAST(String AS DATE) AS "date", CAST(\'2023-01-01 10:01:01\' AS TIMESTAMP) AS "constantTimestamp", ' + + 'CAST(\'2023-01-01 10:01:01.12\' AS TIMESTAMP) AS "constantTimestampMillis", CAST(String AS TIMESTAMP) AS "timestamp" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->project([ @@ -862,17 +888,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | parseDate('2023-01-01T10:01:01.12'), 'constantTimestampMillis'), col(row:TDSRow[1] | parseDate($row.getString('String')), 'timestamp') ]) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testNullCasts():Boolean[1] { - let sqlString = 'SELECT CAST(NULL AS VARCHAR) AS "string", CAST(NULL AS TEXT) AS "text", CAST(NULL AS DATE) AS "date",' - + ' CAST(NULL AS INTEGER) AS "integer", CAST(NULL AS BOOLEAN) AS "boolean", CAST(NULL AS DOUBLE PRECISION) AS "double", CAST(NULL AS NUMERIC) AS "numeric", CAST(NULL AS TIMESTAMP) AS "timestamp" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test('SELECT CAST(NULL AS VARCHAR) AS "string", CAST(NULL AS TEXT) AS "text", CAST(NULL AS DATE) AS "date", ' + + 'CAST(NULL AS INTEGER) AS "integer", CAST(NULL AS BOOLEAN) AS "boolean", CAST(NULL AS DOUBLE PRECISION) AS "double", CAST(NULL AS NUMERIC) AS "numeric", CAST(NULL AS TIMESTAMP) AS "timestamp" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ] )->project([ @@ -885,17 +909,17 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | cast([], @Decimal), 'numeric'), col(row:TDSRow[1] | cast([], @DateTime), 'timestamp') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //CASE function <> meta::external::query::sql::transformation::queryToPure::tests::testMultiSearchedCaseWhen():Boolean[1] { - let sqlString = 'SELECT CASE WHEN String = \'abc\' THEN 1 WHEN String = \'def\' THEN NULL ELSE 3 END AS "Number" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT CASE WHEN String = \'abc\' THEN 1 WHEN String = \'def\' THEN NULL ELSE 3 END AS "Number" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->project( @@ -903,15 +927,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | if ($row.getString('String') == 'abc', | 1, | if ($row.getString('String') == 'def', | [], | 3)), 'Number') ] ) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testCaseWhen():Boolean[1] { - let sqlString = 'SELECT CASE String WHEN \'abc\' THEN 1 WHEN \'def\' THEN 2 ELSE 3 END AS "Number" FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + test( + 'SELECT CASE String WHEN \'abc\' THEN 1 WHEN \'def\' THEN 2 ELSE 3 END AS "Number" FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->project( @@ -919,18 +943,17 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | if ($row.getString('String') == 'abc', | 1, | if ($row.getString('String') == 'def', | 2, | 3)), 'Number') ] ) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } ///ARITHMETIC function <> meta::external::query::sql::transformation::queryToPure::tests::testArithmetic():Boolean[1] { - let sqlString = 'SELECT 1 + 2 AS "plus", 1 - 2 AS "minus", 1 * 2 AS "multiply", 1 / 2 AS "divide", 1 % 2 AS "mod", 1 ^ 2 AS "pow", Integer + Float AS "column plus" FROM service."/service/service1"'; + test( + 'SELECT 1 + 2 AS "plus", 1 - 2 AS "minus", 1 * 2 AS "multiply", 1 / 2 AS "divide", 1 % 2 AS "mod", 1 ^ 2 AS "pow", Integer + Float AS "column plus" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->project( @@ -944,16 +967,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | $row.getInteger('Integer') + $row.getFloat('Float'), 'column plus') ] ) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testArithmeticNullable():Boolean[1] { - let sqlString = 'SELECT 1 + NULL AS "plus", 1 - NULL AS "minus", 1 * NULL AS "multiply", 1 / NULL AS "divide", 1 % NULL AS "mod", 1 ^ NULL AS "pow" FROM service."/service/service1"'; + test( + 'SELECT 1 + NULL AS "plus", 1 - NULL AS "minus", 1 * NULL AS "multiply", 1 / NULL AS "divide", 1 % NULL AS "mod", 1 ^ NULL AS "pow" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ]) ->project( @@ -966,17 +988,16 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | []->cast(@Number), 'pow') ] ) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //JOIN function <> meta::external::query::sql::transformation::queryToPure::tests::testLeftJoinSubqueryOn():Boolean[1] { - let sqlString = 'SELECT table1.Integer FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT Integer FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."Integer"'; + test( + 'SELECT table1.Integer FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT Integer FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."Integer"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->renameColumns([ pair('Boolean', 'Boolean_table1'), @@ -995,17 +1016,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: pair('Integer', 'Integer_table2') ]), meta::relational::metamodel::join::JoinType.LEFT_OUTER, {row1:TDSRow[1], row2:TDSRow[1] | $row1.getInteger('Integer_table1') == $row2.getInteger('Integer_table2')} )->restrict('Integer_table1')->renameColumns(pair('Integer_table1', 'Integer')) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testLeftJoinSubqueryUsing():Boolean[1] { - let sqlString = 'SELECT table1.Integer AS "Int", table2.String AS "Str" FROM service."/service/service1" AS table1 LEFT OUTER JOIN (SELECT Integer, String FROM service."/service/service1") AS table2 USING (Integer)'; + test( + 'SELECT table1.Integer AS "Int", table2.String AS "Str" FROM service."/service/service1" AS table1 LEFT OUTER JOIN (SELECT Integer, String FROM service."/service/service1") AS table2 USING (Integer)', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->renameColumns([ pair('Boolean', 'Boolean_table1'), @@ -1024,20 +1043,21 @@ function <> meta::external::query::sql::transformation::queryToPure:: pair('Integer', 'Integer_table2'), pair('String', 'String_table2') ]), meta::relational::metamodel::join::JoinType.LEFT_OUTER, ['Integer'] - )->renameColumns([ + )->restrict([ + 'Integer_table1', 'String_table2' + ])->renameColumns([ pair('Integer_table1', 'Int'), pair('String_table2', 'Str') - ])->restrict(['Int', 'Str']) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + ]) + }); } function <> meta::external::query::sql::transformation::queryToPure::tests::testCrossJoin():Boolean[1] { - let sqlString = 'SELECT table1.Integer AS "Int", table2.String AS "Str" FROM service."/service/service1" table1 CROSS JOIN service."/service/service2" table2'; + test( + 'SELECT table1.Integer AS "Int", table2.String AS "Str" FROM service."/service/service1" table1 CROSS JOIN service."/service/service2" table2', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->renameColumns([ pair('Boolean', 'Boolean_table1'), @@ -1055,21 +1075,19 @@ function <> meta::external::query::sql::transformation::queryToPure:: pair('Integer', 'Integer_table2'), pair('String', 'String_table2') ]), meta::relational::metamodel::join::JoinType.INNER, {row1:TDSRow[1], row2:TDSRow[1] | true} - )->renameColumns([ + )->restrict(['Integer_table1', 'String_table2'])->renameColumns([ pair('Integer_table1', 'Int'), pair('String_table2', 'Str') - ])->restrict(['Int', 'Str']) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + ]) + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testImplicitJoin():Boolean[1] { - let sqlString = 'SELECT table1.Integer AS "Int", table2.String AS "Str" FROM service."/service/service1" table1, service."/service/service2" table2 where table1.Integer = table2.Integer'; + test( + 'SELECT table1.Integer AS "Int", table2.String AS "Str" FROM service."/service/service1" table1, service."/service/service2" table2 where table1.Integer = table2.Integer', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->renameColumns([ pair('Boolean', 'Boolean_table1'), @@ -1087,21 +1105,20 @@ function <> meta::external::query::sql::transformation::queryToPure:: pair('Integer', 'Integer_table2'), pair('String', 'String_table2') ]), meta::relational::metamodel::join::JoinType.INNER, {row1:TDSRow[1], row2:TDSRow[1] | true} - )->filter({row|($row.getInteger('Integer_table1') == $row.getInteger('Integer_table2'))})->renameColumns([ + )->filter({row|($row.getInteger('Integer_table1') == $row.getInteger('Integer_table2'))}) + ->restrict(['Integer_table1', 'String_table2'])->renameColumns([ pair('Integer_table1', 'Int'), pair('String_table2', 'Str') - ])->restrict(['Int', 'Str']) - }; - - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + ]) + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testLeftJoinRelationOn():Boolean[1] { - let sqlString = 'SELECT table1.Integer AS Integer, table2.String AS String FROM service."/service/service1" AS table1 LEFT OUTER JOIN service."/service/service2" AS table2 ON (table1.Integer = table2.Integer)'; + test( + 'SELECT table1.Integer AS Integer, table2.String AS String FROM service."/service/service1" AS table1 LEFT OUTER JOIN service."/service/service2" AS table2 ON (table1.Integer = table2.Integer)', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->renameColumns([ pair('Boolean', 'Boolean_table1'), @@ -1119,72 +1136,66 @@ function <> meta::external::query::sql::transformation::queryToPure:: pair('Integer', 'Integer_table2'), pair('String', 'String_table2') ]), meta::relational::metamodel::join::JoinType.LEFT_OUTER, {row1:TDSRow[1], row2:TDSRow[1] | $row1.getInteger('Integer_table1') == $row2.getInteger('Integer_table2')}) + ->restrict(['Integer_table1', 'String_table2']) ->renameColumns([ pair('Integer_table1', 'Integer'), pair('String_table2', 'String') ]) - ->restrict(['Integer', 'String']) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testMultiJoin():Boolean[1] { - let sqlString = 'SELECT * FROM (SELECT * FROM service(\'/service/service1\')) "table1" LEFT JOIN (SELECT * FROM service(\'/service/service2\')) "table2" ON ("table1"."String" = "table2"."ID") LEFT JOIN (SELECT * FROM service(\'/service/service2\')) "table3" ON ("table1"."String" = "table3"."String") WHERE "table2"."String" = \'ABC\''; + test( + 'SELECT * FROM (SELECT * FROM service(\'/service/service1\')) "table1" LEFT JOIN (SELECT * FROM service(\'/service/service2\')) "table2" ON ("table1"."String" = "table2"."ID") ' + + 'LEFT JOIN (SELECT * FROM service(\'/service/service2\')) "table3" ON ("table1"."String" = "table3"."String") WHERE "table2"."String" = \'ABC\'', - let sqlTransformContext = $sqlString->processQuery(); - - let expected = {|FlatInput.all() - ->project([{x|$x.booleanIn}, {x|$x.integerIn}, {x|$x.floatIn}, {x|$x.decimalIn}, {x|$x.strictDateIn}, {x|$x.dateTimeIn}, {x|$x.stringIn}], ['Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String']) - ->renameColumns([ - pair('Boolean', 'Boolean_table1'), - pair('Integer', 'Integer_table1'), - pair('Float', 'Float_table1'), - pair('Decimal', 'Decimal_table1'), - pair('StrictDate', 'StrictDate_table1'), - pair('DateTime', 'DateTime_table1'), - pair('String', 'String_table1') - ]) - ->join(FlatInput.all() - ->project([{x|$x.idIn}, {x|$x.integerIn}, {x|$x.stringIn}], ['ID', 'Integer', 'String']) - ->renameColumns([ - pair('ID', 'ID_table2'), - pair('Integer', 'Integer_table2'), - pair('String', 'String_table2') - ]), - meta::relational::metamodel::join::JoinType.LEFT_OUTER, - {row1, row2|$row1.getString('String_table1') == $row2.getInteger('ID_table2')}) - ->join(meta::external::query::sql::transformation::queryToPure::tests::FlatInput.all() - ->project([{x|$x.idIn}, {x|$x.integerIn}, {x|$x.stringIn}], ['ID', 'Integer', 'String']) - ->renameColumns([ - pair('ID', 'ID_table3'), - pair('Integer', 'Integer_table3'), - pair('String', 'String_table3') - ]), - meta::relational::metamodel::join::JoinType.LEFT_OUTER, - {row1, row2|$row1.getString('String_table1') == $row2.getString('String_table3')}) - ->filter({row|$row.getString('String_table2') == 'ABC'}) - ->renameColumns([ - pair('Boolean_table1', 'Boolean'), - pair('Float_table1', 'Float'), - pair('Decimal_table1', 'Decimal'), - pair('StrictDate_table1', 'StrictDate'), - pair('DateTime_table1', 'DateTime') - ]) - }; - - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + {|FlatInput.all() + ->project([{x|$x.booleanIn}, {x|$x.integerIn}, {x|$x.floatIn}, {x|$x.decimalIn}, {x|$x.strictDateIn}, {x|$x.dateTimeIn}, {x|$x.stringIn}], ['Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String']) + ->renameColumns([ + pair('Boolean', 'Boolean_table1'), + pair('Integer', 'Integer_table1'), + pair('Float', 'Float_table1'), + pair('Decimal', 'Decimal_table1'), + pair('StrictDate', 'StrictDate_table1'), + pair('DateTime', 'DateTime_table1'), + pair('String', 'String_table1') + ]) + ->join(FlatInput.all() + ->project([{x|$x.idIn}, {x|$x.integerIn}, {x|$x.stringIn}], ['ID', 'Integer', 'String']) + ->renameColumns([ + pair('ID', 'ID_table2'), + pair('Integer', 'Integer_table2'), + pair('String', 'String_table2') + ]), + meta::relational::metamodel::join::JoinType.LEFT_OUTER, + {row1, row2|$row1.getString('String_table1') == $row2.getInteger('ID_table2')}) + ->join(meta::external::query::sql::transformation::queryToPure::tests::FlatInput.all() + ->project([{x|$x.idIn}, {x|$x.integerIn}, {x|$x.stringIn}], ['ID', 'Integer', 'String']) + ->renameColumns([ + pair('ID', 'ID_table3'), + pair('Integer', 'Integer_table3'), + pair('String', 'String_table3') + ]), + meta::relational::metamodel::join::JoinType.LEFT_OUTER, + {row1, row2|$row1.getString('String_table1') == $row2.getString('String_table3')}) + ->filter({row|$row.getString('String_table2') == 'ABC'}) + ->renameColumns([ + pair('Boolean_table1', 'Boolean'), + pair('Float_table1', 'Float'), + pair('Decimal_table1', 'Decimal'), + pair('StrictDate_table1', 'StrictDate'), + pair('DateTime_table1', 'DateTime') + ])}, false) } //UNION function <> meta::external::query::sql::transformation::queryToPure::tests::testUnion():Boolean[1] { - let sqlString = 'SELECT Integer FROM service."/service/service1" UNION SELECT Integer FROM service."/service/service2"'; + test('SELECT Integer FROM service."/service/service1" UNION SELECT Integer FROM service."/service/service2"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1197,17 +1208,16 @@ function <> meta::external::query::sql::transformation::queryToPure:: [ 'ID', 'Integer', 'String' ]) ->restrict('Integer') ) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //CURRENT TIME function <> meta::external::query::sql::transformation::queryToPure::tests::testCurrentTime():Boolean[1] { - let sqlString = 'SELECT CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_DATE, CURRENT_TIME AS time, CURRENT_TIMESTAMP AS timestamp, CURRENT_DATE AS date FROM service."/service/service1"'; + test( + 'SELECT CURRENT_TIME, CURRENT_TIMESTAMP, CURRENT_DATE, CURRENT_TIME AS time, CURRENT_TIMESTAMP AS timestamp, CURRENT_DATE AS date FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1220,18 +1230,17 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | now(), 'timestamp'), col(row:TDSRow[1] | today(), 'date') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //DATE_TRUNC function <> meta::external::query::sql::transformation::queryToPure::tests::testDateTrunc():Boolean[1] { - let sqlString = 'SELECT date_trunc(\'year\', StrictDate) AS "YEAR", date_trunc(\'quarter\', CAST(\'2023-01-01\' AS DATE)) AS "QUARTER", date_trunc(\'month\', StrictDate) AS "MONTH", date_trunc(\'week\', StrictDate) AS "WEEK", ' + - 'date_trunc(\'day\', StrictDate) AS "DAY", date_trunc(\'hour\', StrictDate) AS "HOUR", date_trunc(\'minute\', StrictDate) AS "MINUTE", date_trunc(\'second\', StrictDate) AS "SECOND" FROM service."/service/service1"'; + test( + 'SELECT date_trunc(\'year\', StrictDate) AS "YEAR", date_trunc(\'quarter\', CAST(\'2023-01-01\' AS DATE)) AS "QUARTER", date_trunc(\'month\', StrictDate) AS "MONTH", date_trunc(\'week\', StrictDate) AS "WEEK", ' + + 'date_trunc(\'day\', StrictDate) AS "DAY", date_trunc(\'hour\', StrictDate) AS "HOUR", date_trunc(\'minute\', StrictDate) AS "MINUTE", date_trunc(\'second\', StrictDate) AS "SECOND" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1246,19 +1255,17 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | firstSecondOfMinute($row.getStrictDate('StrictDate')), 'MINUTE'), col(row:TDSRow[1] | firstMillisecondOfSecond($row.getStrictDate('StrictDate')), 'SECOND') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //DATE_PART function <> meta::external::query::sql::transformation::queryToPure::tests::testDatePart():Boolean[1] { - let sqlString = 'SELECT date_part(\'year\', StrictDate) AS "YEAR", date_part(\'quarter\', CAST(\'2023-01-01\' AS DATE)) AS "QUARTER", date_part(\'month\', StrictDate) AS "MONTH", date_part(\'week\', StrictDate) AS "WEEK", ' + + test('SELECT date_part(\'year\', StrictDate) AS "YEAR", date_part(\'quarter\', CAST(\'2023-01-01\' AS DATE)) AS "QUARTER", date_part(\'month\', StrictDate) AS "MONTH", date_part(\'week\', StrictDate) AS "WEEK", ' + 'date_part(\'dow\', StrictDate) AS "DOW", date_part(\'day\', StrictDate) AS "DAY", date_part(\'doy\', StrictDate) AS "DOY", date_part(\'hour\', StrictDate) AS "HOUR", date_part(\'minute\', StrictDate) AS "MINUTE", ' + - 'date_part(\'second\', StrictDate) AS "SECOND", date_part(\'epoch\', StrictDate) AS "EPOCH" FROM service."/service/service1"'; + 'date_part(\'second\', StrictDate) AS "SECOND", date_part(\'epoch\', StrictDate) AS "EPOCH" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1276,19 +1283,18 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | second($row.getStrictDate('StrictDate')), 'SECOND'), col(row:TDSRow[1] | toEpochValue($row.getStrictDate('StrictDate')), 'EPOCH') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //EXTRACT function <> meta::external::query::sql::transformation::queryToPure::tests::testExtract():Boolean[1] { - let sqlString = 'SELECT EXTRACT(\'year\' FROM StrictDate) AS "YEAR", EXTRACT(\'quarter\' FROM CAST(\'2023-01-01\' AS DATE)) AS "QUARTER", EXTRACT(\'month\' FROM StrictDate) AS "MONTH", EXTRACT(\'week\' FROM StrictDate) AS "WEEK", ' + - 'EXTRACT(\'dow\' FROM StrictDate) AS "DOW", EXTRACT(\'day\' FROM StrictDate) AS "DAY", EXTRACT(\'doy\' FROM StrictDate) AS "DOY", EXTRACT(\'hour\' FROM StrictDate) AS "HOUR", EXTRACT(\'minute\' FROM StrictDate) AS "MINUTE", ' + - 'EXTRACT(\'second\' FROM StrictDate) AS "SECOND", EXTRACT(\'epoch\' FROM StrictDate) AS "EPOCH" FROM service."/service/service1"'; + test( + 'SELECT EXTRACT(\'year\' FROM StrictDate) AS "YEAR", EXTRACT(\'quarter\' FROM CAST(\'2023-01-01\' AS DATE)) AS "QUARTER", EXTRACT(\'month\' FROM StrictDate) AS "MONTH", EXTRACT(\'week\' FROM StrictDate) AS "WEEK", ' + + 'EXTRACT(\'dow\' FROM StrictDate) AS "DOW", EXTRACT(\'day\' FROM StrictDate) AS "DAY", EXTRACT(\'doy\' FROM StrictDate) AS "DOY", EXTRACT(\'hour\' FROM StrictDate) AS "HOUR", EXTRACT(\'minute\' FROM StrictDate) AS "MINUTE", ' + + 'EXTRACT(\'second\' FROM StrictDate) AS "SECOND", EXTRACT(\'epoch\' FROM StrictDate) AS "EPOCH" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1306,27 +1312,26 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | second($row.getStrictDate('StrictDate')), 'SECOND'), col(row:TDSRow[1] | toEpochValue($row.getStrictDate('StrictDate')), 'EPOCH') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //INTERVAL function <> meta::external::query::sql::transformation::queryToPure::tests::testIntervalArithmetic():Boolean[1] { - let sqlString = 'SELECT ' + - 'StrictDate + INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_ADD", ' + - 'StrictDate + 1 + INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_MULTI_ADD", ' + - 'StrictDate + 1 AS "NUMERIC_ADD", ' + - 'StrictDate + 6 * INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_TIMES", ' + - 'StrictDate + NULL * INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_TIMES_NULL", ' + - 'StrictDate + NULL + INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_ADD_NULL", ' + - '(CAST(\'2023-01-01\' AS DATE) + 2 * INTERVAL \'1 DAY\') + 3 * INTERVAL \'2 DAY\' AS "INTERVAL_MIX", ' + - 'StrictDate + EXTRACT(\'year\' FROM StrictDate) * INTERVAL \'2 YEAR 3 DAYS\' AS "INTERVAL_MIX2", ' + - 'CAST((DATE_TRUNC( \'DAY\', CAST("StrictDate" AS DATE) ) + (EXTRACT(DOW FROM "StrictDate") * INTERVAL \'1 DAY\')) AS DATE) AS "INTERVAL_MIX3" ' + - 'FROM service."/service/service1"'; - - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + test( + 'SELECT ' + + 'StrictDate + INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_ADD", ' + + 'StrictDate + 1 + INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_MULTI_ADD", ' + + 'StrictDate + 1 AS "NUMERIC_ADD", ' + + 'StrictDate + 6 * INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_TIMES", ' + + 'StrictDate + NULL * INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_TIMES_NULL", ' + + 'StrictDate + NULL + INTERVAL \'1 YEAR 3 WEEKS 2 DAYS\' AS "INTERVAL_ADD_NULL", ' + + '(CAST(\'2023-01-01\' AS DATE) + 2 * INTERVAL \'1 DAY\') + 3 * INTERVAL \'2 DAY\' AS "INTERVAL_MIX", ' + + 'StrictDate + EXTRACT(\'year\' FROM StrictDate) * INTERVAL \'2 YEAR 3 DAYS\' AS "INTERVAL_MIX2", ' + + 'CAST((DATE_TRUNC( \'DAY\', CAST("StrictDate" AS DATE) ) + (EXTRACT(DOW FROM "StrictDate") * INTERVAL \'1 DAY\')) AS DATE) AS "INTERVAL_MIX3" ' + + 'FROM service."/service/service1"', + + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1342,19 +1347,20 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | $row.getStrictDate('StrictDate')->adjust(year($row.getStrictDate('StrictDate')) * 2, DurationUnit.YEARS)->adjust(year($row.getStrictDate('StrictDate')) * 3, DurationUnit.DAYS), 'INTERVAL_MIX2'), col(row:TDSRow[1] | $row.getStrictDate('StrictDate')->firstHourOfDay()->adjust(($row.getStrictDate('StrictDate')->dayOfWeekNumber() * 1), DurationUnit.DAYS), 'INTERVAL_MIX3') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //STRING FUNCTIONS function <> meta::external::query::sql::transformation::queryToPure::tests::testStringFunctions():Boolean[1] { - let sqlString = 'SELECT ascii(String) AS "ASCII", chr(Integer) AS "CHR", regexp_like(String, \'test\') AS "MATCH", char_length(String) AS "CHAR_LENGTH", length(String) AS "LENGTH", ltrim(String) AS "LTRIM", ltrim(String, \' \') AS "LTRIM2", md5(String) AS "MD5", upper(String) AS "UPPER", ' + - 'lower(String) AS "LOWER", repeat(String, 2) AS "REPEAT", replace(String, \'A\', \'a\') AS "REPLACE", starts_with(String, \'a\') AS "STARTSWITH", strpos(String, \'abc\') AS "STRPOS", reverse(String) AS "REVERSE", rtrim(String) AS "RTRIM", rtrim(String, \' \') AS "RTRIM2", ' + - 'sha256(String) AS "SHA256", substring(String, 1) AS "SUBSTRING", substr(String, 1, 2) AS "SUBSTR", btrim(String) AS "TRIM", btrim(String, \' \') AS "TRIM2" FROM service."/service/service1"'; + test( + 'SELECT ascii(String) AS "ASCII", chr(Integer) AS "CHR", concat(String, \'abc\') AS "CONCAT", String || \'abc\' AS "CONCAT2", regexp_like(String, \'test\') AS "MATCH", ' + + 'char_length(String) AS "CHAR_LENGTH", length(String) AS "LENGTH", ltrim(String) AS "LTRIM", ltrim(String, \' \') AS "LTRIM2", md5(String) AS "MD5", upper(String) AS "UPPER", ' + + 'lower(String) AS "LOWER", repeat(String, 2) AS "REPEAT", replace(String, \'A\', \'a\') AS "REPLACE", starts_with(String, \'a\') AS "STARTSWITH", strpos(String, \'abc\') AS "STRPOS",' + + 'reverse(String) AS "REVERSE", rtrim(String) AS "RTRIM", rtrim(String, \' \') AS "RTRIM2", sha256(String) AS "SHA256", split_part(String, \',\', 1) AS "SPLITPART", ' + + 'split_part(String, \',\', Integer) AS "SPLITPART2", substring(String, 1) AS "SUBSTRING", substr(String, 1, 2) AS "SUBSTR", btrim(String) AS "TRIM", btrim(String, \' \') AS "TRIM2" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1362,6 +1368,8 @@ function <> meta::external::query::sql::transformation::queryToPure:: ->project([ col(row:TDSRow[1] | ascii($row.getString('String')), 'ASCII'), col(row:TDSRow[1] | char($row.getInteger('Integer')), 'CHR'), + col(row:TDSRow[1] | $row.getString('String') + 'abc', 'CONCAT'), + col(row:TDSRow[1] | $row.getString('String') + 'abc', 'CONCAT2'), col(row:TDSRow[1] | matches($row.getString('String'), 'test'), 'MATCH'), col(row:TDSRow[1] | length($row.getString('String')), 'CHAR_LENGTH'), col(row:TDSRow[1] | length($row.getString('String')), 'LENGTH'), @@ -1378,23 +1386,25 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | rtrim($row.getString('String')), 'RTRIM'), col(row:TDSRow[1] | rtrim($row.getString('String')), 'RTRIM2'), col(row:TDSRow[1] | hash($row.getString('String'), HashType.SHA256), 'SHA256'), + col(row:TDSRow[1] | splitPart($row.getString('String'), ',', 2), 'SPLITPART'), + col(row:TDSRow[1] | splitPart($row.getString('String'), ',', $row.getInteger('Integer') + 1), 'SPLITPART2'), col(row:TDSRow[1] | substring($row.getString('String'), 1), 'SUBSTRING'), col(row:TDSRow[1] | substring($row.getString('String'), 1, 2), 'SUBSTR'), col(row:TDSRow[1] | trim($row.getString('String')), 'TRIM'), col(row:TDSRow[1] | trim($row.getString('String')), 'TRIM2') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testStringFunctionsNullable():Boolean[1] { - let sqlString = 'SELECT ascii(NULL) AS "ASCII", chr(NULL) AS "CHR", regexp_like(NULL, \'test\') AS "MATCH", char_length(NULL) AS "CHAR_LENGTH", length(NULL) AS "LENGTH", ltrim(NULL) AS "LTRIM", ltrim(NULL, \' \') AS "LTRIM2", md5(NULL) AS "MD5", upper(NULL) AS "UPPER", ' + - 'lower(NULL) AS "LOWER", replace(NULL, \'A\', \'a\') AS "REPLACE", starts_with(NULL, \'a\') AS "STARTSWITH", strpos(NULL, \'abc\') AS "STRPOS", reverse(NULL) AS "REVERSE", rtrim(NULL) AS "RTRIM", rtrim(NULL, \' \') AS "RTRIM2", ' + - 'sha256(NULL) AS "SHA256", substring(NULL, 1) AS "SUBSTRING", substr(NULL, 1, 2) AS "SUBSTR", btrim(NULL) AS "TRIM", btrim(NULL, \' \') AS "TRIM2" FROM service."/service/service1"'; + test( + 'SELECT ascii(NULL) AS "ASCII", chr(NULL) AS "CHR", regexp_like(NULL, \'test\') AS "MATCH", char_length(NULL) AS "CHAR_LENGTH", length(NULL) AS "LENGTH", ltrim(NULL) AS "LTRIM", ' + + 'ltrim(NULL, \' \') AS "LTRIM2", md5(NULL) AS "MD5", upper(NULL) AS "UPPER", lower(NULL) AS "LOWER", replace(NULL, \'A\', \'a\') AS "REPLACE", starts_with(NULL, \'a\') AS "STARTSWITH", ' + + 'strpos(NULL, \'abc\') AS "STRPOS", reverse(NULL) AS "REVERSE", rtrim(NULL) AS "RTRIM", rtrim(NULL, \' \') AS "RTRIM2", sha256(NULL) AS "SHA256", substring(NULL, 1) AS "SUBSTRING", ' + + 'substr(NULL, 1, 2) AS "SUBSTR", btrim(NULL) AS "TRIM", btrim(NULL, \' \') AS "TRIM2" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1422,21 +1432,21 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | []->cast(@String), 'TRIM'), col(row:TDSRow[1] | []->cast(@String), 'TRIM2') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //MATH FUNCTIONS function <> meta::external::query::sql::transformation::queryToPure::tests::testMathFunctions():Boolean[1] { - let sqlString = 'SELECT abs(Integer) AS "ABS", acos(Integer) AS "ACOS", asin(Integer) AS "ASIN", atan(Integer) AS "ATAN", atan2(Integer, 1) AS "ATAN2", ' + - 'ceil(Integer) AS "CEIL", ceiling(Integer) AS "CEILING", cos(Integer) AS "COS", cot(Integer) AS "COT", degrees(Integer) AS "DEGREES", div(Integer, 1) AS "DIV", exp(Integer) AS "EXP", floor(Float) AS "FLOOR", ' + - 'log(Integer) AS "LOG", ln(Integer) AS "LN", mod(Integer, 1) AS "MOD", pi() AS "PI", power(Integer, 2) AS "POWER", radians(Integer) AS "RADIANS", round(Integer) AS "ROUND", round(Decimal, 1) AS "ROUND_DEC", sign(Integer) AS "SIGN", ' + - 'sin(Integer) AS "SIN", sqrt(Integer) AS "SQRT", tan(Integer) AS "TAN", trunc(Integer) AS "TRUNC" FROM service."/service/service1"'; + test( + 'SELECT abs(Integer) AS "ABS", acos(Integer) AS "ACOS", asin(Integer) AS "ASIN", atan(Integer) AS "ATAN", atan2(Integer, 1) AS "ATAN2", ' + + 'ceil(Integer) AS "CEIL", ceiling(Integer) AS "CEILING", cos(Integer) AS "COS", cot(Integer) AS "COT", degrees(Integer) AS "DEGREES", div(Integer, 1) AS "DIV", ' + + 'exp(Integer) AS "EXP", floor(Float) AS "FLOOR", log(Integer) AS "LOG", ln(Integer) AS "LN", mod(Integer, 1) AS "MOD", pi() AS "PI", power(Integer, 2) AS "POWER", ' + + 'radians(Integer) AS "RADIANS", round(Integer) AS "ROUND", round(Decimal, 1) AS "ROUND_DEC", sign(Integer) AS "SIGN", ' + + 'sin(Integer) AS "SIN", sqrt(Integer) AS "SQRT", tan(Integer) AS "TAN", trunc(Integer) AS "TRUNC" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1469,19 +1479,18 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | tan($row.getInteger('Integer')), 'TAN'), col(row:TDSRow[1] | if ($row.getInteger('Integer') > 0, | floor($row.getInteger('Integer')), | ceiling($row.getInteger('Integer'))), 'TRUNC') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testMathFunctionsNullable():Boolean[1] { - let sqlString = 'SELECT abs(NULL) AS "ABS", acos(NULL) AS "ACOS", asin(NULL) AS "ASIN", atan(NULL) AS "ATAN", atan2(NULL, 1) AS "ATAN2", ' + - 'ceil(NULL) AS "CEIL", ceiling(NULL) AS "CEILING", cos(NULL) AS "COS", cot(NULL) AS "COT", degrees(NULL) AS "DEGREES", div(NULL, 1) AS "DIV", exp(NULL) AS "EXP", floor(NULL) AS "FLOOR", ' + - 'log(NULL) AS "LOG", ln(NULL) AS "LN", mod(NULL, 1) AS "MOD", power(NULL, 2) AS "POWER", radians(NULL) AS "RADIANS", round(NULL) AS "ROUND", round(NULL, 1) AS "ROUND_DEC", sign(NULL) AS "SIGN", ' + - 'sin(NULL) AS "SIN", sqrt(NULL) AS "SQRT", tan(NULL) AS "TAN", trunc(NULL) AS "TRUNC" FROM service."/service/service1"'; + test( + 'SELECT abs(NULL) AS "ABS", acos(NULL) AS "ACOS", asin(NULL) AS "ASIN", atan(NULL) AS "ATAN", atan2(NULL, 1) AS "ATAN2", ' + + 'ceil(NULL) AS "CEIL", ceiling(NULL) AS "CEILING", cos(NULL) AS "COS", cot(NULL) AS "COT", degrees(NULL) AS "DEGREES", div(NULL, 1) AS "DIV", exp(NULL) AS "EXP", floor(NULL) AS "FLOOR", ' + + 'log(NULL) AS "LOG", ln(NULL) AS "LN", mod(NULL, 1) AS "MOD", power(NULL, 2) AS "POWER", radians(NULL) AS "RADIANS", round(NULL) AS "ROUND", round(NULL, 1) AS "ROUND_DEC", sign(NULL) AS "SIGN", ' + + 'sin(NULL) AS "SIN", sqrt(NULL) AS "SQRT", tan(NULL) AS "TAN", trunc(NULL) AS "TRUNC" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1513,17 +1522,16 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | []->cast(@Float), 'TAN'), col(row:TDSRow[1] | []->cast(@Integer), 'TRUNC') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //COLLECTION FUNCTIONS function <> meta::external::query::sql::transformation::queryToPure::tests::testCollectionFunctions():Boolean[1] { - let sqlString = 'SELECT coalesce(NULL, Integer, 1) AS "COALESCE" FROM service."/service/service1"'; + test( + 'SELECT coalesce(NULL, Integer, 1) AS "COALESCE" FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1531,28 +1539,21 @@ function <> meta::external::query::sql::transformation::queryToPure:: ->project([ col(row:TDSRow[1] | meta::pure::tds::extensions::firstNotNull([$row.getInteger('Integer'), 1]), 'COALESCE') ]) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); - - let plan = $sqlTransformContext->getPlan($sqlTransformContext.sources, $sqlTransformContext.extensions); - let sql = $plan.rootExecutionNode->cast(@meta::relational::mapping::RelationalTdsInstantiationExecutionNode).executionNodes->cast(@meta::relational::mapping::SQLExecutionNode).sqlQuery; - - assertEquals('select coalesce("root".integer, 1) as "COALESCE" from flat as "root"', $sql); + }) } //GROUP FUNCTIONS function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupFunctions():Boolean[1] { - let sqlString = 'SELECT String, ' + - 'percentile_cont(0.1) WITHIN GROUP (ORDER BY Integer ASC) AS "PERCENTILE_CONT_ASC", ' + - 'percentile_cont(0.2) WITHIN GROUP (ORDER BY Integer DESC) AS "PERCENTILE_CONT_DESC", ' + - 'percentile_disc(0.3) WITHIN GROUP (ORDER BY Integer ASC) AS "PERCENTILE_DISC_ASC", ' + - 'percentile_disc(0.4) WITHIN GROUP (ORDER BY Integer DESC) AS "PERCENTILE_DISC_DESC" ' + - 'FROM service."/service/service1" GROUP BY String'; + test( + 'SELECT String, ' + + 'percentile_cont(0.1) WITHIN GROUP (ORDER BY Integer ASC) AS "PERCENTILE_CONT_ASC", ' + + 'percentile_cont(0.2) WITHIN GROUP (ORDER BY Integer DESC) AS "PERCENTILE_CONT_DESC", ' + + 'percentile_disc(0.3) WITHIN GROUP (ORDER BY Integer ASC) AS "PERCENTILE_DISC_ASC", ' + + 'percentile_disc(0.4) WITHIN GROUP (ORDER BY Integer DESC) AS "PERCENTILE_DISC_DESC" ' + + 'FROM service."/service/service1" GROUP BY String', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1563,22 +1564,20 @@ function <> meta::external::query::sql::transformation::queryToPure:: agg('PERCENTILE_DISC_ASC', row | $row.getInteger('Integer'), y | $y->percentile(0.3, true, false)), agg('PERCENTILE_DISC_DESC', row | $row.getInteger('Integer'), y | $y->percentile(0.4, false, false)) ]) - }; - - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } //WINDOW FUNCTIONS function <> meta::external::query::sql::transformation::queryToPure::tests::testWindowFunctions():Boolean[1] { - let sqlString = 'SELECT String AS "string", Integer, ' + - 'row_number() OVER (PARTITION BY String ORDER BY Integer ASC) AS "ROW", ' + - 'dense_rank() OVER (PARTITION BY String ORDER BY Integer DESC) AS "DENSE RANK", ' + - 'rank() OVER (PARTITION BY String ORDER BY Integer ASC) AS "RANK" ' + - 'FROM service."/service/service1"'; + test( + 'SELECT String AS "string", Integer, ' + + 'row_number() OVER (PARTITION BY String ORDER BY Integer ASC) AS "ROW", ' + + 'dense_rank() OVER (PARTITION BY String ORDER BY Integer DESC) AS "DENSE RANK", ' + + 'rank() OVER (PARTITION BY String ORDER BY Integer ASC) AS "RANK" ' + + 'FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1588,21 +1587,19 @@ function <> meta::external::query::sql::transformation::queryToPure:: ->olapGroupBy(['String'], desc('Integer'), y | $y->meta::pure::functions::math::olap::denseRank(), 'DENSE RANK') ->olapGroupBy(['String'], asc('Integer'), y | $y->meta::pure::functions::math::olap::rank(), 'RANK') ->restrict(['string', 'Integer', 'ROW', 'DENSE RANK', 'RANK']) - }; - - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testProjectWithWindowFunctions():Boolean[1] { - let sqlString = 'SELECT String, abs(Integer) AS "ABS", ' + - 'row_number() OVER (PARTITION BY upper(String) ORDER BY EXTRACT(YEAR FROM StrictDate) + 10 ASC) AS "ROW", ' + - 'dense_rank() OVER (PARTITION BY String ORDER BY Integer DESC) AS "DENSE RANK", ' + - 'rank() OVER (PARTITION BY String ORDER BY Integer ASC) AS "RANK" ' + - 'FROM service."/service/service1"'; + test( + 'SELECT String, abs(Integer) AS "ABS", ' + + 'row_number() OVER (PARTITION BY upper(String) ORDER BY EXTRACT(YEAR FROM StrictDate) + 10 ASC) AS "ROW", ' + + 'dense_rank() OVER (PARTITION BY String ORDER BY Integer DESC) AS "DENSE RANK", ' + + 'rank() OVER (PARTITION BY String ORDER BY Integer ASC) AS "RANK" ' + + 'FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn], @@ -1616,21 +1613,19 @@ function <> meta::external::query::sql::transformation::queryToPure:: ->olapGroupBy(['String'], desc('Integer'), y | $y->meta::pure::functions::math::olap::denseRank(), 'DENSE RANK') ->olapGroupBy(['String'], asc('Integer'), y | $y->meta::pure::functions::math::olap::rank(), 'RANK') ->restrict(['String', 'ABS', 'ROW', 'DENSE RANK', 'RANK']) - }; - - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + }, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testGroupByWithWindowFunctions():Boolean[1] { - let sqlString = 'SELECT String, Integer, sum(Integer) AS "SUM", ' + - 'row_number() OVER (PARTITION BY String ORDER BY Integer ASC) AS "ROW", ' + - 'dense_rank() OVER (PARTITION BY String ORDER BY Integer DESC) AS "DENSE RANK", ' + - 'rank() OVER (PARTITION BY String ORDER BY Integer ASC) AS "RANK" ' + - 'FROM service."/service/service1" GROUP BY String, Integer'; + test( + 'SELECT String, Integer, sum(Integer) AS "SUM", ' + + 'row_number() OVER (PARTITION BY String ORDER BY Integer ASC) AS "ROW", ' + + 'dense_rank() OVER (PARTITION BY String ORDER BY Integer DESC) AS "DENSE RANK", ' + + 'rank() OVER (PARTITION BY String ORDER BY Integer ASC) AS "RANK" ' + + 'FROM service."/service/service1" GROUP BY String, Integer', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1639,23 +1634,21 @@ function <> meta::external::query::sql::transformation::queryToPure: ->olapGroupBy(['String'], desc('Integer'), y | $y->meta::pure::functions::math::olap::denseRank(), 'DENSE RANK') ->olapGroupBy(['String'], asc('Integer'), y | $y->meta::pure::functions::math::olap::rank(), 'RANK') ->groupBy(['String', 'Integer'], agg('SUM', row | $row.getInteger('Integer'), y | $y->sum())) - }; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } //LIKE function <> meta::external::query::sql::transformation::queryToPure::tests::testLike():Boolean[1] { - let sqlString = 'SELECT ' + - 'String like \'b\' AS "EQUAL", ' + - 'String like \'b%\' AS "STARTS_WITH", ' + - 'String like \'%b\' AS "ENDS_WITH", ' + - 'String like \'%b%\' AS "CONTAINS"' + - 'FROM service."/service/service1"'; + test( + 'SELECT ' + + 'String like \'b\' AS "EQUAL", ' + + 'String like \'b%\' AS "STARTS_WITH", ' + + 'String like \'%b\' AS "ENDS_WITH", ' + + 'String like \'%b%\' AS "CONTAINS"' + + 'FROM service."/service/service1"', - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1664,53 +1657,53 @@ function <> meta::external::query::sql::transformation::queryToPure:: col(row:TDSRow[1] | equal($row.getString('String'), 'b'), 'EQUAL'), col(row:TDSRow[1] | startsWith($row.getString('String'), 'b'), 'STARTS_WITH'), col(row:TDSRow[1] | endsWith($row.getString('String'), 'b'), 'ENDS_WITH'), - col(row:TDSRow[1] | contains($row.getString('String'), 'b'), 'CONTAINS') + col(row:TDSRow[1] | contains($row.getString('String'), 'b'), 'CONTAINS') ]) - }; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + }) } function <> meta::external::query::sql::transformation::queryToPure::tests::testFromScopingSimple():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1"'; - let sqlTransformContext = $sqlString->processQuery(true); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM service."/service/service1"', + + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->from(dummyMapping, dummyRuntime()) - }->meta::pure::router::preeval::preval(sqlExtensions()); - - assertLambdaJSONEquals($expected, $sqlTransformContext.lambda()); + }->meta::pure::router::preeval::preval(sqlExtensions()), testSources(), true, false, true) } function <> meta::external::query::sql::transformation::queryToPure::tests::testFromScopingNoMapping():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service1"'; let mapping = meta::external::query::sql::transformation::queryToPure::emptyMapping(); + let sources = serviceToSource(createService('/service/service1', | FlatInput.all()->project(x | $x.stringIn, 'String'), [], dummyRuntime())); - let sqlTransformContext = $sqlString->processQuery(serviceToSource(createService('/service/service1', | FlatInput.all()->project(x | $x.stringIn, 'String'), [], dummyRuntime())), true); - let expected = {| FlatInput.all()->project(x | $x.stringIn, 'String')->from($mapping, dummyRuntime())}->meta::pure::router::preeval::preval(sqlExtensions()); + test( + 'SELECT * FROM service."/service/service1"', - assertLambdaJSONEquals($expected, $sqlTransformContext.lambda()); + {| + FlatInput.all()->project(x | $x.stringIn, 'String')->from($mapping, dummyRuntime()) + }->meta::pure::router::preeval::preval(sqlExtensions()), $sources, true, false, true); } function <> meta::external::query::sql::transformation::queryToPure::tests::testFromScopingSimpleNested():Boolean[1] { - let sqlString = 'SELECT * FROM (SELECT Integer FROM service."/service/service1")'; - let sqlTransformContext = $sqlString->processQuery(true); - let expected = {| FlatInput.all()->project( + test( + 'SELECT * FROM (SELECT Integer FROM service."/service/service1")', + + {| + FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->restrict('Integer')->from(dummyMapping, dummyRuntime()) - }->meta::pure::router::preeval::preval(sqlExtensions()); - - assertLambdaJSONEquals($expected, $sqlTransformContext.lambda()); + }->meta::pure::router::preeval::preval(sqlExtensions()), testSources(), true, false, true) } function <> meta::external::query::sql::transformation::queryToPure::tests::testFromScopingJoin():Boolean[1] { - let sqlString = 'SELECT table1.Integer FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT Integer AS "int" FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."int"'; + test( + 'SELECT table1.Integer FROM service."/service/service1" "table1" LEFT OUTER JOIN (SELECT Integer AS "int" FROM service."/service/service2") AS "table2" ON "table1"."Integer" = "table2"."int"', - let sqlTransformContext = $sqlString->processQuery(true); - let expected = {| FlatInput.all()->project( + {| FlatInput.all()->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], [ 'Boolean', 'Integer', 'Float', 'Decimal', 'StrictDate', 'DateTime', 'String' ])->restrict('Integer')->from(dummyMapping, dummyRuntime()) ->join(FlatInput.all()->project( @@ -1718,17 +1711,15 @@ function <> meta::external::query::sql::transformation::queryToPure: ['ID', 'Integer', 'String'] )->renameColumns(pair('Integer', 'int'))->restrict(['int'])->from(dummyMapping, dummyRuntime()), meta::relational::metamodel::join::JoinType.LEFT_OUTER, {row1:TDSRow[1], row2:TDSRow[1] | $row1.getInteger('Integer') == $row2.getInteger('int')} )->restrict('Integer') - }->meta::pure::router::preeval::preval(sqlExtensions()); - - assertLambdaJSONEquals($expected, $sqlTransformContext.lambda()); + }->meta::pure::router::preeval::preval(sqlExtensions())) } function <> meta::external::query::sql::transformation::queryToPure::tests::testFromScopingUnion():Boolean[1] { - let sqlString = 'SELECT Integer FROM service."/service/service1" UNION SELECT Integer FROM service."/service/service2"'; + test( + 'SELECT Integer FROM service."/service/service1" UNION SELECT Integer FROM service."/service/service2"', - let sqlTransformContext = $sqlString->processQuery(true); - let expected = {| + {| FlatInput.all() ->project( [ x | $x.booleanIn, x | $x.integerIn, x | $x.floatIn, x | $x.decimalIn, x | $x.strictDateIn, x | $x.dateTimeIn, x | $x.stringIn ], @@ -1741,16 +1732,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: [ 'ID', 'Integer', 'String' ]) ->restrict('Integer')->from(dummyMapping, dummyRuntime()) ) - }->meta::pure::router::preeval::preval(sqlExtensions()); - - assertLambdaJSONEquals($expected, $sqlTransformContext.lambda()); + }->meta::pure::router::preeval::preval(sqlExtensions()), testSources(), true, false, true) } function <> meta::external::query::sql::transformation::queryToPure::tests::testParameterizedAllSupplied():Boolean[1] { - let sqlString = 'SELECT * FROM service(\'/service/service4/{id}\', id => \'abc\', ints => [1,2,3], date => \'2023-01-01\') LIMIT 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {id:String[1], ints:Integer[*], date:StrictDate[0..1]| + test( + 'SELECT * FROM service(\'/service/service4/{id}\', id => \'abc\', ints => [1,2,3], date => \'2023-01-01\') LIMIT 1', + + {id:String[1], ints:Integer[*], date:StrictDate[0..1]| FlatInput.all()->filter(f | $f.idIn == $id && $f.integerIn->in($ints) && ($f.strictDateIn > $date))->project( [ x | $x.idIn, x | $x.integerIn, x | $x.enumVal @@ -1758,16 +1748,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: [ 'ID', 'Integer', 'Enum' ] - )->limit(1);}; - - assertLambdaEquals($expected, $sqlTransformContext.lambda()); + )->limit(1)}, false) } function <> meta::external::query::sql::transformation::queryToPure::tests::testParameterizedNotSupplied():Boolean[1] { - let sqlString = 'SELECT * FROM service(\'/service/service4/{id}\', id => \'abc\') LIMIT 1'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {id:String[1], ints:Integer[*], date:StrictDate[0..1]| + test( + 'SELECT * FROM service(\'/service/service4/{id}\', id => \'abc\') LIMIT 1', + + {id:String[1], ints:Integer[*], date:StrictDate[0..1]| FlatInput.all()->filter(f | $f.idIn == $id && $f.integerIn->in($ints) && ($f.strictDateIn > $date))->project( [ @@ -1776,15 +1765,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: [ 'ID', 'Integer', 'Enum' ] - )->limit(1);}; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + )->limit(1)}) } function <> meta::external::query::sql::transformation::queryToPure::tests::testMultiLineQuery():Boolean[1] { - let sqlString = 'SELECT * FROM service."/service/service3"'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {| + test( + 'SELECT * FROM service."/service/service3"', + + {| let const = 123; FlatInput.all()->project( [ @@ -1793,15 +1782,15 @@ function <> meta::external::query::sql::transformation::queryToPure:: [ 'ID', 'Integer', 'The Enum Value', 'The Type', 'Const' ] - );}; - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + );}) } function <> meta::external::query::sql::transformation::queryToPure::tests::testParameterizedMultiLineJoin():Boolean[1] { - let sqlString = 'SELECT * FROM service(\'/service/service4/{id}\', id => \'abc\') union select ID, Integer, "The Enum Value" from service(\'/service/service3\')'; - let sqlTransformContext = $sqlString->processQuery(); - let expected = {id_1:String[1], ints_1:Integer[*], date_1:StrictDate[0..1]| + test( + 'SELECT * FROM service(\'/service/service4/{id}\', id => \'abc\') union select ID, Integer, "The Enum Value" from service(\'/service/service3\')', + + {id_1:String[1], ints_1:Integer[*], date_1:StrictDate[0..1]| let const_3 = 123; @@ -1819,9 +1808,7 @@ function <> meta::external::query::sql::transformation::queryToPure:: [ 'ID', 'Integer', 'The Enum Value', 'The Type', 'Const' ] - )->restrict(['ID', 'Integer', 'The Enum Value']));}; - - assertLambdaAndJSONEquals($expected, $sqlTransformContext.lambda()); + )->restrict(['ID', 'Integer', 'The Enum Value']));}) } @@ -1873,13 +1860,13 @@ function <> meta::external::query::sql::transformation::queryToPure:: let actualSchema = $sqlString->processQuery(true).columns()->meta::external::query::sql::tdsColsToSchema(); - let expectedCol1 = ^PrimitiveValueSchemaColumn(name='Boolean', type=meta::external::query::sql::PrimitiveType.Boolean); - let expectedCol2 = ^PrimitiveValueSchemaColumn(name='Integer', type=meta::external::query::sql::PrimitiveType.Integer); - let expectedCol3 = ^PrimitiveValueSchemaColumn(name='Float', type=meta::external::query::sql::PrimitiveType.Float); - let expectedCol4 = ^PrimitiveValueSchemaColumn(name='Decimal', type=meta::external::query::sql::PrimitiveType.Decimal); - let expectedCol5 = ^PrimitiveValueSchemaColumn(name='StrictDate', type=meta::external::query::sql::PrimitiveType.StrictDate); - let expectedCol6 = ^PrimitiveValueSchemaColumn(name='DateTime', type=meta::external::query::sql::PrimitiveType.DateTime); - let expectedCol7 = ^PrimitiveValueSchemaColumn(name='String', type=meta::external::query::sql::PrimitiveType.String); + let expectedCol1 = ^PrimitiveSchemaColumn(name='Boolean', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Boolean); + let expectedCol2 = ^PrimitiveSchemaColumn(name='Integer', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Integer); + let expectedCol3 = ^PrimitiveSchemaColumn(name='Float', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Float); + let expectedCol4 = ^PrimitiveSchemaColumn(name='Decimal', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Decimal); + let expectedCol5 = ^PrimitiveSchemaColumn(name='StrictDate', type=meta::external::query::sql::schema::metamodel::PrimitiveType.StrictDate); + let expectedCol6 = ^PrimitiveSchemaColumn(name='DateTime', type=meta::external::query::sql::schema::metamodel::PrimitiveType.DateTime); + let expectedCol7 = ^PrimitiveSchemaColumn(name='String', type=meta::external::query::sql::schema::metamodel::PrimitiveType.String); let expectedSchema = ^Schema(columns=[$expectedCol1, $expectedCol2, $expectedCol3, $expectedCol4, $expectedCol5, $expectedCol6, $expectedCol7]); assertEquals($expectedSchema, $actualSchema); @@ -1891,9 +1878,9 @@ function <> meta::external::query::sql::transformation::queryToPure:: let actualSchema = $sqlString->processQuery(true).columns()->meta::external::query::sql::tdsColsToSchema(); - let expectedCol1 = ^PrimitiveValueSchemaColumn(name='ID', type=meta::external::query::sql::PrimitiveType.Integer); - let expectedCol2 = ^PrimitiveValueSchemaColumn(name='Integer', type=meta::external::query::sql::PrimitiveType.Integer); - let expectedCol3 = ^PrimitiveValueSchemaColumn(name='String', type=meta::external::query::sql::PrimitiveType.String); + let expectedCol1 = ^PrimitiveSchemaColumn(name='ID', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Integer); + let expectedCol2 = ^PrimitiveSchemaColumn(name='Integer', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Integer); + let expectedCol3 = ^PrimitiveSchemaColumn(name='String', type=meta::external::query::sql::schema::metamodel::PrimitiveType.String); let expectedSchema = ^Schema(columns=[$expectedCol1, $expectedCol2, $expectedCol3]); assertEquals($expectedSchema, $actualSchema); @@ -1905,13 +1892,13 @@ function <> meta::external::query::sql::transformation::queryToPure:: let actualSchema = $sqlString->processQuery(true).columns()->meta::external::query::sql::tdsColsToSchema(); - let expectedCol1 = ^PrimitiveValueSchemaColumn(name='ID', type=meta::external::query::sql::PrimitiveType.Integer); - let expectedCol2 = ^PrimitiveValueSchemaColumn(name='Integer', type=meta::external::query::sql::PrimitiveType.Integer); - let expectedCol3 = ^EnumValueSchemaColumn(name='The Enum Value', type=MyEnum->elementToPath()); - let expectedCol4 = ^EnumValueSchemaColumn(name='The Type', type=FieldType->elementToPath()); - let expectedCol5 = ^PrimitiveValueSchemaColumn(name='Const', type=meta::external::query::sql::PrimitiveType.Integer); - let expectedEnum1 = ^meta::external::query::sql::Enum(type=MyEnum->elementToPath(), values=MyEnum->enumValues()->map(v:meta::pure::metamodel::type::Enum[1] | $v->id())); - let expectedEnum2 = ^meta::external::query::sql::Enum(type=FieldType->elementToPath(), values=FieldType->enumValues()->map(v:meta::pure::metamodel::type::Enum[1] | $v->id())); + let expectedCol1 = ^PrimitiveSchemaColumn(name='ID', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Integer); + let expectedCol2 = ^PrimitiveSchemaColumn(name='Integer', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Integer); + let expectedCol3 = ^EnumSchemaColumn(name='The Enum Value', type=MyEnum->elementToPath()); + let expectedCol4 = ^EnumSchemaColumn(name='The Type', type=FieldType->elementToPath()); + let expectedCol5 = ^PrimitiveSchemaColumn(name='Const', type=meta::external::query::sql::schema::metamodel::PrimitiveType.Integer); + let expectedEnum1 = ^meta::external::query::sql::schema::metamodel::Enum(type=MyEnum->elementToPath(), values=MyEnum->enumValues()->map(v:meta::pure::metamodel::type::Enum[1] | $v->id())); + let expectedEnum2 = ^meta::external::query::sql::schema::metamodel::Enum(type=FieldType->elementToPath(), values=FieldType->enumValues()->map(v:meta::pure::metamodel::type::Enum[1] | $v->id())); let expectedSchema = ^Schema(columns=[$expectedCol1, $expectedCol2, $expectedCol3, $expectedCol4,$expectedCol5], enums=[$expectedEnum1, $expectedEnum2]); assertEquals($expectedSchema, $actualSchema); @@ -1960,16 +1947,16 @@ function <> meta::external::query::sql::transformation::queryToPure:: doNameTest(^ArrayLiteral(values = [^IntegerLiteral(value = 1), ^IntegerLiteral(value = 2)]), '1,2'); doNameTest(^NullLiteral(), 'NULL'); - doNameTest(^LikePredicate(value = ^StringLiteral(value = 'a', quoted = false), + doNameTest(^LikePredicate(value = ^StringLiteral(value = 'a', quoted = false), pattern = ^StringLiteral(value = 'b', quoted = false), ignoreCase = false), 'a LIKE b'); - doNameTest(^LikePredicate(value = ^StringLiteral(value = 'a', quoted = false), + doNameTest(^LikePredicate(value = ^StringLiteral(value = 'a', quoted = false), + pattern = ^StringLiteral(value = 'b', quoted = false), + ignoreCase = true), 'a ILIKE b'); + doNameTest(^LikePredicate(value = ^StringLiteral(value = 'a', quoted = false), pattern = ^StringLiteral(value = 'b', quoted = false), - ignoreCase = true), 'a ILIKE b'); - doNameTest(^LikePredicate(value = ^StringLiteral(value = 'a', quoted = false), - pattern = ^StringLiteral(value = 'b', quoted = false), escape = ^StringLiteral(value = '!', quoted = false), - ignoreCase = false), 'a LIKE b ESCAPE !'); + ignoreCase = false), 'a LIKE b ESCAPE !'); doNameTest(^LogicalBinaryExpression(left = ^BooleanLiteral(value = true), right = ^BooleanLiteral(value = false),type = LogicalBinaryType.AND), 'true AND false'); doNameTest(^LogicalBinaryExpression(left = ^BooleanLiteral(value = true), right = ^BooleanLiteral(value = false),type = LogicalBinaryType.OR), 'true OR false'); @@ -1985,6 +1972,36 @@ function <> meta::external::query::sql::transformation::queryToPure:: defaultValue = ^IntegerLiteral(value = 1)), 'CASE WHEN true = false THEN 2 ELSE 1 END'); } + +function meta::external::query::sql::transformation::queryToPure::tests::testSources():SQLSource[*] +{ + [ + serviceToSource(Service1()), + serviceToSource(Service2()), + serviceToSource(Service3()), + serviceToSource(Service4()) + ] +} + +function meta::external::query::sql::transformation::queryToPure::tests::test(sql:String[1], expected:FunctionDefinition[1]):Boolean[1] +{ + test($sql, $expected, true); +} + +function meta::external::query::sql::transformation::queryToPure::tests::test(sql:String[1], expected:FunctionDefinition[1], assertJSON:Boolean[1]):Boolean[1] +{ + test($sql, $expected, testSources(), false, true, $assertJSON); +} + +function meta::external::query::sql::transformation::queryToPure::tests::test(sql:String[1], expected:FunctionDefinition[1], sources:SQLSource[*], scopeWithFrom:Boolean[1], assertLambda:Boolean[1], assertJSON:Boolean[1]):Boolean[1] +{ + let sqlTransformContext = $sql->processQuery($sources, $scopeWithFrom); + let actual = $sqlTransformContext.lambda(); + + if ($assertLambda, | assertLambdaEquals($expected, $actual), | true); + if ($assertJSON, | assertLambdaJSONEquals($expected, $actual), | true); +} + function meta::external::query::sql::transformation::queryToPure::tests::processQuery(sql: String[1]): SqlTransformContext[1] { processQuery($sql, false); @@ -2008,9 +2025,4 @@ function meta::external::query::sql::transformation::queryToPure::tests::process let context = rootContext($sources, $extensions); $query->processRootQuery(^$context(scopeWithFrom = $scopeWithFrom)); -} - -function meta::external::query::sql::transformation::queryToPure::tests::forceGetStringType(col: TDSColumn[1]): String[1] -{ - $col.type->toOne()->elementToPath(); -} +} \ No newline at end of file diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/model.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/model.pure deleted file mode 100644 index 899ee07cf30..00000000000 --- a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/model.pure +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2023 Goldman Sachs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - - -Class meta::external::query::sql::SchemaColumn -{ - <> name: String[1]; -} - -Class meta::external::query::sql::PrimitiveValueSchemaColumn extends meta::external::query::sql::SchemaColumn -{ - <> type: meta::external::query::sql::PrimitiveType[1]; -} - -Class meta::external::query::sql::EnumValueSchemaColumn extends meta::external::query::sql::SchemaColumn -{ - <> type: String[1]; -} - - -Class meta::external::query::sql::Schema -{ - <> columns: meta::external::query::sql::SchemaColumn[*]; - <> enums: meta::external::query::sql::Enum[*]; -} - -Enum meta::external::query::sql::PrimitiveType -{ - Boolean, - StrictDate, - Number, - String, - LatestDate, - Float, - DateTime, - Date, - Integer, - Decimal -} - -Class meta::external::query::sql::Enum -{ - <> type: String[1]; - <> values: String[*]; -} - -function meta::external::query::sql::stringToPrimitiveType(dt : String[1]) : meta::external::query::sql::PrimitiveType[1] -{ - let type = newMap( - [ - pair('Boolean', meta::external::query::sql::PrimitiveType.Boolean), - pair('StrictDate', meta::external::query::sql::PrimitiveType.StrictDate), - pair('Number', meta::external::query::sql::PrimitiveType.Number), - pair('String', meta::external::query::sql::PrimitiveType.String), - pair('LatestDate', meta::external::query::sql::PrimitiveType.LatestDate), - pair('Float', meta::external::query::sql::PrimitiveType.Float), - pair('DateTime', meta::external::query::sql::PrimitiveType.DateTime), - pair('Date', meta::external::query::sql::PrimitiveType.Date), - pair('Integer', meta::external::query::sql::PrimitiveType.Integer), - pair('Decimal', meta::external::query::sql::PrimitiveType.Decimal) - ] - )->get($dt); - if($type->isEmpty(), |fail('Unknown primitive type: ' + $dt), |[]); - $type->toOne(); -} - -function meta::external::query::sql::tdsColToSchemaCol(col: TDSColumn[1]): meta::external::query::sql::SchemaColumn[1] -{ - let columnName = $col.name; - if($col.type->isEmpty(), |fail('Column type is empty for col: ' + $columnName), |[]); - let tdsType = $col.type->toOne(); - let columnPrimitiveType = $col.type->match([ - s:PrimitiveType[1] | ^meta::external::query::sql::PrimitiveValueSchemaColumn(type=$tdsType->meta::pure::functions::meta::elementToPath()->meta::external::query::sql::stringToPrimitiveType(),name=$columnName), - e:Enumeration[1] | ^meta::external::query::sql::EnumValueSchemaColumn(type=$tdsType->meta::pure::functions::meta::elementToPath(),name=$columnName), - a:Any[*] | fail('Unsupported type on column: ' + $columnName + ' (' + $tdsType->meta::pure::functions::meta::elementToPath() + '), only primitive types and enums are supported'); - ])->cast(@meta::external::query::sql::SchemaColumn); -} - -function meta::external::query::sql::tdsColsToSchema(cols: TDSColumn[*]): meta::external::query::sql::Schema[1] -{ - let schemaCols = $cols->map(v:TDSColumn[1] | $v->meta::external::query::sql::tdsColToSchemaCol()); - let enums = $cols - ->map(c:TDSColumn[1] | $c.type->match([ - e:Enumeration[1] | ^meta::external::query::sql::Enum(type=$e->meta::pure::functions::meta::elementToPath(), values=$e->enumValues()->map( v:Enum[1] | $v->id())), - a:Any[*] | [] - ])) - ->distinct(); - ^meta::external::query::sql::Schema(columns=$schemaCols, enums=$enums); -} diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure new file mode 100644 index 00000000000..f75ada02117 --- /dev/null +++ b/legend-engine-xts-sql/legend-engine-xt-sql-pure/src/main/resources/core_external_query_sql/binding/schema.pure @@ -0,0 +1,58 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +function meta::external::query::sql::schema::stringToPrimitiveType(dt : String[1]) : meta::external::query::sql::schema::metamodel::PrimitiveType[1] +{ + let type = newMap( + [ + pair('Boolean', meta::external::query::sql::schema::metamodel::PrimitiveType.Boolean), + pair('StrictDate', meta::external::query::sql::schema::metamodel::PrimitiveType.StrictDate), + pair('Number', meta::external::query::sql::schema::metamodel::PrimitiveType.Number), + pair('String', meta::external::query::sql::schema::metamodel::PrimitiveType.String), + pair('LatestDate', meta::external::query::sql::schema::metamodel::PrimitiveType.LatestDate), + pair('Float', meta::external::query::sql::schema::metamodel::PrimitiveType.Float), + pair('DateTime', meta::external::query::sql::schema::metamodel::PrimitiveType.DateTime), + pair('Date', meta::external::query::sql::schema::metamodel::PrimitiveType.Date), + pair('Integer', meta::external::query::sql::schema::metamodel::PrimitiveType.Integer), + pair('Decimal', meta::external::query::sql::schema::metamodel::PrimitiveType.Decimal) + ] + )->get($dt); + if($type->isEmpty(), |fail('Unknown primitive type: ' + $dt), |[]); + $type->toOne(); +} + +function meta::external::query::sql::schema::tdsColToSchemaCol(col: TDSColumn[1]): meta::external::query::sql::schema::metamodel::SchemaColumn[1] +{ + let columnName = $col.name; + if($col.type->isEmpty(), |fail('Column type is empty for col: ' + $columnName), |[]); + let tdsType = $col.type->toOne(); + let columnPrimitiveType = $col.type->match([ + s:PrimitiveType[1] | ^meta::external::query::sql::schema::metamodel::PrimitiveSchemaColumn(type=$tdsType->meta::pure::functions::meta::elementToPath()->meta::external::query::sql::schema::stringToPrimitiveType(),name=$columnName), + e:Enumeration[1] | ^meta::external::query::sql::schema::metamodel::EnumSchemaColumn(type=$tdsType->meta::pure::functions::meta::elementToPath(),name=$columnName), + a:Any[*] | fail('Unsupported type on column: ' + $columnName + ' (' + $tdsType->meta::pure::functions::meta::elementToPath() + '), only primitive types and enums are supported'); + ])->cast(@meta::external::query::sql::schema::metamodel::SchemaColumn); +} + +function meta::external::query::sql::tdsColsToSchema(cols: TDSColumn[*]): meta::external::query::sql::schema::metamodel::Schema[1] +{ + let schemaCols = $cols->map(v:TDSColumn[1] | $v->meta::external::query::sql::schema::tdsColToSchemaCol()); + let enums = $cols + ->map(c:TDSColumn[1] | $c.type->match([ + e:Enumeration[1] | ^meta::external::query::sql::schema::metamodel::Enum(type=$e->meta::pure::functions::meta::elementToPath(), values=$e->enumValues()->map( v:Enum[1] | $v->id())), + a:Any[*] | [] + ])) + ->distinct(); + ^meta::external::query::sql::schema::metamodel::Schema(columns=$schemaCols, enums=$enums); +} diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-query/pom.xml b/legend-engine-xts-sql/legend-engine-xt-sql-query/pom.xml index b69688d8689..18db6143fc2 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-query/pom.xml +++ b/legend-engine-xts-sql/legend-engine-xt-sql-query/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-sql - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 @@ -33,10 +33,6 @@ org.finos.legend.pure legend-pure-m3-core - - org.finos.legend.pure - legend-pure-runtime-java-engine-compiled - diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/SQLExecutor.java b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/SQLExecutor.java new file mode 100644 index 00000000000..798afbdf56b --- /dev/null +++ b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/SQLExecutor.java @@ -0,0 +1,293 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package org.finos.legend.engine.query.sql.api; + +import org.eclipse.collections.api.RichIterable; +import org.eclipse.collections.api.block.function.Function; +import org.eclipse.collections.api.block.function.Function3; +import org.eclipse.collections.api.collection.MutableCollection; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Maps; +import org.eclipse.collections.api.list.MutableList; +import org.eclipse.collections.api.map.MutableMap; +import org.eclipse.collections.api.multimap.MutableMultimap; +import org.eclipse.collections.api.tuple.Pair; +import org.eclipse.collections.impl.map.mutable.UnifiedMap; +import org.eclipse.collections.impl.tuple.Tuples; +import org.eclipse.collections.impl.utility.Iterate; +import org.eclipse.collections.impl.utility.ListIterate; +import org.eclipse.collections.impl.utility.internal.IterableIterate; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.modelManager.ModelManager; +import org.finos.legend.engine.plan.execution.PlanExecutor; +import org.finos.legend.engine.plan.execution.result.ConstantResult; +import org.finos.legend.engine.plan.execution.result.Result; +import org.finos.legend.engine.plan.generation.transformers.PlanTransformer; +import org.finos.legend.engine.plan.platform.PlanPlatform; +import org.finos.legend.engine.protocol.pure.PureClientVersions; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextPointer; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.protocol.sql.metamodel.ProtocolToMetamodelTranslator; +import org.finos.legend.engine.protocol.sql.metamodel.Query; +import org.finos.legend.engine.protocol.sql.schema.metamodel.MetamodelToProtocolTranslator; +import org.finos.legend.engine.protocol.sql.schema.metamodel.Schema; +import org.finos.legend.engine.query.sql.api.sources.SQLContext; +import org.finos.legend.engine.query.sql.api.sources.SQLSource; +import org.finos.legend.engine.query.sql.api.sources.SQLSourceProvider; +import org.finos.legend.engine.query.sql.api.sources.SQLSourceResolvedContext; +import org.finos.legend.engine.query.sql.api.sources.SQLSourceTranslator; +import org.finos.legend.engine.query.sql.api.sources.TableSource; +import org.finos.legend.engine.query.sql.api.sources.TableSourceExtractor; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; +import org.finos.legend.engine.shared.core.operational.logs.LogInfo; +import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType; +import org.finos.legend.engine.shared.core.operational.prometheus.MetricsHandler; +import org.finos.legend.pure.generated.Root_meta_external_query_sql_metamodel_Query; +import org.finos.legend.pure.generated.Root_meta_external_query_sql_schema_metamodel_Schema; +import org.finos.legend.pure.generated.Root_meta_external_query_sql_transformation_queryToPure_PlanGenerationResult; +import org.finos.legend.pure.generated.Root_meta_external_query_sql_transformation_queryToPure_SQLSource; +import org.finos.legend.pure.generated.Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext; +import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; +import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; +import org.finos.legend.pure.generated.core_external_format_json_toJSON; +import org.finos.legend.pure.generated.core_external_query_sql_binding_fromPure_fromPure; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; +import org.finos.legend.pure.m3.execution.ExecutionSupport; +import org.pac4j.core.profile.CommonProfile; +import org.slf4j.Logger; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.finos.legend.engine.plan.generation.PlanGenerator.transformExecutionPlan; + +public class SQLExecutor +{ + private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(SQLExecutor.class); + private final ModelManager modelManager; + private final PlanExecutor planExecutor; + private final Function> routerExtensions; + private final Iterable transformers; + private final MutableMap providers; + + public SQLExecutor(ModelManager modelManager, PlanExecutor planExecutor, + Function> routerExtensions, + List providers, + Iterable transformers) + { + this.modelManager = modelManager; + this.planExecutor = planExecutor; + this.routerExtensions = routerExtensions; + this.transformers = transformers; + this.providers = ListIterate.groupByUniqueKey(providers, SQLSourceProvider::getType); + } + + + public Result execute(Query query, String user, SQLContext context, MutableList profiles) + { + return process(query, (transformedContext, pureModel, sources) -> + { + long start = System.currentTimeMillis(); + LOGGER.info(new LogInfo(profiles, LoggingEventType.EXECUTE_INTERACTIVE_STOP, (double) System.currentTimeMillis() - start).toString()); + + Root_meta_external_query_sql_transformation_queryToPure_PlanGenerationResult plans = planResult(transformedContext, pureModel, sources); + + Map arguments = UnifiedMap.newMapWith(IterableIterate.collectIf(plans._arguments(), p -> p._value() != null || p._plan() != null, p -> + { + Result result; + + if (p._value() != null) + { + Object value = p._value() instanceof org.finos.legend.pure.m3.coreinstance.meta.pure.functions.collection.List + ? ((org.finos.legend.pure.m3.coreinstance.meta.pure.functions.collection.List) p._value())._values() + : p._value(); + result = new ConstantResult(value); + } + else + { + Root_meta_pure_executionPlan_ExecutionPlan l = PlanPlatform.JAVA.bindPlan(p._plan(), null, pureModel, routerExtensions.apply(pureModel)); + SingleExecutionPlan m = transformExecutionPlan(l, pureModel, PureClientVersions.production, profiles, routerExtensions.apply(pureModel), transformers); + result = planExecutor.execute(m, Maps.mutable.empty(), "pentej", profiles); + } + + return Tuples.pair(p._name(), result); + })); + + Root_meta_pure_executionPlan_ExecutionPlan plan = plans._plan(); + plan = PlanPlatform.JAVA.bindPlan(plan, null, pureModel, routerExtensions.apply(pureModel)); + SingleExecutionPlan transformedPlan = transformExecutionPlan(plan, pureModel, PureClientVersions.production, profiles, routerExtensions.apply(pureModel), transformers); + + Result result = planExecutor.execute(transformedPlan, arguments, user, profiles); + + MetricsHandler.observe("execute", start, System.currentTimeMillis()); + + return result; + }, context, profiles); + } + + public Lambda lambda(Query query, SQLContext context, MutableList profiles) + { + return process(query, + (transformedContext, pureModel, sources) -> + { + LambdaFunction lambda = transformedContext.lambda(pureModel.getExecutionSupport()); + return transformLambda(lambda, pureModel); + }, + (sources, extensions, pureModel) -> core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_rootContext_SQLSource_MANY__Extension_MANY__SqlTransformContext_1_(sources, extensions, pureModel.getExecutionSupport())._scopeWithFrom(false), + context, profiles); + } + + public SingleExecutionPlan plan(Query query, SQLContext context, MutableList profiles) + { + return process(query, (transformedContext, pureModel, sources) -> transformExecutionPlan(planResult(transformedContext, pureModel, sources)._plan(), pureModel, PureClientVersions.production, profiles, routerExtensions.apply(pureModel), transformers), context, profiles); + } + + public Schema schema(Query query, MutableList profiles) + { + SQLContext context = new SQLContext(query, Maps.mutable.of()); + return process(query, (t, pm, sources) -> + { + Root_meta_external_query_sql_schema_metamodel_Schema schema = core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_getSchema_SqlTransformContext_1__Schema_1_(t, pm.getExecutionSupport()); + return new MetamodelToProtocolTranslator().translate(schema); + }, context, profiles); + } + + private Root_meta_external_query_sql_transformation_queryToPure_PlanGenerationResult planResult(Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext transformedContext, PureModel pureModel, RichIterable sources) + { + return core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_getPlanResult_SqlTransformContext_1__SQLSource_MANY__Extension_MANY__PlanGenerationResult_1_(transformedContext, sources, routerExtensions.apply(pureModel), pureModel.getExecutionSupport()); + } + + private T process(Query query, Function3, T> func, SQLContext context, MutableList profiles) + { + return process(query, func, (sources, extensions, pureModel) -> core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_rootContext_SQLSource_MANY__Extension_MANY__SqlTransformContext_1_(sources, extensions, pureModel.getExecutionSupport()), context, profiles); + } + + private T process(Query query, + Function3, T> func, + Function3, RichIterable, PureModel, Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext> transformContextFunc, + SQLContext context, + MutableList profiles) + { + Pair, PureModelContext> sqlSourcesAndPureModel = getSourcesAndModel(query, context, profiles); + RichIterable sources = sqlSourcesAndPureModel.getOne(); + PureModelContext pureModelContext = sqlSourcesAndPureModel.getTwo(); + + PureModel pureModel = modelManager.loadModel(pureModelContext, PureClientVersions.production, profiles, ""); + + Root_meta_external_query_sql_metamodel_Query compiledQuery = new ProtocolToMetamodelTranslator().translate(query, pureModel); + + RichIterable compiledSources = new SQLSourceTranslator().translate(sources, pureModel); + LOGGER.info("{}", new LogInfo(profiles, LoggingEventType.GENERATE_PLAN_START)); + + Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext transformedContext = core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_processRootQuery_Query_1__SqlTransformContext_1__SqlTransformContext_1_( + compiledQuery, transformContextFunc.value(compiledSources, routerExtensions.apply(pureModel), pureModel), pureModel.getExecutionSupport()); + + return func.value(transformedContext, pureModel, compiledSources); + } + + private Pair, PureModelContext> getSourcesAndModel(Query query, SQLContext context, MutableList profiles) + { + Set tables = new TableSourceExtractor().visit(query); + + MutableMultimap grouped = Iterate.groupBy(tables, TableSource::getType); + + boolean schemasValid = Iterate.allSatisfy(grouped.keySet(), providers::containsKey); + + if (!schemasValid) + { + throw new IllegalArgumentException("Unsupported schema types " + String.join(", ", grouped.keySet().select(k -> !providers.containsKey(k)))); + } + + RichIterable resolved = grouped.keySet().collect(k -> resolve(grouped.get(k), context, providers.get(k), profiles)); + + MutableList allContexts = IterableIterate.flatCollect(resolved, SQLSourceResolvedContext::getPureModelContexts); + boolean allCompatiblePointers = allContexts.allSatisfy(p -> p instanceof PureModelContextPointer && allContexts.allSatisfy(p2 -> ((PureModelContextPointer) p).safeEqual(p, p2))); + + PureModelContext pureModelContext; + + //this means all pointers are from same source, so we can combine to utilise model cache. + if (allCompatiblePointers) + { + pureModelContext = allContexts.collectIf(p -> p instanceof PureModelContextPointer, p -> ((PureModelContextPointer) p)) + .injectInto(null, (d, e) -> e.combine((PureModelContextPointer) d)); + } + else + { + pureModelContext = resolved.injectInto(PureModelContextData.newPureModelContextData(), (p, p2) -> PureModelContextData.combine(p, PureModelContextData.newPureModelContextData(), ListIterate.collect(p2.getPureModelContexts(), c -> modelManager.loadData(c, PureClientVersions.production, profiles)).toArray(new PureModelContextData[]{}))); + } + RichIterable sources = resolved.flatCollect(SQLSourceResolvedContext::getSources); + return Tuples.pair(sources, pureModelContext); + } + + + private SQLSourceResolvedContext resolve(MutableCollection tables, SQLContext context, SQLSourceProvider extension, MutableList profiles) + { + return extension.resolve(tables.toList(), context, profiles); + } + + static String serializeToJSON(Object pureObject, PureModel pureModel, Boolean alloyJSON) + { + return alloyJSON + ? org.finos.legend.pure.generated.core_pure_protocol_protocol.Root_meta_alloy_metadataServer_alloyToJSON_Any_1__String_1_(pureObject, pureModel.getExecutionSupport()) + : core_external_format_json_toJSON.Root_meta_json_toJSON_Any_MANY__Integer_$0_1$__Config_1__String_1_( + Lists.mutable.with(pureObject), + 1000L, + core_external_format_json_toJSON.Root_meta_json_config_Boolean_1__Boolean_1__Boolean_1__Boolean_1__Config_1_(true, false, false, false, pureModel.getExecutionSupport()), + pureModel.getExecutionSupport() + ); + } + + private Lambda transformLambda(LambdaFunction lambda, PureModel pureModel) + { + Object protocol = transformToVersionedModel(lambda, PureClientVersions.production, routerExtensions.apply(pureModel), pureModel.getExecutionSupport()); + return transform(protocol, Lambda.class, pureModel); + } + + private T transform(Object object, java.lang.Class clazz, PureModel pureModel) + { + String json = serializeToJSON(object, pureModel, true); + try + { + return ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().readValue(json, clazz); + } + catch (Exception e) + { + throw new EngineException(e.getMessage()); + } + } + + public Object transformToVersionedModel(FunctionDefinition lambda, String version, RichIterable extensions, ExecutionSupport executionSupport) + { + try + { + Class cl = Class.forName("org.finos.legend.pure.generated.core_pure_protocol_" + version + "_transfers_valueSpecification"); + Method method = cl.getMethod("Root_meta_protocols_pure_" + version + "_transformation_fromPureGraph_transformLambda_FunctionDefinition_1__Extension_MANY__Lambda_1_", FunctionDefinition.class, RichIterable.class, org.finos.legend.pure.m3.execution.ExecutionSupport.class); + return method.invoke(null, lambda, extensions, executionSupport); + } + catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException | IllegalAccessException e) + { + throw new RuntimeException(e); + } + } +} diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/execute/SqlExecute.java b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/execute/SqlExecute.java index 220aba69785..9d463443298 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/execute/SqlExecute.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/main/java/org/finos/legend/engine/query/sql/api/execute/SqlExecute.java @@ -21,74 +21,29 @@ import io.swagger.annotations.ApiParam; import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.block.function.Function; -import org.eclipse.collections.api.block.function.Function3; -import org.eclipse.collections.api.collection.MutableCollection; -import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Maps; import org.eclipse.collections.api.list.MutableList; -import org.eclipse.collections.api.map.MutableMap; -import org.eclipse.collections.api.multimap.MutableMultimap; -import org.eclipse.collections.api.tuple.Pair; -import org.eclipse.collections.impl.map.mutable.UnifiedMap; -import org.eclipse.collections.impl.tuple.Tuples; -import org.eclipse.collections.impl.utility.Iterate; -import org.eclipse.collections.impl.utility.ListIterate; -import org.eclipse.collections.impl.utility.internal.IterableIterate; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.modelManager.ModelManager; import org.finos.legend.engine.language.sql.grammar.from.SQLGrammarParser; import org.finos.legend.engine.plan.execution.PlanExecutor; -import org.finos.legend.engine.plan.execution.result.ConstantResult; import org.finos.legend.engine.plan.execution.result.Result; import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; import org.finos.legend.engine.plan.generation.transformers.PlanTransformer; -import org.finos.legend.engine.plan.platform.PlanPlatform; -import org.finos.legend.engine.protocol.pure.PureClientVersions; -import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContext; -import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; -import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextPointer; -import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.SingleExecutionPlan; +import org.finos.legend.engine.protocol.pure.v1.model.executionPlan.ExecutionPlan; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; -import org.finos.legend.engine.protocol.sql.metamodel.ProtocolToMetamodelTranslator; import org.finos.legend.engine.protocol.sql.metamodel.Query; -import org.finos.legend.engine.protocol.sql.metamodel.Statement; +import org.finos.legend.engine.protocol.sql.schema.metamodel.Schema; +import org.finos.legend.engine.query.sql.api.SQLExecutor; import org.finos.legend.engine.query.sql.api.sources.SQLContext; -import org.finos.legend.engine.query.sql.api.sources.SQLSource; import org.finos.legend.engine.query.sql.api.sources.SQLSourceProvider; -import org.finos.legend.engine.query.sql.api.sources.SQLSourceResolvedContext; -import org.finos.legend.engine.query.sql.api.sources.SQLSourceTranslator; -import org.finos.legend.engine.query.sql.api.sources.TableSource; -import org.finos.legend.engine.query.sql.api.sources.TableSourceExtractor; -import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.finos.legend.engine.shared.core.kerberos.ProfileManagerHelper; -import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; -import org.finos.legend.engine.shared.core.operational.logs.LogInfo; import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType; -import org.finos.legend.engine.shared.core.operational.prometheus.MetricsHandler; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_Schema; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_metamodel_Node; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_metamodel_Query; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_transformation_queryToPure_PlanGenerationResult; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_transformation_queryToPure_SQLSource; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext; -import org.finos.legend.pure.generated.Root_meta_pure_executionPlan_ExecutionPlan; import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; -import org.finos.legend.pure.generated.core_external_format_json_toJSON; -import org.finos.legend.pure.generated.core_external_query_sql_binding_fromPure_fromPure; -import org.finos.legend.pure.generated.core_pure_tools_tools_extension; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.FunctionDefinition; -import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; -import org.finos.legend.pure.m3.execution.ExecutionSupport; import org.pac4j.core.profile.CommonProfile; import org.pac4j.core.profile.ProfileManager; import org.pac4j.jax.rs.annotations.Pac4JProfileManager; -import org.slf4j.Logger; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.List; -import java.util.Map; -import java.util.Set; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.Consumes; import javax.ws.rs.DefaultValue; @@ -100,34 +55,31 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import java.util.List; import static org.finos.legend.engine.plan.execution.api.result.ResultManager.manageResult; -import static org.finos.legend.engine.plan.generation.PlanGenerator.transformExecutionPlan; @Api(tags = "SQL - Execution") @Path("sql/v1/execution") @Produces(MediaType.APPLICATION_JSON) public class SqlExecute { + private static final SQLGrammarParser PARSER = SQLGrammarParser.newInstance(); - private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(SqlExecute.class); - private static final SQLGrammarParser parser = SQLGrammarParser.newInstance(); - private final ModelManager modelManager; - private final PlanExecutor planExecutor; - private final Function> routerExtensions; - private final Iterable transformers; - private final MutableMap providers; + private final SQLExecutor executor; + @Deprecated public SqlExecute(ModelManager modelManager, PlanExecutor planExecutor, Function> routerExtensions, List providers, Iterable transformers) { - this.modelManager = modelManager; - this.planExecutor = planExecutor; - this.routerExtensions = routerExtensions; - this.transformers = transformers; - this.providers = ListIterate.groupByUniqueKey(providers, SQLSourceProvider::getType); + this(new SQLExecutor(modelManager, planExecutor, routerExtensions, providers, transformers)); + } + + public SqlExecute(SQLExecutor executor) + { + this.executor = executor; } @POST @@ -137,7 +89,7 @@ public SqlExecute(ModelManager modelManager, PlanExecutor planExecutor, public Response executeSql(@Context HttpServletRequest request, String sql, @DefaultValue(SerializationFormat.defaultFormatString) @QueryParam("serializationFormat")SerializationFormat format, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { - Query query = (Query) parser.parseStatement(sql); + Query query = parseSQL(sql); return executeSql(request, query, format, pm, uriInfo); } @@ -149,14 +101,10 @@ public Response executeSql(@Context HttpServletRequest request, Query query, @De @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { MutableList profiles = ProfileManagerHelper.extractProfiles(pm); - SQLContext context = new SQLContext(query, Maps.mutable.of()); - long start = System.currentTimeMillis(); - LOGGER.info(new LogInfo(profiles, LoggingEventType.EXECUTE_INTERACTIVE_STOP, (double) System.currentTimeMillis() - start).toString()); - Result result = execute(query, request.getRemoteUser(), context, profiles); + Result result = this.executor.execute(query, request.getRemoteUser(), context, profiles); - MetricsHandler.observe("execute", start, System.currentTimeMillis()); try (Scope ignored = GlobalTracer.get().buildSpan("Manage Results").startActive(true)) { return manageResult(profiles, result, format, LoggingEventType.EXECUTE_INTERACTIVE_ERROR); @@ -169,7 +117,7 @@ public Response executeSql(@Context HttpServletRequest request, Query query, @De @Consumes({MediaType.TEXT_PLAIN}) public Lambda generateLambda(@Context HttpServletRequest request, String sql, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { - Query query = (Query) parser.parseStatement(sql); + Query query = parseSQL(sql); return generateLambda(request, query, pm, uriInfo); } @@ -182,16 +130,16 @@ public Lambda generateLambda(@Context HttpServletRequest request, Query query, @ MutableList profiles = ProfileManagerHelper.extractProfiles(pm); SQLContext context = new SQLContext(query, Maps.mutable.of()); - return lambda(query, context, profiles); + return executor.lambda(query, context, profiles); } @POST @ApiOperation(value = "Generate plans for a SQL query using sql string") @Path("generatePlanQueryString") @Consumes({MediaType.TEXT_PLAIN}) - public Response generatePlan(@Context HttpServletRequest request, String sql, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) + public ExecutionPlan generatePlan(@Context HttpServletRequest request, String sql, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { - Query query = (Query) parser.parseStatement(sql); + Query query = parseSQL(sql); return generatePlan(request, query, pm, uriInfo); } @@ -199,210 +147,45 @@ public Response generatePlan(@Context HttpServletRequest request, String sql, @A @ApiOperation(value = "Generate plans for a SQL query using protocol model") @Path("generatePlanQuery") @Consumes({MediaType.APPLICATION_JSON}) - public Response generatePlan(@Context HttpServletRequest request, Query query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) + public ExecutionPlan generatePlan(@Context HttpServletRequest request, Query query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { MutableList profiles = ProfileManagerHelper.extractProfiles(pm); SQLContext context = new SQLContext(query, Maps.mutable.of()); - SingleExecutionPlan singleExecutionPlan = plan(query, context, profiles); - return Response.ok().type(MediaType.APPLICATION_JSON_TYPE).entity(singleExecutionPlan).build(); + return this.executor.plan(query, context, profiles); } @POST - @ApiOperation(value = "Get schema for a SQL query in the context of a Mapping and a Runtime from a SDLC project") + @ApiOperation(value = "Get schema for a SQL query") @Path("getSchemaFromQueryString") @Consumes({MediaType.TEXT_PLAIN}) - public Response getSchema(@Context HttpServletRequest request, String sql, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) + public Schema getSchema(@Context HttpServletRequest request, String sql, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { - Query query = (Query) parser.parseStatement(sql); + Query query = parseSQL(sql); return getSchema(request, query, pm, uriInfo); } @POST - @ApiOperation(value = "Get schema for a SQL query in the context of a Mapping and a Runtime from a SDLC project") + @ApiOperation(value = "Get schema for a SQL query") @Path("getSchemaFromQuery") @Consumes({MediaType.APPLICATION_JSON}) - public Response getSchema(@Context HttpServletRequest request, Query query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) + public Schema getSchema(@Context HttpServletRequest request, Query query, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { MutableList profiles = ProfileManagerHelper.extractProfiles(pm); - - String schema = schema(query, profiles); - return Response.ok().type(MediaType.APPLICATION_JSON_TYPE).entity(schema).build(); - } - - private Lambda lambda(Query query, SQLContext context, MutableList profiles) - { - return process(query, - (transformedContext, pureModel, sources) -> transformLambda(transformedContext.lambda(pureModel.getExecutionSupport()), pureModel), - (sources, extensions, pureModel) -> core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_rootContext_SQLSource_MANY__Extension_MANY__SqlTransformContext_1_(sources, extensions, pureModel.getExecutionSupport())._scopeWithFrom(false), - context, profiles); + return this.executor.schema(query, profiles); } - private String schema(Query query, MutableList profiles) - { - SQLContext context = new SQLContext(query, Maps.mutable.of()); - return process(query, (t, pm, sources) -> - { - Root_meta_external_query_sql_Schema schema = core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_getSchema_SqlTransformContext_1__Schema_1_(t, pm.getExecutionSupport()); - return serializeToJSON(schema, pm, false); - }, context, profiles); - } - - private Root_meta_external_query_sql_transformation_queryToPure_PlanGenerationResult planResult(Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext transformedContext, PureModel pureModel, RichIterable sources) - { - return core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_getPlanResult_SqlTransformContext_1__SQLSource_MANY__Extension_MANY__PlanGenerationResult_1_(transformedContext, sources, routerExtensions.apply(pureModel), pureModel.getExecutionSupport()); - } - - private SingleExecutionPlan plan(Query query, SQLContext context, MutableList profiles) - { - return process(query, (transformedContext, pureModel, sources) -> transformExecutionPlan(planResult(transformedContext, pureModel, sources)._plan(), pureModel, PureClientVersions.production, profiles, routerExtensions.apply(pureModel), transformers), context, profiles); - } - - private Result execute(Query query, String user, SQLContext context, MutableList profiles) - { - return process(query, (transformedContext, pureModel, sources) -> - { - Root_meta_external_query_sql_transformation_queryToPure_PlanGenerationResult plans = planResult(transformedContext, pureModel, sources); - - Map arguments = UnifiedMap.newMapWith(IterableIterate.collectIf(plans._arguments(), p -> p._value() != null || p._plan() != null, p -> - { - Result result; - - if (p._value() != null) - { - Object value = p._value() instanceof org.finos.legend.pure.m3.coreinstance.meta.pure.functions.collection.List - ? ((org.finos.legend.pure.m3.coreinstance.meta.pure.functions.collection.List) p._value())._values() - : p._value(); - result = new ConstantResult(value); - } - else - { - Root_meta_pure_executionPlan_ExecutionPlan l = PlanPlatform.JAVA.bindPlan(p._plan(), null, pureModel, routerExtensions.apply(pureModel)); - SingleExecutionPlan m = transformExecutionPlan(l, pureModel, PureClientVersions.production, profiles, routerExtensions.apply(pureModel), transformers); - result = planExecutor.execute(m, Maps.mutable.empty(), "pentej", profiles); - } - - return Tuples.pair(p._name(), result); - })); - - Root_meta_pure_executionPlan_ExecutionPlan plan = plans._plan(); - plan = PlanPlatform.JAVA.bindPlan(plan, null, pureModel, routerExtensions.apply(pureModel)); - SingleExecutionPlan transformedPlan = transformExecutionPlan(plan, pureModel, PureClientVersions.production, profiles, routerExtensions.apply(pureModel), transformers); - - return planExecutor.execute(transformedPlan, arguments, user, profiles); - }, context, profiles); - } - - private T process(Query query, Function3, T> func, SQLContext context, MutableList profiles) - { - return process(query, func, (sources, extensions, pureModel) -> core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_rootContext_SQLSource_MANY__Extension_MANY__SqlTransformContext_1_(sources, extensions, pureModel.getExecutionSupport()), context, profiles); - } - - private T process(Query query, - Function3, T> func, - Function3, RichIterable, PureModel, Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext> transformContextFunc, - SQLContext context, - MutableList profiles) - { - Pair, PureModelContext> sqlSourcesAndPureModel = getSourcesAndModel(query, context, profiles); - RichIterable sources = sqlSourcesAndPureModel.getOne(); - PureModelContext pureModelContext = sqlSourcesAndPureModel.getTwo(); - - PureModel pureModel = modelManager.loadModel(pureModelContext, PureClientVersions.production, profiles, ""); - - Root_meta_external_query_sql_metamodel_Query compiledQuery = new ProtocolToMetamodelTranslator().translate(query, pureModel); - - RichIterable compiledSources = new SQLSourceTranslator().translate(sources, pureModel); - LOGGER.info("{}", new LogInfo(profiles, LoggingEventType.GENERATE_PLAN_START)); - - Root_meta_external_query_sql_transformation_queryToPure_SqlTransformContext transformedContext = core_external_query_sql_binding_fromPure_fromPure.Root_meta_external_query_sql_transformation_queryToPure_processRootQuery_Query_1__SqlTransformContext_1__SqlTransformContext_1_( - compiledQuery, transformContextFunc.value(compiledSources, routerExtensions.apply(pureModel), pureModel), pureModel.getExecutionSupport()); - - return func.value(transformedContext, pureModel, compiledSources); - } - - private Pair, PureModelContext> getSourcesAndModel(Query query, SQLContext context, MutableList profiles) - { - Set tables = new TableSourceExtractor().visit(query); - - MutableMultimap grouped = Iterate.groupBy(tables, TableSource::getType); - - boolean schemasValid = Iterate.allSatisfy(grouped.keySet(), providers::containsKey); - - if (!schemasValid) - { - throw new IllegalArgumentException("Unsupported schema types " + String.join(", ", grouped.keySet().select(k -> !providers.containsKey(k)))); - } - - RichIterable resolved = grouped.keySet().collect(k -> resolve(grouped.get(k), context, providers.get(k), profiles)); - - MutableList allContexts = IterableIterate.flatCollect(resolved, SQLSourceResolvedContext::getPureModelContexts); - boolean allCompatiblePointers = allContexts.allSatisfy(p -> p instanceof PureModelContextPointer && allContexts.allSatisfy(p2 -> ((PureModelContextPointer) p).safeEqual(p, p2))); - - PureModelContext pureModelContext; - - //this means all pointers are from same source, so we can combine to utilise model cache. - if (allCompatiblePointers) - { - pureModelContext = allContexts.collectIf(p -> p instanceof PureModelContextPointer, p -> ((PureModelContextPointer) p)) - .injectInto(null, (d, e) -> e.combine((PureModelContextPointer) d)); - } - else - { - pureModelContext = resolved.injectInto(PureModelContextData.newPureModelContextData(), (p, p2) -> PureModelContextData.combine(p, PureModelContextData.newPureModelContextData(), ListIterate.collect(p2.getPureModelContexts(), c -> modelManager.loadData(c, PureClientVersions.production, profiles)).toArray(new PureModelContextData[]{}))); - } - RichIterable sources = resolved.flatCollect(SQLSourceResolvedContext::getSources); - return Tuples.pair(sources, pureModelContext); - } - - - private SQLSourceResolvedContext resolve(MutableCollection tables, SQLContext context, SQLSourceProvider extension, MutableList profiles) - { - return extension.resolve(tables.toList(), context, profiles); - } - - static String serializeToJSON(Object pureObject, PureModel pureModel, Boolean alloyJSON) - { - return alloyJSON - ? org.finos.legend.pure.generated.core_pure_protocol_protocol.Root_meta_alloy_metadataServer_alloyToJSON_Any_1__String_1_(pureObject, pureModel.getExecutionSupport()) - : core_external_format_json_toJSON.Root_meta_json_toJSON_Any_MANY__Integer_$0_1$__Config_1__String_1_( - Lists.mutable.with(pureObject), - 1000L, - core_external_format_json_toJSON.Root_meta_json_config_Boolean_1__Boolean_1__Boolean_1__Boolean_1__Config_1_(true, false, false, false, pureModel.getExecutionSupport()), - pureModel.getExecutionSupport() - ); - } - - private Lambda transformLambda(LambdaFunction lambda, PureModel pureModel) - { - Object protocol = transformToVersionedModel(lambda, PureClientVersions.production, routerExtensions.apply(pureModel), pureModel.getExecutionSupport()); - return transform(protocol, Lambda.class, pureModel); - } - - private T transform(Object object, java.lang.Class clazz, PureModel pureModel) + @POST + @ApiOperation(value = "Parse SQL to metamodel") + @Path("parseToMetamodel") + @Consumes({MediaType.TEXT_PLAIN}) + public Query parseToMetamodel(@Context HttpServletRequest request, String sql, @ApiParam(hidden = true) @Pac4JProfileManager ProfileManager pm, @Context UriInfo uriInfo) { - String json = serializeToJSON(object, pureModel, true); - try - { - return ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports().readValue(json, clazz); - } - catch (Exception e) - { - throw new EngineException(e.getMessage()); - } + return parseSQL(sql); } - public Object transformToVersionedModel(FunctionDefinition lambda, String version, RichIterable extensions, ExecutionSupport executionSupport) + private Query parseSQL(String sql) { - try - { - Class cl = Class.forName("org.finos.legend.pure.generated.core_pure_protocol_" + version + "_transfers_valueSpecification"); - Method method = cl.getMethod("Root_meta_protocols_pure_" + version + "_transformation_fromPureGraph_transformLambda_FunctionDefinition_1__Extension_MANY__Lambda_1_", FunctionDefinition.class, RichIterable.class, org.finos.legend.pure.m3.execution.ExecutionSupport.class); - return method.invoke(null, lambda, extensions, executionSupport); - } - catch (ClassNotFoundException | NoSuchMethodException | InvocationTargetException | IllegalAccessException e) - { - throw new RuntimeException(e); - } + return (Query) PARSER.parseStatement(sql); } } \ No newline at end of file diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/java/org/finos/legend/engine/query/sql/api/execute/SqlExecuteTest.java b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/java/org/finos/legend/engine/query/sql/api/execute/SqlExecuteTest.java index 4dc4d7f0815..408501ec7c3 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/java/org/finos/legend/engine/query/sql/api/execute/SqlExecuteTest.java +++ b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/java/org/finos/legend/engine/query/sql/api/execute/SqlExecuteTest.java @@ -30,21 +30,18 @@ import org.finos.legend.engine.plan.execution.result.serialization.SerializationFormat; import org.finos.legend.engine.plan.generation.extension.PlanGeneratorExtension; import org.finos.legend.engine.protocol.pure.PureClientVersions; -import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; +import org.finos.legend.engine.protocol.sql.schema.metamodel.Enum; +import org.finos.legend.engine.protocol.sql.schema.metamodel.EnumSchemaColumn; +import org.finos.legend.engine.protocol.sql.schema.metamodel.PrimitiveSchemaColumn; +import org.finos.legend.engine.protocol.sql.schema.metamodel.PrimitiveType; +import org.finos.legend.engine.protocol.sql.schema.metamodel.Schema; +import org.finos.legend.engine.pure.code.core.PureCoreExtensionLoader; import org.finos.legend.engine.query.sql.api.CatchAllExceptionMapper; import org.finos.legend.engine.query.sql.api.MockPac4jFeature; import org.finos.legend.engine.query.sql.api.sources.TestSQLSourceProvider; import org.finos.legend.engine.shared.core.api.grammar.RenderStyle; import org.finos.legend.engine.shared.core.deployment.DeploymentMode; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_Enum; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_EnumValueSchemaColumn_Impl; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_Enum_Impl; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_PrimitiveValueSchemaColumn_Impl; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_Schema; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_SchemaColumn; -import org.finos.legend.pure.generated.Root_meta_external_query_sql_Schema_Impl; -import org.finos.legend.pure.generated.Root_meta_pure_metamodel_type_Enum_Impl; import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory; import org.junit.Assert; import org.junit.ClassRule; @@ -201,52 +198,88 @@ public void testExecuteWithCSVFormat() } @Test - public void getSchemaForQueryWithDuplicateSources() + public void getSchemaForQueryWithDuplicateSources() throws JsonProcessingException { String actualSchema = resources.target("sql/v1/execution/getSchemaFromQueryString") .request() .post(Entity.text("SELECT Id FROM service.\"/testService\" UNION SELECT Id FROM service.\"/testService\"")).readEntity(String.class); - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum integerType = new Root_meta_pure_metamodel_type_Enum_Impl("Integer"); - Root_meta_external_query_sql_SchemaColumn idColumn = new Root_meta_external_query_sql_PrimitiveValueSchemaColumn_Impl((String) null)._name("Id")._type(integerType); - Root_meta_external_query_sql_Schema schema = new Root_meta_external_query_sql_Schema_Impl((String) null)._columnsAdd(idColumn); - String expectedSchema = SqlExecute.serializeToJSON(schema, pureModel, false); - Assert.assertEquals(expectedSchema, actualSchema); + Schema schema = new Schema(); + schema.columns = FastList.newListWith( + primitiveColumn("Id", PrimitiveType.Integer) + ); + + Assert.assertEquals(new ObjectMapper().writeValueAsString(schema), actualSchema); } @Test - public void getSchemaFromQueryString() + public void getSchemaFromQueryString() throws JsonProcessingException { String actualSchema = resources.target("sql/v1/execution/getSchemaFromQueryString") .request() .post(Entity.text("SELECT * FROM service.\"/testService\"")).readEntity(String.class); - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum integerType = new Root_meta_pure_metamodel_type_Enum_Impl("Integer"); - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum stringType = new Root_meta_pure_metamodel_type_Enum_Impl("String"); - Root_meta_external_query_sql_SchemaColumn idColumn = new Root_meta_external_query_sql_PrimitiveValueSchemaColumn_Impl((String) null)._name("Id")._type(integerType); - Root_meta_external_query_sql_SchemaColumn nameColumn = new Root_meta_external_query_sql_PrimitiveValueSchemaColumn_Impl((String) null)._name("Name")._type(stringType); - Root_meta_external_query_sql_SchemaColumn typeColumn = new Root_meta_external_query_sql_EnumValueSchemaColumn_Impl((String) null)._name("Employee Type")._type("demo::employeeType"); - Root_meta_external_query_sql_Enum employeeType = new Root_meta_external_query_sql_Enum_Impl((String) null)._type("demo::employeeType")._valuesAdd("Type1")._valuesAdd("Type2"); - Root_meta_external_query_sql_Schema schema = new Root_meta_external_query_sql_Schema_Impl((String) null)._columnsAdd(idColumn)._columnsAdd(nameColumn)._columnsAdd(typeColumn)._enumsAdd(employeeType); - String expectedSchema = SqlExecute.serializeToJSON(schema, pureModel, false); - Assert.assertEquals(expectedSchema, actualSchema); + + Schema schema = new Schema(); + schema.columns = FastList.newListWith( + primitiveColumn("Id", PrimitiveType.Integer), + primitiveColumn("Name", PrimitiveType.String), + enumColumn("Employee Type", "demo::employeeType") + ); + + schema.enums = FastList.newListWith( + enumValue("demo::employeeType", "Type1", "Type2") + ); + + Assert.assertEquals(new ObjectMapper().writeValueAsString(schema), actualSchema); } @Test - public void getSchemaFromQuery() + public void getSchemaFromQuery() throws JsonProcessingException { String actualSchema = resources.target("sql/v1/execution/getSchemaFromQuery") .request() .post(Entity.json("{ \"_type\": \"query\", \"orderBy\": [], \"queryBody\": { \"_type\": \"querySpecification\", \"from\": [ { \"_type\": \"table\", \"name\": { \"parts\": [ \"service\", \"/testService\" ] } } ], \"groupBy\": [], \"orderBy\": [], \"select\": { \"_type\": \"select\", \"distinct\": false, \"selectItems\": [ { \"_type\": \"allColumns\" } ] } } }")).readEntity(String.class); - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum integerType = new Root_meta_pure_metamodel_type_Enum_Impl("Integer"); - org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Enum stringType = new Root_meta_pure_metamodel_type_Enum_Impl("String"); - Root_meta_external_query_sql_SchemaColumn idColumn = new Root_meta_external_query_sql_PrimitiveValueSchemaColumn_Impl((String) null)._name("Id")._type(integerType); - Root_meta_external_query_sql_SchemaColumn nameColumn = new Root_meta_external_query_sql_PrimitiveValueSchemaColumn_Impl((String) null)._name("Name")._type(stringType); - Root_meta_external_query_sql_SchemaColumn typeColumn = new Root_meta_external_query_sql_EnumValueSchemaColumn_Impl((String) null)._name("Employee Type")._type("demo::employeeType"); - Root_meta_external_query_sql_Enum employeeType = new Root_meta_external_query_sql_Enum_Impl((String) null)._type("demo::employeeType")._valuesAdd("Type1")._valuesAdd("Type2"); - Root_meta_external_query_sql_Schema schema = new Root_meta_external_query_sql_Schema_Impl((String) null)._columnsAdd(idColumn)._columnsAdd(nameColumn)._columnsAdd(typeColumn)._enumsAdd(employeeType); - String expectedSchema = SqlExecute.serializeToJSON(schema, pureModel, false); - Assert.assertEquals(expectedSchema, actualSchema); + + Schema schema = new Schema(); + schema.columns = FastList.newListWith( + primitiveColumn("Id", PrimitiveType.Integer), + primitiveColumn("Name", PrimitiveType.String), + enumColumn("Employee Type", "demo::employeeType") + ); + + schema.enums = FastList.newListWith( + enumValue("demo::employeeType", "Type1", "Type2") + ); + + Assert.assertEquals(new ObjectMapper().writeValueAsString(schema), actualSchema); + } + + private static PrimitiveSchemaColumn primitiveColumn(String name, PrimitiveType type) + { + PrimitiveSchemaColumn c = new PrimitiveSchemaColumn(); + c.name = name; + c.type = type; + + return c; + } + + private static EnumSchemaColumn enumColumn(String name, String type) + { + EnumSchemaColumn c = new EnumSchemaColumn(); + c.name = name; + c.type = type;; + + return c; + } + + private static Enum enumValue(String type, String... values) + { + Enum e = new Enum(); + e.type = type; + e.values = FastList.newListWith(values); + + return e; } } diff --git a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/resources/proj-1.pure b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/resources/proj-1.pure index 3f5795bebfa..3119c464447 100644 --- a/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/resources/proj-1.pure +++ b/legend-engine-xts-sql/legend-engine-xt-sql-query/src/test/resources/proj-1.pure @@ -86,12 +86,16 @@ Service demo::H2PersonService [ x|$x.id, x|$x.name, - x|$x.type + x|$x.type, + x|$x.name + ' Smith', + x|$x.derivedName ], [ 'Id', 'Name', - 'Employee Type' + 'Employee Type', + 'Full Name', + 'Derived Name' ] ); mapping: demo::DemoRelationalMapping; @@ -171,6 +175,7 @@ Class demo::employee name: String[1]; type: demo::employeeType[1]; startDate: Date[1]; + derivedName() {$this.name+'_derived'} : String[1]; } diff --git a/legend-engine-xts-sql/pom.xml b/legend-engine-xts-sql/pom.xml index 5ea7d804956..44c49166147 100644 --- a/legend-engine-xts-sql/pom.xml +++ b/legend-engine-xts-sql/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-text/legend-engine-xt-text-compiler/pom.xml b/legend-engine-xts-text/legend-engine-xt-text-compiler/pom.xml index 661417ff9df..bae7fab17da 100644 --- a/legend-engine-xts-text/legend-engine-xt-text-compiler/pom.xml +++ b/legend-engine-xts-text/legend-engine-xt-text-compiler/pom.xml @@ -18,7 +18,7 @@ legend-engine-xts-text org.finos.legend.engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-text/legend-engine-xt-text-grammar/pom.xml b/legend-engine-xts-text/legend-engine-xt-text-grammar/pom.xml index ae1d2b9f7e8..7e8247c1120 100644 --- a/legend-engine-xts-text/legend-engine-xt-text-grammar/pom.xml +++ b/legend-engine-xts-text/legend-engine-xt-text-grammar/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-text - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-text/legend-engine-xt-text-protocol/pom.xml b/legend-engine-xts-text/legend-engine-xt-text-protocol/pom.xml index d71b32e16cc..722df9430b7 100644 --- a/legend-engine-xts-text/legend-engine-xt-text-protocol/pom.xml +++ b/legend-engine-xts-text/legend-engine-xt-text-protocol/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine-xts-text - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-text/legend-engine-xt-text-pure-metamodel/pom.xml b/legend-engine-xts-text/legend-engine-xt-text-pure-metamodel/pom.xml index b50d2762fff..63ad090a887 100644 --- a/legend-engine-xts-text/legend-engine-xt-text-pure-metamodel/pom.xml +++ b/legend-engine-xts-text/legend-engine-xt-text-pure-metamodel/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-text - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-text/pom.xml b/legend-engine-xts-text/pom.xml index f6593f65a6e..8a6d5ba0806 100644 --- a/legend-engine-xts-text/pom.xml +++ b/legend-engine-xts-text/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/pom.xml b/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/pom.xml index 5ec9e029a67..7dd29f6af5b 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/pom.xml +++ b/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-xml - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/src/main/resources/core_external_format_xml_java_platform_binding/legendJavaPlatformBinding/internalize.pure b/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/src/main/resources/core_external_format_xml_java_platform_binding/legendJavaPlatformBinding/internalize.pure index 42357fa5b83..bfc86c58a7f 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/src/main/resources/core_external_format_xml_java_platform_binding/legendJavaPlatformBinding/internalize.pure +++ b/legend-engine-xts-xml/legend-engine-xt-xml-javaPlatformBinding-pure/src/main/resources/core_external_format_xml_java_platform_binding/legendJavaPlatformBinding/internalize.pure @@ -216,7 +216,7 @@ function <> meta::external::format::xml::executionPlan::platform {| let addToType = javaParameterizedType($conventions->className(_AddEnumToObject), [$dataClass, $itemType]); let simpleTypeHandler = $ctxParam->j_invoke('stringSimpleTypeHandler', [], javaParameterizedType($conventions->className(_SimpleTypeHandler), javaString())); - j_new($addToType, [$getAdder, $simpleTypeHandler, $itemType->j_field('class')]); + j_new($addToType, [$getAdder, $simpleTypeHandler, $itemType->j_field('class'), j_string($propType->elementToPath())]); }, | if($itemType == javaBoolean(), diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-model/pom.xml b/legend-engine-xts-xml/legend-engine-xt-xml-model/pom.xml index 6584a2285f6..083caa700ab 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-model/pom.xml +++ b/legend-engine-xts-xml/legend-engine-xt-xml-model/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-xml - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXsdToModelGeneration.java b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXsdToModelGeneration.java index 476f5f2dff4..48b3644f030 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXsdToModelGeneration.java +++ b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXsdToModelGeneration.java @@ -107,7 +107,7 @@ public void testAttributes() String expected = ">>>test::gen::LinkId\n" + "Class {meta::pure::profiles::doc.doc = 'The data type used for link identifiers.'} test::gen::LinkId\n" + "[\n" + - " c1_length: $this.value->length() <= 255\n" + + " c1_length_LinkId: $this.value->length() <= 255\n" + "]\n" + "{\n" + " id: String[0..1];\n" + @@ -208,20 +208,20 @@ public void testValidations() String expected = ">>>test::gen::ShowValidations\n" + "Class test::gen::ShowValidations\n" + "[\n" + - " c1_length: $this.fixedLengthString->forAll(x: String[1]|$x->length() == 12),\n" + - " c2_length: $this.minLengthString->length() >= 5,\n" + - " c3_length: $this.maxLengthString->forAll(x: String[1]|$x->length() <= 20),\n" + - " c4_length: $this.rangeLengthString->forAll(x: String[1]|$x->length() >= 5),\n" + - " c5_length: $this.rangeLengthString->forAll(x: String[1]|$x->length() <= 20),\n" + - " c6_range: $this.minValInteger >= 1,\n" + - " c7_range: $this.minValFloat > 2.4,\n" + - " c8_range: $this.maxValInteger <= 100,\n" + - " c9_range: $this.maxValFloat < 10.12,\n" + - " c10_range: $this.rangeValInteger > 10,\n" + - " c11_range: $this.rangeValInteger <= 100,\n" + - " c12_range: $this.rangeValFloat > 2.7,\n" + - " c13_range: $this.rangeValFloat < 10.99,\n" + - " c14_values: $this.fixedValuesString->in(['AUD', 'USD'])\n" + + " c1_length_ShowValidations: $this.fixedLengthString->forAll(x: String[1]|$x->length() == 12),\n" + + " c2_length_ShowValidations: $this.minLengthString->length() >= 5,\n" + + " c3_length_ShowValidations: $this.maxLengthString->forAll(x: String[1]|$x->length() <= 20),\n" + + " c4_length_ShowValidations: $this.rangeLengthString->forAll(x: String[1]|$x->length() >= 5),\n" + + " c5_length_ShowValidations: $this.rangeLengthString->forAll(x: String[1]|$x->length() <= 20),\n" + + " c6_range_ShowValidations: $this.minValInteger >= 1,\n" + + " c7_range_ShowValidations: $this.minValFloat > 2.4,\n" + + " c8_range_ShowValidations: $this.maxValInteger <= 100,\n" + + " c9_range_ShowValidations: $this.maxValFloat < 10.12,\n" + + " c10_range_ShowValidations: $this.rangeValInteger > 10,\n" + + " c11_range_ShowValidations: $this.rangeValInteger <= 100,\n" + + " c12_range_ShowValidations: $this.rangeValFloat > 2.7,\n" + + " c13_range_ShowValidations: $this.rangeValFloat < 10.99,\n" + + " c14_values_ShowValidations: $this.fixedValuesString->in(['AUD', 'USD'])\n" + "]\n" + "{\n" + " fixedLengthString: String[*];\n" + @@ -260,7 +260,7 @@ public void testChoice() String expected = ">>>test::gen::Choice\n" + "Class test::gen::Choice\n" + "[\n" + - " c1_choice: ($this.optionOne->isNotEmpty() && $this.optionTwo->isEmpty()) || ($this.optionOne->isEmpty() && $this.optionTwo->isNotEmpty())\n" + + " c1_choice_Choice: ($this.optionOne->isNotEmpty() && $this.optionTwo->isEmpty()) || ($this.optionOne->isEmpty() && $this.optionTwo->isNotEmpty())\n" + "]\n" + "{\n" + " optionOne: Integer[0..1];\n" + @@ -298,7 +298,7 @@ public void testChoiceWithCommonProperty() String expected = ">>>test::gen::Account\n" + "Class test::gen::Account\n" + "[\n" + - " c1_choice: $this.beneficiaryId->isNotEmpty() || ($this.beneficiaryId->isEmpty() && $this.servicerId->isNotEmpty())\n" + + " c1_choice_Account: $this.beneficiaryId->isNotEmpty() || ($this.beneficiaryId->isEmpty() && $this.servicerId->isNotEmpty())\n" + "]\n" + "{\n" + " accountId: String[1];\n" + @@ -454,8 +454,8 @@ public void testAttributeWithInlineType() String expected = ">>>test::gen::AttributeWithInlineType\n" + "Class test::gen::AttributeWithInlineType\n" + "[\n" + - " c1_values: $this.fullOrDelta->forAll(x: String[1]|$x->in(['FULL', 'DELTA'])),\n" + - " c2_length: $this.notTooLongString->forAll(x: String[1]|$x->length() <= 500)\n" + + " c1_values_AttributeWithInlineType: $this.fullOrDelta->forAll(x: String[1]|$x->in(['FULL', 'DELTA'])),\n" + + " c2_length_AttributeWithInlineType: $this.notTooLongString->forAll(x: String[1]|$x->length() <= 500)\n" + "]\n" + "{\n" + " fullOrDelta: String[0..1];\n" + @@ -709,7 +709,7 @@ public void testExtendComplexTypeWithComplexTypePartiallyResolved() ">>>test::gen::ExceptionType\n" + "Class test::gen::ExceptionType\n" + "[\n" + - " c1_values: $this.type->in(['INFO', 'WARN', 'ERROR'])\n" + + " c1_values_ExceptionType: $this.type->in(['INFO', 'WARN', 'ERROR'])\n" + "]\n" + "{\n" + " type: String[1];\n" + @@ -847,8 +847,8 @@ public void testInheritedLengthRestriction() String expected = ">>>test::gen::AccountId\n" + "Class test::gen::AccountId\n" + "[\n" + - " c1_length: $this.value->length() >= 1,\n" + - " c2_length: $this.value->length() <= 255\n" + + " c1_length_AccountId: $this.value->length() >= 1,\n" + + " c2_length_AccountId: $this.value->length() <= 255\n" + "]\n" + "{\n" + " value: String[1];\n" + @@ -969,6 +969,71 @@ public void testNgm() assertModelTexts(modelTextsFromResource("ngm-sample/exchangeGenResult.txt"), modelTextsFromContextData(exchModel)); } + @Test + public void testContraintName() + { + String schemaCode = newExternalSchemaSetGrammarBuilder("test::simpleWithConstraint", "XSD") + .withSchemaText(null, "simpleWithConstraint.xsd", "\n" + + "\n" + + " \n" + + " \n" + + " \n" + + " A type defining a number specified as non negative decimal greater than 0 inclusive.\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " One of two dimensions of a rectangle\n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + " \n" + + "\n") + .build(); + + PureModelContextData model = generateModel(schemaCode, config("test::simpleWithConstraint", "test::gen", true, true)); + + String expected = ">>>test::gen::Rectangle\n" + + "Class test::gen::Rectangle extends test::gen::Shape\n" + + "[\n" + + " c1_range_Rectangle: $this.height->forAll(x: Decimal[1]|$x >= 0),\n" + + " c2_range_Rectangle: $this.width->forAll(x: Decimal[1]|$x >= 0)\n" + + "]\n" + + "{\n" + + " {meta::pure::profiles::doc.doc = 'One of two dimensions of a rectangle'} height: Decimal[1..*];\n" + + " width: Decimal[*];\n" + + "}\n" + + "\n" + + ">>>test::gen::Shape\n" + + "Class test::gen::Shape\n" + + "[\n" + + " c1_range_Shape: $this.area->forAll(x: Decimal[1]|$x >= 0)\n" + + "]\n" + + "{\n" + + " area: Decimal[*];\n" + + "}"; + + Assert.assertEquals(modelTextsFromString(expected), modelTextsFromContextData(model)); + } + private XsdToModelConfiguration config(String sourceSchemaSet, String targetPackage) { return config(sourceSchemaSet, targetPackage, false, false); diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/fpml-sample/genResult.txt b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/fpml-sample/genResult.txt index 7b8f24336ac..73d8aaafae3 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/fpml-sample/genResult.txt +++ b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/fpml-sample/genResult.txt @@ -1,7 +1,7 @@ >>>test::gen::Account Class {meta::pure::profiles::doc.doc = 'A generic account that represents any party\'s account at another party. Parties may be identified by the account at another party.'} test::gen::Account [ - c1_choice: $this.accountBeneficiary->isNotEmpty() || ($this.accountBeneficiary->isEmpty() && $this.servicingParty->isNotEmpty()) + c1_choice_Account: $this.accountBeneficiary->isNotEmpty() || ($this.accountBeneficiary->isEmpty() && $this.servicingParty->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'An account identifier. For example an Account number.'} accountId: test::gen::AccountId[1..*]; @@ -15,9 +15,9 @@ Class {meta::pure::profiles::doc.doc = 'A generic account that represents any pa >>>test::gen::AccountId Class {meta::pure::profiles::doc.doc = 'The data type used for account identifiers.'} test::gen::AccountId [ - c1_length: $this.accountIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_AccountId: $this.accountIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_AccountId: $this.value->length() >= 1, + c3_length_AccountId: $this.value->length() <= 255 ] { {meta::pure::profiles::doc.doc = 'The identifier scheme used with this accountId. A unique URI to determine the authoritative issuer of these identifiers.'} accountIdScheme: String[0..1]; @@ -27,8 +27,8 @@ Class {meta::pure::profiles::doc.doc = 'The data type used for account identifie >>>test::gen::AccountName Class {meta::pure::profiles::doc.doc = 'The data type used for the name of the account.'} test::gen::AccountName [ - c1_length: $this.accountNameScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_AccountName: $this.accountNameScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_AccountName: $this.value->length() <= 255 ] { {meta::pure::profiles::doc.doc = 'The identifier scheme used with this accountName. A unique URI to determine the source of the account name.'} accountNameScheme: String[0..1]; @@ -44,8 +44,8 @@ Class {meta::pure::profiles::doc.doc = 'Reference to an account.'} test::gen::Ac >>>test::gen::AccountType Class {meta::pure::profiles::doc.doc = 'The data type used for account type.'} test::gen::AccountType [ - c1_length: $this.accountTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_AccountType: $this.accountTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_AccountType: $this.value->length() <= 255 ] { {meta::pure::profiles::doc.doc = 'The identifier scheme used with this accountType. A unique URI to determine the the type of account.'} accountTypeScheme: String[0..1]; @@ -66,7 +66,7 @@ Enum {meta::pure::profiles::doc.doc = 'The type of automatic adjustment that a L >>>test::gen::ActionType Class {meta::pure::profiles::doc.doc = 'The data type used for ESMA action type.'} test::gen::ActionType [ - c1_length: $this.value->length() <= 255 + c1_length_ActionType: $this.value->length() <= 255 ] { {meta::pure::profiles::doc.doc = 'The identifier scheme used with this actionType. A unique URI to determine the the type of action.'} actionTypeScheme: String[0..1]; @@ -76,9 +76,9 @@ Class {meta::pure::profiles::doc.doc = 'The data type used for ESMA action type. >>>test::gen::Address Class {meta::pure::profiles::doc.doc = 'A type that represents a physical postal address.'} test::gen::Address [ - c1_length: $this.city->forAll(x: String[1]|$x->length() <= 255), - c2_length: $this.state->forAll(x: String[1]|$x->length() <= 255), - c3_length: $this.postalCode->forAll(x: String[1]|$x->length() <= 255) + c1_length_Address: $this.city->forAll(x: String[1]|$x->length() <= 255), + c2_length_Address: $this.state->forAll(x: String[1]|$x->length() <= 255), + c3_length_Address: $this.postalCode->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'The set of street and building number information that identifies a postal address within a city.'} streetAddress: test::gen::StreetAddress[0..1]; @@ -100,7 +100,7 @@ Class {meta::pure::profiles::doc.doc = 'A type for defining a date that shall be >>>test::gen::AdjustableDate2 Class {meta::pure::profiles::doc.doc = 'A type that is different from AdjustableDate in two regards. First, date adjustments can be specified with either a dateAdjustments element or a reference to an existing dateAdjustments element. Second, it does not require the specification of date adjustments.'} test::gen::AdjustableDate2 [ - c1_choice: ($this.dateAdjustments->isNotEmpty() && $this.dateAdjustmentsReference->isEmpty()) || ($this.dateAdjustments->isEmpty() && $this.dateAdjustmentsReference->isNotEmpty()) + c1_choice_AdjustableDate2: ($this.dateAdjustments->isNotEmpty() && $this.dateAdjustmentsReference->isEmpty()) || ($this.dateAdjustments->isEmpty() && $this.dateAdjustmentsReference->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A date subject to adjustment.'} unadjustedDate: test::gen::IdentifiedDate[1]; @@ -122,7 +122,7 @@ Class {meta::pure::profiles::doc.doc = 'A type for defining a series of dates th >>>test::gen::AdjustableDatesOrRelativeDateOffset Class {meta::pure::profiles::doc.doc = 'A type for defining a series of dates, either as a list of adjustable dates, or a as a repeating sequence from a base date'} test::gen::AdjustableDatesOrRelativeDateOffset [ - c1_choice: ($this.adjustableDates->isNotEmpty() && $this.relativeDate->isEmpty()) || ($this.adjustableDates->isEmpty() && $this.relativeDate->isNotEmpty()) + c1_choice_AdjustableDatesOrRelativeDateOffset: ($this.adjustableDates->isNotEmpty() && $this.relativeDate->isEmpty()) || ($this.adjustableDates->isEmpty() && $this.relativeDate->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A series of adjustable dates'} adjustableDates: test::gen::AdjustableDates[0..1]; @@ -132,7 +132,7 @@ Class {meta::pure::profiles::doc.doc = 'A type for defining a series of dates, e >>>test::gen::AdjustableOrAdjustedDate Class {meta::pure::profiles::doc.doc = 'A type for defining a date that shall be subject to adjustment if it would otherwise fall on a day that is not a business day in the specified business centers, together with the convention for adjusting the date.'} test::gen::AdjustableOrAdjustedDate [ - c1_choice: ($this.unadjustedDate->isNotEmpty() && $this.dateAdjustments->isNotEmpty()) || ($this.unadjustedDate->isEmpty() && ($this.dateAdjustments->isEmpty() && $this.adjustedDate->isNotEmpty())) + c1_choice_AdjustableOrAdjustedDate: ($this.unadjustedDate->isNotEmpty() && $this.dateAdjustments->isNotEmpty()) || ($this.unadjustedDate->isEmpty() && ($this.dateAdjustments->isEmpty() && $this.adjustedDate->isNotEmpty())) ] { {meta::pure::profiles::doc.doc = 'A date subject to adjustment.'} unadjustedDate: test::gen::IdentifiedDate[0..1]; @@ -144,7 +144,7 @@ Class {meta::pure::profiles::doc.doc = 'A type for defining a date that shall be >>>test::gen::AdjustableOrRelativeDate Class {meta::pure::profiles::doc.doc = 'A type giving the choice between defining a date as an explicit date together with applicable adjustments or as relative to some other (anchor) date.'} test::gen::AdjustableOrRelativeDate [ - c1_choice: ($this.adjustableDate->isNotEmpty() && $this.relativeDate->isEmpty()) || ($this.adjustableDate->isEmpty() && $this.relativeDate->isNotEmpty()) + c1_choice_AdjustableOrRelativeDate: ($this.adjustableDate->isNotEmpty() && $this.relativeDate->isEmpty()) || ($this.adjustableDate->isEmpty() && $this.relativeDate->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A date that shall be subject to adjustment if it would otherwise fall on a day that is not a business day in the specified business centers, together with the convention for adjusting the date.'} adjustableDate: test::gen::AdjustableDate[0..1]; @@ -155,7 +155,7 @@ Class {meta::pure::profiles::doc.doc = 'A type giving the choice between definin >>>test::gen::AdjustableOrRelativeDates Class {meta::pure::profiles::doc.doc = 'A type giving the choice between defining a series of dates as an explicit list of dates together with applicable adjustments or as relative to some other series of (anchor) dates.'} test::gen::AdjustableOrRelativeDates [ - c1_choice: ($this.adjustableDates->isNotEmpty() && $this.relativeDates->isEmpty()) || ($this.adjustableDates->isEmpty() && $this.relativeDates->isNotEmpty()) + c1_choice_AdjustableOrRelativeDates: ($this.adjustableDates->isNotEmpty() && $this.relativeDates->isEmpty()) || ($this.adjustableDates->isEmpty() && $this.relativeDates->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A series of dates that shall be subject to adjustment if they would otherwise fall on a day that is not a business day in the specified business centers, together with the convention for adjusting the date.'} adjustableDates: test::gen::AdjustableDates[0..1]; @@ -166,7 +166,7 @@ Class {meta::pure::profiles::doc.doc = 'A type giving the choice between definin >>>test::gen::AdjustableRelativeOrPeriodicDates Class test::gen::AdjustableRelativeOrPeriodicDates [ - c1_choice: ($this.adjustableDates->isNotEmpty() && ($this.relativeDateSequence->isEmpty() && $this.periodicDates->isEmpty())) || (($this.adjustableDates->isEmpty() && ($this.relativeDateSequence->isNotEmpty() && $this.periodicDates->isEmpty())) || ($this.adjustableDates->isEmpty() && ($this.relativeDateSequence->isEmpty() && $this.periodicDates->isNotEmpty()))) + c1_choice_AdjustableRelativeOrPeriodicDates: ($this.adjustableDates->isNotEmpty() && ($this.relativeDateSequence->isEmpty() && $this.periodicDates->isEmpty())) || (($this.adjustableDates->isEmpty() && ($this.relativeDateSequence->isNotEmpty() && $this.periodicDates->isEmpty())) || ($this.adjustableDates->isEmpty() && ($this.relativeDateSequence->isEmpty() && $this.periodicDates->isNotEmpty()))) ] { {meta::pure::profiles::doc.doc = 'A series of dates that shall be subject to adjustment if they would otherwise fall on a day that is not a business day in the specified business centers, together with the convention for adjusting the date.'} adjustableDates: test::gen::AdjustableDates[0..1]; @@ -178,7 +178,7 @@ Class test::gen::AdjustableRelativeOrPeriodicDates >>>test::gen::AdjustableRelativeOrPeriodicDates2 Class {meta::pure::profiles::doc.doc = 'A type giving the choice between defining a series of dates as an explicit list of dates together with applicable adjustments, or as relative to some other series of (anchor) dates, or as a set of factors to specify periodic occurences.'} test::gen::AdjustableRelativeOrPeriodicDates2 [ - c1_choice: ($this.adjustableDates->isNotEmpty() && ($this.relativeDates->isEmpty() && $this.periodicDates->isEmpty())) || (($this.adjustableDates->isEmpty() && ($this.relativeDates->isNotEmpty() && $this.periodicDates->isEmpty())) || ($this.adjustableDates->isEmpty() && ($this.relativeDates->isEmpty() && $this.periodicDates->isNotEmpty()))) + c1_choice_AdjustableRelativeOrPeriodicDates2: ($this.adjustableDates->isNotEmpty() && ($this.relativeDates->isEmpty() && $this.periodicDates->isEmpty())) || (($this.adjustableDates->isEmpty() && ($this.relativeDates->isNotEmpty() && $this.periodicDates->isEmpty())) || ($this.adjustableDates->isEmpty() && ($this.relativeDates->isEmpty() && $this.periodicDates->isNotEmpty()))) ] { {meta::pure::profiles::doc.doc = 'A series of dates that shall be subject to adjustment if they would otherwise fall on a day that is not a business day in the specified business centers, together with the convention for adjusting the date.'} adjustableDates: test::gen::AdjustableDates[0..1]; @@ -228,8 +228,8 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a currency amount or a c >>>test::gen::AssetClass Class test::gen::AssetClass [ - c1_length: $this.assetClassScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_AssetClass: $this.assetClassScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_AssetClass: $this.value->length() <= 255 ] { assetClassScheme: String[0..1]; @@ -245,8 +245,8 @@ Class {meta::pure::profiles::doc.doc = 'A type to define automatic exercise of a >>>test::gen::AverageDailyTradingVolumeLimit Class {meta::pure::profiles::doc.doc = 'To indicate the limitation percentage and limitation period.'} test::gen::AverageDailyTradingVolumeLimit [ - c1_range: $this.limitationPercentage >= 0, - c2_range: $this.limitationPercentage <= 1 + c1_range_AverageDailyTradingVolumeLimit: $this.limitationPercentage >= 0, + c2_range_AverageDailyTradingVolumeLimit: $this.limitationPercentage <= 1 ] { {meta::pure::profiles::doc.doc = 'Specifies the limitation percentage in Average Daily trading volume.'} limitationPercentage: Decimal[1]; @@ -271,7 +271,7 @@ Enum {meta::pure::profiles::doc.doc = 'The method of calculation to be used when >>>test::gen::Beneficiary Class {meta::pure::profiles::doc.doc = 'A type defining the beneficiary of the funds.'} test::gen::Beneficiary [ - c1_choice: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) + c1_choice_Beneficiary: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) ] { {meta::pure::profiles::doc.doc = 'A set of unique identifiers for a party, eachone identifying the party within a payment system. The assumption is that each party will not have more than one identifier within the same payment system.'} routingIds: test::gen::RoutingIds[0..1]; @@ -308,8 +308,8 @@ Class {meta::pure::profiles::doc.doc = 'Identifies the market sector in which th >>>test::gen::BrokerConfirmationType Class {meta::pure::profiles::doc.doc = 'Identifies the market sector in which the trade has been arranged.'} test::gen::BrokerConfirmationType [ - c1_length: $this.brokerConfirmationTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_BrokerConfirmationType: $this.brokerConfirmationTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_BrokerConfirmationType: $this.value->length() <= 255 ] { brokerConfirmationTypeScheme: String[0..1]; @@ -333,8 +333,8 @@ Enum {meta::pure::profiles::doc.doc = 'Defines which type of bullion is applicab >>>test::gen::BusinessCenter Class {meta::pure::profiles::doc.doc = 'A code identifying a business day calendar location. A business day calendar location is drawn from the list identified by the business day calendar location scheme.'} test::gen::BusinessCenter [ - c1_length: $this.businessCenterScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_BusinessCenter: $this.businessCenterScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_BusinessCenter: $this.value->length() <= 255 ] { businessCenterScheme: String[0..1]; @@ -373,7 +373,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a range of contiguous bu >>>test::gen::BusinessDayAdjustments Class {meta::pure::profiles::doc.doc = 'A type defining the business day convention and financial business centers used for adjusting any relevant date if it would otherwise fall on a day that is not a business day in the specified business centers.'} test::gen::BusinessDayAdjustments [ - c1_choice: ($this.businessCentersReference->isNotEmpty() && $this.businessCenters->isEmpty()) || ($this.businessCentersReference->isEmpty() && $this.businessCenters->isNotEmpty()) + c1_choice_BusinessDayAdjustments: ($this.businessCentersReference->isNotEmpty() && $this.businessCenters->isEmpty()) || ($this.businessCentersReference->isEmpty() && $this.businessCenters->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The convention for adjusting a date if it would otherwise fall on a day that is not a business day.'} businessDayConvention: test::gen::BusinessDayConventionEnum[1]; @@ -404,7 +404,7 @@ Enum {meta::pure::profiles::doc.doc = 'The convention for adjusting any relevant >>>test::gen::BusinessUnit Class {meta::pure::profiles::doc.doc = 'A type that represents information about a unit within an organization.'} test::gen::BusinessUnit [ - c1_length: $this.name->forAll(x: String[1]|$x->length() <= 255) + c1_length_BusinessUnit: $this.name->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'A name used to describe the organization unit'} name: String[0..1]; @@ -423,8 +423,8 @@ Class {meta::pure::profiles::doc.doc = 'Reference to an organizational unit.'} t >>>test::gen::BusinessUnitRole Class {meta::pure::profiles::doc.doc = 'A type describing a role played by a unit in one or more transactions. Examples include roles such as Trader, Collateral, Confirmation, Settlement, etc. This can be extended to provide custom roles.'} test::gen::BusinessUnitRole [ - c1_length: $this.unitRoleScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_BusinessUnitRole: $this.unitRoleScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_BusinessUnitRole: $this.value->length() <= 255 ] { unitRoleScheme: String[0..1]; @@ -441,7 +441,7 @@ Enum test::gen::BuySellEnum >>>test::gen::CalculationAgent Class {meta::pure::profiles::doc.doc = 'A type defining the ISDA calculation agent responsible for performing duties as defined in the applicable product definitions.'} test::gen::CalculationAgent [ - c1_choice: ($this.calculationAgentPartyReference->isNotEmpty() && $this.calculationAgentParty->isEmpty()) || ($this.calculationAgentPartyReference->isEmpty() && $this.calculationAgentParty->isNotEmpty()) + c1_choice_CalculationAgent: ($this.calculationAgentPartyReference->isNotEmpty() && $this.calculationAgentParty->isEmpty()) || ($this.calculationAgentPartyReference->isEmpty() && $this.calculationAgentParty->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A pointer style reference to a party identifier defined elsewhere in the document. The party referenced is the ISDA Calculation Agent for the trade. If more than one party is referenced then the parties are assumed to be co-calculation agents, i.e. they have joint responsibility.'} calculationAgentPartyReference: test::gen::PartyReference[*]; @@ -490,9 +490,9 @@ Class {meta::pure::profiles::doc.doc = 'A type defining the list of reference in >>>test::gen::CashflowId Class {meta::pure::profiles::doc.doc = 'An identifier used to identify a single component cashflow.'} test::gen::CashflowId [ - c1_length: $this.cashflowIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_CashflowId: $this.cashflowIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_CashflowId: $this.value->length() >= 1, + c3_length_CashflowId: $this.value->length() <= 255 ] { cashflowIdScheme: String[0..1]; @@ -502,8 +502,8 @@ Class {meta::pure::profiles::doc.doc = 'An identifier used to identify a single >>>test::gen::CashflowNotional Class {meta::pure::profiles::doc.doc = 'The notional/principal value/quantity/volume used to compute the cashflow.'} test::gen::CashflowNotional [ - c1_length: $this.units->forAll(x: String[1]|$x->length() >= 0), - c2_choice: ($this.currency->isNotEmpty() && $this.units->isEmpty()) || ($this.currency->isEmpty() && $this.units->isNotEmpty()) + c1_length_CashflowNotional: $this.units->forAll(x: String[1]|$x->length() >= 0), + c2_choice_CashflowNotional: ($this.currency->isNotEmpty() && $this.units->isEmpty()) || ($this.currency->isEmpty() && $this.units->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The currency in which an amount is denominated.'} currency: test::gen::Currency[0..1]; @@ -515,8 +515,8 @@ Class {meta::pure::profiles::doc.doc = 'The notional/principal value/quantity/vo >>>test::gen::CashflowType Class {meta::pure::profiles::doc.doc = 'A coding scheme used to describe the type or purpose of a cash flow or cash flow component.'} test::gen::CashflowType [ - c1_length: $this.cashflowTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_CashflowType: $this.cashflowTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_CashflowType: $this.value->length() <= 255 ] { cashflowTypeScheme: String[0..1]; @@ -526,8 +526,8 @@ Class {meta::pure::profiles::doc.doc = 'A coding scheme used to describe the typ >>>test::gen::ClearanceSystem Class {meta::pure::profiles::doc.doc = 'Unless otherwise specified, the principal clearance system customarily used for settling trades in the relevant underlying.'} test::gen::ClearanceSystem [ - c1_length: $this.clearanceSystemScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_ClearanceSystem: $this.clearanceSystemScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ClearanceSystem: $this.value->length() <= 255 ] { clearanceSystemScheme: String[0..1]; @@ -647,7 +647,7 @@ Enum {meta::pure::profiles::doc.doc = 'Defines whether conditions precedent have >>>test::gen::ContactInformation Class {meta::pure::profiles::doc.doc = 'A type that represents how to contact an individual or organization.'} test::gen::ContactInformation [ - c1_length: $this.email->forAll(x: String[1]|$x->length() >= 0) + c1_length_ContactInformation: $this.email->forAll(x: String[1]|$x->length() >= 0) ] { {meta::pure::profiles::doc.doc = 'A telephonic contact.'} telephone: test::gen::TelephoneNumber[*]; @@ -658,8 +658,8 @@ Class {meta::pure::profiles::doc.doc = 'A type that represents how to contact an >>>test::gen::ContractualDefinitions Class {meta::pure::profiles::doc.doc = 'The definitions, such as those published by ISDA, that will define the terms of the trade.'} test::gen::ContractualDefinitions [ - c1_length: $this.contractualDefinitionsScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_ContractualDefinitions: $this.contractualDefinitionsScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ContractualDefinitions: $this.value->length() <= 255 ] { contractualDefinitionsScheme: String[0..1]; @@ -677,8 +677,8 @@ Class test::gen::ContractualMatrix >>>test::gen::ContractualSupplement Class {meta::pure::profiles::doc.doc = 'A contractual supplement (such as those published by ISDA) that will apply to the trade.'} test::gen::ContractualSupplement [ - c1_length: $this.contractualSupplementScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_ContractualSupplement: $this.contractualSupplementScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ContractualSupplement: $this.value->length() <= 255 ] { contractualSupplementScheme: String[0..1]; @@ -695,7 +695,7 @@ Class {meta::pure::profiles::doc.doc = 'A contractual supplement (such as those >>>test::gen::CorrespondentInformation Class {meta::pure::profiles::doc.doc = 'A type that describes the information to identify a correspondent bank that will make delivery of the funds on the paying bank\'s behalf in the country where the payment is to be made.'} test::gen::CorrespondentInformation [ - c1_choice: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) + c1_choice_CorrespondentInformation: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) ] { {meta::pure::profiles::doc.doc = 'A set of unique identifiers for a party, eachone identifying the party within a payment system. The assumption is that each party will not have more than one identifier within the same payment system.'} routingIds: test::gen::RoutingIds[0..1]; @@ -707,8 +707,8 @@ Class {meta::pure::profiles::doc.doc = 'A type that describes the information to >>>test::gen::CountryCode Class {meta::pure::profiles::doc.doc = 'The code representation of a country or an area of special sovereignty. By default it is a valid 2 character country code as defined by the ISO standard 3166-1 alpha-2 - Codes for representation of countries http://www.niso.org/standards/resources/3166.html.'} test::gen::CountryCode [ - c1_length: $this.countryScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 0 + c1_length_CountryCode: $this.countryScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_CountryCode: $this.value->length() >= 0 ] { countryScheme: String[0..1]; @@ -718,8 +718,8 @@ Class {meta::pure::profiles::doc.doc = 'The code representation of a country or >>>test::gen::CreditRating Class {meta::pure::profiles::doc.doc = 'A party\'s credit rating.'} test::gen::CreditRating [ - c1_length: $this.creditRatingScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_CreditRating: $this.creditRatingScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_CreditRating: $this.value->length() <= 255 ] { creditRatingScheme: String[0..1]; @@ -729,8 +729,8 @@ Class {meta::pure::profiles::doc.doc = 'A party\'s credit rating.'} test::gen::C >>>test::gen::CreditSeniority Class {meta::pure::profiles::doc.doc = 'The repayment precedence of a debt instrument.'} test::gen::CreditSeniority [ - c1_length: $this.creditSeniorityScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_CreditSeniority: $this.creditSeniorityScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_CreditSeniority: $this.value->length() <= 255 ] { {meta::pure::profiles::doc.doc = 'creditSeniorityTradingScheme overrides creditSeniorityScheme when the underlyer defines the reference obligation used in a single name credit default swap trade.'} creditSeniorityScheme: String[0..1]; @@ -748,9 +748,9 @@ Class {meta::pure::profiles::doc.doc = 'The agreement executed between the parti >>>test::gen::CreditSupportAgreementIdentifier Class test::gen::CreditSupportAgreementIdentifier [ - c1_length: $this.creditSupportAgreementIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_CreditSupportAgreementIdentifier: $this.creditSupportAgreementIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_CreditSupportAgreementIdentifier: $this.value->length() >= 1, + c3_length_CreditSupportAgreementIdentifier: $this.value->length() <= 255 ] { creditSupportAgreementIdScheme: String[0..1]; @@ -760,8 +760,8 @@ Class test::gen::CreditSupportAgreementIdentifier >>>test::gen::CreditSupportAgreementType Class test::gen::CreditSupportAgreementType [ - c1_length: $this.creditSupportAgreementTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_CreditSupportAgreementType: $this.creditSupportAgreementTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_CreditSupportAgreementType: $this.value->length() <= 255 ] { creditSupportAgreementTypeScheme: String[0..1]; @@ -771,8 +771,8 @@ Class test::gen::CreditSupportAgreementType >>>test::gen::Currency Class {meta::pure::profiles::doc.doc = 'The code representation of a currency or fund. By default it is a valid currency code as defined by the ISO standard 4217 - Codes for representation of currencies and funds http://www.iso.org/iso/en/prods-services/popstds/currencycodeslist.html.'} test::gen::Currency [ - c1_length: $this.currencyScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_Currency: $this.currencyScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_Currency: $this.value->length() <= 255 ] { currencyScheme: String[0..1]; @@ -813,8 +813,8 @@ Class {meta::pure::profiles::doc.doc = 'List of DateTimes'} test::gen::DateTimeL >>>test::gen::DayCountFraction Class {meta::pure::profiles::doc.doc = 'The specification for how the number of days between two dates is calculated for purposes of calculation of a fixed or floating payment amount and the basis for how many days are assumed to be in a year. Day Count Fraction is an ISDA term. The equivalent AFB (Association Francaise de Banques) term is Calculation Basis.'} test::gen::DayCountFraction [ - c1_length: $this.dayCountFractionScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_DayCountFraction: $this.dayCountFractionScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_DayCountFraction: $this.value->length() <= 255 ] { dayCountFractionScheme: String[0..1]; @@ -1001,8 +1001,8 @@ Enum test::gen::DeliveryTypeEnum >>>test::gen::DeterminationMethod Class {meta::pure::profiles::doc.doc = 'Coding scheme that specifies the method according to which an amount or a date is determined.'} test::gen::DeterminationMethod [ - c1_length: $this.determinationMethodScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_DeterminationMethod: $this.determinationMethodScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_DeterminationMethod: $this.value->length() <= 255 ] { determinationMethodScheme: String[0..1]; @@ -1115,7 +1115,7 @@ Enum {meta::pure::profiles::doc.doc = 'Defines the First Period or the Second Pe >>>test::gen::Documentation Class {meta::pure::profiles::doc.doc = 'An entity for defining the definitions that govern the document and should include the year and type of definitions referenced, along with any relevant documentation (such as master agreement) and the date it was signed.'} test::gen::Documentation [ - c1_choice: ($this.masterConfirmation->isNotEmpty() && $this.brokerConfirmation->isEmpty()) || ($this.masterConfirmation->isEmpty() && $this.brokerConfirmation->isNotEmpty()) + c1_choice_Documentation: ($this.masterConfirmation->isNotEmpty() && $this.brokerConfirmation->isEmpty()) || ($this.masterConfirmation->isEmpty() && $this.brokerConfirmation->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The agreement executed between the parties and intended to govern all OTC derivatives transactions between those parties.'} masterAgreement: test::gen::MasterAgreement[0..1]; @@ -1160,9 +1160,9 @@ Class {meta::pure::profiles::doc.doc = 'A special type meant to be used for elem >>>test::gen::EntityId Class {meta::pure::profiles::doc.doc = 'A legal entity identifier (e.g. RED entity code).'} test::gen::EntityId [ - c1_length: $this.entityIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_EntityId: $this.entityIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_EntityId: $this.value->length() >= 1, + c3_length_EntityId: $this.value->length() <= 255 ] { entityIdScheme: String[0..1]; @@ -1172,8 +1172,8 @@ Class {meta::pure::profiles::doc.doc = 'A legal entity identifier (e.g. RED enti >>>test::gen::EntityName Class {meta::pure::profiles::doc.doc = 'The name of the reference entity. A free format string. FpML does not define usage rules for this element.'} test::gen::EntityName [ - c1_length: $this.entityNameScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_EntityName: $this.entityNameScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_EntityName: $this.value->length() <= 255 ] { entityNameScheme: String[0..1]; @@ -1235,9 +1235,9 @@ Class {meta::pure::profiles::doc.doc = 'A type defining the exercise period for >>>test::gen::ExchangeId Class {meta::pure::profiles::doc.doc = 'A short form unique identifier for an exchange. If the element is not present then the exchange shall be the primary exchange on which the underlying is listed. The term "Exchange" is assumed to have the meaning as defined in the ISDA 2002 Equity Derivatives Definitions.'} test::gen::ExchangeId [ - c1_length: $this.exchangeIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_ExchangeId: $this.exchangeIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ExchangeId: $this.value->length() >= 1, + c3_length_ExchangeId: $this.value->length() <= 255 ] { exchangeIdScheme: String[0..1]; @@ -1261,7 +1261,7 @@ Enum {meta::pure::profiles::doc.doc = 'The specification of whether an OTC optio >>>test::gen::ExerciseFee Class {meta::pure::profiles::doc.doc = 'A type defining the fee payable on exercise of an option. This fee may be defined as an amount or a percentage of the notional exercised.'} test::gen::ExerciseFee [ - c1_choice: ($this.feeAmount->isNotEmpty() && $this.feeRate->isEmpty()) || ($this.feeAmount->isEmpty() && $this.feeRate->isNotEmpty()) + c1_choice_ExerciseFee: ($this.feeAmount->isNotEmpty() && $this.feeRate->isEmpty()) || ($this.feeAmount->isEmpty() && $this.feeRate->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A reference to the party responsible for making the payments defined by this structure.'} payerPartyReference: test::gen::PartyReference[1]; @@ -1277,7 +1277,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining the fee payable on exerc >>>test::gen::ExerciseFeeSchedule Class {meta::pure::profiles::doc.doc = 'A type to define a fee or schedule of fees to be payable on the exercise of an option. This fee may be defined as an amount or a percentage of the notional exercised.'} test::gen::ExerciseFeeSchedule [ - c1_choice: ($this.feeAmountSchedule->isNotEmpty() && $this.feeRateSchedule->isEmpty()) || ($this.feeAmountSchedule->isEmpty() && $this.feeRateSchedule->isNotEmpty()) + c1_choice_ExerciseFeeSchedule: ($this.feeAmountSchedule->isNotEmpty() && $this.feeRateSchedule->isEmpty()) || ($this.feeAmountSchedule->isEmpty() && $this.feeRateSchedule->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A reference to the party responsible for making the payments defined by this structure.'} payerPartyReference: test::gen::PartyReference[1]; @@ -1301,7 +1301,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining to whom and where notice >>>test::gen::ExerciseProcedure Class {meta::pure::profiles::doc.doc = 'A type describing how notice of exercise should be given. This can be either manual or automatic.'} test::gen::ExerciseProcedure [ - c1_choice: ($this.manualExercise->isNotEmpty() && $this.automaticExercise->isEmpty()) || ($this.manualExercise->isEmpty() && $this.automaticExercise->isNotEmpty()) + c1_choice_ExerciseProcedure: ($this.manualExercise->isNotEmpty() && $this.automaticExercise->isEmpty()) || ($this.manualExercise->isEmpty() && $this.automaticExercise->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'Specifies that the notice of exercise must be given by the buyer to the seller or seller\'s agent.'} manualExercise: test::gen::ManualExercise[0..1]; @@ -1314,7 +1314,7 @@ Class {meta::pure::profiles::doc.doc = 'A type describing how notice of exercise >>>test::gen::ExerciseProcedureOption Class {meta::pure::profiles::doc.doc = 'A type describing how notice of exercise should be given. This can be either manual or automatic.'} test::gen::ExerciseProcedureOption [ - c1_choice: ($this.manualExercise->isNotEmpty() && $this.automaticExercise->isEmpty()) || ($this.manualExercise->isEmpty() && $this.automaticExercise->isNotEmpty()) + c1_choice_ExerciseProcedureOption: ($this.manualExercise->isNotEmpty() && $this.automaticExercise->isEmpty()) || ($this.manualExercise->isEmpty() && $this.automaticExercise->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'Specifies that the notice of exercise must be given by the buyer to the seller or seller\'s agent.'} manualExercise: test::gen::Empty[0..1]; @@ -1386,8 +1386,8 @@ Class {meta::pure::profiles::doc.doc = 'A type defining the floating rate and de >>>test::gen::FloatingRateIndex Class {meta::pure::profiles::doc.doc = 'The ISDA Floating Rate Option, i.e. the floating rate index.'} test::gen::FloatingRateIndex [ - c1_length: $this.floatingRateIndexScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_FloatingRateIndex: $this.floatingRateIndexScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_FloatingRateIndex: $this.value->length() <= 255 ] { floatingRateIndexScheme: String[0..1]; @@ -1404,7 +1404,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a rate index.'} test::ge >>>test::gen::Formula Class {meta::pure::profiles::doc.doc = 'A type describing a financial formula, with its description and components.'} test::gen::Formula [ - c1_length: $this.formulaDescription->forAll(x: String[1]|$x->length() <= 255) + c1_length_Formula: $this.formulaDescription->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'Text description of the formula'} formulaDescription: String[0..1]; @@ -1415,8 +1415,8 @@ Class {meta::pure::profiles::doc.doc = 'A type describing a financial formula, w >>>test::gen::FormulaComponent Class {meta::pure::profiles::doc.doc = 'Elements describing the components of the formula. The name attribute points to a value used in the math element. The href attribute points to a numeric value defined elsewhere in the document that is used by the formula component.'} test::gen::FormulaComponent [ - c1_length: $this.componentDescription->length() <= 255, - c2_length: $this.name->forAll(x: String[1]|$x->length() >= 0) + c1_length_FormulaComponent: $this.componentDescription->length() <= 255, + c2_length_FormulaComponent: $this.name->forAll(x: String[1]|$x->length() >= 0) ] { {meta::pure::profiles::doc.doc = 'Text description of the component'} componentDescription: String[1]; @@ -1509,7 +1509,7 @@ Enum {meta::pure::profiles::doc.doc = 'Defines the outcome in the event that the >>>test::gen::FxCashSettlement Class {meta::pure::profiles::doc.doc = 'A type that is used for describing cash settlement of an option / non deliverable forward. It includes the currency to settle into together with the fixings required to calculate the currency amount.'} test::gen::FxCashSettlement [ - c1_choice: ($this.fixing->isNotEmpty() && $this.rateSourceFixing->isEmpty()) || ($this.fixing->isEmpty() && $this.rateSourceFixing->isNotEmpty()) + c1_choice_FxCashSettlement: ($this.fixing->isNotEmpty() && $this.rateSourceFixing->isEmpty()) || ($this.fixing->isEmpty() && $this.rateSourceFixing->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The currency in which cash settlement occurs for non-deliverable forwards and cash-settled options (non-deliverable or otherwise).'} settlementCurrency: test::gen::Currency[1]; @@ -1523,7 +1523,7 @@ Class {meta::pure::profiles::doc.doc = 'A type that is used for describing cash >>>test::gen::FxCashSettlementSimple Class {meta::pure::profiles::doc.doc = 'A type that is used for describing cash settlement of a variance or volatility swap option. It includes the settlement currency together with the spot currency exchange required to calculate the settlement currency amount.'} test::gen::FxCashSettlementSimple [ - c1_choice: ($this.fixing->isNotEmpty() && $this.rateSourceFixing->isEmpty()) || ($this.fixing->isEmpty() && $this.rateSourceFixing->isNotEmpty()) + c1_choice_FxCashSettlementSimple: ($this.fixing->isNotEmpty() && $this.rateSourceFixing->isEmpty()) || ($this.fixing->isEmpty() && $this.rateSourceFixing->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The currency in which cash settlement occurs.'} settlementCurrency: test::gen::Currency[1]; @@ -1591,7 +1591,7 @@ Enum {meta::pure::profiles::doc.doc = 'Target specific settlement adjustment met >>>test::gen::FxSettlementRateSource Class test::gen::FxSettlementRateSource [ - c1_choice: ($this.settlementRateOption->isNotEmpty() && $this.nonstandardSettlementRate->isEmpty()) || ($this.settlementRateOption->isEmpty() && $this.nonstandardSettlementRate->isNotEmpty()) + c1_choice_FxSettlementRateSource: ($this.settlementRateOption->isNotEmpty() && $this.nonstandardSettlementRate->isEmpty()) || ($this.settlementRateOption->isEmpty() && $this.nonstandardSettlementRate->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'Indicates that an officially defined rate settlement rate option will be the used for the fixing.'} settlementRateOption: test::gen::SettlementRateOption[0..1]; @@ -1656,8 +1656,8 @@ Enum {meta::pure::profiles::doc.doc = 'The type of gas product.'} test::gen::Gas >>>test::gen::GoverningLaw Class {meta::pure::profiles::doc.doc = 'Identification of the law governing the transaction.'} test::gen::GoverningLaw [ - c1_length: $this.governingLawScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_GoverningLaw: $this.governingLawScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_GoverningLaw: $this.value->length() <= 255 ] { governingLawScheme: String[0..1]; @@ -1732,8 +1732,8 @@ Enum {meta::pure::profiles::doc.doc = 'The specification of the consequences of >>>test::gen::IndustryClassification Class {meta::pure::profiles::doc.doc = 'A party\'s industry sector classification.'} test::gen::IndustryClassification [ - c1_length: $this.industryClassificationScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_IndustryClassification: $this.industryClassificationScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_IndustryClassification: $this.value->length() <= 255 ] { industryClassificationScheme: String[0..1]; @@ -1743,8 +1743,8 @@ Class {meta::pure::profiles::doc.doc = 'A party\'s industry sector classificatio >>>test::gen::InformationProvider Class test::gen::InformationProvider [ - c1_length: $this.informationProviderScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_InformationProvider: $this.informationProviderScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_InformationProvider: $this.value->length() <= 255 ] { informationProviderScheme: String[0..1]; @@ -1754,7 +1754,7 @@ Class test::gen::InformationProvider >>>test::gen::InformationSource Class {meta::pure::profiles::doc.doc = 'A type defining the source for a piece of information (e.g. a rate refix or an fx fixing).'} test::gen::InformationSource [ - c1_length: $this.rateSourcePageHeading->forAll(x: String[1]|$x->length() <= 255) + c1_length_InformationSource: $this.rateSourcePageHeading->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'An information source for obtaining a market rate. For example Bloomberg, Reuters, Telerate etc.'} rateSource: test::gen::InformationProvider[1]; @@ -1765,9 +1765,9 @@ Class {meta::pure::profiles::doc.doc = 'A type defining the source for a piece o >>>test::gen::InstrumentId Class {meta::pure::profiles::doc.doc = 'A short form unique identifier for a security.'} test::gen::InstrumentId [ - c1_length: $this.instrumentIdScheme->length() >= 1, - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_InstrumentId: $this.instrumentIdScheme->length() >= 1, + c2_length_InstrumentId: $this.value->length() >= 1, + c3_length_InstrumentId: $this.value->length() <= 255 ] { instrumentIdScheme: String[1]; @@ -1783,7 +1783,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining the way in which interes >>>test::gen::InterestAccrualsMethod Class {meta::pure::profiles::doc.doc = 'A type describing the method for accruing interests on dividends. Can be either a fixed rate reference or a floating rate reference.'} test::gen::InterestAccrualsMethod [ - c1_choice: ($this.floatingRateCalculation->isNotEmpty() && $this.fixedRate->isEmpty()) || ($this.floatingRateCalculation->isEmpty() && $this.fixedRate->isNotEmpty()) + c1_choice_InterestAccrualsMethod: ($this.floatingRateCalculation->isNotEmpty() && $this.fixedRate->isEmpty()) || ($this.floatingRateCalculation->isEmpty() && $this.fixedRate->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The floating rate calculation definitions'} floatingRateCalculation: test::gen::FloatingRateCalculation[0..1]; @@ -1807,7 +1807,7 @@ Enum {meta::pure::profiles::doc.doc = 'The specification of the interest shortfa >>>test::gen::IntermediaryInformation Class {meta::pure::profiles::doc.doc = 'A type that describes the information to identify an intermediary through which payment will be made by the correspondent bank to the ultimate beneficiary of the funds.'} test::gen::IntermediaryInformation [ - c1_choice: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) + c1_choice_IntermediaryInformation: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) ] { {meta::pure::profiles::doc.doc = 'A set of unique identifiers for a party, eachone identifying the party within a payment system. The assumption is that each party will not have more than one identifier within the same payment system.'} routingIds: test::gen::RoutingIds[0..1]; @@ -1820,8 +1820,8 @@ Class {meta::pure::profiles::doc.doc = 'A type that describes the information to >>>test::gen::InterpolationMethod Class {meta::pure::profiles::doc.doc = 'The type of interpolation used.'} test::gen::InterpolationMethod [ - c1_length: $this.interpolationMethodScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_InterpolationMethod: $this.interpolationMethodScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_InterpolationMethod: $this.value->length() <= 255 ] { interpolationMethodScheme: String[0..1]; @@ -1840,9 +1840,9 @@ Enum {meta::pure::profiles::doc.doc = 'Defines applicable periods for interpolat >>>test::gen::IssuerId Class {meta::pure::profiles::doc.doc = 'The data type used for issuer identifiers.'} test::gen::IssuerId [ - c1_length: $this.issuerIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_IssuerId: $this.issuerIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_IssuerId: $this.value->length() >= 1, + c3_length_IssuerId: $this.value->length() <= 255 ] { issuerIdScheme: String[0..1]; @@ -1859,8 +1859,8 @@ Class {meta::pure::profiles::doc.doc = 'A complex type for a two part identifier >>>test::gen::Language Class {meta::pure::profiles::doc.doc = 'The data type used for indicating the language of the resource, described using the ISO 639-2/T Code.'} test::gen::Language [ - c1_length: $this.languageScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_Language: $this.languageScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_Language: $this.value->length() <= 255 ] { languageScheme: String[0..1]; @@ -1884,9 +1884,9 @@ Class <> {meta::pure::profiles::do >>>test::gen::LegId Class {meta::pure::profiles::doc.doc = 'Leg identity.'} test::gen::LegId [ - c1_length: $this.legIdScheme->length() >= 1, - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 60 + c1_length_LegId: $this.legIdScheme->length() >= 1, + c2_length_LegId: $this.value->length() >= 1, + c3_length_LegId: $this.value->length() <= 60 ] { legIdScheme: String[1]; @@ -1904,7 +1904,7 @@ Class {meta::pure::profiles::doc.doc = 'Version aware identification of a leg.'} >>>test::gen::LegalEntity Class {meta::pure::profiles::doc.doc = 'A type defining a legal entity.'} test::gen::LegalEntity [ - c1_choice: $this.entityName->isNotEmpty() || ($this.entityName->isEmpty() && $this.entityId->isNotEmpty()) + c1_choice_LegalEntity: $this.entityName->isNotEmpty() || ($this.entityName->isEmpty() && $this.entityId->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The name of the reference entity. A free format string. FpML does not define usage rules for this element.'} entityName: test::gen::EntityName[0..1]; @@ -2040,8 +2040,8 @@ Enum {meta::pure::profiles::doc.doc = 'A list of standard market rules for the t >>>test::gen::MainPublication Class {meta::pure::profiles::doc.doc = 'A type to define the main publication source.'} test::gen::MainPublication [ - c1_length: $this.mainPublicationScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MainPublication: $this.mainPublicationScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MainPublication: $this.value->length() <= 255 ] { mainPublicationScheme: String[0..1]; @@ -2083,9 +2083,9 @@ Class {meta::pure::profiles::doc.doc = 'An entity for defining the agreement exe >>>test::gen::MasterAgreementId Class {meta::pure::profiles::doc.doc = 'A master agreement identifier allocated by a party. FpML does not define the domain values associated with this element. Note that the domain values for this element are not strictly an enumerated list.'} test::gen::MasterAgreementId [ - c1_length: $this.masterAgreementIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_MasterAgreementId: $this.masterAgreementIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MasterAgreementId: $this.value->length() >= 1, + c3_length_MasterAgreementId: $this.value->length() <= 255 ] { masterAgreementIdScheme: String[0..1]; @@ -2095,8 +2095,8 @@ Class {meta::pure::profiles::doc.doc = 'A master agreement identifier allocated >>>test::gen::MasterAgreementType Class test::gen::MasterAgreementType [ - c1_length: $this.masterAgreementTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MasterAgreementType: $this.masterAgreementTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MasterAgreementType: $this.value->length() <= 255 ] { masterAgreementTypeScheme: String[0..1]; @@ -2106,8 +2106,8 @@ Class test::gen::MasterAgreementType >>>test::gen::MasterAgreementVersion Class test::gen::MasterAgreementVersion [ - c1_length: $this.masterAgreementVersionScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MasterAgreementVersion: $this.masterAgreementVersionScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MasterAgreementVersion: $this.value->length() <= 255 ] { masterAgreementVersionScheme: String[0..1]; @@ -2126,8 +2126,8 @@ Class {meta::pure::profiles::doc.doc = 'An entity for defining the master confir >>>test::gen::MasterConfirmationAnnexType Class test::gen::MasterConfirmationAnnexType [ - c1_length: $this.masterConfirmationAnnexTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MasterConfirmationAnnexType: $this.masterConfirmationAnnexTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MasterConfirmationAnnexType: $this.value->length() <= 255 ] { masterConfirmationAnnexTypeScheme: String[0..1]; @@ -2137,8 +2137,8 @@ Class test::gen::MasterConfirmationAnnexType >>>test::gen::MasterConfirmationType Class test::gen::MasterConfirmationType [ - c1_length: $this.masterConfirmationTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MasterConfirmationType: $this.masterConfirmationTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MasterConfirmationType: $this.value->length() <= 255 ] { masterConfirmationTypeScheme: String[0..1]; @@ -2148,9 +2148,9 @@ Class test::gen::MasterConfirmationType >>>test::gen::MatchId Class {meta::pure::profiles::doc.doc = 'An identifier used to identify matched cashflows.'} test::gen::MatchId [ - c1_length: $this.matchIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_MatchId: $this.matchIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MatchId: $this.value->length() >= 1, + c3_length_MatchId: $this.value->length() <= 255 ] { matchIdScheme: String[0..1]; @@ -2165,8 +2165,8 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a mathematical expressio >>>test::gen::MatrixTerm Class test::gen::MatrixTerm [ - c1_length: $this.matrixTermScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MatrixTerm: $this.matrixTermScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MatrixTerm: $this.value->length() <= 255 ] { matrixTermScheme: String[0..1]; @@ -2176,8 +2176,8 @@ Class test::gen::MatrixTerm >>>test::gen::MatrixType Class test::gen::MatrixType [ - c1_length: $this.matrixTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MatrixType: $this.matrixTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MatrixType: $this.value->length() <= 255 ] { matrixTypeScheme: String[0..1]; @@ -2201,8 +2201,8 @@ Enum {meta::pure::profiles::doc.doc = 'Defines how adjustments will be made to t >>>test::gen::MimeType Class {meta::pure::profiles::doc.doc = 'The type that indicates the type of media used to store the content. MimeType is used to determine the software product(s) that can read the content. MIME types are described in RFC 2046.'} test::gen::MimeType [ - c1_length: $this.mimeTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_MimeType: $this.mimeTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_MimeType: $this.value->length() <= 255 ] { mimeTypeScheme: String[0..1]; @@ -2225,9 +2225,9 @@ Class <> {meta::pure::profiles::do >>>test::gen::MultipleExercise Class {meta::pure::profiles::doc.doc = 'A type defining multiple exercises. As defining in the 2000 ISDA Definitions, Section 12.4. Multiple Exercise, the buyer of the option has the right to exercise all or less than all the unexercised notional amount of the underlying swap on one or more days in the exercise period, but on any such day may not exercise less than the minimum notional amount or more than the maximum notional amount, and if an integral multiple amount is specified, the notional exercised must be equal to or, be an integral multiple of, the integral multiple amount.'} test::gen::MultipleExercise [ - c1_range: $this.maximumNumberOfOptions->forAll(x: Decimal[1]|$x >= 0), - c2_choice: ($this.minimumNotionalAmount->isNotEmpty() && $this.minimumNumberOfOptions->isEmpty()) || ($this.minimumNotionalAmount->isEmpty() && $this.minimumNumberOfOptions->isNotEmpty()), - c3_choice: ($this.maximumNotionalAmount->isNotEmpty() && $this.maximumNumberOfOptions->isEmpty()) || ($this.maximumNotionalAmount->isEmpty() && $this.maximumNumberOfOptions->isNotEmpty()) + c1_range_MultipleExercise: $this.maximumNumberOfOptions->forAll(x: Decimal[1]|$x >= 0), + c2_choice_MultipleExercise: ($this.minimumNotionalAmount->isNotEmpty() && $this.minimumNumberOfOptions->isEmpty()) || ($this.minimumNotionalAmount->isEmpty() && $this.minimumNumberOfOptions->isNotEmpty()), + c3_choice_MultipleExercise: ($this.maximumNotionalAmount->isNotEmpty() && $this.maximumNumberOfOptions->isEmpty()) || ($this.maximumNotionalAmount->isEmpty() && $this.maximumNumberOfOptions->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A pointer style reference to the associated notional schedule defined elsewhere in the document. This element has been made optional as part of its integration in the OptionBaseExtended, because not required for the options on securities.'} notionalReference: test::gen::NotionalReference[*]; @@ -2268,7 +2268,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a currency amount or a c >>>test::gen::NonNegativeMoney Class {meta::pure::profiles::doc.doc = 'A type defining a non negative money amount.'} test::gen::NonNegativeMoney extends test::gen::MoneyBase [ - c1_range: $this.amount >= 0 + c1_range_NonNegativeMoney: $this.amount >= 0 ] { {meta::pure::profiles::doc.doc = 'The non negative monetary quantity in currency units.'} amount: Decimal[1]; @@ -2283,7 +2283,7 @@ Class {meta::pure::profiles::doc.doc = 'A complex type to specify non negative p >>>test::gen::NonNegativeSchedule Class {meta::pure::profiles::doc.doc = 'A type defining a schedule of non-negative rates or amounts in terms of an initial value and then a series of step date and value pairs. On each step date the rate or amount changes to the new step value. The series of step date and value pairs are optional. If not specified, this implies that the initial value remains unchanged over time.'} test::gen::NonNegativeSchedule [ - c1_range: $this.initialValue >= 0 + c1_range_NonNegativeSchedule: $this.initialValue >= 0 ] { {meta::pure::profiles::doc.doc = 'The non-negative initial rate or amount, as the case may be. An initial rate of 5% would be represented as 0.05.'} initialValue: Decimal[1]; @@ -2294,7 +2294,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a schedule of non-negati >>>test::gen::NonNegativeStep Class {meta::pure::profiles::doc.doc = 'A type defining a step date and non-negative step value pair. This step definitions are used to define varying rate or amount schedules, e.g. a notional amortization or a step-up coupon schedule.'} test::gen::NonNegativeStep extends test::gen::StepBase [ - c1_range: $this.stepValue >= 0 + c1_range_NonNegativeStep: $this.stepValue >= 0 ] { {meta::pure::profiles::doc.doc = 'The non-negative rate or amount which becomes effective on the associated stepDate. A rate of 5% would be represented as 0.05.'} stepValue: Decimal[1]; @@ -2394,8 +2394,8 @@ Enum {meta::pure::profiles::doc.doc = 'Specifies the type of the option.'} test: >>>test::gen::OrganizationType Class {meta::pure::profiles::doc.doc = 'A code that describes what type of role an organization plays, for example a SwapsDealer, a Major Swaps Participant, or Other'} test::gen::OrganizationType [ - c1_length: $this.organizationTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 0 + c1_length_OrganizationType: $this.organizationTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_OrganizationType: $this.value->length() >= 0 ] { organizationTypeScheme: String[0..1]; @@ -2405,8 +2405,8 @@ Class {meta::pure::profiles::doc.doc = 'A code that describes what type of role >>>test::gen::OriginatingEvent Class test::gen::OriginatingEvent [ - c1_length: $this.originatingEventScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_OriginatingEvent: $this.originatingEventScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_OriginatingEvent: $this.value->length() <= 255 ] { originatingEventScheme: String[0..1]; @@ -2425,9 +2425,9 @@ Class {meta::pure::profiles::doc.doc = 'An entity for defining the an agreement >>>test::gen::OtherAgreementId Class {meta::pure::profiles::doc.doc = 'A agreement identifier allocated by a party. FpML does not define the domain values associated with this element. Note that the domain values for this element are not strictly an enumerated list.'} test::gen::OtherAgreementId [ - c1_length: $this.agreementIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_OtherAgreementId: $this.agreementIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_OtherAgreementId: $this.value->length() >= 1, + c3_length_OtherAgreementId: $this.value->length() <= 255 ] { agreementIdScheme: String[0..1]; @@ -2437,8 +2437,8 @@ Class {meta::pure::profiles::doc.doc = 'A agreement identifier allocated by a pa >>>test::gen::OtherAgreementType Class test::gen::OtherAgreementType [ - c1_length: $this.agreementTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_OtherAgreementType: $this.agreementTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_OtherAgreementType: $this.value->length() <= 255 ] { agreementTypeScheme: String[0..1]; @@ -2448,8 +2448,8 @@ Class test::gen::OtherAgreementType >>>test::gen::OtherAgreementVersion Class test::gen::OtherAgreementVersion [ - c1_length: $this.agreementVersionScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_OtherAgreementVersion: $this.agreementVersionScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_OtherAgreementVersion: $this.value->length() <= 255 ] { agreementVersionScheme: String[0..1]; @@ -2459,7 +2459,7 @@ Class test::gen::OtherAgreementVersion >>>test::gen::PartialExercise Class {meta::pure::profiles::doc.doc = 'A type defining partial exercise. As defined in the 2000 ISDA Definitions, Section 12.3 Partial Exercise, the buyer of the option may exercise all or less than all the notional amount of the underlying swap but may not be less than the minimum notional amount (if specified) and must be an integral multiple of the integral multiple amount if specified.'} test::gen::PartialExercise [ - c1_choice: ($this.minimumNotionalAmount->isNotEmpty() && $this.minimumNumberOfOptions->isEmpty()) || ($this.minimumNotionalAmount->isEmpty() && $this.minimumNumberOfOptions->isNotEmpty()) + c1_choice_PartialExercise: ($this.minimumNotionalAmount->isNotEmpty() && $this.minimumNumberOfOptions->isEmpty()) || ($this.minimumNotionalAmount->isEmpty() && $this.minimumNumberOfOptions->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'A pointer style reference to the associated notional schedule defined elsewhere in the document. This element has been made optional as part of its integration in the OptionBaseExtended, because not required for the options on securities.'} notionalReference: test::gen::NotionalReference[*]; @@ -2471,7 +2471,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining partial exercise. As def >>>test::gen::Party Class {meta::pure::profiles::doc.doc = 'A type defining a legal entity or a subdivision of a legal entity. Parties can perform multiple roles in a trade lifecycle. For example, the principal parties obligated to make payments from time to time during the term of the trade, but may include other parties involved in, or incidental to, the trade, such as parties acting in the role of novation transferor/transferee, broker, calculation agent, etc. In FpML roles are defined in multiple places within a document.'} test::gen::Party [ - c1_choice: ($this.partyId->isNotEmpty() && ($this.groupType->isEmpty() && $this.partyReference->isEmpty())) || ($this.partyId->isEmpty() && ($this.partyName->isEmpty() && ($this.classification->isEmpty() && ($this.creditRating->isEmpty() && ($this.country->isEmpty() && ($this.region->isEmpty() && ($this.jurisdiction->isEmpty() && ($this.organizationType->isEmpty() && ($this.contactInfo->isEmpty() && ($this.businessUnit->isEmpty() && ($this.person->isEmpty() && ($this.groupType->isNotEmpty() && $this.partyReference->isNotEmpty())))))))))))) + c1_choice_Party: ($this.partyId->isNotEmpty() && ($this.groupType->isEmpty() && $this.partyReference->isEmpty())) || ($this.partyId->isEmpty() && ($this.partyName->isEmpty() && ($this.classification->isEmpty() && ($this.creditRating->isEmpty() && ($this.country->isEmpty() && ($this.region->isEmpty() && ($this.jurisdiction->isEmpty() && ($this.organizationType->isEmpty() && ($this.contactInfo->isEmpty() && ($this.businessUnit->isEmpty() && ($this.person->isEmpty() && ($this.groupType->isNotEmpty() && $this.partyReference->isNotEmpty())))))))))))) ] { {meta::pure::profiles::doc.doc = 'A party identifier, e.g. a S.W.I.F.T. bank identifier code (BIC).'} partyId: test::gen::PartyId[*]; @@ -2493,8 +2493,8 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a legal entity or a subd >>>test::gen::PartyGroupType Class {meta::pure::profiles::doc.doc = 'The data type used for party group classification.'} test::gen::PartyGroupType [ - c1_length: $this.partyGroupTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_PartyGroupType: $this.partyGroupTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_PartyGroupType: $this.value->length() <= 255 ] { partyGroupTypeScheme: String[0..1]; @@ -2504,9 +2504,9 @@ Class {meta::pure::profiles::doc.doc = 'The data type used for party group class >>>test::gen::PartyId Class {meta::pure::profiles::doc.doc = 'The data type used for party identifiers.'} test::gen::PartyId [ - c1_length: $this.partyIdScheme->length() >= 1, - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_PartyId: $this.partyIdScheme->length() >= 1, + c2_length_PartyId: $this.value->length() >= 1, + c3_length_PartyId: $this.value->length() <= 255 ] { partyIdScheme: String[1]; @@ -2516,8 +2516,8 @@ Class {meta::pure::profiles::doc.doc = 'The data type used for party identifiers >>>test::gen::PartyName Class {meta::pure::profiles::doc.doc = 'The data type used for the legal name of an organization.'} test::gen::PartyName [ - c1_length: $this.partyNameScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_PartyName: $this.partyNameScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_PartyName: $this.value->length() <= 255 ] { partyNameScheme: String[0..1]; @@ -2533,8 +2533,8 @@ Class {meta::pure::profiles::doc.doc = 'Reference to a party.'} test::gen::Party >>>test::gen::PartyRole Class {meta::pure::profiles::doc.doc = 'A type describing a role played by a party in one or more transactions. Examples include roles such as guarantor, custodian, confirmation service provider, etc. This can be extended to provide custom roles.'} test::gen::PartyRole [ - c1_length: $this.partyRoleScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_PartyRole: $this.partyRoleScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_PartyRole: $this.value->length() <= 255 ] { partyRoleScheme: String[0..1]; @@ -2544,8 +2544,8 @@ Class {meta::pure::profiles::doc.doc = 'A type describing a role played by a par >>>test::gen::PartyRoleType Class {meta::pure::profiles::doc.doc = 'A type refining the role a role played by a party in one or more transactions. Examples include "AllPositions" and "SomePositions" for Guarantor. This can be extended to provide custom types.'} test::gen::PartyRoleType [ - c1_length: $this.partyRoleTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_PartyRoleType: $this.partyRoleTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_PartyRoleType: $this.value->length() <= 255 ] { partyRoleTypeScheme: String[0..1]; @@ -2637,8 +2637,8 @@ Class <> {meta::pure::profiles::do >>>test::gen::PaymentType Class test::gen::PaymentType [ - c1_length: $this.paymentTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_PaymentType: $this.paymentTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_PaymentType: $this.value->length() <= 255 ] { paymentTypeScheme: String[0..1]; @@ -2703,13 +2703,13 @@ Class test::gen::PeriodicDates >>>test::gen::Person Class {meta::pure::profiles::doc.doc = 'A type that represents information about a person connected with a trade or business process.'} test::gen::Person [ - c1_length: $this.honorific->forAll(x: String[1]|$x->length() >= 0), - c2_length: $this.firstName->length() >= 0, - c3_length: $this.middleName->forAll(x: String[1]|$x->length() >= 0), - c4_length: $this.initial->forAll(x: String[1]|$x->length() == 1), - c5_length: $this.surname->length() >= 0, - c6_length: $this.suffix->forAll(x: String[1]|$x->length() >= 0), - c7_choice: ($this.middleName->isNotEmpty() && $this.initial->isEmpty()) || ($this.middleName->isEmpty() && $this.initial->isNotEmpty()) + c1_length_Person: $this.honorific->forAll(x: String[1]|$x->length() >= 0), + c2_length_Person: $this.firstName->length() >= 0, + c3_length_Person: $this.middleName->forAll(x: String[1]|$x->length() >= 0), + c4_length_Person: $this.initial->forAll(x: String[1]|$x->length() == 1), + c5_length_Person: $this.surname->length() >= 0, + c6_length_Person: $this.suffix->forAll(x: String[1]|$x->length() >= 0), + c7_choice_Person: ($this.middleName->isNotEmpty() && $this.initial->isEmpty()) || ($this.middleName->isEmpty() && $this.initial->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'An honorific title, such as Mr., Ms., Dr. etc.'} honorific: String[0..1]; @@ -2729,9 +2729,9 @@ Class {meta::pure::profiles::doc.doc = 'A type that represents information about >>>test::gen::PersonId Class {meta::pure::profiles::doc.doc = 'An identifier used to identify an individual person.'} test::gen::PersonId [ - c1_length: $this.personIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_PersonId: $this.personIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_PersonId: $this.value->length() >= 1, + c3_length_PersonId: $this.value->length() <= 255 ] { personIdScheme: String[0..1]; @@ -2747,8 +2747,8 @@ Class {meta::pure::profiles::doc.doc = 'Reference to an individual.'} test::gen: >>>test::gen::PersonRole Class {meta::pure::profiles::doc.doc = 'A type describing a role played by a person in one or more transactions. Examples include roles such as Trader, Broker, MiddleOffice, Legal, etc. This can be extended to provide custom roles.'} test::gen::PersonRole [ - c1_length: $this.personRoleScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_PersonRole: $this.personRoleScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_PersonRole: $this.value->length() <= 255 ] { personRoleScheme: String[0..1]; @@ -2758,7 +2758,7 @@ Class {meta::pure::profiles::doc.doc = 'A type describing a role played by a per >>>test::gen::PositiveMoney Class {meta::pure::profiles::doc.doc = 'A type defining a positive money amount'} test::gen::PositiveMoney extends test::gen::MoneyBase [ - c1_range: $this.amount > 0 + c1_range_PositiveMoney: $this.amount > 0 ] { {meta::pure::profiles::doc.doc = 'The positive monetary quantity in currency units.'} amount: Decimal[1]; @@ -2800,7 +2800,7 @@ Enum {meta::pure::profiles::doc.doc = 'The mode of expression of a price.'} test >>>test::gen::PricingStructure Class <> {meta::pure::profiles::doc.doc = 'An abstract pricing structure base type. Used as a base for structures such as yield curves and volatility matrices.'} test::gen::PricingStructure [ - c1_length: $this.name->forAll(x: String[1]|$x->length() >= 0) + c1_length_PricingStructure: $this.name->forAll(x: String[1]|$x->length() >= 0) ] { {meta::pure::profiles::doc.doc = 'The name of the structure, e.g "USDLIBOR-3M EOD Curve".'} name: String[0..1]; @@ -2837,9 +2837,9 @@ Class <> {meta::pure::profiles::do >>>test::gen::ProductId Class test::gen::ProductId [ - c1_length: $this.productIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 1023 + c1_length_ProductId: $this.productIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ProductId: $this.value->length() >= 1, + c3_length_ProductId: $this.value->length() <= 1023 ] { productIdScheme: String[0..1]; @@ -2855,9 +2855,9 @@ Class {meta::pure::profiles::doc.doc = 'Reference to a full FpML product.'} test >>>test::gen::ProductType Class test::gen::ProductType [ - c1_length: $this.productTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 1023 + c1_length_ProductType: $this.productTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ProductType: $this.value->length() >= 1, + c3_length_ProductType: $this.value->length() <= 1023 ] { productTypeScheme: String[0..1]; @@ -2948,8 +2948,8 @@ Class {meta::pure::profiles::doc.doc = 'Reference to any rate (floating, inflati >>>test::gen::RateSourcePage Class test::gen::RateSourcePage [ - c1_length: $this.rateSourcePageScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_RateSourcePage: $this.rateSourcePageScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_RateSourcePage: $this.value->length() <= 255 ] { rateSourcePageScheme: String[0..1]; @@ -2979,8 +2979,8 @@ Class <> {meta::pure::profiles::do >>>test::gen::ReferenceAmount Class {meta::pure::profiles::doc.doc = 'Specifies the reference amount using a scheme.'} test::gen::ReferenceAmount [ - c1_length: $this.referenceAmountScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_ReferenceAmount: $this.referenceAmountScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ReferenceAmount: $this.value->length() <= 255 ] { referenceAmountScheme: String[0..1]; @@ -2990,7 +2990,7 @@ Class {meta::pure::profiles::doc.doc = 'Specifies the reference amount using a s >>>test::gen::ReferenceBank Class {meta::pure::profiles::doc.doc = 'A type to describe an institution (party) identified by means of a coding scheme and an optional name.'} test::gen::ReferenceBank [ - c1_length: $this.referenceBankName->forAll(x: String[1]|$x->length() <= 255) + c1_length_ReferenceBank: $this.referenceBankName->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'An institution (party) identifier, e.g. a bank identifier code (BIC).'} referenceBankId: test::gen::ReferenceBankId[1]; @@ -3000,9 +3000,9 @@ Class {meta::pure::profiles::doc.doc = 'A type to describe an institution (party >>>test::gen::ReferenceBankId Class test::gen::ReferenceBankId [ - c1_length: $this.referenceBankIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_ReferenceBankId: $this.referenceBankIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ReferenceBankId: $this.value->length() >= 1, + c3_length_ReferenceBankId: $this.value->length() <= 255 ] { referenceBankIdScheme: String[0..1]; @@ -3012,8 +3012,8 @@ Class test::gen::ReferenceBankId >>>test::gen::Region Class {meta::pure::profiles::doc.doc = 'A code that describes the world region of a counterparty. For example, NorthAmerica, Europe, Asia.'} test::gen::Region [ - c1_length: $this.regionScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_Region: $this.regionScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_Region: $this.value->length() <= 255 ] { regionScheme: String[0..1]; @@ -3056,7 +3056,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining a date (referred to as t >>>test::gen::RelativeDateSequence Class {meta::pure::profiles::doc.doc = 'A type describing a date when this date is defined in reference to another date through one or several date offsets.'} test::gen::RelativeDateSequence [ - c1_choice: ($this.businessCentersReference->isNotEmpty() && $this.businessCenters->isEmpty()) || ($this.businessCentersReference->isEmpty() && $this.businessCenters->isNotEmpty()) + c1_choice_RelativeDateSequence: ($this.businessCentersReference->isNotEmpty() && $this.businessCenters->isEmpty()) || ($this.businessCentersReference->isEmpty() && $this.businessCenters->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'Specifies the anchor as an href attribute. The href attribute value is a pointer style reference to the element or component elsewhere in the document where the anchor date is defined.'} dateRelativeTo: test::gen::DateReference[1]; @@ -3082,8 +3082,8 @@ Enum {meta::pure::profiles::doc.doc = 'A duration code for a Repo (or Securities >>>test::gen::ReportingRegimeName Class {meta::pure::profiles::doc.doc = 'An identifier of an reporting regime or format used for regulatory reporting, for example DoddFrankAct, MiFID, HongKongOTCDRepository, etc.'} test::gen::ReportingRegimeName [ - c1_length: $this.reportingRegimeNameScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_ReportingRegimeName: $this.reportingRegimeNameScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ReportingRegimeName: $this.value->length() <= 255 ] { reportingRegimeNameScheme: String[0..1]; @@ -3093,8 +3093,8 @@ Class {meta::pure::profiles::doc.doc = 'An identifier of an reporting regime or >>>test::gen::RequestedAction Class test::gen::RequestedAction [ - c1_length: $this.requestedActionScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_RequestedAction: $this.requestedActionScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_RequestedAction: $this.value->length() <= 255 ] { requestedActionScheme: String[0..1]; @@ -3124,11 +3124,11 @@ Enum {meta::pure::profiles::doc.doc = 'The specification of whether resets occur >>>test::gen::Resource Class {meta::pure::profiles::doc.doc = 'Describes the resource that contains the media representation of a business event (i.e used for stating the Publicly Available Information). For example, can describe a file or a URL that represents the event. This type is an extended version of a type defined by RIXML (www.rixml.org).'} test::gen::Resource [ - c1_length: $this.name->forAll(x: String[1]|$x->length() >= 0), - c2_length: $this.comments->forAll(x: String[1]|$x->length() <= 255), - c3_length: $this.string->forAll(x: String[1]|$x->length() <= 255), - c4_length: $this.url->forAll(x: String[1]|$x->length() >= 1), - c5_choice: ($this.string->isNotEmpty() && ($this.hexadecimalBinary->isEmpty() && ($this.base64Binary->isEmpty() && $this.url->isEmpty()))) || (($this.string->isEmpty() && ($this.hexadecimalBinary->isNotEmpty() && ($this.base64Binary->isEmpty() && $this.url->isEmpty()))) || (($this.string->isEmpty() && ($this.hexadecimalBinary->isEmpty() && ($this.base64Binary->isNotEmpty() && $this.url->isEmpty()))) || ($this.string->isEmpty() && ($this.hexadecimalBinary->isEmpty() && ($this.base64Binary->isEmpty() && $this.url->isNotEmpty()))))) + c1_length_Resource: $this.name->forAll(x: String[1]|$x->length() >= 0), + c2_length_Resource: $this.comments->forAll(x: String[1]|$x->length() <= 255), + c3_length_Resource: $this.string->forAll(x: String[1]|$x->length() <= 255), + c4_length_Resource: $this.url->forAll(x: String[1]|$x->length() >= 1), + c5_choice_Resource: ($this.string->isNotEmpty() && ($this.hexadecimalBinary->isEmpty() && ($this.base64Binary->isEmpty() && $this.url->isEmpty()))) || (($this.string->isEmpty() && ($this.hexadecimalBinary->isNotEmpty() && ($this.base64Binary->isEmpty() && $this.url->isEmpty()))) || (($this.string->isEmpty() && ($this.hexadecimalBinary->isEmpty() && ($this.base64Binary->isNotEmpty() && $this.url->isEmpty()))) || ($this.string->isEmpty() && ($this.hexadecimalBinary->isEmpty() && ($this.base64Binary->isEmpty() && $this.url->isNotEmpty()))))) ] { {meta::pure::profiles::doc.doc = 'The unique identifier of the resource within the event.'} resourceId: test::gen::ResourceId[1]; @@ -3148,9 +3148,9 @@ Class {meta::pure::profiles::doc.doc = 'Describes the resource that contains the >>>test::gen::ResourceId Class {meta::pure::profiles::doc.doc = 'The data type used for resource identifiers.'} test::gen::ResourceId [ - c1_length: $this.resourceIdScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_ResourceId: $this.resourceIdScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ResourceId: $this.value->length() >= 1, + c3_length_ResourceId: $this.value->length() <= 255 ] { resourceIdScheme: String[0..1]; @@ -3167,8 +3167,8 @@ Class {meta::pure::profiles::doc.doc = 'The type that indicates the length of th >>>test::gen::ResourceType Class {meta::pure::profiles::doc.doc = 'The data type used for describing the type or purpose of a resource, e.g. "Confirmation".'} test::gen::ResourceType [ - c1_length: $this.resourceTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_ResourceType: $this.resourceTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_ResourceType: $this.value->length() <= 255 ] { resourceTypeScheme: String[0..1]; @@ -3258,7 +3258,7 @@ Enum {meta::pure::profiles::doc.doc = 'The method of rounding a fractional numbe >>>test::gen::Routing Class {meta::pure::profiles::doc.doc = 'A type that provides three alternative ways of identifying a party involved in the routing of a payment. The identification may use payment system identifiers only; actual name, address and other reference information; or a combination of both.'} test::gen::Routing [ - c1_choice: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) + c1_choice_Routing: ($this.routingIds->isNotEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || (($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isNotEmpty() && $this.routingIdsAndExplicitDetails->isEmpty())) || ($this.routingIds->isEmpty() && ($this.routingExplicitDetails->isEmpty() && $this.routingIdsAndExplicitDetails->isNotEmpty()))) ] { {meta::pure::profiles::doc.doc = 'A set of unique identifiers for a party, eachone identifying the party within a payment system. The assumption is that each party will not have more than one identifier within the same payment system.'} routingIds: test::gen::RoutingIds[0..1]; @@ -3269,9 +3269,9 @@ Class {meta::pure::profiles::doc.doc = 'A type that provides three alternative w >>>test::gen::RoutingExplicitDetails Class {meta::pure::profiles::doc.doc = 'A type that models name, address and supplementary textual information for the purposes of identifying a party involved in the routing of a payment.'} test::gen::RoutingExplicitDetails [ - c1_length: $this.routingName->length() <= 255, - c2_length: $this.routingAccountNumber->forAll(x: String[1]|$x->length() <= 255), - c3_length: $this.routingReferenceText->forAll(x: String[1]|$x->length() <= 255) + c1_length_RoutingExplicitDetails: $this.routingName->length() <= 255, + c2_length_RoutingExplicitDetails: $this.routingAccountNumber->forAll(x: String[1]|$x->length() <= 255), + c3_length_RoutingExplicitDetails: $this.routingReferenceText->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'A real name that is used to identify a party involved in the routing of a payment.'} routingName: String[1]; @@ -3283,8 +3283,8 @@ Class {meta::pure::profiles::doc.doc = 'A type that models name, address and sup >>>test::gen::RoutingId Class test::gen::RoutingId [ - c1_length: $this.routingIdCodeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_RoutingId: $this.routingIdCodeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_RoutingId: $this.value->length() <= 255 ] { routingIdCodeScheme: String[0..1]; @@ -3300,9 +3300,9 @@ Class {meta::pure::profiles::doc.doc = 'A type that provides for identifying a p >>>test::gen::RoutingIdsAndExplicitDetails Class {meta::pure::profiles::doc.doc = 'A type that provides a combination of payment system identification codes with physical postal address details, for the purposes of identifying a party involved in the routing of a payment.'} test::gen::RoutingIdsAndExplicitDetails [ - c1_length: $this.routingName->length() <= 255, - c2_length: $this.routingAccountNumber->forAll(x: String[1]|$x->length() <= 255), - c3_length: $this.routingReferenceText->forAll(x: String[1]|$x->length() <= 255) + c1_length_RoutingIdsAndExplicitDetails: $this.routingName->length() <= 255, + c2_length_RoutingIdsAndExplicitDetails: $this.routingAccountNumber->forAll(x: String[1]|$x->length() <= 255), + c3_length_RoutingIdsAndExplicitDetails: $this.routingReferenceText->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'A set of unique identifiers for a party, eachone identifying the party within a payment system. The assumption is that each party will not have more than one identifier within the same payment system.'} routingIds: test::gen::RoutingIds[1..*]; @@ -3329,7 +3329,7 @@ Class {meta::pure::profiles::doc.doc = 'Reference to a schedule of rates or amou >>>test::gen::SettlementInformation Class {meta::pure::profiles::doc.doc = 'A type that represents the choice of methods for settling a potential currency payment resulting from a trade: by means of a standard settlement instruction, by netting it out with other payments, or with an explicit settlement instruction.'} test::gen::SettlementInformation [ - c1_choice: ($this.standardSettlementStyle->isNotEmpty() && $this.settlementInstruction->isEmpty()) || ($this.standardSettlementStyle->isEmpty() && $this.settlementInstruction->isNotEmpty()) + c1_choice_SettlementInformation: ($this.standardSettlementStyle->isNotEmpty() && $this.settlementInstruction->isEmpty()) || ($this.standardSettlementStyle->isEmpty() && $this.settlementInstruction->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'An optional element used to describe how a trade will settle. This defines a scheme and is used for identifying trades that are identified as settling standard and/or flagged for settlement netting.'} standardSettlementStyle: test::gen::StandardSettlementStyleEnum[0..1]; @@ -3351,8 +3351,8 @@ Class {meta::pure::profiles::doc.doc = 'A type that models a complete instructio >>>test::gen::SettlementMethod Class test::gen::SettlementMethod [ - c1_length: $this.settlementMethodScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_SettlementMethod: $this.settlementMethodScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_SettlementMethod: $this.value->length() <= 255 ] { settlementMethodScheme: String[0..1]; @@ -3371,8 +3371,8 @@ Enum {meta::pure::profiles::doc.doc = 'Defines the Settlement Period Duration fo >>>test::gen::SettlementPriceDefaultElection Class {meta::pure::profiles::doc.doc = 'Coding scheme that specifies the settlement price default election.'} test::gen::SettlementPriceDefaultElection [ - c1_length: $this.settlementPriceDefaultElectionScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_SettlementPriceDefaultElection: $this.settlementPriceDefaultElectionScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_SettlementPriceDefaultElection: $this.value->length() <= 255 ] { settlementPriceDefaultElectionScheme: String[0..1]; @@ -3382,8 +3382,8 @@ Class {meta::pure::profiles::doc.doc = 'Coding scheme that specifies the settlem >>>test::gen::SettlementPriceSource Class {meta::pure::profiles::doc.doc = 'The source from which the settlement price is to be obtained, e.g. a Reuters page, Prezzo di Riferimento, etc.'} test::gen::SettlementPriceSource [ - c1_length: $this.settlementPriceSourceScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_SettlementPriceSource: $this.settlementPriceSourceScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_SettlementPriceSource: $this.value->length() <= 255 ] { settlementPriceSourceScheme: String[0..1]; @@ -3393,8 +3393,8 @@ Class {meta::pure::profiles::doc.doc = 'The source from which the settlement pri >>>test::gen::SettlementRateOption Class {meta::pure::profiles::doc.doc = 'A type defining the settlement rate options through a scheme reflecting the terms of the Annex A to the 1998 FX and Currency Option Definitions.'} test::gen::SettlementRateOption [ - c1_length: $this.settlementRateOptionScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_SettlementRateOption: $this.settlementRateOptionScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_SettlementRateOption: $this.value->length() <= 255 ] { settlementRateOptionScheme: String[0..1]; @@ -3404,7 +3404,7 @@ Class {meta::pure::profiles::doc.doc = 'A type defining the settlement rate opti >>>test::gen::SettlementRateSource Class {meta::pure::profiles::doc.doc = 'A type describing the method for obtaining a settlement rate.'} test::gen::SettlementRateSource [ - c1_choice: ($this.informationSource->isNotEmpty() && $this.cashSettlementReferenceBanks->isEmpty()) || ($this.informationSource->isEmpty() && $this.cashSettlementReferenceBanks->isNotEmpty()) + c1_choice_SettlementRateSource: ($this.informationSource->isNotEmpty() && $this.cashSettlementReferenceBanks->isEmpty()) || ($this.informationSource->isEmpty() && $this.cashSettlementReferenceBanks->isNotEmpty()) ] { {meta::pure::profiles::doc.doc = 'The information source where a published or displayed market rate will be obtained, e.g. Telerate Page 3750.'} informationSource: test::gen::InformationSource[0..1]; @@ -3502,8 +3502,8 @@ Class {meta::pure::profiles::doc.doc = 'Provides a reference to a spread schedul >>>test::gen::SpreadScheduleType Class {meta::pure::profiles::doc.doc = 'Defines a Spread Type Scheme to identify a long or short spread value.'} test::gen::SpreadScheduleType [ - c1_length: $this.spreadScheduleTypeScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_SpreadScheduleType: $this.spreadScheduleTypeScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_SpreadScheduleType: $this.value->length() <= 255 ] { spreadScheduleTypeScheme: String[0..1]; @@ -3541,7 +3541,7 @@ Enum {meta::pure::profiles::doc.doc = 'The specification of whether a percentage >>>test::gen::StreetAddress Class {meta::pure::profiles::doc.doc = 'A type that describes the set of street and building number information that identifies a postal address within a city.'} test::gen::StreetAddress [ - c1_length: $this.streetLine->forAll(x: String[1]|$x->length() <= 255) + c1_length_StreetAddress: $this.streetLine->forAll(x: String[1]|$x->length() <= 255) ] { {meta::pure::profiles::doc.doc = 'An individual line of street and building number information, forming part of a postal address.'} streetLine: String[1..*]; @@ -3601,7 +3601,7 @@ Enum {meta::pure::profiles::doc.doc = 'Element to define how to deal with a none >>>test::gen::StubValue Class {meta::pure::profiles::doc.doc = 'A type defining how a stub calculation period amount is calculated. A single floating rate tenor different to that used for the regular part of the calculation periods schedule may be specified, or two floating rate tenors many be specified. If two floating rate tenors are specified then Linear Interpolation (in accordance with the 2000 ISDA Definitions, Section 8.3 Interpolation) is assumed to apply. Alternatively, an actual known stub rate or stub amount may be specified.'} test::gen::StubValue [ - c1_choice: ($this.floatingRate->isNotEmpty() && ($this.stubRate->isEmpty() && $this.stubAmount->isEmpty())) || (($this.floatingRate->isEmpty() && ($this.stubRate->isNotEmpty() && $this.stubAmount->isEmpty())) || ($this.floatingRate->isEmpty() && ($this.stubRate->isEmpty() && $this.stubAmount->isNotEmpty()))) + c1_choice_StubValue: ($this.floatingRate->isNotEmpty() && ($this.stubRate->isEmpty() && $this.stubAmount->isEmpty())) || (($this.floatingRate->isEmpty() && ($this.stubRate->isNotEmpty() && $this.stubAmount->isEmpty())) || ($this.floatingRate->isEmpty() && ($this.stubRate->isEmpty() && $this.stubAmount->isNotEmpty()))) ] { {meta::pure::profiles::doc.doc = 'The rates to be applied to the initial or final stub may be the linear interpolation of two different rates. While the majority of the time, the rate indices will be the same as that specified in the stream and only the tenor itself will be different, it is possible to specift two different rates. For example, a 2 month stub period may use the linear interpolation of a 1 month and 3 month rate. The different rates would be specified in this component. Note that a maximum of two rates can be specified. If a stub period uses the same floating rate index, including tenor, as the regular calculation periods then this should not be specified again within this component, i.e. the stub calculation period amount component may not need to be specified even if there is an initial or final stub period. If a stub period uses a different floating rate index compared to the regular calculation periods then this should be specified within this component. If specified here, they are likely to have id attributes, allowing them to be referenced from within the cashflows component.'} floatingRate: test::gen::StubFloatingRate[0..2]; @@ -3612,8 +3612,8 @@ Class {meta::pure::profiles::doc.doc = 'A type defining how a stub calculation p >>>test::gen::SupervisoryBody Class {meta::pure::profiles::doc.doc = 'An identifier of an organization that supervises or regulates trading activity, e.g. CFTC, SEC, FSA, ODRF, etc.'} test::gen::SupervisoryBody [ - c1_length: $this.supervisoryBodyScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_SupervisoryBody: $this.supervisoryBodyScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_SupervisoryBody: $this.value->length() <= 255 ] { supervisoryBodyScheme: String[0..1]; @@ -3630,7 +3630,7 @@ Class test::gen::SwaptionPhysicalSettlement >>>test::gen::TelephoneNumber Class {meta::pure::profiles::doc.doc = 'A type that represents a telephonic contact.'} test::gen::TelephoneNumber [ - c1_length: $this.number->length() <= 255 + c1_length_TelephoneNumber: $this.number->length() <= 255 ] { {meta::pure::profiles::doc.doc = 'The type of telephone number (work, personal, mobile).'} type: test::gen::TelephoneTypeEnum[0..1]; @@ -3661,8 +3661,8 @@ Enum {meta::pure::profiles::doc.doc = 'Defines points in the day when equity opt >>>test::gen::TimezoneLocation Class {meta::pure::profiles::doc.doc = 'A geophraphic location for the purposes of defining a prevailing time according to the tz database.'} test::gen::TimezoneLocation [ - c1_length: $this.timezoneLocationScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_TimezoneLocation: $this.timezoneLocationScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_TimezoneLocation: $this.value->length() <= 255 ] { timezoneLocationScheme: String[0..1]; @@ -3679,9 +3679,9 @@ Enum {meta::pure::profiles::doc.doc = 'The specification of, for American-style >>>test::gen::TradeId Class {meta::pure::profiles::doc.doc = 'A trade reference identifier allocated by a party. FpML does not define the domain values associated with this element. Note that the domain values for this element are not strictly an enumerated list.'} test::gen::TradeId [ - c1_length: $this.tradeIdScheme->length() >= 1, - c2_length: $this.value->length() >= 1, - c3_length: $this.value->length() <= 255 + c1_length_TradeId: $this.tradeIdScheme->length() >= 1, + c2_length_TradeId: $this.value->length() >= 1, + c3_length_TradeId: $this.value->length() <= 255 ] { tradeIdScheme: String[1]; @@ -3725,8 +3725,8 @@ Class {meta::pure::profiles::doc.doc = 'A type describing interest payments asso >>>test::gen::Unit Class {meta::pure::profiles::doc.doc = 'A type used to record information about a unit, subdivision, desk, or other similar business entity.'} test::gen::Unit [ - c1_length: $this.unitScheme->forAll(x: String[1]|$x->length() >= 1), - c2_length: $this.value->length() <= 255 + c1_length_Unit: $this.unitScheme->forAll(x: String[1]|$x->length() >= 1), + c2_length_Unit: $this.value->length() <= 255 ] { unitScheme: String[0..1]; diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/rdu-sample/genResult.txt b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/rdu-sample/genResult.txt index 26c1c06f5e4..9c32daa4661 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/rdu-sample/genResult.txt +++ b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/rdu-sample/genResult.txt @@ -206,7 +206,7 @@ Class {meta::pure::profiles::doc.doc = 'Derivative trailer information'} test::g >>>test::gen::ExceptionType Class test::gen::ExceptionType [ - c1_values: $this.type->in(['INFO', 'WARN', 'ERROR']) + c1_values_ExceptionType: $this.type->in(['INFO', 'WARN', 'ERROR']) ] { type: String[1]; @@ -349,7 +349,7 @@ Class test::gen::NcaMifidNcaMifidsContractType >>>test::gen::ParamInfoType Class test::gen::ParamInfoType [ - c1_values: $this.fullOrDelta->forAll(x: String[1]|$x->in(['FULL', 'DELTA'])) + c1_values_ParamInfoType: $this.fullOrDelta->forAll(x: String[1]|$x->in(['FULL', 'DELTA'])) ] { assetTypeFilter: String[*]; diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/xetra-sample/genResult.txt b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/xetra-sample/genResult.txt index cc1f1848525..b15fc0ad1d0 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/xetra-sample/genResult.txt +++ b/legend-engine-xts-xml/legend-engine-xt-xml-model/src/test/resources/xetra-sample/genResult.txt @@ -123,7 +123,7 @@ Class {meta::pure::profiles::doc.doc = ''} test::gen::EvntGrpBlockT >>>test::gen::Fixml Class test::gen::Fixml [ - c1_choice: ($this.message->isNotEmpty() && $this.batch->isEmpty()) || ($this.message->isEmpty() && $this.batch->isNotEmpty()) + c1_choice_Fixml: ($this.message->isNotEmpty() && $this.batch->isEmpty()) || ($this.message->isEmpty() && $this.batch->isNotEmpty()) ] { message: test::gen::AbstractMessageT[0..1]; diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/pom.xml b/legend-engine-xts-xml/legend-engine-xt-xml-pure/pom.xml index deb0840ef3e..0d6f5492533 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-pure/pom.xml +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-xml - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/constraints.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/constraints.pure new file mode 100644 index 00000000000..42f6b83d63c --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/constraints.pure @@ -0,0 +1,167 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::xsd::tests::toPure::*; +import meta::external::format::xml::executionPlan::test::*; +import meta::external::format::xml::metamodel::xml::*; +import meta::external::format::xml::metamodel::xsd::*; +import meta::external::format::shared::binding::*; +import meta::external::shared::format::executionPlan::*; +import meta::external::format::shared::functions::*; +import meta::external::shared::functions::*; +import meta::external::shared::format::metamodel::*; +import meta::pure::executionPlan::*; +import meta::pure::executionPlan::toString::*; +import meta::pure::graphFetch::execution::*; +import meta::pure::mapping::modelToModel::*; +import meta::pure::model::unit::*; +import meta::pure::runtime::*; +import meta::legend::*; +import meta::pure::dataQuality::*; + +function <> meta::external::format::xml::executionPlan::test::constraints::testXmlConstraintViolation(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint{firstName, lastName, age, dateOfBirth, firm{legalName, firmAddress{street}, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/constraint.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint->internalize($binding, $data)->checked()->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' -1\n'+ + ' 2000-01-01\n'+ + ' \n'+ + ' FirmName\n'+ + ' \n' + + ' Mapletree\n'+ + ' \n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' Anson\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + // TODO [internalize] should only fetch primitive properties + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[{"id":"0","externalId":null,"message":"Constraint :[0] violated in the Class PersonWithFirmConstraint","enforcementLevel":"Error","ruleType":"ClassConstraint","ruleDefinerPath":"meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint","path":[]}],"source":{"number":1,"record":"JohnDoe-12000-01-01FirmNameMapletreetrueAnson"},"value":{"firstName":"John","lastName":"Doe","age":-1,"dateOfBirth":"2000-01-01","firm":{"legalName":"FirmName","firmAddress":[{"street":"Mapletree"}],"active":true},"addresses":[{"street":"Anson"}]}}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::constraints::testXmlConstraintViolationUnexpandedInTree(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint{firstName, lastName, age, dateOfBirth, firm{legalName, firmAddress{street}, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/constraint.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint->internalize($binding, $data)->graphFetchCheckedUnexpanded($tree)->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' -1\n'+ + ' 2000-01-01\n'+ + ' \n'+ + ' FirmName\n'+ + ' \n' + + ' Mapletree\n'+ + ' \n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' Anson\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[{"id":"0","externalId":null,"message":"Constraint :[0] violated in the Class PersonWithFirmConstraint","enforcementLevel":"Error","ruleType":"ClassConstraint","ruleDefinerPath":"meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint","path":[]}],"source":{"number":1,"record":"JohnDoe-12000-01-01FirmNameMapletreetrueAnson"},"value":{"firstName":"John","lastName":"Doe","age":-1,"dateOfBirth":"2000-01-01","firm":{"legalName":"FirmName","firmAddress":[{"street":"Mapletree"}],"active":true},"addresses":[{"street":"Anson"}]}}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::constraints::testXmlConstraintViolationUnexpandedNotInTree(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint{firstName, lastName, dateOfBirth, firm{legalName, firmAddress{street}, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/constraint.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint->internalize($binding, $data)->graphFetchCheckedUnexpanded($tree)->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' -1\n'+ + ' 2000-01-01\n'+ + ' \n'+ + ' FirmName\n'+ + ' \n' + + ' Mapletree\n'+ + ' \n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' Anson\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[],"source":{"number":1,"record":"JohnDoe-12000-01-01FirmNameMapletreetrueAnson"},"value":{"firstName":"John","lastName":"Doe","dateOfBirth":"2000-01-01","firm":{"legalName":"FirmName","firmAddress":[{"street":"Mapletree"}],"active":true},"addresses":[{"street":"Anson"}]}}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::constraints::testXmlConstraintViolationCheckedInTree(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint{firstName, lastName, age, dateOfBirth, firm{legalName, firmAddress{street}, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/constraint.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint->internalize($binding, $data)->graphFetchChecked($tree)->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' -1\n'+ + ' 2000-01-01\n'+ + ' \n'+ + ' FirmName\n'+ + ' \n' + + ' Mapletree\n'+ + ' \n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' Anson\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[{"id":"0","externalId":null,"message":"Constraint :[0] violated in the Class PersonWithFirmConstraint","enforcementLevel":"Error","ruleType":"ClassConstraint","ruleDefinerPath":"meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint","path":[]}],"source":{"number":1,"record":"JohnDoe-12000-01-01FirmNameMapletreetrueAnson"},"value":{"firstName":"John","lastName":"Doe","age":-1,"dateOfBirth":"2000-01-01","firm":{"legalName":"FirmName","firmAddress":[{"street":"Mapletree"}],"active":true},"addresses":[{"street":"Anson"}]}}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::constraints::testXmlConstraintViolationCheckedNotInTree(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint{firstName, lastName, dateOfBirth, firm{legalName, firmAddress{street}, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/constraint.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint->internalize($binding, $data)->graphFetchChecked($tree)->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' -1\n'+ + ' 2000-01-01\n'+ + ' \n'+ + ' FirmName\n'+ + ' \n' + + ' Mapletree\n'+ + ' \n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' Anson\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[{"id":"0","externalId":null,"message":"Constraint :[0] violated in the Class PersonWithFirmConstraint","enforcementLevel":"Error","ruleType":"ClassConstraint","ruleDefinerPath":"meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint","path":[]}],"source":{"number":1,"record":"JohnDoe-12000-01-01FirmNameMapletreetrueAnson"},"value":{"firstName":"John","lastName":"Doe","dateOfBirth":"2000-01-01","firm":{"legalName":"FirmName","firmAddress":[{"street":"Mapletree"}],"active":true},"addresses":[{"street":"Anson"}]}}}', $result); +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/dataTypes.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/dataTypes.pure new file mode 100644 index 00000000000..93a38e51d19 --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/dataTypes.pure @@ -0,0 +1,85 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::external::format::xml::executionPlan::test::*; +import meta::external::format::xml::executionPlan::test::types::*; +import meta::external::format::xml::functions::*; + +import meta::external::shared::format::binding::*; +import meta::external::shared::format::functions::*; + +import meta::legend::*; + +import meta::pure::dataQuality::*; +import meta::pure::model::unit::*; +import meta::pure::graphFetch::execution::*; + +function <> meta::external::format::xml::executionPlan::test::types::testAllTypes(): Boolean[1] +{ + let tree = #{AllTypes{string,optionalString,repeatedString,integer,optionalInteger,repeatedInteger,float,optionalFloat,repeatedFloat,decimal,optionalDecimal,repeatedDecimal,boolean,optionalBoolean,repeatedBoolean,date,optionalDate,repeatedDate,strictDate,optionalStrictDate,repeatedStrictDate,dateTime,optionalDateTime,repeatedDateTime,enum,optionalEnum,repeatedEnum}}#; + let query = {data:String[1] | AllTypes->fromXml($data)->checked()->serialize($tree)}; + + let input = readFile('/core_external_format_xml/executionPlan/tests/resources/allTypes1.xml')->toOne(); + let result = executeLegendQuery($query, pair('data', $input), xsdExtensions()); + + assertEquals('{"builder":{"_type":"json"},"values":[{"defects":[],"source":{"number":1,"record":"StringOptionalStringRepeatedString1RepeatedString212121.02.51.12.21.234567892.35967842361.235936181812.15456123618truefalsetruefalse2022-12-122022-12-12T01:00:052022-12-122023-12-12T12:00:002022-12-122019-12-122022-12-122020-12-122023-12-12T12:00:002020-12-12T12:00:002020-12-12T12:00:002020-12-12T12:00:00MALEmeta::external::format::xml::executionPlan::test::types::Gender.FEMALEMALEmeta::external::format::xml::executionPlan::test::types::Gender.OTHER"},"value":{"string":"String","optionalString":"OptionalString","repeatedString":["RepeatedString1","RepeatedString2"],"integer":1,"optionalInteger":2,"repeatedInteger":[1,2],"float":1.0,"optionalFloat":2.5,"repeatedFloat":[1.1,2.2],"decimal":1.23456789,"optionalDecimal":2.3596784236,"repeatedDecimal":[1.2359361818,12.15456123618],"boolean":true,"optionalBoolean":false,"repeatedBoolean":[true,false],"date":"2022-12-12T00:00:00.000","optionalDate":"2022-12-12T01:00:05.000","repeatedDate":["2022-12-12T00:00:00.000","2023-12-12T12:00:00.000"],"strictDate":"2022-12-12","optionalStrictDate":"2019-12-12","repeatedStrictDate":["2022-12-12","2020-12-12"],"dateTime":"2023-12-12T12:00:00.000","optionalDateTime":"2020-12-12T12:00:00.000","repeatedDateTime":["2020-12-12T12:00:00.000","2020-12-12T12:00:00.000"],"enum":"MALE","optionalEnum":"FEMALE","repeatedEnum":["MALE","OTHER"]}},{"defects":[],"source":{"number":2,"record":"String11.01.23456789true2022-12-122022-12-122023-12-12T12:00:00MALE"},"value":{"string":"String","optionalString":null,"repeatedString":[],"integer":1,"optionalInteger":null,"repeatedInteger":[],"float":1.0,"optionalFloat":null,"repeatedFloat":[],"decimal":1.23456789,"optionalDecimal":null,"repeatedDecimal":[],"boolean":true,"optionalBoolean":null,"repeatedBoolean":[],"date":"2022-12-12T00:00:00.000","optionalDate":null,"repeatedDate":[],"strictDate":"2022-12-12","optionalStrictDate":null,"repeatedStrictDate":[],"dateTime":"2023-12-12T12:00:00.000","optionalDateTime":null,"repeatedDateTime":[],"enum":"MALE","optionalEnum":null,"repeatedEnum":[]}}]}', $result); +} + +###Pure +import meta::pure::unit::*; +import meta::external::format::xml::executionPlan::test::types::*; + +Class meta::external::format::xml::executionPlan::test::types::AllTypes +{ + string : String[1]; + optionalString : String[0..1]; + repeatedString : String[*]; + + integer : Integer[1]; + optionalInteger : Integer[0..1]; + repeatedInteger : Integer[*]; + + float : Float[1]; + optionalFloat : Float[0..1]; + repeatedFloat : Float[*]; + + decimal : Decimal[1]; + optionalDecimal : Decimal[0..1]; + repeatedDecimal : Decimal[*]; + + boolean : Boolean[1]; + optionalBoolean : Boolean[0..1]; + repeatedBoolean : Boolean[*]; + + date : Date[1]; + optionalDate : Date[0..1]; + repeatedDate : Date[*]; + + strictDate : StrictDate[1]; + optionalStrictDate : StrictDate[0..1]; + repeatedStrictDate : StrictDate[*]; + + dateTime : DateTime[1]; + optionalDateTime : DateTime[0..1]; + repeatedDateTime : DateTime[*]; + + enum : Gender[1]; + optionalEnum : Gender[0..1]; + repeatedEnum : Gender[*]; +} + +Enum meta::external::format::xml::executionPlan::test::types::Gender +{ + MALE, FEMALE, OTHER +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/executionPlanTest.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/executionPlanTest.pure index 8e09765dcd8..5768ebec3b6 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/executionPlanTest.pure +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/executionPlanTest.pure @@ -11,7 +11,6 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - import meta::xsd::tests::toPure::*; import meta::external::format::xml::executionPlan::test::*; import meta::external::format::xml::metamodel::xml::*; @@ -34,6 +33,41 @@ Class meta::external::format::xml::executionPlan::test::Person lastName : String[1]; } +Class meta::external::format::xml::executionPlan::test::PersonWithFirm +{ + firstName : String[1]; + lastName : String[1]; + age : Integer[0..1]; + dateOfBirth : StrictDate[0..1]; + firm : Firm[1]; + addresses : Address[1..*]; +} + +Class meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint +[ + $this.age > 0 +] +{ + firstName : String[1]; + lastName : String[1]; + age : Integer[0..1]; + dateOfBirth : StrictDate[0..1]; + firm : Firm[1]; + addresses : Address[1..*]; +} + +Class meta::external::format::xml::executionPlan::test::Firm +{ + legalName : String[1]; + firmAddress: Address[*]; + active : Boolean[1]; +} + +Class meta::external::format::xml::executionPlan::test::Address +{ + street : String[1]; +} + Class meta::external::format::xml::executionPlan::test::TargetPerson { fullName : String[1]; @@ -78,66 +112,6 @@ function meta::external::format::xml::executionPlan::test::getTestBinding(): Bin ); } -function <> meta::external::format::xml::executionPlan::test::xmlNodes(): Boolean[1] -{ - let extensions = meta::external::format::shared::externalFormatExtension()->concatenate(meta::external::format::xml::extension::xsdFormatExtension()); - - let binding = getTestBinding(); - - let tree = #{Person {firstName, lastName}}#; - - let plan = executionPlan( - data:Byte[*]|Person->internalize($binding, $data)->graphFetchChecked($tree)->externalize($binding, checked($tree, $binding)), - ^ExecutionContext(), - $extensions, - noDebug() - ); - - let planString = $plan->planToString(false, $extensions); - - let expected = 'Sequence\n' + - '(\n' + - ' type = String\n' + - ' resultSizeRange = 1\n' + - ' (\n' + - ' FunctionParametersValidationNode\n' + - ' (\n' + - ' functionParameters = [data:Byte[*]]\n' + - ' )\n' + - ' ExternalFormat_Externalize\n' + - ' (\n' + - ' type = String\n' + - ' resultSizeRange = 1\n' + - ' checked = true\n' + - ' binding = meta::external::format::xml::executionPlan::test::TestBinding\n' + - '\n' + - ' (\n' + - ' ExternalFormat_Internalize\n' + - ' (\n' + - ' type = Class[impls=]\n' + - ' resultSizeRange = *\n' + - ' binding = meta::external::format::xml::executionPlan::test::TestBinding[application/xml]\n' + - ' enableConstraints = true\n' + - ' checked = true\n' + - ' graphFetchTree = #{meta::external::format::xml::executionPlan::test::Person{firstName,lastName}}#\n' + - '\n' + - ' (\n' + - ' VariableResolution\n' + - ' (\n' + - ' type = Byte\n' + - ' varName = data\n' + - '\n' + - ' )\n' + - ' )\n' + - ' )\n' + - ' )\n' + - ' )\n' + - ' )\n' + - ')\n'; - - assertEquals($expected, $planString); -} - function meta::external::format::xml::executionPlan::test::xsType(type: String[1]): QName[1] { ^QName(localPart=$type, namespace='http://www.w3.org/2001/XMLSchema'); @@ -145,7 +119,6 @@ function meta::external::format::xml::executionPlan::test::xsType(type: String[1 ###Mapping import meta::external::format::xml::executionPlan::test::*; - Mapping meta::external::format::xml::executionPlan::test::M2MMapping ( TargetPerson: Pure diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/allTypes1.xml b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/allTypes1.xml new file mode 100644 index 00000000000..32409d21586 --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/allTypes1.xml @@ -0,0 +1,51 @@ + + + String + OptionalString + RepeatedString1 + RepeatedString2 + 1 + 2 + 1 + 2 + 1.0 + 2.5 + 1.1 + 2.2 + 1.23456789 + 2.3596784236 + 1.2359361818 + 12.15456123618 + true + false + true + false + 2022-12-12 + 2022-12-12T01:00:05 + 2022-12-12 + 2023-12-12T12:00:00 + 2022-12-12 + 2019-12-12 + 2022-12-12 + 2020-12-12 + 2023-12-12T12:00:00 + 2020-12-12T12:00:00 + 2020-12-12T12:00:00 + 2020-12-12T12:00:00 + MALE + meta::external::format::xml::executionPlan::test::types::Gender.FEMALE + MALE + meta::external::format::xml::executionPlan::test::types::Gender.OTHER + + + String + 1 + 1.0 + 1.23456789 + true + 2022-12-12 + 2022-12-12 + 2023-12-12T12:00:00 + MALE + + \ No newline at end of file diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/composition.txt b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/composition.txt new file mode 100644 index 00000000000..2a969eaca79 --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/composition.txt @@ -0,0 +1,23 @@ +###ExternalFormat +SchemaSet meta::external::format::xml::executionPlan::test::customSchema::PersonWithFirmSchema +{ + format: XSD; + schemas: [ + { + id: PersonWithFirm; + location: 'PersonWithFirm.xsd'; + content: '\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n'; + } + ]; +} + +Binding meta::external::format::xml::executionPlan::test::customSchema::PersonWithFirmBinding +{ + schemaSet: meta::external::format::xml::executionPlan::test::customSchema::PersonWithFirmSchema; + contentType: 'application/xml'; + modelIncludes: [ + meta::external::format::xml::executionPlan::test::PersonWithFirm, + meta::external::format::xml::executionPlan::test::Firm, + meta::external::format::xml::executionPlan::test::Address + ]; +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/constraint.txt b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/constraint.txt new file mode 100644 index 00000000000..6b3e6202312 --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/resources/constraint.txt @@ -0,0 +1,23 @@ +###ExternalFormat +SchemaSet meta::external::format::xml::executionPlan::test::customSchema::PersonWithFirmSchema +{ + format: XSD; + schemas: [ + { + id: PersonWithFirm; + location: 'PersonWithFirm.xsd'; + content: '\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n'; + } + ]; +} + +Binding meta::external::format::xml::executionPlan::test::customSchema::PersonWithFirmBinding +{ + schemaSet: meta::external::format::xml::executionPlan::test::customSchema::PersonWithFirmSchema; + contentType: 'application/xml'; + modelIncludes: [ + meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint, + meta::external::format::xml::executionPlan::test::Firm, + meta::external::format::xml::executionPlan::test::Address + ]; +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/simple.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/simple.pure new file mode 100644 index 00000000000..bf3fe9318ee --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/simple.pure @@ -0,0 +1,143 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::xsd::tests::toPure::*; +import meta::external::format::shared::binding::*; +import meta::external::format::shared::functions::*; +import meta::external::format::xml::executionPlan::test::*; +import meta::external::format::xml::metamodel::xml::*; +import meta::external::format::xml::metamodel::xsd::*; +import meta::external::shared::format::executionPlan::*; +import meta::external::shared::functions::*; +import meta::external::shared::format::metamodel::*; +import meta::pure::executionPlan::*; +import meta::pure::executionPlan::toString::*; +import meta::pure::graphFetch::execution::*; +import meta::pure::mapping::modelToModel::*; +import meta::pure::model::unit::*; +import meta::pure::runtime::*; +import meta::legend::*; +import meta::pure::dataQuality::*; + +function <> meta::external::format::xml::executionPlan::test::testXmlMultiplicityError(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirm{firstName, lastName, firm{legalName}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/composition.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirm->internalize($binding, $data)->checked()->serialize($tree)}; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', '')); + + // TODO [internalize] should only fetch primitive properties + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[{"id":null,"externalId":null,"message":"Invalid multiplicity for firm: expected [1] found [0]","enforcementLevel":"Critical","ruleType":"ClassStructure","ruleDefinerPath":"meta::external::format::xml::executionPlan::test::PersonWithFirm","path":[]},{"id":null,"externalId":null,"message":"Invalid multiplicity for addresses: expected [1..*] found [0]","enforcementLevel":"Critical","ruleType":"ClassStructure","ruleDefinerPath":"meta::external::format::xml::executionPlan::test::PersonWithFirm","path":[]}],"source":{"number":1,"record":""},"value":null}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::testXmlNoOptionalAttributes(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirm{firstName, lastName, firm{legalName, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/composition.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirm->internalize($binding, $data)->checked()->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' \n'+ + ' FirmName\n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' StreetName\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + // TODO [internalize] should only fetch primitive properties + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[],"source":{"number":1,"record":"JohnDoeFirmNametrueStreetName"},"value":{"firstName":"John","lastName":"Doe","firm":{"legalName":"FirmName","active":true},"addresses":[{"street":"StreetName"}]}}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::testXmlSingleQueryAttributes(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirm{firstName}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/composition.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirm->internalize($binding, $data)->checked()->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' \n'+ + ' FirmName\n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' StreetName\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[],"source":{"number":1,"record":"JohnDoeFirmNametrueStreetName"},"value":{"firstName":"John"}}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::testXmlFewerQueryAttributes(): Boolean[1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirm{firstName, firm{legalName, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/composition.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirm->internalize($binding, $data)->checked()->serialize($tree)}; + let data = '\n'+ + ' John\n'+ + ' Doe\n'+ + ' \n'+ + ' FirmName\n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' StreetName\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + // TODO [internalize] should only fetch primitive properties + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[],"source":{"number":1,"record":"JohnDoeFirmNametrueStreetName"},"value":{"firstName":"John","firm":{"legalName":"FirmName","active":true},"addresses":[{"street":"StreetName"}]}}}', $result); +} + +function <> meta::external::format::xml::executionPlan::test::testXmlConstraintPass(): Boolean[0..1] +{ + let tree = #{meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint{firstName, lastName, age, dateOfBirth, firm{legalName, firmAddress{street}, active}, addresses{street}}}#; + let binding = compileLegendGrammar(readFile('/core_external_format_xml/executionPlan/tests/resources/constraint.txt')->toOne())->filter(e | $e->instanceOf(Binding))->cast(@Binding)->toOne(); + let query = {data:String[1]| meta::external::format::xml::executionPlan::test::PersonWithFirmConstraint->internalize($binding, $data)->checked()->serialize($tree)}; + let data = '\n'+ + ' 23\n'+ + ' 2000-01-01\n'+ + ' \n'+ + ' FirmName\n'+ + ' \n' + + ' Mapletree\n'+ + ' \n'+ + ' \n' + + ' Anson\n'+ + ' \n'+ + ' true\n'+ + ' \n'+ + ' \n'+ + ' Raffles\n'+ + ' \n'+ + ' \n'+ + ' Link\n'+ + ' \n'+ + ''; + + let result = meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery($query, pair('data', $data)); + + // TODO [internalize] should only fetch primitive properties + assertEquals('{"builder":{"_type":"json"},"values":{"defects":[],"source":{"number":1,"record":"232000-01-01FirmNameMapletreeAnsontrueRafflesLink"},"value":{"firstName":"John","lastName":"Doe","age":23,"dateOfBirth":"2000-01-01","firm":{"legalName":"FirmName","firmAddress":[{"street":"Mapletree"},{"street":"Anson"}],"active":true},"addresses":[{"street":"Raffles"},{"street":"Link"}]}}}', $result); +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/utils.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/utils.pure new file mode 100644 index 00000000000..4342f10beee --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/executionPlan/tests/utils.pure @@ -0,0 +1,36 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Pure +import meta::legend::*; +import meta::pure::extension::*; +import meta::pure::runtime::*; + +function meta::external::format::xml::executionPlan::test::convertStringToBytes(data:String[1]): InstanceValue[1] +{ + ^InstanceValue(genericType=^GenericType(rawType=Byte), multiplicity=ZeroMany, values=$data) +} + +function meta::external::format::xml::executionPlan::test::executeXsdSchemaBindingQuery(f: FunctionDefinition[1], vars: Pair[*]): String[1] +{ + executeLegendQuery($f, $vars, ^ExecutionContext(), meta::external::format::xml::executionPlan::test::xsdExtensions()); +} + +function meta::external::format::xml::executionPlan::test::xsdExtensions(): Extension[*] +{ + [ + meta::external::format::shared::externalFormatExtension(), + meta::external::format::xml::extension::xsdFormatExtension() + ] +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/functions/functions.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/functions/functions.pure new file mode 100644 index 00000000000..a7b28246d59 --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/functions/functions.pure @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import meta::external::format::shared::functions::*; +import meta::pure::graphFetch::*; + +function <> meta::external::format::xml::functions::fromXml(class:Class[1], data:String[1]): T[*] +{ + internalize($class, 'application/xml', $data) +} + +function <> meta::external::format::xml::functions::fromXml(class:Class[1], data:Byte[*]): T[*] +{ + internalize($class, 'application/xml', $data) +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdAnnotatedDigram.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdAnnotatedDigram.pure new file mode 100644 index 00000000000..55b873c5b26 --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdAnnotatedDigram.pure @@ -0,0 +1,162 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Diagram +Diagram meta::external::format::xml::metamodel::diagram::XsdAnnotatedDiagram(width=0.0, height=0.0) +{ + TypeView cview_1( + type=meta::external::format::xml::metamodel::xsd::XsdObject, + position=(228.04744, 356.93176), + width=97.38477, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_2( + type=meta::external::format::xml::metamodel::xsd::XsdAnnotationItem, + position=(354.35795, 661.73578), + width=129.99805, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_3( + type=meta::external::format::xml::metamodel::xsd::XsdAppInfo, + position=(281.35795, 744.72443), + width=135.74023, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_4( + type=meta::external::format::xml::metamodel::xsd::XsdDocumentation, + position=(443.35795, 744.69031), + width=135.74023, + height=86.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_5( + type=meta::external::format::xml::metamodel::xsd::XsdAnnotation, + position=(406.35795, 565.74713), + width=169.73535, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_6( + type=meta::external::format::xml::metamodel::xsd::XsdAnnotated, + position=(683.35795, 503.76990), + width=188.66309, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + GeneralizationView gview_0( + source=cview_2, + target=cview_1, + points=[(372.68949,666.23010),(371.03424,587.81104),(242.68949,587.23010),(241.68949,405.23010)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_1( + source=cview_3, + target=cview_2, + points=[(400.68949,752.23010),(400.68949,697.23010)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_2( + source=cview_4, + target=cview_2, + points=[(457.68949,753.23010),(457.68949,695.23010)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_3( + source=cview_5, + target=cview_1, + points=[(491.22562,587.74713),(489.03424,511.81104),(277.68949,510.23010),(276.73982,385.93176)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_4( + source=cview_6, + target=cview_1, + points=[(777.68949,532.76990),(777.03424,465.81104),(311.68949,468.23010),(312.68949,405.23010)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + PropertyView pview_0( + property=meta::external::format::xml::metamodel::xsd::XsdAnnotation.items, + source=cview_5, + target=cview_2, + points=[(554.68949,605.23010),(553.68949,684.23010),(419.35697,683.73578)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_1( + property=meta::external::format::xml::metamodel::xsd::XsdAnnotated.annotation, + source=cview_6, + target=cview_5, + points=[(700.68949,557.23010),(700.68949,588.23010),(491.22562,587.74713)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdSchemaDiagram.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdSchemaDiagram.pure new file mode 100644 index 00000000000..6ab822765aa --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdSchemaDiagram.pure @@ -0,0 +1,596 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Diagram +Diagram meta::external::format::xml::metamodel::diagram::XsdSchemaDiagram(width=0.0, height=0.0) +{ + TypeView cview_1( + type=meta::external::format::xml::metamodel::xsd::XsdAnnotated, + position=(445.36698, 59.33624), + width=188.66309, + height=72.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_2( + type=meta::external::format::xml::metamodel::xsd::XsdObject, + position=(490.28229, -38.70528), + width=97.38477, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_3( + type=meta::external::format::xml::metamodel::xsd::XsdAttribute, + position=(852.61698, 523.67226), + width=150.80762, + height=100.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_4( + type=meta::external::format::xml::metamodel::xsd::XsdParticle, + position=(61.21533, 481.84088), + width=150.83447, + height=72.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_5( + type=meta::external::format::xml::metamodel::xsd::XsdAny, + position=(44.36698, 612.30215), + width=274.27832, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_6( + type=meta::external::format::xml::metamodel::xsd::XsdElement, + position=(330.00000, 612.70166), + width=188.67920, + height=184.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_7( + type=meta::external::format::xml::metamodel::xsd::XsdAnyAttribute, + position=(549.61698, 646.78591), + width=274.27832, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_8( + type=meta::external::format::xml::metamodel::xsd::XsdAttributeGroup, + position=(681.61698, 523.66088), + width=159.97070, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_9( + type=meta::external::format::xml::metamodel::xsd::XsdAttributeItem, + position=(778.61698, 436.64956), + width=129.34180, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_10( + type=meta::external::format::xml::metamodel::xsd::XsdGroupParticle, + position=(-166.84705, 745.76317), + width=121.36133, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_11( + type=meta::external::format::xml::metamodel::xsd::XsdGroup, + position=(-208.63302, 614.16579), + width=207.04297, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_12( + type=meta::external::format::xml::metamodel::xsd::XsdAll, + position=(-225.63302, 840.34762), + width=57.34180, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_13( + type=meta::external::format::xml::metamodel::xsd::XsdChoice, + position=(-151.63302, 841.25668), + width=82.01563, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_14( + type=meta::external::format::xml::metamodel::xsd::XsdSequence, + position=(-54.63302, 839.10897), + width=98.69727, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_15( + type=meta::external::format::xml::metamodel::xsd::XsdSchema, + position=(173.04596, 192.46164), + width=219.06885, + height=156.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_16( + type=meta::external::format::xml::metamodel::xsd::XsdImport, + position=(-93.13302, 401.35810), + width=144.09570, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_17( + type=meta::external::format::xml::metamodel::xsd::XsdInclude, + position=(58.86698, 402.34668), + width=144.09570, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_18( + type=meta::external::format::xml::metamodel::xsd::XsdRedefine, + position=(-246.13302, 402.30127), + width=144.09570, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_19( + type=meta::external::format::xml::metamodel::xsd::XsdExternalSchema, + position=(-107.13302, 255.39215), + width=172.81299, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_20( + type=meta::external::format::xml::metamodel::xsd::XsdComplexType, + position=(1073.00000, 320.72443), + width=219.22461, + height=142.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_21( + type=meta::external::format::xml::metamodel::xsd::XsdSimpleType, + position=(830.00000, 320.65625), + width=241.69189, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_22( + type=meta::external::format::xml::metamodel::xsd::XsdAnyType, + position=(1011.36698, 159.29079), + width=97.38477, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + GeneralizationView gview_0( + source=cview_1, + target=cview_2, + points=[(539.69852,95.33624),(538.97468,-9.70528)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_1( + source=cview_4, + target=cview_1, + points=[(136.63257,517.84088),(134.25903,465.87966),(534.50903,465.70185),(535.00903,126.49819)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_2( + source=cview_5, + target=cview_4, + points=[(136.25903,617.87966),(136.63257,517.84088)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_3( + source=cview_6, + target=cview_4, + points=[(424.33960,704.70166),(423.25903,587.87966),(190.25903,588.87966),(191.00903,524.49819)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_4( + source=cview_7, + target=cview_1, + points=[(568.25903,653.87966),(561.25903,126.87966)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_5( + source=cview_9, + target=cview_1, + points=[(843.28788,458.64956),(842.25903,399.87966),(596.25903,399.87966),(591.25903,124.87966)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_6( + source=cview_3, + target=cview_9, + points=[(887.25903,527.87966),(883.25903,522.87966),(883.25903,474.87966)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_7( + source=cview_8, + target=cview_9, + points=[(818.25903,528.87966),(816.25903,476.87966)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_8( + source=cview_10, + target=cview_4, + points=[(-59.41807,748.87966),(-58.41807,719.87966),(17.58193,718.87966),(19.58193,600.87966),(100.58193,598.87966),(99.58193,527.87966)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_9( + source=cview_11, + target=cview_4, + points=[(-105.11153,643.16579),(-105.41807,589.87966),(74.15295,588.87966),(75.15296,546.87966)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_10( + source=cview_12, + target=cview_10, + points=[(-196.96212,855.34762),(-106.16638,767.76317)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_11( + source=cview_13, + target=cview_10, + points=[(-110.62521,856.25668),(-106.16638,767.76317)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_12( + source=cview_14, + target=cview_10, + points=[(-5.28439,854.10897),(-106.16638,767.76317)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_13( + source=cview_15, + target=cview_1, + points=[(361.90327,241.45351),(463.90327,241.45351),(462.00903,118.49819)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_14( + source=cview_19, + target=cview_1, + points=[(59.90327,306.45351),(121.90327,306.45351),(123.90327,372.45351),(484.90327,372.45351),(485.50903,114.70185)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_15( + source=cview_16, + target=cview_19, + points=[(-32.09673,410.45351),(-32.49097,307.70185)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_16( + source=cview_17, + target=cview_19, + points=[(80.50903,406.70185),(79.90327,380.45351),(15.90327,380.45351),(15.90327,305.45351)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_17( + source=cview_18, + target=cview_19, + points=[(-137.49097,407.70185),(-138.09673,381.45351),(-83.09673,381.45351),(-82.49097,307.70185)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_18( + source=cview_22, + target=cview_1, + points=[(1060.05936,181.29079),(624.25903,179.87966),(624.00903,120.49819)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_19( + source=cview_20, + target=cview_22, + points=[(1098.86682,363.87966),(1093.90425,182.45351)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_20( + source=cview_21, + target=cview_22, + points=[(1044.01826,339.45351),(1036.90425,192.45351)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + PropertyView pview_0( + property=meta::external::format::xml::metamodel::xsd::XsdAttributeGroup.anyAttribute, + source=cview_8, + target=cview_7, + points=[(686.25903,557.87966),(654.25903,557.87966),(654.25903,652.87966)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_1( + property=meta::external::format::xml::metamodel::xsd::XsdAttributeGroup.items, + source=cview_8, + target=cview_9, + points=[(686.25903,533.87966),(653.25903,533.87966),(650.25903,457.87966),(843.28788,458.64956)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_2( + property=meta::external::format::xml::metamodel::xsd::XsdGroupParticle.items, + source=cview_10, + target=cview_4, + points=[(-106.16638,767.76317),(-229.41807,766.87966),(-229.41807,517.87966),(136.63257,517.84088)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_3( + property=meta::external::format::xml::metamodel::xsd::XsdGroup.particle, + source=cview_11, + target=cview_10, + points=[(-105.11153,643.16579),(-106.16638,767.76317)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_4( + property=meta::external::format::xml::metamodel::xsd::XsdSchema.items, + source=cview_15, + target=cview_2, + points=[(191.50903,218.20185),(132.90327,217.45351),(131.50903,11.70185),(514.50903,13.70185)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_5( + property=meta::external::format::xml::metamodel::xsd::XsdRedefine.items, + source=cview_18, + target=cview_2, + points=[(-174.08517,424.30127),(-174.09673,-34.54649),(501.50903,-32.29815)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_6( + property=meta::external::format::xml::metamodel::xsd::XsdSchema.externals, + source=cview_15, + target=cview_19, + points=[(278.39093,270.46164),(55.90327,271.45351)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_7( + property=meta::external::format::xml::metamodel::xsd::XsdAttribute.type, + source=cview_3, + target=cview_21, + points=[(975.25903,586.87966),(1022.25903,587.87966),(1025.92090,368.87966)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_8( + property=meta::external::format::xml::metamodel::xsd::XsdElement.type, + source=cview_6, + target=cview_22, + points=[(507.00903,763.49819),(1194.00903,765.49819),(1190.25903,179.87966),(1060.05936,181.29079)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdTypeDiagram.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdTypeDiagram.pure new file mode 100644 index 00000000000..8cf065be187 --- /dev/null +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/metamodel/xsdTypeDiagram.pure @@ -0,0 +1,604 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +###Diagram +Diagram meta::external::format::xml::metamodel::diagram::XsdTypeDiagram(width=0.0, height=0.0) +{ + TypeView cview_1( + type=meta::external::format::xml::metamodel::xsd::XsdPattern, + position=(421.37825, 1170.06599), + width=83.35156, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_2( + type=meta::external::format::xml::metamodel::xsd::XsdWhiteSpace, + position=(519.22859, 1169.54998), + width=110.02344, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_3( + type=meta::external::format::xml::metamodel::xsd::XsdLength, + position=(647.22859, 1169.86212), + width=81.99805, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_5( + type=meta::external::format::xml::metamodel::xsd::XsdMaxInclusive, + position=(516.22859, 1238.82806), + width=116.70898, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_6( + type=meta::external::format::xml::metamodel::xsd::XsdMaxExclusive, + position=(645.22859, 1238.83942), + width=120.72266, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_7( + type=meta::external::format::xml::metamodel::xsd::XsdTotalDigits, + position=(786.22859, 1239.02728), + width=103.11523, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_8( + type=meta::external::format::xml::metamodel::xsd::XsdFractionDigits, + position=(902.22859, 1238.27728), + width=122.67969, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_9( + type=meta::external::format::xml::metamodel::xsd::XsdMinInclusive, + position=(253.22859, 1239.79395), + width=114.02539, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_10( + type=meta::external::format::xml::metamodel::xsd::XsdMaxLength, + position=(883.22859, 1173.81671), + width=105.34180, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_13( + type=meta::external::format::xml::metamodel::xsd::XsdContentDerivation, + position=(628.45624, 752.83588), + width=146.01172, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_11( + type=meta::external::format::xml::metamodel::xsd::XsdFacet, + position=(541.37825, 1027.80464), + width=123.30225, + height=72.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_12( + type=meta::external::format::xml::metamodel::xsd::XsdContentModel, + position=(960.40558, 743.93816), + width=210.08301, + height=58.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_14( + type=meta::external::format::xml::metamodel::xsd::XsdMinLength, + position=(754.22859, 1172.78259), + width=102.65820, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_15( + type=meta::external::format::xml::metamodel::xsd::XsdRestriction, + position=(499.45624, 913.51770), + width=205.79688, + height=86.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_16( + type=meta::external::format::xml::metamodel::xsd::XsdExtension, + position=(725.84242, 915.74497), + width=205.79688, + height=86.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_17( + type=meta::external::format::xml::metamodel::xsd::XsdSimpleContent, + position=(941.45624, 844.11993), + width=126.67578, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_18( + type=meta::external::format::xml::metamodel::xsd::XsdComplexContent, + position=(1093.45624, 842.49497), + width=138.00781, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_19( + type=meta::external::format::xml::metamodel::xsd::XsdComplexType, + position=(959.40558, 586.04041), + width=205.79688, + height=128.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_20( + type=meta::external::format::xml::metamodel::xsd::XsdEnumeration, + position=(285.37825, 1172.46369), + width=115.34961, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_21( + type=meta::external::format::xml::metamodel::xsd::XsdSimpleTypeDerivation, + position=(401.77750, 751.14013), + width=167.15234, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_22( + type=meta::external::format::xml::metamodel::xsd::XsdList, + position=(308.92016, 940.23101), + width=170.18652, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_23( + type=meta::external::format::xml::metamodel::xsd::XsdUnion, + position=(262.86608, 865.71830), + width=182.41650, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_24( + type=meta::external::format::xml::metamodel::xsd::XsdSimpleType, + position=(163.35795, 572.59943), + width=204.36816, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_25( + type=meta::external::format::xml::metamodel::xsd::XsdAnyType, + position=(548.35795, 440.79260), + width=97.38477, + height=44.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + TypeView cview_4( + type=meta::external::format::xml::metamodel::xsd::XsdMinExclusive, + position=(384.22859, 1238.80530), + width=118.03906, + height=30.00000, + stereotypesVisible=true, + attributesVisible=true, + attributeStereotypesVisible=true, + attributeTypesVisible=true, + color=#FFFFCC, + lineWidth=1.0) + + GeneralizationView gview_0( + source=cview_1, + target=cview_11, + points=[(463.05403,1185.06599),(463.05403,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_1( + source=cview_2, + target=cview_11, + points=[(574.24031,1184.54998),(573.57527,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_2( + source=cview_3, + target=cview_11, + points=[(688.22762,1184.86212),(688.22762,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_3( + source=cview_4, + target=cview_11, + points=[(443.24812,1253.80530),(443.24812,1223.47332),(264.87064,1223.47332),(264.87064,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_4( + source=cview_5, + target=cview_11, + points=[(574.58308,1253.82806),(574.58308,1223.47332),(264.87064,1223.47332),(264.87064,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_5( + source=cview_6, + target=cview_11, + points=[(705.58992,1253.83942),(705.58992,1223.47332),(264.87064,1223.47332),(264.87064,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_6( + source=cview_7, + target=cview_11, + points=[(837.78621,1254.02728),(837.78621,1223.47332),(264.87064,1223.47332),(264.87064,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_7( + source=cview_8, + target=cview_11, + points=[(963.56844,1253.27728),(963.56844,1223.47332),(264.87064,1223.47332),(264.87064,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_8( + source=cview_9, + target=cview_11, + points=[(310.24129,1254.79395),(310.24129,1223.47332),(264.87064,1223.47332),(264.87064,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_9( + source=cview_10, + target=cview_11, + points=[(935.89949,1188.81671),(935.89949,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_10( + source=cview_14, + target=cview_11, + points=[(805.55770,1187.78259),(805.55769,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_11( + source=cview_15, + target=cview_13, + points=[(666.84242,940.29327),(667.84242,781.29327)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_12( + source=cview_16, + target=cview_13, + points=[(755.84242,944.29327),(752.84242,790.29327)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_13( + source=cview_17, + target=cview_12, + points=[(1004.79414,866.11993),(1004.56823,783.79327)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_14( + source=cview_18, + target=cview_12, + points=[(1136.56823,855.79327),(1136.56823,794.79327)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_15( + source=cview_20, + target=cview_11, + points=[(343.05306,1187.46369),(343.05306,1147.47332),(603.02937,1147.47332),(603.02937,1063.80464)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_16( + source=cview_15, + target=cview_21, + points=[(531.84242,929.29327),(534.84242,791.29327)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_17( + source=cview_22, + target=cview_21, + points=[(453.29655,968.34979),(456.84242,782.29327)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_18( + source=cview_23, + target=cview_21, + points=[(421.90126,875.29327),(419.84242,788.29327)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_19( + source=cview_19, + target=cview_25, + points=[(1062.30401,650.04041),(1058.84242,463.29327),(597.05033,462.79260)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + GeneralizationView gview_20( + source=cview_24, + target=cview_25, + points=[(265.54203,594.59943),(264.84242,463.29327),(597.05033,462.79260)], + label='', + color=#000000, + lineWidth=-1.0, + lineStyle=SIMPLE) + + PropertyView pview_0( + property=meta::external::format::xml::metamodel::xsd::XsdContentModel.derivation, + source=cview_12, + target=cview_13, + points=[(1065.44708,772.93816),(701.46210,774.83588)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_1( + property=meta::external::format::xml::metamodel::xsd::XsdRestriction.facets, + source=cview_15, + target=cview_11, + points=[(602.35468,956.51770),(603.02937,1063.80464)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_2( + property=meta::external::format::xml::metamodel::xsd::XsdComplexType.contentModel, + source=cview_19, + target=cview_12, + points=[(1062.30401,650.04041),(1065.44708,772.93816)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_3( + property=meta::external::format::xml::metamodel::xsd::XsdSimpleType.derivation, + source=cview_24, + target=cview_21, + points=[(344.84242,604.29327),(346.84242,756.29327),(417.84242,756.29327)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_4( + property=meta::external::format::xml::metamodel::xsd::XsdList.itemType, + source=cview_22, + target=cview_24, + points=[(325.31103,964.64545),(114.84242,963.29327),(115.84242,599.29327),(182.50813,599.63028)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_5( + property=meta::external::format::xml::metamodel::xsd::XsdUnion.memberTypes, + source=cview_23, + target=cview_24, + points=[(297.90126,889.29327),(213.84242,888.29327),(211.84242,606.29327)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) + + PropertyView pview_6( + property=meta::external::format::xml::metamodel::xsd::XsdRestriction.baseType, + source=cview_15, + target=cview_25, + points=[(625.06823,976.79327),(604.06823,909.79327),(597.05033,462.79260)], + label='', + propertyPosition=(0.0,0.0), + multiplicityPosition=(0.0,0.0), + color=#000000, + lineWidth=-1.0, + stereotypesVisible=true, + nameVisible=true, + lineStyle=SIMPLE) +} diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/transformation/toPure/xsdToPure.pure b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/transformation/toPure/xsdToPure.pure index b775322c4f8..9e0ac6451e6 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/transformation/toPure/xsdToPure.pure +++ b/legend-engine-xts-xml/legend-engine-xt-xml-pure/src/main/resources/core_external_format_xml/transformation/toPure/xsdToPure.pure @@ -751,7 +751,7 @@ function <> meta::external::format::xml::transformation::toPure: let num = $indexed.first; let category = $indexed.second.first; let expr = $indexed.second.second; - let name = 'c'+toString($num+1)+'_'+$category; + let name = 'c'+toString($num+1)+'_'+$category+'_'+$cls.name->toOne(); ^meta::pure::metamodel::constraint::Constraint(name=$name, functionDefinition=$f->lambda($thisVar, $expr)); }); } diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/pom.xml b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/pom.xml index 89b4c0a1607..9e1dfd4dae2 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/pom.xml +++ b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-xml - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/DeserializeContext.java b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/DeserializeContext.java index 2ca9bc8a94f..8f4a2ad9185 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/DeserializeContext.java +++ b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/DeserializeContext.java @@ -488,6 +488,7 @@ Frame start() Frame finish() { XmlDataRecord source = new XmlDataRecord(recordNumber, reader.endCapture()); + get().checkMultiplicities().forEach(this::addDefect); IChecked checked = getDefects().stream().anyMatch(d -> d.getEnforcementLevel() == EnforcementLevel.Critical) ? (IChecked) BasicChecked.newChecked(null, source, getDefects()) : (IChecked) BasicChecked.newChecked(get(), source, getDefects()); diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/handlers/Element.java b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/handlers/Element.java index 0eab743d748..14f4f44744e 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/handlers/Element.java +++ b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/handlers/Element.java @@ -138,10 +138,15 @@ private void processOne(DeserializeContext context) } while (context.reader.isStartElement()) { - QName name = context.reader.getName(); - String msg = "Unexpected element '" + XmlUtils.toShortString(name) + "'" + context.getPath(); - context.getUnexpectedElementHandling().handle(context, msg); context.reader.skipElement(); + if (textContent != null) + { + textContent.process(context); + } + if (particle != null) + { + particle.process(context); + } } } diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/valueProcessors/AddEnumToObject.java b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/valueProcessors/AddEnumToObject.java index ab671c6992f..4117ce93cb4 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/valueProcessors/AddEnumToObject.java +++ b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/main/java/org/finos/legend/engine/external/format/xml/read/valueProcessors/AddEnumToObject.java @@ -31,8 +31,14 @@ public class AddEnumToObject> implements ValueProcessor private final ExternalDataObjectAdder dataAdder; private final Method getNameMethod; private final V[] enumConstants; + private String typePath; public AddEnumToObject(ExternalDataAdder dataAdder, SimpleTypeHandler handler, Class clazz) + { + this(dataAdder, handler, clazz, ""); + } + + public AddEnumToObject(ExternalDataAdder dataAdder, SimpleTypeHandler handler, Class clazz, String typePath) { this.dataAdder = (ExternalDataObjectAdder) dataAdder; this.handler = handler; @@ -46,18 +52,21 @@ public AddEnumToObject(ExternalDataAdder dataAdder, SimpleTypeHandler throw new RuntimeException("getName does not exist in : " + clazz.getSimpleName(), e); } this.enumConstants = clazz.getEnumConstants(); + this.typePath = typePath; } @Override public void process(DeserializeContext context, String rawValue) { String text = handler.parse(rawValue); + + String textWithoutPath = text.startsWith(typePath) ? text.substring(typePath.length()) : text; V value = null; for (V v : enumConstants) { try { - if (XmlUtils.lenientMatch((String) getNameMethod.invoke(v), text)) + if (XmlUtils.lenientMatch((String) getNameMethod.invoke(v), textWithoutPath)) { value = v; } diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXmlDeserializer.java b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXmlDeserializer.java index dc3b4c2f15c..7b5058f10d4 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXmlDeserializer.java +++ b/legend-engine-xts-xml/legend-engine-xt-xml-runtime/src/test/java/org/finos/legend/engine/external/format/xml/test/TestXmlDeserializer.java @@ -217,7 +217,7 @@ public void testDeserializeFirmWithEmployeesWithAttributes() public void testDeserializeFirmWithEmployeesWithElements() { XmlReader reader = reader( - "", + "", " Acme Inc.", " 2", " ", @@ -259,7 +259,7 @@ public void testDeserializeFirmWithEmployeesWithElements() public void testDeserializeFirmWithEmployeesWithElementsNested() { XmlReader reader = reader( - "", + "", " Acme Inc.", " 2", " ", diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-shared/pom.xml b/legend-engine-xts-xml/legend-engine-xt-xml-shared/pom.xml index 8c79ab139c0..b9cd1dcf840 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-shared/pom.xml +++ b/legend-engine-xts-xml/legend-engine-xt-xml-shared/pom.xml @@ -19,7 +19,7 @@ org.finos.legend.engine legend-engine-xts-xml - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/legend-engine-xts-xml/legend-engine-xt-xml-shared/src/main/java/org/finos/legend/engine/external/format/xml/shared/datatypes/SimpleTypesContext.java b/legend-engine-xts-xml/legend-engine-xt-xml-shared/src/main/java/org/finos/legend/engine/external/format/xml/shared/datatypes/SimpleTypesContext.java index c65721a9182..c4b80c91bf9 100644 --- a/legend-engine-xts-xml/legend-engine-xt-xml-shared/src/main/java/org/finos/legend/engine/external/format/xml/shared/datatypes/SimpleTypesContext.java +++ b/legend-engine-xts-xml/legend-engine-xt-xml-shared/src/main/java/org/finos/legend/engine/external/format/xml/shared/datatypes/SimpleTypesContext.java @@ -1463,6 +1463,16 @@ private class DateTimeHandler extends Handler { private final Pattern WITH_TIMEZONE = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?(Z|[+-]\\d{2}:?\\d{2})"); private final Pattern WITHOUT_TIMEZONE = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(\\.\\d+)?"); + private final Pattern WITH_TIMEZONE_NO_TIME = Pattern.compile("\\d{4}-\\d{2}-\\d{2}(Z|\\+\\d{2}:?\\d{2})"); + private final Pattern WITHOUT_TIMEZONE_NO_TIME = Pattern.compile("\\d{4}-\\d{2}-\\d{2}"); + + private final DateTimeFormatter WITH_TIMEZONE_NO_TIME_FORMATTER = new DateTimeFormatterBuilder() + .append(DateTimeFormatter.ISO_OFFSET_DATE) + .parseDefaulting(ChronoField.HOUR_OF_DAY, 0) + .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0) + .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0) + .parseDefaulting(ChronoField.NANO_OF_SECOND, 0) + .toFormatter(); private DateTimeHandler(QName name) { @@ -1482,6 +1492,14 @@ else if (WITHOUT_TIMEZONE.matcher(text).matches()) { return LocalDateTime.parse(text).atZone(defaultTimezone).toInstant(); } + else if (WITH_TIMEZONE_NO_TIME.matcher(text).matches()) + { + return ZonedDateTime.parse(text, WITH_TIMEZONE_NO_TIME_FORMATTER); + } + else if (WITHOUT_TIMEZONE_NO_TIME.matcher(text).matches()) + { + return LocalDate.parse(text); + } } catch (Exception e) { diff --git a/legend-engine-xts-xml/pom.xml b/legend-engine-xts-xml/pom.xml index 8e8e0de784f..6b6bcc5775a 100644 --- a/legend-engine-xts-xml/pom.xml +++ b/legend-engine-xts-xml/pom.xml @@ -18,7 +18,7 @@ org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT 4.0.0 diff --git a/pom.xml b/pom.xml index dc0dd3e1d23..f0a5691cc30 100644 --- a/pom.xml +++ b/pom.xml @@ -26,7 +26,7 @@ Legend Engine org.finos.legend.engine legend-engine - 4.30.1-SNAPSHOT + 4.30.3-SNAPSHOT pom @@ -967,6 +967,11 @@ legend-engine-xt-relationalStore-sybaseiq-pure ${project.version} + + org.finos.legend.engine + legend-engine-xt-relationalStore-sparksql-pure + ${project.version} + org.finos.legend.engine legend-engine-xt-relationalStore-hive-pure