From b0ed4ff86fc86f1d1fb715c8d5b427e3f4a14b81 Mon Sep 17 00:00:00 2001 From: Mythreyi Date: Tue, 24 Dec 2024 14:35:09 +0800 Subject: [PATCH] Persistence Component: Refactoring DeleteStrategy as interfaces --- .../UnitemporalSnapshotAbstract.java | 4 +-- .../DeleteAllStrategyAbstract.java | 35 +++++++++++++++++++ .../deletestrategy}/DeleteStrategy.java | 7 ++-- .../deletestrategy/DeleteStrategyVisitor.java | 25 +++++++++++++ .../DeleteUpdatedStrategyAbstract.java | 34 ++++++++++++++++++ .../partitioning/PartitioningAbstract.java | 5 +-- .../planner/UnitemporalSnapshotPlanner.java | 6 ++-- .../unitemporal/UnitemporalSnapshotTest.java | 4 +-- ...temporalSnapshotBatchIdBasedScenarios.java | 8 ++--- 9 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteAllStrategyAbstract.java rename legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/{util => ingestmode/deletestrategy}/DeleteStrategy.java (78%) create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteStrategyVisitor.java create mode 100644 legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteUpdatedStrategyAbstract.java diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotAbstract.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotAbstract.java index 372c3863f49..9ce69634dad 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotAbstract.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotAbstract.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.persistence.components.ingestmode; +import org.finos.legend.engine.persistence.components.ingestmode.deletestrategy.DeleteUpdatedStrategy; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.DeleteTargetData; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.EmptyDatasetHandling; import org.finos.legend.engine.persistence.components.ingestmode.partitioning.*; @@ -24,7 +25,6 @@ import org.finos.legend.engine.persistence.components.ingestmode.versioning.MaxVersionStrategyAbstract; import org.finos.legend.engine.persistence.components.ingestmode.versioning.MergeDataVersionResolver; import org.finos.legend.engine.persistence.components.ingestmode.versioning.DigestBasedResolverAbstract; -import org.finos.legend.engine.persistence.components.util.DeleteStrategy; import org.immutables.value.Value; import java.util.*; @@ -71,7 +71,7 @@ default void validate() @Override public Void visitPartitioning(PartitioningAbstract partitionStrategy) { - if (partitionStrategy.deleteStrategy() == DeleteStrategy.DELETE_UPDATED) + if (partitionStrategy.deleteStrategy() instanceof DeleteUpdatedStrategy) { throw new IllegalStateException("Cannot build UnitemporalSnapshot, digestField is mandatory for Partitioning when delete strategy = DELETE_UPDATED"); } diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteAllStrategyAbstract.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteAllStrategyAbstract.java new file mode 100644 index 00000000000..250bb728c41 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteAllStrategyAbstract.java @@ -0,0 +1,35 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode.deletestrategy; + +import org.immutables.value.Value; + +@Value.Immutable +@Value.Style( + typeAbstract = "*Abstract", + typeImmutable = "*", + jdkOnly = true, + optionalAcceptNullable = true, + strictBuilder = true +) +public interface DeleteAllStrategyAbstract extends DeleteStrategy +{ + + @Override + default T accept(DeleteStrategyVisitor visitor) + { + return visitor.visitDeleteAll(this); + } +} \ No newline at end of file diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/DeleteStrategy.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteStrategy.java similarity index 78% rename from legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/DeleteStrategy.java rename to legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteStrategy.java index 3741683e600..0bfeed9030e 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/DeleteStrategy.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteStrategy.java @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -package org.finos.legend.engine.persistence.components.util; +package org.finos.legend.engine.persistence.components.ingestmode.deletestrategy; -public enum DeleteStrategy +public interface DeleteStrategy { - DELETE_ALL, - DELETE_UPDATED + T accept(DeleteStrategyVisitor visitor); } diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteStrategyVisitor.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteStrategyVisitor.java new file mode 100644 index 00000000000..9e0e5d264b4 --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteStrategyVisitor.java @@ -0,0 +1,25 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode.deletestrategy; + +import org.finos.legend.engine.persistence.components.ingestmode.partitioning.NoPartitioningAbstract; +import org.finos.legend.engine.persistence.components.ingestmode.partitioning.PartitioningAbstract; + +public interface DeleteStrategyVisitor +{ + T visitDeleteAll(DeleteAllStrategyAbstract deleteStrategy); + + T visitDeleteUpdated(DeleteUpdatedStrategyAbstract deleteStrategy); +} diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteUpdatedStrategyAbstract.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteUpdatedStrategyAbstract.java new file mode 100644 index 00000000000..b381a8855fa --- /dev/null +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deletestrategy/DeleteUpdatedStrategyAbstract.java @@ -0,0 +1,34 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode.deletestrategy; + +import org.immutables.value.Value; + +@Value.Immutable +@Value.Style( + typeAbstract = "*Abstract", + typeImmutable = "*", + jdkOnly = true, + optionalAcceptNullable = true, + strictBuilder = true +) +public interface DeleteUpdatedStrategyAbstract extends DeleteStrategy +{ + @Override + default T accept(DeleteStrategyVisitor visitor) + { + return visitor.visitDeleteUpdated(this); + } +} \ No newline at end of file diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/partitioning/PartitioningAbstract.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/partitioning/PartitioningAbstract.java index 1307d675a86..a13529c80f4 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/partitioning/PartitioningAbstract.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/partitioning/PartitioningAbstract.java @@ -14,7 +14,8 @@ package org.finos.legend.engine.persistence.components.ingestmode.partitioning; -import org.finos.legend.engine.persistence.components.util.DeleteStrategy; +import org.finos.legend.engine.persistence.components.ingestmode.deletestrategy.DeleteStrategy; +import org.finos.legend.engine.persistence.components.ingestmode.deletestrategy.*; import org.immutables.value.Value; import java.util.List; @@ -34,7 +35,7 @@ public interface PartitioningAbstract extends PartitioningStrategy @Value.Default default DeleteStrategy deleteStrategy() { - return DeleteStrategy.DELETE_UPDATED; + return DeleteUpdatedStrategy.builder().build(); } List partitionFields(); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalSnapshotPlanner.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalSnapshotPlanner.java index cf4068a6577..6d6adc1ce55 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalSnapshotPlanner.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalSnapshotPlanner.java @@ -18,6 +18,7 @@ import org.finos.legend.engine.persistence.components.common.Resources; import org.finos.legend.engine.persistence.components.exception.EmptyBatchException; import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalSnapshot; +import org.finos.legend.engine.persistence.components.ingestmode.deletestrategy.DeleteAllStrategy; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.DeleteTargetDataAbstract; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.EmptyDatasetHandlingVisitor; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.FailEmptyBatchAbstract; @@ -40,7 +41,6 @@ import org.finos.legend.engine.persistence.components.logicalplan.values.Pair; import org.finos.legend.engine.persistence.components.logicalplan.values.Value; import org.finos.legend.engine.persistence.components.util.Capability; -import org.finos.legend.engine.persistence.components.util.DeleteStrategy; import org.finos.legend.engine.persistence.components.util.LogicalPlanUtils; import java.util.*; @@ -139,7 +139,7 @@ protected Insert sqlToUpsertRows() List fieldsToInsert = new ArrayList<>(dataFields); fieldsToInsert.addAll(transactionMilestoningFields()); - if (partitioning.isPresent() && partitioning.get().deleteStrategy() == DeleteStrategy.DELETE_ALL) + if (partitioning.isPresent() && partitioning.get().deleteStrategy() instanceof DeleteAllStrategy) { Dataset selectStage = Selection.builder().source(stagingDataset()).addAllFields(fieldsToSelect).build(); return Insert.of(mainDataset(), selectStage, fieldsToInsert); @@ -212,7 +212,7 @@ protected Update getSqlToMilestoneRows(List> values) { List whereClause = new ArrayList<>(Arrays.asList(openRecordCondition)); - if (!(partitioning.isPresent() && partitioning.get().deleteStrategy() == DeleteStrategy.DELETE_ALL)) + if (!(partitioning.isPresent() && partitioning.get().deleteStrategy() instanceof DeleteAllStrategy)) { Condition notExistsWhereClause = Not.of(Exists.of( Selection.builder() diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java index 3787d1203e2..12d8c21c4df 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalSnapshotTest.java @@ -20,6 +20,7 @@ import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalSnapshot; import org.finos.legend.engine.persistence.components.ingestmode.deduplication.FailOnDuplicates; import org.finos.legend.engine.persistence.components.ingestmode.deduplication.FilterDuplicates; +import org.finos.legend.engine.persistence.components.ingestmode.deletestrategy.DeleteAllStrategy; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.FailEmptyBatch; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.NoOp; import org.finos.legend.engine.persistence.components.ingestmode.partitioning.Partitioning; @@ -31,7 +32,6 @@ import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; import org.finos.legend.engine.persistence.components.logicalplan.datasets.FilteredDataset; import org.finos.legend.engine.persistence.components.planner.PlannerOptions; -import org.finos.legend.engine.persistence.components.util.DeleteStrategy; import org.finos.legend.engine.persistence.components.util.MetadataDataset; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -683,7 +683,7 @@ void testUnitemporalSnapshotMilestoningLogicWithPartitionNoDigest() throws Excep .dateTimeInName(batchTimeInName) .dateTimeOutName(batchTimeOutName) .build()) - .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Collections.singletonList(dateName)).deleteStrategy(DeleteStrategy.DELETE_ALL).build()) + .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Collections.singletonList(dateName)).deleteStrategy(DeleteAllStrategy.builder().build()).build()) .build(); PlannerOptions options = PlannerOptions.builder().collectStatistics(true).build(); diff --git a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/scenarios/UnitemporalSnapshotBatchIdBasedScenarios.java b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/scenarios/UnitemporalSnapshotBatchIdBasedScenarios.java index affba0fe9e8..6df2e375b0b 100644 --- a/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/scenarios/UnitemporalSnapshotBatchIdBasedScenarios.java +++ b/legend-engine-xts-persistence/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-test/src/test/java/org/finos/legend/engine/persistence/components/scenarios/UnitemporalSnapshotBatchIdBasedScenarios.java @@ -17,12 +17,12 @@ import org.finos.legend.engine.persistence.components.BaseTest; import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalSnapshot; import org.finos.legend.engine.persistence.components.ingestmode.deduplication.FailOnDuplicates; +import org.finos.legend.engine.persistence.components.ingestmode.deletestrategy.DeleteAllStrategy; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.DeleteTargetData; import org.finos.legend.engine.persistence.components.ingestmode.emptyhandling.NoOp; import org.finos.legend.engine.persistence.components.ingestmode.partitioning.NoPartitioning; import org.finos.legend.engine.persistence.components.ingestmode.partitioning.Partitioning; import org.finos.legend.engine.persistence.components.ingestmode.transactionmilestoning.BatchId; -import org.finos.legend.engine.persistence.components.util.DeleteStrategy; import java.util.Arrays; @@ -122,7 +122,7 @@ public TestScenario BATCH_ID_BASED__WITH_PARTITIONS__NO_DEDUP__NO_VERSION__NO_DI .batchIdInName(batchIdInField) .batchIdOutName(batchIdOutField) .build()) - .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Arrays.asList(partitionKeys)).deleteStrategy(DeleteStrategy.DELETE_ALL).build()) + .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Arrays.asList(partitionKeys)).deleteStrategy(DeleteAllStrategy.builder().build()).build()) .build(); return new TestScenario(mainTableWithBatchIdBasedSchemaWithoutDigest, stagingTableWithBaseSchema, ingestMode); } @@ -130,7 +130,7 @@ public TestScenario BATCH_ID_BASED__WITH_PARTITIONS__NO_DEDUP__NO_VERSION__NO_DI public TestScenario BATCH_ID_BASED__WITH_PARTITION_FILTER__NO_DEDUP__NO_VERSION__NO_DIGEST() { UnitemporalSnapshot ingestMode = UnitemporalSnapshot.builder() - .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Arrays.asList(partitionKeys)).putAllPartitionValuesByField(partitionFilter).deleteStrategy(DeleteStrategy.DELETE_ALL).build()) + .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Arrays.asList(partitionKeys)).putAllPartitionValuesByField(partitionFilter).deleteStrategy(DeleteAllStrategy.builder().build()).build()) .transactionMilestoning(BatchId.builder() .batchIdInName(batchIdInField) .batchIdOutName(batchIdOutField) @@ -146,7 +146,7 @@ public TestScenario BATCH_ID_BASED__WITH_PARTITION_SPEC_LIST__NO_DEDUP__NO_VERSI .batchIdInName(batchIdInField) .batchIdOutName(batchIdOutField) .build()) - .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Arrays.asList(partitionKeysMulti)).addAllPartitionSpecList(partitionSpecList()).deleteStrategy(DeleteStrategy.DELETE_ALL).build()) + .partitioningStrategy(Partitioning.builder().addAllPartitionFields(Arrays.asList(partitionKeysMulti)).addAllPartitionSpecList(partitionSpecList()).deleteStrategy(DeleteAllStrategy.builder().build()).build()) .emptyDatasetHandling(DeleteTargetData.builder().build()) .build(); return new TestScenario(mainTableMultiPartitionsBasedWithoutDigest, stagingTableWithMultiPartitionsWithoutDigest, ingestMode);