Improve usage of removeDuplicates (#2740)

finos · Apr 2, 2024 · fd619d4 · fd619d4
1 parent 3d4c9a4
commit fd619d4
Show file tree

Hide file tree

Showing 11 changed files with 36 additions and 38 deletions.
diff --git a/...iled-core/src/main/resources/core/pure/executionPlan/platformBinding/platformBinding.pure b/...iled-core/src/main/resources/core/pure/executionPlan/platformBinding/platformBinding.pure
@@ -62,7 +62,7 @@ function meta::pure::executionPlan::platformBinding::extractPlatformBindingById(
    let platformBindings = $extensions.availablePlatformBindings->filter(p | $p.platformId == $platformId);
    assert($platformBindings->size() > 0, | 'Found no platform binding with id: ' +  $platformId);
    let first = $platformBindings->at(0);
-   ^$first(platformBindingExtensions = $platformBindings.platformBindingExtensions->removeDuplicates(x|$x.id, {a,b|$a==$b}));
+   ^$first(platformBindingExtensions = $platformBindings.platformBindingExtensions->removeDuplicatesBy(x|$x.id));
 }
 
 

diff --git a/...end-engine-pure-code-compiled-core/src/main/resources/core/pure/extensions/functions.pure b/...end-engine-pure-code-compiled-core/src/main/resources/core/pure/extensions/functions.pure
@@ -35,7 +35,7 @@ import meta::pure::graphFetch::executionPlan::*;
 function meta::pure::extension::storeContractForSetImplementation(extensions: Extension[*], s:InstanceSetImplementation[1]):StoreContract[1]
 {
   $extensions.availableStores
-    ->removeDuplicates(x|$x.id,{a,b|$a==$b})
+    ->removeDuplicatesBy(x|$x.id)
     ->filter(sc|$sc.supportsSetImplementation->eval($s))
     ->unique(|'for the setImplementation ' + $s->class().name->toOne() + '. For the extensions: [' + $extensions.type->joinStrings(',') + ']');
 }
@@ -47,22 +47,22 @@ function meta::pure::extension::_storeContractForSetImplementation(extensions: E
             $extensions->meta::pure::extension::storeContractForSetImplementation($i->toOne()),
         o: meta::pure::mapping::OperationSetImplementation[1]|
             let roots = $o->resolveInstanceSetImplementations();
-            $roots->map(r|$extensions->meta::pure::extension::storeContractForSetImplementation($r))->removeDuplicates(x|$x.id, {a,b|$a==$b})->toOne();
+            $roots->map(r|$extensions->meta::pure::extension::storeContractForSetImplementation($r))->removeDuplicatesBy(x|$x.id)->toOne();
     ]
   );
 }
 
 function meta::pure::extension::storeContractFromStore(extensions: Extension[*], s: Store[1]):StoreContract[1]
 {
   $extensions.availableStores
-    ->removeDuplicates(x|$x.id,{a,b|$a==$b})
+    ->removeDuplicatesBy(x|$x.id)
     ->filter(sc|$sc.supportsStore->eval($s))
     ->unique(|'for the store ' + $s->class().name->toOne() + '. For the extensions: [' + $extensions.type->joinStrings(',') + ']');
 }
 
 function <<access.private>> meta::pure::extension::unique(storeContracts:StoreContract[*], message:Function<{->String[1]}>[1]):meta::pure::store::StoreContract[1]
 {
-  let list = $storeContracts->removeDuplicates(x|$x.id,{a,b|$a==$b});
+  let list = $storeContracts->removeDuplicatesBy(x|$x.id);
   assert($list->isNotEmpty(), |'The system can\'t find the appropriate StoreContract '+$message->eval());
   assert($list->size() == 1, |'The system found too many StoreContracts ['+$list.id->joinStrings(',')+'] '+$message->eval());
   $list->toOne();

diff --git a/...end-engine-pure-code-compiled-core/src/main/resources/core/pure/router/store/builder.pure b/...end-engine-pure-code-compiled-core/src/main/resources/core/pure/router/store/builder.pure
@@ -301,7 +301,7 @@ function <<access.private>> meta::pure::router::store::routing::permutationSetOn
 
 function meta::pure::router::store::routing::permute(var:PermutationSet[*]):List<PermutationSet>[*]
 {
-   let l = $var->map(p|^List<PermutationSet>(values=$p.sets->removeDuplicates({a,b|$a.id == $b.id})->map(v|^$p(sets=$v))));
+   let l = $var->map(p|^List<PermutationSet>(values=$p.sets->removeDuplicatesBy(x|$x.id)->map(v|^$p(sets=$v))));
    let seed = $l->head().values->map(o|^List<PermutationSet>(values=$o));
    $l->tail()->fold({v,a|$a->combine($v.values)}, $seed);
 }
@@ -333,8 +333,8 @@ function <<access.private>> meta::pure::router::store::routing::unionL(vs:Functi
    let fr = $vs->at(0);
    let i = $fr.value->cast(@InstanceValue);
    let l = $i.values->at(0)->cast(@LambdaFunction<Any>);
-   let res = $vs.value->cast(@InstanceValue).values->cast(@LambdaFunction<Any>).expressionSequence->map(e|pair($e->asString(), $e))->removeDuplicates({a,b|$a.first == $b.first});
-   ^$fr(value=^$i(values=^$l(expressionSequence=doUnion($res.second, $concatenate, $l.expressionSequence->toOne().genericType))));
+   let res = $vs.value->cast(@InstanceValue).values->cast(@LambdaFunction<Any>).expressionSequence->removeDuplicatesBy(e|$e->asString());
+   ^$fr(value=^$i(values=^$l(expressionSequence=doUnion($res, $concatenate, $l.expressionSequence->toOne().genericType))));
 }
 
 function <<access.private>> meta::pure::router::store::routing::doUnion(v:ValueSpecification[*], concatenate:Boolean[1], genericType:GenericType[1]):ValueSpecification[1]

diff --git a/...end-engine-pure-code-compiled-core/src/main/resources/core/pure/router/store/cluster.pure b/...end-engine-pure-code-compiled-core/src/main/resources/core/pure/router/store/cluster.pure
@@ -486,8 +486,8 @@ function meta::pure::router::clustering::storeContractForSetImplementation(setIm
          pair($storeContract, $mapping->resolveStore($store));,
       o: meta::pure::mapping::OperationSetImplementation[1]|
          let roots = $o->resolveInstanceSetImplementations();
-         let storeContract = $roots->map(r|$extensions->meta::pure::extension::storeContractForSetImplementation($r))->removeDuplicates(x|$x.id,{a,b|$a==$b})->toOne();
-         let store = $roots->map(r|$mapping->resolveStore($storeContract.resolveStoreFromSetImplementation->toOne()->eval($r)))->removeDuplicates(x|$x->elementToPath(),{a,b|$a==$b})->toOne();
+         let storeContract = $roots->map(r|$extensions->meta::pure::extension::storeContractForSetImplementation($r))->removeDuplicatesBy(x|$x.id)->toOne();
+         let store = $roots->map(r|$mapping->resolveStore($storeContract.resolveStoreFromSetImplementation->toOne()->eval($r)))->removeDuplicatesBy(x|$x->elementToPath())->toOne();
          pair($storeContract, $mapping->resolveStore($store));
   ]);
 }

diff --git a/...core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure b/...core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/tds/tds.pure
@@ -872,4 +872,4 @@ function meta::pure::tds::groupByWithWindowSubset<K,V,U>(set:K[*], functions:met
 
    meta::pure::tds::groupBy( $set, $subSetfunctions ,$newAggValues , $newIds  );
 
-}
+}
diff --git a/...gend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/flow.pure b/...gend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/flow.pure
@@ -132,7 +132,7 @@ function meta::analytics::lineage::flow::merge(flows:Flow[*]):Flow[1]
       functions = $flows.functions->removeDuplicates(),
       databases = $flows.databases->removeDuplicates(),
       tables = $flows.tables->removeDuplicates(),
-      systems = $flows.systems->removeDuplicates({a,b|$a.appId == $b.appId}),
+      systems = $flows.systems->removeDuplicatesBy(a | $a.appId),
       packages = $flows.packages->removeDuplicates(),
       classes = $flows.classes->removeDuplicates(),
       properties = $flows.properties->removeDuplicates(),

diff --git a/.../resources/core_elasticsearch_seven_metamodel/executionPlan/executionPlan_generation.pure b/.../resources/core_elasticsearch_seven_metamodel/executionPlan/executionPlan_generation.pure
@@ -43,6 +43,6 @@ function meta::external::store::elasticsearch::v7::executionPlan::generation::pl
 
   if ($processOutput.extraNodes->isEmpty(),
     |$esNode,
-    |^SequenceExecutionNode(resultType = $esNode.resultType, executionNodes = $processOutput.extraNodes->removeDuplicates({x, y | $x.varName == $y.varName})->concatenate($esNode))
+    |^SequenceExecutionNode(resultType = $esNode.resultType, executionNodes = $processOutput.extraNodes->removeDuplicatesBy(x | $x.varName)->concatenate($esNode))
   );
 }
diff --git a/...e-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure b/...e-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure
@@ -96,7 +96,7 @@ function <<access.private>> meta::pure::lineage::scanColumns::scanColumns(p:Prop
                              |
                                 let nClassToSets = if($classToSets->keys()->contains($cl), |$classToSets, | $classToSets->keyValues()->concatenate(pair($cl, list($m->_classMappingByClass($cl)->map(s|$s->resolveOperation($m)))))->newMap());
                                 let nsets = $nClassToSets->get($cl).values;
-                                $p.children->map(c|$c->scanColumns($m, $sets->concatenate($nsets)->removeDuplicates({a,b|$a.id == $b.id}), $nClassToSets, $idToSet, false, ^$debug(space=$debug.space+'    ')));,
+                                $p.children->map(c|$c->scanColumns($m, $sets->concatenate($nsets)->removeDuplicatesBy(x | $x.id), $nClassToSets, $idToSet, false, ^$debug(space=$debug.space+'    ')));,
                              |
                                 $p.children->map(c|$c->scanColumns($m, $sets, $classToSets, $idToSet, false, ^$debug(space=$debug.space+'    '))););,
                      a :Any[1]|

diff --git a/...re/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure b/...re/src/main/resources/core_relational/relational/lineage/scanRelations/scanRelations.pure
@@ -187,7 +187,7 @@ function <<access.private>> meta::pure::lineage::scanRelations::scanRelations(p:
                                | [],
                                | if($source->isNotEmpty(),
                                     | $source,
-                                    | $m->rootClassMappingByClass($cl)->toOne()->resolveOperation($m)->removeDuplicates({a,b|$a.id == $b.id})->filter(s | $s->instanceOf(RootRelationalInstanceSetImplementation))->cast(@RootRelationalInstanceSetImplementation);
+                                    | $m->rootClassMappingByClass($cl)->toOne()->resolveOperation($m)->removeDuplicatesBy(x | $x.id)->filter(s | $s->instanceOf(RootRelationalInstanceSetImplementation))->cast(@RootRelationalInstanceSetImplementation);
                                  );
                             );
          print(if($debug.debug,|$debug.space+'Process Class: '+$cl.name->toOne()+', Sets: '+$finalSources.id->joinStrings('[',',',']')+'\n',|''));
Original file line number	Diff line number	Diff line change
Expand Up		@@ -872,4 +872,4 @@ function meta::pure::tds::groupByWithWindowSubset<K,V,U>(set:K[*], functions:met

		meta::pure::tds::groupBy( $set, $subSetfunctions ,$newAggValues , $newIds );

		}
		}