Skip to content

Commit

Permalink
Remove fourth, fifth, and sixth passes from Engine compiler (#3181)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jake-kim1
jake-kim1 authored Oct 15, 2024
1 parent 14a6f84 commit d06d6a7
Show file tree
Hide file tree
Showing 28 changed files with 1,051 additions and 493 deletions.
156 changes: 155 additions & 1 deletion ...java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DependencyManagement.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,21 @@
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.list.FixedSizeList;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.set.MutableSet;
import org.eclipse.collections.api.tuple.Pair;
import org.eclipse.collections.impl.multimap.list.FastListMultimap;
import org.eclipse.collections.impl.tuple.Tuples;
import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement;
import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException;

import java.lang.reflect.Modifier;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;
import java.util.Queue;

public class DependencyManagement
Expand All @@ -38,7 +43,7 @@ public class DependencyManagement
private final MutableMap<Class<? extends PackageableElement>, Class<? extends PackageableElement>> unionFind = Maps.mutable.empty();
private final MutableSet<MutableSet<java.lang.Class<? extends PackageableElement>>> disjointDependencyGraphs = Sets.mutable.empty();

public DependencyManagement(MutableMap<Class<? extends PackageableElement>, Collection<Class<? extends PackageableElement>>> dependencyGraph)
public void processDependencyGraph(MutableMap<Class<? extends PackageableElement>, Collection<Class<? extends PackageableElement>>> dependencyGraph)
{
MutableSet<Class<? extends PackageableElement>> abstractClasses = Sets.mutable.empty();
dependencyGraph.forEach((dependent, dependencies) ->
Expand Down Expand Up @@ -129,6 +134,140 @@ public void detectCircularDependency()
}
}

public FastListMultimap<java.lang.Class<? extends org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement>, PackageableElementsByDependencyLevel> topologicallySortElements(FastListMultimap<java.lang.Class<? extends org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement>, PackageableElementsByDependencyLevel> classToElements, MutableMap<java.lang.Class<? extends org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement>, MutableMap<String, MutableSet<String>>> elementPrerequisitesByClass)
{
FastListMultimap<java.lang.Class<? extends org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement>, PackageableElementsByDependencyLevel> classToElementsSortedByDependencyLevel = new FastListMultimap<>();
classToElements.forEachKeyMultiValues((clazz, elementsByDependencyLevel) ->
{
if (elementPrerequisitesByClass.containsKey(clazz))
{
MutableMap<String, MutableSet<String>> elementPrerequisites = elementPrerequisitesByClass.get(clazz);
MutableMap<String, Integer> inDegrees = Maps.mutable.empty();
MutableMap<String, List<String>> dependencyToDependents = Maps.mutable.empty();
elementPrerequisites.forEachKeyValue((dependentElementFullPath, prerequisiteFullPaths) ->
{
dependencyToDependents.putIfAbsent(dependentElementFullPath, Lists.mutable.empty());
inDegrees.put(dependentElementFullPath, inDegrees.getOrDefault(dependentElementFullPath, 0) + prerequisiteFullPaths.size());
prerequisiteFullPaths.forEach(prerequisiteFullPath ->
{
dependencyToDependents.putIfAbsent(prerequisiteFullPath, Lists.mutable.empty());
dependencyToDependents.get(prerequisiteFullPath).add(dependentElementFullPath);
});
});
MutableMap<String, PackageableElement> packageableElementsIndex = Maps.mutable.empty();
elementsByDependencyLevel.forEach(elementsInCurrentDependencyLevel ->
{
elementsInCurrentDependencyLevel.getIndependentElementAndPathPairs().forEach(elementAndPathPair ->
{
PackageableElement element = elementAndPathPair.getOne();
String elementFullPath = elementAndPathPair.getTwo();
packageableElementsIndex.put(elementFullPath, element);
inDegrees.putIfAbsent(elementFullPath, 0);
});
});

Queue<String> queue = new LinkedList<>();
inDegrees.forEach((elementFullPath, inDegree) ->
{
if (inDegree == 0)
{
queue.offer(elementFullPath);
}
});

int sortedElementCount = 0;
MutableList<PackageableElementsByDependencyLevel> elementsSortedByDependencyLevel = Lists.mutable.empty();
while (!queue.isEmpty())
{
int size = queue.size();
MutableList<Pair<PackageableElement, String>> currentDependencyLevelElements = Lists.mutable.empty();
for (int i = 0; i < size; i++)
{
String elementFullPath = queue.poll();
currentDependencyLevelElements.add(Tuples.pair(packageableElementsIndex.get(elementFullPath), elementFullPath));
sortedElementCount++;
dependencyToDependents.getOrDefault(elementFullPath, Lists.fixedSize.empty()).forEach(dependent ->
{
inDegrees.put(dependent, inDegrees.getOrDefault(dependent, 0) - 1);
if (inDegrees.get(dependent) == 0)
{
queue.offer(dependent);
}
});
}
elementsSortedByDependencyLevel.add(new PackageableElementsByDependencyLevel(currentDependencyLevelElements));
}

if (sortedElementCount != packageableElementsIndex.size())
{
throw new EngineException("Detected a circular dependency in element prerequisites graph in the following metamodel: " + clazz + ".\nCycle: " + getElementsInCircularDependency(elementPrerequisites), EngineErrorType.COMPILATION);
}

classToElementsSortedByDependencyLevel.putAll(clazz, elementsSortedByDependencyLevel);
}

else
{
classToElementsSortedByDependencyLevel.putAll(clazz, elementsByDependencyLevel);
}
});

return classToElementsSortedByDependencyLevel;
}

private String getElementsInCircularDependency(MutableMap<String, MutableSet<String>> elementPrerequisites)
{
StringBuilder cycleBuilder = new StringBuilder();
MutableSet<String> visitedElements = Sets.mutable.empty();
MutableSet<String> stack = Sets.mutable.empty();
MutableList<String> cycle = Lists.mutable.empty();

for (String dependentElementFullPath : elementPrerequisites.keySet())
{
if (!visitedElements.contains(dependentElementFullPath))
{
if (dfs(cycleBuilder, elementPrerequisites, dependentElementFullPath, visitedElements, stack, cycle))
{
break;
}
}
}

return cycleBuilder.toString();
}

private boolean dfs(StringBuilder cycleBuilder, MutableMap<String, MutableSet<String>> elementPrerequisites, String currentElementPath, MutableSet<String> visitedElements, MutableSet<String> stack, MutableList<String> cycle)
{
visitedElements.add(currentElementPath);
stack.add(currentElementPath);
cycle.add(currentElementPath);

for (String prerequisiteFullPath : elementPrerequisites.getOrDefault(currentElementPath, Sets.mutable.empty()))
{
if (!visitedElements.contains(prerequisiteFullPath))
{
if (dfs(cycleBuilder, elementPrerequisites, prerequisiteFullPath, visitedElements, stack, cycle))
{
return true;
}
}
else if (stack.contains(prerequisiteFullPath))
{
int cycleStart = cycle.indexOf(prerequisiteFullPath);
for (int i = cycleStart; i < cycle.size(); i++)
{
cycleBuilder.append(cycle.get(i)).append(" -> ");
}
cycleBuilder.append(cycle.get(cycleStart));
return true;
}
}

stack.remove(currentElementPath);
cycle.remove(cycle.size() - 1);
return false;
}

public MutableMap<Class<? extends PackageableElement>, Collection<Class<? extends PackageableElement>>> getDependentToDependencies()
{
return dependentToDependencies == null ? Maps.fixedSize.empty() : dependentToDependencies;
Expand Down Expand Up @@ -210,4 +349,19 @@ private void union(Class<? extends PackageableElement> p, Class<? extends Packag
rankBySize.put(root1, 0);
}
}

protected class PackageableElementsByDependencyLevel
{
private final FixedSizeList<Pair<PackageableElement, String>> independentElementAndPathPairs;

protected PackageableElementsByDependencyLevel(Collection<Pair<PackageableElement, String>> packageableElementAndPathPairs)
{
this.independentElementAndPathPairs = Lists.fixedSize.withAll(packageableElementAndPathPairs);
}

protected FixedSizeList<Pair<org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement, String>> getIndependentElementAndPathPairs()
{
return this.independentElementAndPathPairs;
}
}
}
57 changes: 7 additions & 50 deletions .../legend/engine/language/pure/compiler/toPureGraph/PackageableElementFifthPassBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@

package org.finos.legend.engine.language.pure.compiler.toPureGraph;

import org.eclipse.collections.api.RichIterable;
import org.eclipse.collections.api.list.MutableList;
import org.eclipse.collections.impl.utility.ListIterate;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElementVisitor;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.data.DataElement;
Expand All @@ -29,21 +26,14 @@
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.runtime.PackageableRuntime;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.section.SectionIndex;
import org.finos.legend.engine.shared.core.identity.Identity;
import org.finos.legend.engine.shared.core.identity.factory.*;
import org.finos.legend.engine.shared.core.operational.logs.LogInfo;
import org.finos.legend.engine.shared.core.operational.logs.LoggingEventType;
import org.finos.legend.pure.generated.Root_meta_pure_runtime_PackageableRuntime;
import org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.AssociationImplementation;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType;
import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.valuespecification.ValueSpecification;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
* @deprecated This will soon be removed as part of Engine compiler cleanup in the next release
*/
@Deprecated
public class PackageableElementFifthPassBuilder implements PackageableElementVisitor<PackageableElement>
{
private static final Logger LOGGER = LoggerFactory.getLogger(PackageableElementFifthPassBuilder.class);
private final CompileContext context;

public PackageableElementFifthPassBuilder(CompileContext context)
Expand All @@ -54,7 +44,6 @@ public PackageableElementFifthPassBuilder(CompileContext context)
@Override
public PackageableElement visit(org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement element)
{
this.context.getExtraProcessorOrThrow(element).processFifthPass(element, this.context);
return null;
}

Expand Down Expand Up @@ -85,26 +74,7 @@ public PackageableElement visit(Association association)
@Override
public PackageableElement visit(Function function)
{
String packageString = this.context.pureModel.buildPackageString(function._package, HelperModelBuilder.getSignature(function));
org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.ConcreteFunctionDefinition<?> targetFunc = this.context.pureModel.getConcreteFunctionDefinition(packageString, function.sourceInformation);
ProcessingContext ctx = new ProcessingContext("Function '" + packageString + "' Fifth Pass");
MutableList<ValueSpecification> body;
try
{
function.parameters.forEach(p -> p.accept(new ValueSpecificationBuilder(this.context, org.eclipse.collections.api.factory.Lists.mutable.empty(), ctx)));
body = ListIterate.collect(function.body, expression -> expression.accept(new ValueSpecificationBuilder(this.context, org.eclipse.collections.api.factory.Lists.mutable.empty(), ctx)));
}
catch (Exception e)
{
LOGGER.warn(new LogInfo(Identity.getAnonymousIdentity().getName(), LoggingEventType.GRAPH_EXPRESSION_ERROR, "Can't build function '" + packageString + "' - stack: " + ctx.getStack()).toString());
throw e;
}
FunctionType fType = ((FunctionType) targetFunc._classifierGenericType()._typeArguments().getFirst()._rawType());
HelperModelBuilder.checkCompatibility(this.context, body.getLast()._genericType()._rawType(), body.getLast()._multiplicity(), fType._returnType()._rawType(), fType._returnMultiplicity(), "Error in function '" + packageString + "'", function.body.get(function.body.size() - 1).sourceInformation);
ctx.pop();
targetFunc._expressionSequence(body);
HelperFunctionBuilder.processFunctionSuites(function, targetFunc, this.context, ctx);
return targetFunc;
return null;
}

@Override
Expand All @@ -116,26 +86,13 @@ public PackageableElement visit(Measure measure)
@Override
public PackageableElement visit(Mapping mapping)
{
final org.finos.legend.pure.m3.coreinstance.meta.pure.mapping.Mapping pureMapping = this.context.pureModel.getMapping(this.context.pureModel.buildPackageString(mapping._package, mapping.name), mapping.sourceInformation);
if (mapping.associationMappings != null)
{
RichIterable<AssociationImplementation> associationImplementations = ListIterate.collect(mapping.associationMappings, cm -> HelperMappingBuilder.processAssociationImplementation(cm, this.context, pureMapping));
pureMapping._associationMappings(associationImplementations);
}
if (mapping.classMappings != null)
{
mapping.classMappings.forEach(cm -> cm.accept(new ClassMappingSecondPassBuilder(this.context, pureMapping)));
}
return pureMapping;
return null;
}

@Override
public PackageableElement visit(PackageableRuntime packageableRuntime)
{
String fullPath = this.context.pureModel.buildPackageString(packageableRuntime._package, packageableRuntime.name);
Root_meta_pure_runtime_PackageableRuntime metamodel = this.context.pureModel.getPackageableRuntime(fullPath, packageableRuntime.sourceInformation);
HelperRuntimeBuilder.buildEngineRuntime(packageableRuntime.runtimeValue, metamodel._runtimeValue(), this.context);
return metamodel;
return null;
}

@Override
Expand Down
Loading

0 comments on commit d06d6a7

Please sign in to comment.