Skip to content

Commit

Permalink
Implement dependency-based (partial) ordering for parallel initializa…
Browse files Browse the repository at this point in the history 
…tion tasks (#123)
  • Loading branch information
@Technici4n
Technici4n authored May 3, 2024
1 parent 0844898 commit 8f01520
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 36 deletions.
54 changes: 46 additions & 8 deletions loader/src/main/java/net/neoforged/fml/ModLoader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand Down Expand Up @@ -190,22 +191,57 @@ public static void waitForTask(String name, Runnable periodicTask, CompletableFu
}
}

/**
* Exception that is fired when a mod loading future cannot be executed because a dependent future failed.
* It is only used for control flow and easy filtering out, but never logged or propagated further.
*/
private static class DependentFutureFailedException extends RuntimeException {}

/**
* Dispatches a task across all mod containers in parallel, with progress displayed on the loading screen.
*/
public static void dispatchParallelTask(String name, Executor parallelExecutor, Runnable periodicTask, Consumer<ModContainer> task) {
var progress = StartupNotificationManager.addProgressBar(name, modList.size());
try {
periodicTask.run();
Map<IModInfo, CompletableFuture<Void>> modFutures = new IdentityHashMap<>(modList.size());
var futureList = modList.getSortedMods().stream()
.map(modContainer -> {
return CompletableFuture.runAsync(() -> {
ModLoadingContext.get().setActiveContainer(modContainer);
task.accept(modContainer);
}, parallelExecutor).whenComplete((result, exception) -> {
progress.increment();
ModLoadingContext.get().setActiveContainer(null);
});
// Collect futures for all dependencies first
var depFutures = LoadingModList.get().getDependencies(modContainer.getModInfo()).stream()
.map(modInfo -> {
var future = modFutures.get(modInfo);
if (future == null) {
throw new IllegalStateException("Dependency future for mod %s which is a dependency of %s not found!".formatted(
modInfo.getModId(), modContainer.getModId()));
}
return future;
})
.toArray(CompletableFuture[]::new);

// Build the future for this container
var future = CompletableFuture.allOf(depFutures)
.<Void>handleAsync((void_, exception) -> {
if (exception != null) {
// If there was any exception, short circuit.
// The exception will already be handled by `waitForFuture` since it comes from another mod.
LOGGER.debug("Skipping {} task for mod {} because a dependency threw an exception.", name, modContainer.getModId());
progress.increment();
// Throw a marker exception to make sure that dependencies of *this* task don't get executed.
throw new DependentFutureFailedException();
}

try {
ModLoadingContext.get().setActiveContainer(modContainer);
task.accept(modContainer);
} finally {
progress.increment();
ModLoadingContext.get().setActiveContainer(null);
}
return null;
}, parallelExecutor);
modFutures.put(modContainer.getModInfo(), future);
return future;
})
.toList();
var singleFuture = ModList.gather(futureList)
Expand All @@ -226,7 +262,9 @@ private static void waitForFuture(String name, Runnable periodicTask, Completabl
// Merge all potential modloading issues
var errorCount = 0;
for (var error : e.getCause().getSuppressed()) {
if (error instanceof ModLoadingException modLoadingException) {
if (error instanceof DependentFutureFailedException) {
continue;
} else if (error instanceof ModLoadingException modLoadingException) {
loadingIssues.addAll(modLoadingException.getIssues());
} else {
loadingIssues.add(ModLoadingIssue.error("fml.modloading.uncaughterror", name).withCause(e));
Expand Down
20 changes: 17 additions & 3 deletions loader/src/main/java/net/neoforged/fml/loading/LoadingModList.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import net.neoforged.fml.loading.moddiscovery.ModFileInfo;
import net.neoforged.fml.loading.moddiscovery.ModInfo;
import net.neoforged.fml.loading.modscan.BackgroundScanHandler;
import net.neoforged.neoforgespi.language.IModInfo;

/**
* Master list of all mods <em>in the loading context. This class cannot refer outside the
Expand All @@ -32,17 +33,19 @@ public class LoadingModList {
private static LoadingModList INSTANCE;
private final List<ModFileInfo> modFiles;
private final List<ModInfo> sortedList;
private final Map<ModInfo, List<ModInfo>> modDependencies;
private final Map<String, ModFileInfo> fileById;
private final List<ModLoadingIssue> modLoadingIssues;

private LoadingModList(final List<ModFile> modFiles, final List<ModInfo> sortedList) {
private LoadingModList(final List<ModFile> modFiles, final List<ModInfo> sortedList, Map<ModInfo, List<ModInfo>> modDependencies) {
this.modFiles = modFiles.stream()
.map(ModFile::getModFileInfo)
.map(ModFileInfo.class::cast)
.collect(Collectors.toList());
this.sortedList = sortedList.stream()
.map(ModInfo.class::cast)
.collect(Collectors.toList());
this.modDependencies = modDependencies;
this.fileById = this.modFiles.stream()
.map(ModFileInfo::getMods)
.flatMap(Collection::stream)
Expand All @@ -51,8 +54,8 @@ private LoadingModList(final List<ModFile> modFiles, final List<ModInfo> sortedL
this.modLoadingIssues = new ArrayList<>();
}

public static LoadingModList of(List<ModFile> modFiles, List<ModInfo> sortedList, List<ModLoadingIssue> issues) {
INSTANCE = new LoadingModList(modFiles, sortedList);
public static LoadingModList of(List<ModFile> modFiles, List<ModInfo> sortedList, List<ModLoadingIssue> issues, Map<ModInfo, List<ModInfo>> modDependencies) {
INSTANCE = new LoadingModList(modFiles, sortedList, modDependencies);
INSTANCE.modLoadingIssues.addAll(issues);
return INSTANCE;
}
Expand Down Expand Up @@ -154,6 +157,17 @@ public List<ModInfo> getMods() {
return this.sortedList;
}

/**
* Returns all direct loading dependencies of the given mod.
*
* <p>This means: all the mods that are directly specified to be loaded before the given mod,
* either because the given mod has an {@link IModInfo.Ordering#AFTER} constraint on the dependency,
* or because the dependency has a {@link IModInfo.Ordering#BEFORE} constraint on the given mod.
*/
public List<ModInfo> getDependencies(IModInfo mod) {
return this.modDependencies.getOrDefault(mod, List.of());
}

public boolean hasErrors() {
return modLoadingIssues.stream().noneMatch(issue -> issue.severity() == ModLoadingIssue.Severity.ERROR);
}
Expand Down
47 changes: 22 additions & 25 deletions loader/src/main/java/net/neoforged/fml/loading/ModSorter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public class ModSorter {
private final UniqueModListBuilder uniqueModListBuilder;
private List<ModFile> modFiles;
private List<ModInfo> sortedList;
private Map<ModInfo, List<ModInfo>> modDependencies;
private Map<String, IModInfo> modIdNameLookup;
private List<ModFile> systemMods;

Expand All @@ -59,7 +60,7 @@ public static LoadingModList sort(List<ModFile> mods, final List<ModLoadingIssue
ms.buildUniqueList();
} catch (ModLoadingException e) {
// We cannot build any list with duped mods. We have to abort immediately and report it
return LoadingModList.of(ms.systemMods, ms.systemMods.stream().map(mf -> (ModInfo) mf.getModInfos().get(0)).collect(toList()), e.getIssues());
return LoadingModList.of(ms.systemMods, ms.systemMods.stream().map(mf -> (ModInfo) mf.getModInfos().get(0)).collect(toList()), e.getIssues(), Map.of());
}

// try and validate dependencies
Expand All @@ -69,7 +70,7 @@ public static LoadingModList sort(List<ModFile> mods, final List<ModLoadingIssue

// if we miss a dependency or detect an incompatibility, we abort now
if (!resolutionResult.versionResolution.isEmpty() || !resolutionResult.incompatibilities.isEmpty()) {
list = LoadingModList.of(ms.systemMods, ms.systemMods.stream().map(mf -> (ModInfo) mf.getModInfos().get(0)).collect(toList()), concat(issues, resolutionResult.buildErrorMessages()));
list = LoadingModList.of(ms.systemMods, ms.systemMods.stream().map(mf -> (ModInfo) mf.getModInfos().get(0)).collect(toList()), concat(issues, resolutionResult.buildErrorMessages()), Map.of());
} else {
// Otherwise, lets try and sort the modlist and proceed
ModLoadingException modLoadingException = null;
Expand All @@ -79,9 +80,9 @@ public static LoadingModList sort(List<ModFile> mods, final List<ModLoadingIssue
modLoadingException = e;
}
if (modLoadingException == null) {
list = LoadingModList.of(ms.modFiles, ms.sortedList, issues);
list = LoadingModList.of(ms.modFiles, ms.sortedList, issues, ms.modDependencies);
} else {
list = LoadingModList.of(ms.modFiles, ms.sortedList, concat(issues, modLoadingException.getIssues()));
list = LoadingModList.of(ms.modFiles, ms.sortedList, concat(issues, modLoadingException.getIssues()), Map.of());
}
}

Expand All @@ -106,12 +107,11 @@ private static <T> List<T> concat(List<T>... lists) {
@SuppressWarnings("UnstableApiUsage")
private void sort() {
// lambdas are identity based, so sorting them is impossible unless you hold reference to them
final MutableGraph<ModFileInfo> graph = GraphBuilder.directed().build();
final MutableGraph<ModInfo> graph = GraphBuilder.directed().build();
AtomicInteger counter = new AtomicInteger();
Map<ModFileInfo, Integer> infos = modFiles.stream()
.map(ModFile::getModFileInfo)
.filter(ModFileInfo.class::isInstance)
.map(ModFileInfo.class::cast)
Map<ModInfo, Integer> infos = modFiles.stream()
.flatMap(mf -> mf.getModInfos().stream())
.map(ModInfo.class::cast)
.collect(toMap(Function.identity(), e -> counter.incrementAndGet()));
infos.keySet().forEach(graph::addNode);
modFiles.stream()
Expand All @@ -120,7 +120,7 @@ private void sort() {
.map(IModInfo::getDependencies).<IModInfo.ModVersion>mapMulti(Iterable::forEach)
.forEach(dep -> addDependency(graph, dep));

final List<ModFileInfo> sorted;
final List<ModInfo> sorted;
try {
sorted = TopologicalSort.topologicalSort(graph, Comparator.comparing(infos::get));
} catch (CyclePresentException e) {
Expand All @@ -136,22 +136,21 @@ private void sort() {
.toList();
throw new ModLoadingException(dataList);
}
this.sortedList = sorted.stream()
.map(ModFileInfo::getMods)
.<IModInfo>mapMulti(Iterable::forEach)
.map(ModInfo.class::cast)
.collect(toList());
this.sortedList = List.copyOf(sorted);
this.modDependencies = sorted.stream()
.collect(Collectors.toMap(modInfo -> modInfo, modInfo -> List.copyOf(graph.predecessors(modInfo))));
this.modFiles = sorted.stream()
.map(ModFileInfo::getFile)
.collect(toList());
.map(mi -> mi.getOwningFile().getFile())
.distinct()
.toList();
}

@SuppressWarnings("UnstableApiUsage")
private void addDependency(MutableGraph<ModFileInfo> topoGraph, IModInfo.ModVersion dep) {
final ModFileInfo self = (ModFileInfo) dep.getOwner().getOwningFile();
private void addDependency(MutableGraph<ModInfo> topoGraph, IModInfo.ModVersion dep) {
final ModInfo self = (ModInfo) dep.getOwner();
final IModInfo targetModInfo = modIdNameLookup.get(dep.getModId());
// soft dep that doesn't exist. Just return. No edge required.
if (targetModInfo == null || !(targetModInfo.getOwningFile() instanceof final ModFileInfo target)) return;
if (!(targetModInfo instanceof ModInfo target)) return;
if (self == target)
return; // in case a jar has two mods that have dependencies between
switch (dep.getOrdering()) {
Expand All @@ -167,11 +166,9 @@ private void buildUniqueList() {

detectSystemMods(uniqueModListData.modFilesByFirstId());

modIdNameLookup = uniqueModListData.modFilesByFirstId().entrySet().stream()
.filter(e -> !e.getValue().get(0).getModInfos().isEmpty())
.collect(Collectors.toMap(
Map.Entry::getKey,
e -> e.getValue().get(0).getModInfos().get(0)));
modIdNameLookup = uniqueModListData.modFiles().stream()
.flatMap(mf -> mf.getModInfos().stream())
.collect(Collectors.toMap(IModInfo::getModId, mi -> mi));
}

private void detectSystemMods(final Map<String, List<ModFile>> modFilesByFirstId) {
Expand Down

0 comments on commit 8f01520

Please sign in to comment.