From bfaddd34fc8654f1165e879b7729e1bc30565559 Mon Sep 17 00:00:00 2001 From: Spottedleaf Date: Tue, 6 Aug 2024 21:15:09 -0700 Subject: [PATCH] Update to ConcurrentUtil 0.0.2, and refactor I/O + worker pool With ConcurrentUtil 0.0.2, we can make all thread pools (worker+I/O) have their thread counts configurable on the fly. The new I/O system splits the compression/decompression work from the I/O. The compression/decompression is ran on the worker pool, while the I/O is ran on the I/O pool. This allows for better cpu utilisation control on systems with low core counts, and allows higher read/write speeds on systems with higher core counts. Additionally, the I/O scheduling for thread counts > 1 is also improved as it longer selects a thread to schedule based on the chunk location. --- ConcurrentUtil | 2 +- build.gradle | 2 +- .../common/config/PostDeserializeHook.java | 7 + .../config/adapter/TypeAdapterRegistry.java | 5 + .../config/moonrise/MoonriseConfig.java | 29 +- .../moonrise/common/misc/LazyRunnable.java | 22 + .../moonrise/common/util/ChunkSystem.java | 12 +- .../moonrise/common/util/MoonriseCommon.java | 105 +- .../mixin/chunk_system/ChunkBufferMixin.java | 58 + .../mixin/chunk_system/ChunkMapMixin.java | 46 +- .../chunk_system/MinecraftServerMixin.java | 37 +- .../mixin/chunk_system/PoiManagerMixin.java | 19 +- .../mixin/chunk_system/RegionFileMixin.java | 54 +- .../chunk_system/RegionFileStorageMixin.java | 97 ++ .../chunk_system/ServerChunkCacheMixin.java | 6 +- .../mixin/chunk_system/ServerLevelMixin.java | 33 +- .../render/SectionRenderDispatcherMixin.java | 10 +- .../io/ChunkSystemRegionFileStorage.java | 17 + ...OThread.java => MoonriseRegionFileIO.java} | 1151 +++++++++++------ .../datacontroller/ChunkDataController.java | 50 +- .../datacontroller/EntityDataController.java | 48 +- .../io/datacontroller/PoiDataController.java | 28 +- .../level/ChunkSystemChunkMap.java | 10 + .../level/ChunkSystemServerLevel.java | 18 +- .../player/RegionizedPlayerChunkLoader.java | 4 +- .../scheduling/ChunkHolderManager.java | 69 +- .../scheduling/ChunkTaskScheduler.java | 141 +- .../scheduling/NewChunkHolder.java | 135 +- .../scheduling/PriorityHolder.java | 38 +- .../RadiusAwarePrioritisedExecutor.java | 103 +- .../scheduling/task/ChunkFullTask.java | 19 +- .../scheduling/task/ChunkLightTask.java | 26 +- .../scheduling/task/ChunkLoadTask.java | 43 +- .../scheduling/task/ChunkProgressionTask.java | 10 +- .../task/ChunkUpgradeGenericStatusTask.java | 21 +- .../scheduling/task/GenericDataLoadTask.java | 69 +- .../storage/ChunkSystemChunkBuffer.java | 12 + .../storage/ChunkSystemRegionFile.java | 12 + .../stream/ExternalChunkStreamMarker.java | 37 + .../starlight/light/StarLightInterface.java | 27 +- src/main/resources/moonrise.accesswidener | 10 +- src/main/resources/moonrise.mixins.json | 1 + 42 files changed, 1728 insertions(+), 915 deletions(-) create mode 100644 src/main/java/ca/spottedleaf/moonrise/common/config/PostDeserializeHook.java create mode 100644 src/main/java/ca/spottedleaf/moonrise/common/misc/LazyRunnable.java create mode 100644 src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkBufferMixin.java rename src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/{RegionFileIOThread.java => MoonriseRegionFileIO.java} (51%) create mode 100644 src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemChunkMap.java create mode 100644 src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemChunkBuffer.java create mode 100644 src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/storage/ChunkSystemRegionFile.java create mode 100644 src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/util/stream/ExternalChunkStreamMarker.java diff --git a/ConcurrentUtil b/ConcurrentUtil index e49c7f60..7733f5fe 160000 --- a/ConcurrentUtil +++ b/ConcurrentUtil @@ -1 +1 @@ -Subproject commit e49c7f609b2c63a938ad15ff94fb3e21151ee7af +Subproject commit 7733f5fef43faf505d62bc8a341728a10f938da7 diff --git a/build.gradle b/build.gradle index bf7bb2a1..6a958d5a 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,7 @@ dependencies { mappings loom.officialMojangMappings() modImplementation "net.fabricmc:fabric-loader:${project.loader_version}" - shadow('ca.spottedleaf:concurrentutil:0.0.1-SNAPSHOT') + shadow('ca.spottedleaf:concurrentutil:0.0.2-SNAPSHOT') shadow('org.yaml:snakeyaml:2.2') } diff --git a/src/main/java/ca/spottedleaf/moonrise/common/config/PostDeserializeHook.java b/src/main/java/ca/spottedleaf/moonrise/common/config/PostDeserializeHook.java new file mode 100644 index 00000000..a84b3170 --- /dev/null +++ b/src/main/java/ca/spottedleaf/moonrise/common/config/PostDeserializeHook.java @@ -0,0 +1,7 @@ +package ca.spottedleaf.moonrise.common.config; + +public interface PostDeserializeHook { + + public void deserialize(); + +} diff --git a/src/main/java/ca/spottedleaf/moonrise/common/config/adapter/TypeAdapterRegistry.java b/src/main/java/ca/spottedleaf/moonrise/common/config/adapter/TypeAdapterRegistry.java index fb89e118..8822c931 100644 --- a/src/main/java/ca/spottedleaf/moonrise/common/config/adapter/TypeAdapterRegistry.java +++ b/src/main/java/ca/spottedleaf/moonrise/common/config/adapter/TypeAdapterRegistry.java @@ -1,5 +1,6 @@ package ca.spottedleaf.moonrise.common.config.adapter; +import ca.spottedleaf.moonrise.common.config.PostDeserializeHook; import ca.spottedleaf.moonrise.common.config.adapter.collection.CollectionTypeAdapter; import ca.spottedleaf.moonrise.common.config.adapter.collection.ListTypeAdapter; import ca.spottedleaf.moonrise.common.config.adapter.collection.SortedMapTypeAdapter; @@ -228,6 +229,10 @@ public T deserialize(final TypeAdapterRegistry registry, final Object input, fin field.field.set(ret, field.adapter.deserialize(registry, fieldValue, field.field.getGenericType())); } + if (ret instanceof PostDeserializeHook hook) { + hook.deserialize(); + } + return ret; } catch (final Exception ex) { throw new RuntimeException(ex); diff --git a/src/main/java/ca/spottedleaf/moonrise/common/config/moonrise/MoonriseConfig.java b/src/main/java/ca/spottedleaf/moonrise/common/config/moonrise/MoonriseConfig.java index 97dd334e..d5d4f777 100644 --- a/src/main/java/ca/spottedleaf/moonrise/common/config/moonrise/MoonriseConfig.java +++ b/src/main/java/ca/spottedleaf/moonrise/common/config/moonrise/MoonriseConfig.java @@ -1,9 +1,12 @@ package ca.spottedleaf.moonrise.common.config.moonrise; +import ca.spottedleaf.moonrise.common.config.PostDeserializeHook; import ca.spottedleaf.moonrise.common.config.annotation.Adaptable; import ca.spottedleaf.moonrise.common.config.annotation.Serializable; import ca.spottedleaf.moonrise.common.config.moonrise.type.DefaultedValue; import ca.spottedleaf.moonrise.common.config.type.Duration; +import ca.spottedleaf.moonrise.common.util.MoonriseCommon; +import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; @Adaptable public final class MoonriseConfig { @@ -125,18 +128,33 @@ public static final class ChunkSaving { @Serializable( comment = """ + Configuration options which control the behavior of the common threadpool workers. + """ + ) + public WorkerPool workerPool = new WorkerPool(); + + @Adaptable + public static final class WorkerPool implements PostDeserializeHook { + @Serializable( + comment = """ Set the number of shared worker threads to be used by chunk rendering, chunk loading, chunk generation. If the value is <= 0, then the number of threads will automatically be determined. """ - ) - public int workerThreads = -1; + ) + public int workerThreads = -1; + + @Override + public void deserialize() { + MoonriseCommon.adjustWorkerThreads(this); + } + } @Serializable public ChunkSystem chunkSystem = new ChunkSystem(); @Adaptable - public static final class ChunkSystem { + public static final class ChunkSystem implements PostDeserializeHook { @Serializable( comment = """ @@ -157,6 +175,11 @@ gen and are saturating the population generation (~10 threads of the worker pool """ ) public boolean populationGenParallelism = false; + + @Override + public void deserialize() { + ChunkTaskScheduler.init(this); + } } @Serializable diff --git a/src/main/java/ca/spottedleaf/moonrise/common/misc/LazyRunnable.java b/src/main/java/ca/spottedleaf/moonrise/common/misc/LazyRunnable.java new file mode 100644 index 00000000..c2d917c2 --- /dev/null +++ b/src/main/java/ca/spottedleaf/moonrise/common/misc/LazyRunnable.java @@ -0,0 +1,22 @@ +package ca.spottedleaf.moonrise.common.misc; + +import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import java.lang.invoke.VarHandle; + +public final class LazyRunnable implements Runnable { + + private volatile Runnable toRun; + private static final VarHandle TO_RUN_HANDLE = ConcurrentUtil.getVarHandle(LazyRunnable.class, "toRun", Runnable.class); + + public void setRunnable(final Runnable run) { + final Runnable prev = (Runnable)TO_RUN_HANDLE.compareAndExchange(this, (Runnable)null, run); + if (prev != null) { + throw new IllegalStateException("Runnable already set"); + } + } + + @Override + public void run() { + ((Runnable)TO_RUN_HANDLE.getVolatile(this)).run(); + } +} diff --git a/src/main/java/ca/spottedleaf/moonrise/common/util/ChunkSystem.java b/src/main/java/ca/spottedleaf/moonrise/common/util/ChunkSystem.java index 79ea42ff..cd7da64c 100644 --- a/src/main/java/ca/spottedleaf/moonrise/common/util/ChunkSystem.java +++ b/src/main/java/ca/spottedleaf/moonrise/common/util/ChunkSystem.java @@ -1,6 +1,6 @@ package ca.spottedleaf.moonrise.common.util; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.level.chunk.ChunkSystemLevelChunk; import ca.spottedleaf.moonrise.patches.chunk_system.player.RegionizedPlayerChunkLoader; @@ -23,27 +23,27 @@ public final class ChunkSystem { private static final Logger LOGGER = LogUtils.getLogger(); public static void scheduleChunkTask(final ServerLevel level, final int chunkX, final int chunkZ, final Runnable run) { - scheduleChunkTask(level, chunkX, chunkZ, run, PrioritisedExecutor.Priority.NORMAL); + scheduleChunkTask(level, chunkX, chunkZ, run, Priority.NORMAL); } - public static void scheduleChunkTask(final ServerLevel level, final int chunkX, final int chunkZ, final Runnable run, final PrioritisedExecutor.Priority priority) { + public static void scheduleChunkTask(final ServerLevel level, final int chunkX, final int chunkZ, final Runnable run, final Priority priority) { ((ChunkSystemServerLevel)level).moonrise$getChunkTaskScheduler().scheduleChunkTask(chunkX, chunkZ, run, priority); } public static void scheduleChunkLoad(final ServerLevel level, final int chunkX, final int chunkZ, final boolean gen, - final ChunkStatus toStatus, final boolean addTicket, final PrioritisedExecutor.Priority priority, + final ChunkStatus toStatus, final boolean addTicket, final Priority priority, final Consumer onComplete) { ((ChunkSystemServerLevel)level).moonrise$getChunkTaskScheduler().scheduleChunkLoad(chunkX, chunkZ, gen, toStatus, addTicket, priority, onComplete); } public static void scheduleChunkLoad(final ServerLevel level, final int chunkX, final int chunkZ, final ChunkStatus toStatus, - final boolean addTicket, final PrioritisedExecutor.Priority priority, final Consumer onComplete) { + final boolean addTicket, final Priority priority, final Consumer onComplete) { ((ChunkSystemServerLevel)level).moonrise$getChunkTaskScheduler().scheduleChunkLoad(chunkX, chunkZ, toStatus, addTicket, priority, onComplete); } public static void scheduleTickingState(final ServerLevel level, final int chunkX, final int chunkZ, final FullChunkStatus toStatus, final boolean addTicket, - final PrioritisedExecutor.Priority priority, final Consumer onComplete) { + final Priority priority, final Consumer onComplete) { ((ChunkSystemServerLevel)level).moonrise$getChunkTaskScheduler().scheduleTickingState(chunkX, chunkZ, toStatus, addTicket, priority, onComplete); } diff --git a/src/main/java/ca/spottedleaf/moonrise/common/util/MoonriseCommon.java b/src/main/java/ca/spottedleaf/moonrise/common/util/MoonriseCommon.java index 6d69c50f..63449105 100644 --- a/src/main/java/ca/spottedleaf/moonrise/common/util/MoonriseCommon.java +++ b/src/main/java/ca/spottedleaf/moonrise/common/util/MoonriseCommon.java @@ -1,6 +1,6 @@ package ca.spottedleaf.moonrise.common.util; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedThreadPool; +import ca.spottedleaf.concurrentutil.executor.thread.PrioritisedThreadPool; import ca.spottedleaf.moonrise.common.config.adapter.TypeAdapterRegistry; import ca.spottedleaf.moonrise.common.config.moonrise.MoonriseConfig; import ca.spottedleaf.moonrise.common.config.config.YamlConfig; @@ -9,11 +9,77 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.File; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; public final class MoonriseCommon { private static final Logger LOGGER = LoggerFactory.getLogger(MoonriseCommon.class); + public static final PrioritisedThreadPool WORKER_POOL = new PrioritisedThreadPool( + new Consumer() { + private final AtomicInteger idGenerator = new AtomicInteger(); + + @Override + public void accept(Thread thread) { + thread.setDaemon(true); + thread.setName("Moonrise Common Worker #" + this.idGenerator.getAndIncrement()); + thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(final Thread thread, final Throwable throwable) { + LOGGER.error("Uncaught exception in thread " + thread.getName(), throwable); + } + }); + } + } + ); + public static final long WORKER_QUEUE_HOLD_TIME = (long)(20.0e6); // 20ms + public static final int CLIENT_DIVISION = 0; + public static final PrioritisedThreadPool.ExecutorGroup RENDER_EXECUTOR_GROUP = MoonriseCommon.WORKER_POOL.createExecutorGroup(CLIENT_DIVISION, 0); + public static final int SERVER_DIVISION = 1; + public static final PrioritisedThreadPool.ExecutorGroup PARALLEL_GEN_GROUP = MoonriseCommon.WORKER_POOL.createExecutorGroup(SERVER_DIVISION, 0); + public static final PrioritisedThreadPool.ExecutorGroup RADIUS_AWARE_GROUP = MoonriseCommon.WORKER_POOL.createExecutorGroup(SERVER_DIVISION, 0); + public static final PrioritisedThreadPool.ExecutorGroup LOAD_GROUP = MoonriseCommon.WORKER_POOL.createExecutorGroup(SERVER_DIVISION, 0); + + public static void adjustWorkerThreads(final MoonriseConfig.WorkerPool config) { + int defaultWorkerThreads = Runtime.getRuntime().availableProcessors() / 2; + if (defaultWorkerThreads <= 4) { + defaultWorkerThreads = defaultWorkerThreads <= 3 ? 1 : 2; + } else { + defaultWorkerThreads = defaultWorkerThreads / 2; + } + defaultWorkerThreads = Integer.getInteger("Moonrise.WorkerThreadCount", Integer.valueOf(defaultWorkerThreads)); + + int workerThreads = config.workerThreads; + + if (workerThreads <= 0) { + workerThreads = defaultWorkerThreads; + } + + WORKER_POOL.adjustThreadCount(workerThreads); + } + + public static final PrioritisedThreadPool IO_POOL = new PrioritisedThreadPool( + new Consumer() { + private final AtomicInteger idGenerator = new AtomicInteger(); + + @Override + public void accept(Thread thread) { + thread.setDaemon(true); + thread.setName("Moonrise I/O Worker #" + this.idGenerator.getAndIncrement()); + thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(final Thread thread, final Throwable throwable) { + LOGGER.error("Uncaught exception in thread " + thread.getName(), throwable); + } + }); + } + } + ); + public static final long IO_QUEUE_HOLD_TIME = (long)(100.0e6); // 100ms + public static final PrioritisedThreadPool.ExecutorGroup SERVER_REGION_IO_GROUP = IO_POOL.createExecutorGroup(SERVER_DIVISION, 0); + private static final File CONFIG_FILE = new File(System.getProperty("Moonrise.ConfigFile", "moonrise.yml")); private static final TypeAdapterRegistry CONFIG_ADAPTERS = new TypeAdapterRegistry(); private static final YamlConfig CONFIG; @@ -78,35 +144,20 @@ public static boolean saveConfig() { } } - public static final PrioritisedThreadPool WORKER_POOL; - public static final int WORKER_THREADS; - static { - int defaultWorkerThreads = Runtime.getRuntime().availableProcessors() / 2; - if (defaultWorkerThreads <= 4) { - defaultWorkerThreads = defaultWorkerThreads <= 3 ? 1 : 2; - } else { - defaultWorkerThreads = defaultWorkerThreads / 2; + public static void haltExecutors() { + MoonriseCommon.WORKER_POOL.shutdown(false); + LOGGER.info("Awaiting termination of worker pool for up to 60s..."); + if (!MoonriseCommon.WORKER_POOL.join(TimeUnit.SECONDS.toMillis(60L))) { + LOGGER.error("Worker pool did not shut down in time!"); + MoonriseCommon.WORKER_POOL.halt(false); } - defaultWorkerThreads = Integer.getInteger("Moonrise.WorkerThreadCount", Integer.valueOf(defaultWorkerThreads)); - - int workerThreads = MoonriseCommon.getConfig().workerThreads; - if (workerThreads <= 0) { - workerThreads = defaultWorkerThreads; + MoonriseCommon.IO_POOL.shutdown(false); + LOGGER.info("Awaiting termination of I/O pool for up to 60s..."); + if (!MoonriseCommon.IO_POOL.join(TimeUnit.SECONDS.toMillis(60L))) { + LOGGER.error("I/O pool did not shut down in time!"); + MoonriseCommon.IO_POOL.halt(false); } - - WORKER_POOL = new PrioritisedThreadPool( - "Moonrise Worker Pool", workerThreads, - (final Thread thread, final Integer id) -> { - thread.setName("Moonrise Common Worker #" + id.intValue()); - thread.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(final Thread thread, final Throwable throwable) { - LOGGER.error("Uncaught exception in thread " + thread.getName(), throwable); - } - }); - }, (long)(20.0e6)); // 20ms - WORKER_THREADS = workerThreads; } private MoonriseCommon() {} diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkBufferMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkBufferMixin.java new file mode 100644 index 00000000..c763e9ec --- /dev/null +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkBufferMixin.java @@ -0,0 +1,58 @@ +package ca.spottedleaf.moonrise.mixin.chunk_system; + +import ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemChunkBuffer; +import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.chunk.storage.RegionFile; +import org.spongepowered.asm.mixin.Final; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.Shadow; +import org.spongepowered.asm.mixin.Unique; +import org.spongepowered.asm.mixin.injection.At; +import org.spongepowered.asm.mixin.injection.Inject; +import org.spongepowered.asm.mixin.injection.Redirect; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; + +@Mixin(RegionFile.ChunkBuffer.class) +public abstract class ChunkBufferMixin extends ByteArrayOutputStream implements ChunkSystemChunkBuffer { + + @Shadow + @Final + private ChunkPos pos; + + @Unique + private boolean writeOnClose = true; + + @Override + public final boolean moonrise$getWriteOnClose() { + return this.writeOnClose; + } + + @Override + public final void moonrise$setWriteOnClose(final boolean value) { + this.writeOnClose = value; + } + + @Override + public final void moonrise$write(final RegionFile regionFile) throws IOException { + regionFile.write(this.pos, ByteBuffer.wrap(this.buf, 0, this.count)); + } + + /** + * @reason Allow delaying write I/O until later + * @author Spottedleaf + */ + @Redirect( + method = "close", + at = @At( + value = "INVOKE", + target = "Lnet/minecraft/world/level/chunk/storage/RegionFile;write(Lnet/minecraft/world/level/ChunkPos;Ljava/nio/ByteBuffer;)V" + ) + ) + private void redirectClose(final RegionFile instance, final ChunkPos chunkPos, final ByteBuffer byteBuffer) throws IOException { + if (this.writeOnClose) { + instance.write(chunkPos, byteBuffer); + } + } +} diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkMapMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkMapMixin.java index 7fc88c05..29d58c80 100644 --- a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkMapMixin.java +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ChunkMapMixin.java @@ -2,7 +2,8 @@ import ca.spottedleaf.moonrise.common.util.MoonriseConstants; import ca.spottedleaf.moonrise.common.util.ChunkSystem; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; +import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemChunkMap; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.level.chunk.ChunkSystemChunkHolder; import ca.spottedleaf.moonrise.patches.chunk_system.player.RegionizedPlayerChunkLoader; @@ -51,7 +52,7 @@ import java.util.function.IntSupplier; @Mixin(ChunkMap.class) -public abstract class ChunkMapMixin extends ChunkStorage implements ChunkHolder.PlayerProvider, GeneratingChunkMap { +public abstract class ChunkMapMixin extends ChunkStorage implements ChunkSystemChunkMap, ChunkHolder.PlayerProvider, GeneratingChunkMap { @Shadow @Final @@ -88,6 +89,12 @@ public ChunkMapMixin(RegionStorageInfo regionStorageInfo, Path path, DataFixer d super(regionStorageInfo, path, dataFixer, bl); } + @Override + public final void moonrise$writeFinishCallback(final ChunkPos pos) throws IOException { + // see ChunkStorage#write + this.handleLegacyStructureIndex(pos); + } + /** * @reason Destroy old chunk system hooks * @author Spottedleaf @@ -433,38 +440,31 @@ public void dumpChunks(final Writer writer) throws IOException { @Override public CompletableFuture> read(final ChunkPos pos) { - if (!RegionFileIOThread.isRegionFileThread()) { - try { - return CompletableFuture.completedFuture( - Optional.ofNullable( - RegionFileIOThread.loadData( - this.level, pos.x, pos.z, RegionFileIOThread.RegionFileType.CHUNK_DATA, - RegionFileIOThread.getIOBlockingPriorityForCurrentThread() - ) - ) - ); - } catch (final Throwable thr) { - return CompletableFuture.failedFuture(thr); - } + try { + return CompletableFuture.completedFuture( + Optional.ofNullable( + MoonriseRegionFileIO.loadData( + this.level, pos.x, pos.z, MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, + MoonriseRegionFileIO.getIOBlockingPriorityForCurrentThread() + ) + ) + ); + } catch (final Throwable thr) { + return CompletableFuture.failedFuture(thr); } - return super.read(pos); } @Override public CompletableFuture write(final ChunkPos pos, final CompoundTag tag) { - if (!RegionFileIOThread.isRegionFileThread()) { - RegionFileIOThread.scheduleSave( + MoonriseRegionFileIO.scheduleSave( this.level, pos.x, pos.z, tag, - RegionFileIOThread.RegionFileType.CHUNK_DATA); - return null; - } - super.write(pos, tag); + MoonriseRegionFileIO.RegionFileType.CHUNK_DATA); return null; } @Override public void flushWorker() { - RegionFileIOThread.flush(); + MoonriseRegionFileIO.flush(this.level); } /** diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/MinecraftServerMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/MinecraftServerMixin.java index cea99780..e9804848 100644 --- a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/MinecraftServerMixin.java +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/MinecraftServerMixin.java @@ -1,17 +1,19 @@ package ca.spottedleaf.moonrise.mixin.chunk_system; +import ca.spottedleaf.moonrise.common.util.MoonriseCommon; import ca.spottedleaf.moonrise.common.util.TickThread; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; -import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import ca.spottedleaf.moonrise.patches.chunk_system.server.ChunkSystemMinecraftServer; import net.minecraft.commands.CommandSource; import net.minecraft.server.MinecraftServer; import net.minecraft.server.ServerInfo; import net.minecraft.server.ServerTickRateManager; import net.minecraft.server.TickTask; +import net.minecraft.server.dedicated.DedicatedServer; import net.minecraft.server.level.ServerLevel; import net.minecraft.util.thread.ReentrantBlockableEventLoop; +import org.slf4j.Logger; import org.spongepowered.asm.mixin.Final; import org.spongepowered.asm.mixin.Mixin; import org.spongepowered.asm.mixin.Overwrite; @@ -21,9 +23,6 @@ import org.spongepowered.asm.mixin.injection.Inject; import org.spongepowered.asm.mixin.injection.Redirect; import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; -import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; -import java.util.Iterator; -import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Stream; @@ -43,6 +42,10 @@ public abstract class MinecraftServerMixin extends ReentrantBlockableEventLoop void initHook(Function function, CallbackInfoReturnable cir) { - // TODO better place? - ChunkTaskScheduler.init(); - } - /** * @reason Make server thread an instance of TickThread for thread checks * @author Spottedleaf @@ -257,7 +245,7 @@ private boolean markClosed(final MinecraftServer instance, boolean bl, boolean b private void noOpClose(final ServerLevel instance) {} /** - * @reason Halt regionfile threads after everything is closed + * @reason Halt all executors * @author Spottedleaf */ @Inject( @@ -267,7 +255,10 @@ private void noOpClose(final ServerLevel instance) {} ) ) private void closeIOThreads(final CallbackInfo ci) { - // TODO reinit code needs to be put somewhere - RegionFileIOThread.deinit(); + LOGGER.info("Waiting for I/O tasks to complete..."); + MoonriseRegionFileIO.flush((MinecraftServer)(Object)this); + if ((Object)this instanceof DedicatedServer) { + MoonriseCommon.haltExecutors(); + } } } diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/PoiManagerMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/PoiManagerMixin.java index 6be00f0a..cd7d62cc 100644 --- a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/PoiManagerMixin.java +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/PoiManagerMixin.java @@ -4,7 +4,7 @@ import ca.spottedleaf.moonrise.common.util.CoordinateUtils; import ca.spottedleaf.moonrise.common.util.TickThread; import ca.spottedleaf.moonrise.common.util.WorldUtil; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.level.poi.ChunkSystemPoiManager; import ca.spottedleaf.moonrise.patches.chunk_system.level.poi.ChunkSystemPoiSection; @@ -261,21 +261,14 @@ private Stream skipLoadedSet(final Stream instance, final Predicate cir) { + final DataInputStream is = cir.getReturnValue(); + if (is == null) { + return; + } + cir.setReturnValue(new ExternalChunkStreamMarker(is)); + } } diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/RegionFileStorageMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/RegionFileStorageMixin.java index 3c69e621..c6d7fc15 100644 --- a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/RegionFileStorageMixin.java +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/RegionFileStorageMixin.java @@ -1,10 +1,14 @@ package ca.spottedleaf.moonrise.mixin.chunk_system; import ca.spottedleaf.moonrise.patches.chunk_system.io.ChunkSystemRegionFileStorage; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; +import ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemRegionFile; +import ca.spottedleaf.moonrise.patches.chunk_system.util.stream.ExternalChunkStreamMarker; import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap; import it.unimi.dsi.fastutil.longs.LongLinkedOpenHashSet; import net.minecraft.FileUtil; import net.minecraft.nbt.CompoundTag; +import net.minecraft.nbt.NbtIo; import net.minecraft.nbt.StreamTagVisitor; import net.minecraft.util.ExceptionCollector; import net.minecraft.world.level.ChunkPos; @@ -22,6 +26,8 @@ import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; import org.spongepowered.asm.mixin.injection.callback.LocalCapture; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -170,6 +176,97 @@ public final RegionFile getRegionFile(final ChunkPos chunkPos) throws IOExceptio } } + @Override + public final MoonriseRegionFileIO.RegionDataController.WriteData moonrise$startWrite( + final int chunkX, final int chunkZ, final CompoundTag compound + ) throws IOException { + if (compound == null) { + return new MoonriseRegionFileIO.RegionDataController.WriteData( + compound, MoonriseRegionFileIO.RegionDataController.WriteData.WriteResult.DELETE, + null, null + ); + } + + final ChunkPos pos = new ChunkPos(chunkX, chunkZ); + final RegionFile regionFile = this.getRegionFile(pos); + + // note: not required to keep regionfile loaded after this call, as the write param takes a regionfile as input + // (and, the regionfile parameter is unused for writing until the write call) + final MoonriseRegionFileIO.RegionDataController.WriteData writeData = ((ChunkSystemRegionFile)regionFile).moonrise$startWrite(compound, pos); + + try { + NbtIo.write(compound, writeData.output()); + } finally { + writeData.output().close(); + } + + return writeData; + } + + @Override + public final void moonrise$finishWrite( + final int chunkX, final int chunkZ, final MoonriseRegionFileIO.RegionDataController.WriteData writeData + ) throws IOException { + final ChunkPos pos = new ChunkPos(chunkX, chunkZ); + if (writeData.result() == MoonriseRegionFileIO.RegionDataController.WriteData.WriteResult.DELETE) { + final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); + if (regionFile != null) { + regionFile.clear(pos); + } // else: didn't exist + + return; + } + + writeData.write().run(this.getRegionFile(pos)); + } + + @Override + public final MoonriseRegionFileIO.RegionDataController.ReadData moonrise$readData( + final int chunkX, final int chunkZ + ) throws IOException { + final RegionFile regionFile = this.moonrise$getRegionFileIfExists(chunkX, chunkZ); + + final DataInputStream input = regionFile == null ? null : regionFile.getChunkDataInputStream(new ChunkPos(chunkX, chunkZ)); + + if (input == null) { + return new MoonriseRegionFileIO.RegionDataController.ReadData( + MoonriseRegionFileIO.RegionDataController.ReadData.ReadResult.NO_DATA, null, null + ); + } + + final MoonriseRegionFileIO.RegionDataController.ReadData ret = new MoonriseRegionFileIO.RegionDataController.ReadData( + MoonriseRegionFileIO.RegionDataController.ReadData.ReadResult.HAS_DATA, input, null + ); + + if (!(input instanceof ExternalChunkStreamMarker)) { + // internal stream, which is fully read + return ret; + } + + final CompoundTag syncRead = this.moonrise$finishRead(chunkX, chunkZ, ret); + + if (syncRead == null) { + // need to try again + return this.moonrise$readData(chunkX, chunkZ); + } + + return new MoonriseRegionFileIO.RegionDataController.ReadData( + MoonriseRegionFileIO.RegionDataController.ReadData.ReadResult.SYNC_READ, null, syncRead + ); + } + + // if the return value is null, then the caller needs to re-try with a new call to readData() + @Override + public final CompoundTag moonrise$finishRead( + final int chunkX, final int chunkZ, final MoonriseRegionFileIO.RegionDataController.ReadData readData + ) throws IOException { + try { + return NbtIo.read(readData.input()); + } finally { + readData.input().close(); + } + } + /** * @reason Make this method thread-safe * @author Spottedleaf diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerChunkCacheMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerChunkCacheMixin.java index c17034c1..03a7fc51 100644 --- a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerChunkCacheMixin.java +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerChunkCacheMixin.java @@ -1,7 +1,7 @@ package ca.spottedleaf.moonrise.mixin.chunk_system; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.map.ConcurrentLong2ReferenceChainedHashTable; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; import ca.spottedleaf.moonrise.common.util.TickThread; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; @@ -73,7 +73,7 @@ private ChunkAccess syncLoad(final int chunkX, final int chunkZ, final ChunkStat final ChunkTaskScheduler chunkTaskScheduler = ((ChunkSystemServerLevel)this.level).moonrise$getChunkTaskScheduler(); final CompletableFuture completable = new CompletableFuture<>(); chunkTaskScheduler.scheduleChunkLoad( - chunkX, chunkZ, toStatus, true, PrioritisedExecutor.Priority.BLOCKING, + chunkX, chunkZ, toStatus, true, Priority.BLOCKING, completable::complete ); @@ -182,7 +182,7 @@ public CompletableFuture> getChunkFutureMainThread(fina ((ChunkSystemServerLevel)this.level).moonrise$getChunkTaskScheduler().scheduleChunkLoad( chunkX, chunkZ, toStatus, true, - PrioritisedExecutor.Priority.HIGHER, + Priority.HIGHER, complete ); diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerLevelMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerLevelMixin.java index 4de7bfad..04d4f289 100644 --- a/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerLevelMixin.java +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/chunk_system/ServerLevelMixin.java @@ -1,11 +1,10 @@ package ca.spottedleaf.moonrise.mixin.chunk_system; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.list.ReferenceList; import ca.spottedleaf.moonrise.common.misc.NearbyPlayers; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; -import ca.spottedleaf.moonrise.common.util.MoonriseCommon; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.io.datacontroller.ChunkDataController; import ca.spottedleaf.moonrise.patches.chunk_system.io.datacontroller.EntityDataController; import ca.spottedleaf.moonrise.patches.chunk_system.io.datacontroller.PoiDataController; @@ -34,7 +33,6 @@ import net.minecraft.world.level.ChunkPos; import net.minecraft.world.level.CustomSpawner; import net.minecraft.world.level.Level; -import net.minecraft.world.level.LevelAccessor; import net.minecraft.world.level.WorldGenLevel; import net.minecraft.world.level.chunk.ChunkAccess; import net.minecraft.world.level.chunk.LevelChunk; @@ -98,10 +96,10 @@ protected ServerLevelMixin(WritableLevelData writableLevelData, ResourceKey> onLoad) { this.moonrise$loadChunksAsync( (pos.getX() - radiusBlocks) >> 4, @@ -246,7 +247,7 @@ private void init(MinecraftServer minecraftServer, Executor executor, @Override public final void moonrise$loadChunksAsync(final BlockPos pos, final int radiusBlocks, - final ChunkStatus chunkStatus, final PrioritisedExecutor.Priority priority, + final ChunkStatus chunkStatus, final Priority priority, final Consumer> onLoad) { this.moonrise$loadChunksAsync( (pos.getX() - radiusBlocks) >> 4, @@ -259,14 +260,14 @@ private void init(MinecraftServer minecraftServer, Executor executor, @Override public final void moonrise$loadChunksAsync(final int minChunkX, final int maxChunkX, final int minChunkZ, final int maxChunkZ, - final PrioritisedExecutor.Priority priority, + final Priority priority, final Consumer> onLoad) { this.moonrise$loadChunksAsync(minChunkX, maxChunkX, minChunkZ, maxChunkZ, ChunkStatus.FULL, priority, onLoad); } @Override public final void moonrise$loadChunksAsync(final int minChunkX, final int maxChunkX, final int minChunkZ, final int maxChunkZ, - final ChunkStatus chunkStatus, final PrioritisedExecutor.Priority priority, + final ChunkStatus chunkStatus, final Priority priority, final Consumer> onLoad) { final ChunkTaskScheduler chunkTaskScheduler = this.moonrise$getChunkTaskScheduler(); final ChunkHolderManager chunkHolderManager = chunkTaskScheduler.chunkHolderManager; diff --git a/src/main/java/ca/spottedleaf/moonrise/mixin/render/SectionRenderDispatcherMixin.java b/src/main/java/ca/spottedleaf/moonrise/mixin/render/SectionRenderDispatcherMixin.java index 68f94f56..7c019a6f 100644 --- a/src/main/java/ca/spottedleaf/moonrise/mixin/render/SectionRenderDispatcherMixin.java +++ b/src/main/java/ca/spottedleaf/moonrise/mixin/render/SectionRenderDispatcherMixin.java @@ -1,6 +1,8 @@ package ca.spottedleaf.moonrise.mixin.render; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.thread.PrioritisedThreadPool; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.MoonriseCommon; import net.minecraft.client.renderer.chunk.SectionRenderDispatcher; import org.spongepowered.asm.mixin.Mixin; @@ -15,8 +17,8 @@ public abstract class SectionRenderDispatcherMixin { @Unique - private static final PrioritisedExecutor RENDER_EXECUTOR = MoonriseCommon.WORKER_POOL.createExecutor( - "Moonrise Render Executor", 1, MoonriseCommon.WORKER_THREADS + private static final PrioritisedThreadPool.ExecutorGroup.ThreadPoolExecutor RENDER_EXECUTOR = MoonriseCommon.RENDER_EXECUTOR_GROUP.createExecutor( + -1, MoonriseCommon.WORKER_QUEUE_HOLD_TIME, 0 ); /** @@ -36,7 +38,7 @@ private CompletableFuture changeExecutor(final Supplier supplier, fina return CompletableFuture.supplyAsync( supplier, (final Runnable task) -> { - RENDER_EXECUTOR.queueRunnable(task, PrioritisedExecutor.Priority.NORMAL); + RENDER_EXECUTOR.queueTask(task, Priority.NORMAL); } ); } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java index 73df26b2..a814512f 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/ChunkSystemRegionFileStorage.java @@ -1,5 +1,6 @@ package ca.spottedleaf.moonrise.patches.chunk_system.io; +import net.minecraft.nbt.CompoundTag; import net.minecraft.world.level.chunk.storage.RegionFile; import java.io.IOException; @@ -11,4 +12,20 @@ public interface ChunkSystemRegionFileStorage { public RegionFile moonrise$getRegionFileIfExists(final int chunkX, final int chunkZ) throws IOException; + public MoonriseRegionFileIO.RegionDataController.WriteData moonrise$startWrite( + final int chunkX, final int chunkZ, final CompoundTag compound + ) throws IOException; + + public void moonrise$finishWrite( + final int chunkX, final int chunkZ, final MoonriseRegionFileIO.RegionDataController.WriteData writeData + ) throws IOException; + + public MoonriseRegionFileIO.RegionDataController.ReadData moonrise$readData( + final int chunkX, final int chunkZ + ) throws IOException; + + // if the return value is null, then the caller needs to re-try with a new call to readData() + public CompoundTag moonrise$finishRead( + final int chunkX, final int chunkZ, final MoonriseRegionFileIO.RegionDataController.ReadData readData + ) throws IOException; } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java similarity index 51% rename from src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java rename to src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java index a73e97c1..9966e1ed 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/RegionFileIOThread.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/MoonriseRegionFileIO.java @@ -2,52 +2,43 @@ import ca.spottedleaf.concurrentutil.collection.MultiThreadedQueue; import ca.spottedleaf.concurrentutil.executor.Cancellable; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedQueueExecutorThread; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedThreadedTaskQueue; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.queue.PrioritisedTaskQueue; import ca.spottedleaf.concurrentutil.function.BiLong1Function; import ca.spottedleaf.concurrentutil.map.ConcurrentLong2ReferenceChainedHashTable; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; import ca.spottedleaf.moonrise.common.util.TickThread; import ca.spottedleaf.moonrise.common.util.WorldUtil; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import net.minecraft.nbt.CompoundTag; +import net.minecraft.server.MinecraftServer; import net.minecraft.server.level.ServerLevel; -import net.minecraft.world.level.ChunkPos; import net.minecraft.world.level.chunk.storage.RegionFile; import net.minecraft.world.level.chunk.storage.RegionFileStorage; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.lang.invoke.VarHandle; +import java.nio.ByteBuffer; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.BiConsumer; import java.util.function.Consumer; -import java.util.function.Function; - -/** - * Prioritised RegionFile I/O executor, responsible for all RegionFile access. - *

- * All functions provided are MT-Safe, however certain ordering constraints are recommended: - *

  • - * Chunk saves may not occur for unloaded chunks. - *
    • - *
  • - * Tasks must be scheduled on the chunk scheduler thread. - *
    • - * By following these constraints, no chunk data loss should occur with the exception of underlying I/O problems. - *

    - */ -public final class RegionFileIOThread extends PrioritisedQueueExecutorThread { - - private static final Logger LOGGER = LoggerFactory.getLogger(RegionFileIOThread.class); +import java.util.function.Supplier; + +public final class MoonriseRegionFileIO { + + private static final int REGION_FILE_SHIFT = 5; + private static final Logger LOGGER = LoggerFactory.getLogger(MoonriseRegionFileIO.class); /** - * The kinds of region files controlled by the region file thread. Add more when needed, and ensure - * getControllerFor is updated. + * The types of RegionFiles controlled by the I/O thread(s). */ public static enum RegionFileType { CHUNK_DATA, @@ -55,9 +46,7 @@ public static enum RegionFileType { ENTITY_DATA; } - private static final RegionFileType[] CACHED_REGIONFILE_TYPES = RegionFileType.values(); - - public static ChunkDataController getControllerFor(final ServerLevel world, final RegionFileType type) { + public static RegionDataController getControllerFor(final ServerLevel world, final RegionFileType type) { switch (type) { case CHUNK_DATA: return ((ChunkSystemServerLevel)world).moonrise$getChunkDataController(); @@ -70,8 +59,10 @@ public static ChunkDataController getControllerFor(final ServerLevel world, fina } } + private static final RegionFileType[] CACHED_REGIONFILE_TYPES = RegionFileType.values(); + /** - * Collects regionfile data for a certain chunk. + * Collects RegionFile data for a certain chunk. */ public static final class RegionFileData { @@ -80,13 +71,13 @@ public static final class RegionFileData { private final Throwable[] throwables = new Throwable[CACHED_REGIONFILE_TYPES.length]; /** - * Sets the result associated with the specified regionfile type. Note that - * results can only be set once per regionfile type. + * Sets the result associated with the specified RegionFile type. Note that + * results can only be set once per RegionFile type. * - * @param type The regionfile type. + * @param type The RegionFile type. * @param data The result to set. */ - public void setData(final RegionFileType type, final CompoundTag data) { + public void setData(final MoonriseRegionFileIO.RegionFileType type, final CompoundTag data) { final int index = type.ordinal(); if (this.hasResult[index]) { @@ -97,13 +88,13 @@ public void setData(final RegionFileType type, final CompoundTag data) { } /** - * Sets the result associated with the specified regionfile type. Note that - * results can only be set once per regionfile type. + * Sets the result associated with the specified RegionFile type. Note that + * results can only be set once per RegionFile type. * - * @param type The regionfile type. + * @param type The RegionFile type. * @param throwable The result to set. */ - public void setThrowable(final RegionFileType type, final Throwable throwable) { + public void setThrowable(final MoonriseRegionFileIO.RegionFileType type, final Throwable throwable) { final int index = type.ordinal(); if (this.hasResult[index]) { @@ -114,26 +105,26 @@ public void setThrowable(final RegionFileType type, final Throwable throwable) { } /** - * Returns whether there is a result for the specified regionfile type. + * Returns whether there is a result for the specified RegionFile type. * - * @param type Specified regionfile type. + * @param type Specified RegionFile type. * * @return Whether a result exists for {@code type}. */ - public boolean hasResult(final RegionFileType type) { + public boolean hasResult(final MoonriseRegionFileIO.RegionFileType type) { return this.hasResult[type.ordinal()]; } /** - * Returns the data result for the regionfile type. + * Returns the data result for the RegionFile type. * - * @param type Specified regionfile type. + * @param type Specified RegionFile type. * * @throws IllegalArgumentException If the result has not been set for {@code type}. * @return The data result for the specified type. If the result is a {@code Throwable}, * then returns {@code null}. */ - public CompoundTag getData(final RegionFileType type) { + public CompoundTag getData(final MoonriseRegionFileIO.RegionFileType type) { final int index = type.ordinal(); if (!this.hasResult[index]) { @@ -144,15 +135,15 @@ public CompoundTag getData(final RegionFileType type) { } /** - * Returns the throwable result for the regionfile type. + * Returns the throwable result for the RegionFile type. * - * @param type Specified regionfile type. + * @param type Specified RegionFile type. * * @throws IllegalArgumentException If the result has not been set for {@code type}. * @return The throwable result for the specified type. If the result is an {@code CompoundTag}, * then returns {@code null}. */ - public Throwable getThrowable(final RegionFileType type) { + public Throwable getThrowable(final MoonriseRegionFileIO.RegionFileType type) { final int index = type.ordinal(); if (!this.hasResult[index]) { @@ -163,134 +154,55 @@ public Throwable getThrowable(final RegionFileType type) { } } - private static final Object INIT_LOCK = new Object(); - - static RegionFileIOThread[] threads; - - /* needs to be consistent given a set of parameters */ - static RegionFileIOThread selectThread(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type) { - if (threads == null) { - throw new IllegalStateException("Threads not initialised"); - } - - final int regionX = chunkX >> 5; - final int regionZ = chunkZ >> 5; - final int typeOffset = type.ordinal(); - - return threads[(System.identityHashCode(world) + regionX + regionZ + typeOffset) % threads.length]; - } - - /** - * Shuts down the I/O executor(s). Watis for all tasks to complete if specified. - * Tasks queued during this call might not be accepted, and tasks queued after will not be accepted. - * - * @param wait Whether to wait until all tasks have completed. - */ - public static void close(final boolean wait) { - for (int i = 0, len = threads.length; i < len; ++i) { - threads[i].close(false, true); - } - if (wait) { - RegionFileIOThread.flush(); - } - } - - public static long[] getExecutedTasks() { - final long[] ret = new long[threads.length]; - for (int i = 0, len = threads.length; i < len; ++i) { - ret[i] = threads[i].getTotalTasksExecuted(); + public static void flushRegionStorages(final ServerLevel world) throws IOException { + for (final RegionFileType type : CACHED_REGIONFILE_TYPES) { + flushRegionStorages(world, type); } - - return ret; } - public static long[] getTasksScheduled() { - final long[] ret = new long[threads.length]; - for (int i = 0, len = threads.length; i < len; ++i) { - ret[i] = threads[i].getTotalTasksScheduled(); - } - return ret; + public static void flushRegionStorages(final ServerLevel world, final RegionFileType type) throws IOException { + getControllerFor(world, type).getCache().flush(); } - public static void flush() { - for (int i = 0, len = threads.length; i < len; ++i) { - threads[i].waitUntilAllExecuted(); + public static void flush(final MinecraftServer server) { + for (final ServerLevel world : server.getAllLevels()) { + flush(world); } } - public static void flushRegionStorages(final ServerLevel world) throws IOException { - for (final RegionFileType type : CACHED_REGIONFILE_TYPES) { - getControllerFor(world, type).getCache().flush(); + public static void flush(final ServerLevel world) { + for (final RegionFileType regionFileType : CACHED_REGIONFILE_TYPES) { + flush(world, regionFileType); } } - public static void partialFlush(final int totalTasksRemaining) { - long failures = 1L; // start out at 0.25ms - - for (;;) { - final long[] executed = getExecutedTasks(); - final long[] scheduled = getTasksScheduled(); + public static void flush(final ServerLevel world, final RegionFileType type) { + final RegionDataController taskController = getControllerFor(world, type); - long sum = 0; - for (int i = 0; i < executed.length; ++i) { - sum += scheduled[i] - executed[i]; - } + long failures = 1L; // start at 0.13ms - if (sum <= totalTasksRemaining) { - break; - } - - failures = ConcurrentUtil.linearLongBackoff(failures, 250_000L, 5_000_000L); // 500us, 5ms + while (taskController.hasTasks()) { + Thread.yield(); + failures = ConcurrentUtil.linearLongBackoff(failures, 125_000L, 5_000_000L); // 125us, 5ms } } - /** - * Inits the executor with the specified number of threads. - * - * @param threads Specified number of threads. - */ - public static void init(final int threads) { - synchronized (INIT_LOCK) { - if (RegionFileIOThread.threads != null) { - throw new IllegalStateException("Already initialised threads"); + public static void partialFlush(final ServerLevel world, final int tasksRemaining) { + for (long failures = 1L;;) { // start at 0.13ms + long totalTasks = 0L; + for (final RegionFileType regionFileType : CACHED_REGIONFILE_TYPES) { + totalTasks += getControllerFor(world, regionFileType).getTotalWorkingTasks(); } - RegionFileIOThread.threads = new RegionFileIOThread[threads]; - - for (int i = 0; i < threads; ++i) { - RegionFileIOThread.threads[i] = new RegionFileIOThread(i); - RegionFileIOThread.threads[i].start(); + if (totalTasks > (long)tasksRemaining) { + Thread.yield(); + failures = ConcurrentUtil.linearLongBackoff(failures, 125_000L, 5_000_000L); // 125us, 5ms + } else { + return; } } } - public static void deinit() { - if (false) { - // TODO does this cause issues with mods? how to implement - close(true); - synchronized (INIT_LOCK) { - RegionFileIOThread.threads = null; - } - } else { RegionFileIOThread.flush(); } - } - - private RegionFileIOThread(final int threadNumber) { - super(new PrioritisedThreadedTaskQueue(), (int)(1.0e6)); // 1.0ms spinwait time - this.setName("RegionFile I/O Thread #" + threadNumber); - this.setPriority(Thread.NORM_PRIORITY - 2); // we keep priority close to normal because threads can wait on us - this.setUncaughtExceptionHandler((final Thread thread, final Throwable thr) -> { - LOGGER.error("Uncaught exception thrown from I/O thread, report this! Thread: " + thread.getName(), thr); - }); - } - - /** - * Returns whether the current thread is a regionfile I/O executor. - * @return Whether the current thread is a regionfile I/O executor. - */ - public static boolean isRegionFileThread() { - return Thread.currentThread() instanceof RegionFileIOThread; - } - /** * Returns the priority associated with blocking I/O based on the current thread. The goal is to avoid * dumb plugins from taking away priority from threads we consider crucial. @@ -315,13 +227,8 @@ public static Priority getIOBlockingPriorityForCurrentThread() { * @return The compound tag associated for the specified chunk. {@code null} if no write was pending, or if {@code null} is the write pending. */ public static CompoundTag getPendingWrite(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type) { - final RegionFileIOThread thread = RegionFileIOThread.selectThread(world, chunkX, chunkZ, type); - return thread.getPendingWriteInternal(world, chunkX, chunkZ, type); - } - - CompoundTag getPendingWriteInternal(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type) { - final ChunkDataController taskController = getControllerFor(world, type); - final ChunkDataTask task = taskController.tasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); + final RegionDataController taskController = getControllerFor(world, type); + final ChunkIOTask task = taskController.chunkTasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); if (task == null) { return null; @@ -329,7 +236,7 @@ CompoundTag getPendingWriteInternal(final ServerLevel world, final int chunkX, f final CompoundTag ret = task.inProgressWrite; - return ret == ChunkDataTask.NOTHING_TO_WRITE ? null : ret; + return ret == ChunkIOTask.NOTHING_TO_WRITE ? null : ret; } /** @@ -341,19 +248,14 @@ CompoundTag getPendingWriteInternal(final ServerLevel world, final int chunkX, f * @return The priority for the chunk */ public static Priority getPriority(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type) { - final RegionFileIOThread thread = RegionFileIOThread.selectThread(world, chunkX, chunkZ, type); - return thread.getPriorityInternal(world, chunkX, chunkZ, type); - } - - Priority getPriorityInternal(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type) { - final ChunkDataController taskController = getControllerFor(world, type); - final ChunkDataTask task = taskController.tasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); + final RegionDataController taskController = getControllerFor(world, type); + final ChunkIOTask task = taskController.chunkTasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); if (task == null) { return Priority.COMPLETING; } - return task.prioritisedTask.getPriority(); + return task.getPriority(); } /** @@ -374,7 +276,7 @@ Priority getPriorityInternal(final ServerLevel world, final int chunkX, final in public static void setPriority(final ServerLevel world, final int chunkX, final int chunkZ, final Priority priority) { for (final RegionFileType type : CACHED_REGIONFILE_TYPES) { - RegionFileIOThread.setPriority(world, chunkX, chunkZ, type, priority); + MoonriseRegionFileIO.setPriority(world, chunkX, chunkZ, type, priority); } } @@ -396,17 +298,11 @@ public static void setPriority(final ServerLevel world, final int chunkX, final */ public static void setPriority(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, final Priority priority) { - final RegionFileIOThread thread = RegionFileIOThread.selectThread(world, chunkX, chunkZ, type); - thread.setPriorityInternal(world, chunkX, chunkZ, type, priority); - } - - void setPriorityInternal(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, - final Priority priority) { - final ChunkDataController taskController = getControllerFor(world, type); - final ChunkDataTask task = taskController.tasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); + final RegionDataController taskController = getControllerFor(world, type); + final ChunkIOTask task = taskController.chunkTasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); if (task != null) { - task.prioritisedTask.setPriority(priority); + task.setPriority(priority); } } @@ -426,7 +322,7 @@ void setPriorityInternal(final ServerLevel world, final int chunkX, final int ch public static void raisePriority(final ServerLevel world, final int chunkX, final int chunkZ, final Priority priority) { for (final RegionFileType type : CACHED_REGIONFILE_TYPES) { - RegionFileIOThread.raisePriority(world, chunkX, chunkZ, type, priority); + MoonriseRegionFileIO.raisePriority(world, chunkX, chunkZ, type, priority); } } @@ -446,17 +342,11 @@ public static void raisePriority(final ServerLevel world, final int chunkX, fina */ public static void raisePriority(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, final Priority priority) { - final RegionFileIOThread thread = RegionFileIOThread.selectThread(world, chunkX, chunkZ, type); - thread.raisePriorityInternal(world, chunkX, chunkZ, type, priority); - } - - void raisePriorityInternal(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, - final Priority priority) { - final ChunkDataController taskController = getControllerFor(world, type); - final ChunkDataTask task = taskController.tasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); + final RegionDataController taskController = getControllerFor(world, type); + final ChunkIOTask task = taskController.chunkTasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); if (task != null) { - task.prioritisedTask.raisePriority(priority); + task.raisePriority(priority); } } @@ -476,7 +366,7 @@ void raisePriorityInternal(final ServerLevel world, final int chunkX, final int public static void lowerPriority(final ServerLevel world, final int chunkX, final int chunkZ, final Priority priority) { for (final RegionFileType type : CACHED_REGIONFILE_TYPES) { - RegionFileIOThread.lowerPriority(world, chunkX, chunkZ, type, priority); + MoonriseRegionFileIO.lowerPriority(world, chunkX, chunkZ, type, priority); } } @@ -496,17 +386,11 @@ public static void lowerPriority(final ServerLevel world, final int chunkX, fina */ public static void lowerPriority(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, final Priority priority) { - final RegionFileIOThread thread = RegionFileIOThread.selectThread(world, chunkX, chunkZ, type); - thread.lowerPriorityInternal(world, chunkX, chunkZ, type, priority); - } - - void lowerPriorityInternal(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, - final Priority priority) { - final ChunkDataController taskController = getControllerFor(world, type); - final ChunkDataTask task = taskController.tasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); + final RegionDataController taskController = getControllerFor(world, type); + final ChunkIOTask task = taskController.chunkTasks.get(CoordinateUtils.getChunkKey(chunkX, chunkZ)); if (task != null) { - task.prioritisedTask.lowerPriority(priority); + task.lowerPriority(priority); } } @@ -533,7 +417,7 @@ void lowerPriorityInternal(final ServerLevel world, final int chunkX, final int */ public static void scheduleSave(final ServerLevel world, final int chunkX, final int chunkZ, final CompoundTag data, final RegionFileType type) { - RegionFileIOThread.scheduleSave(world, chunkX, chunkZ, data, type, Priority.NORMAL); + MoonriseRegionFileIO.scheduleSave(world, chunkX, chunkZ, data, type, Priority.NORMAL); } /** @@ -560,23 +444,18 @@ public static void scheduleSave(final ServerLevel world, final int chunkX, final */ public static void scheduleSave(final ServerLevel world, final int chunkX, final int chunkZ, final CompoundTag data, final RegionFileType type, final Priority priority) { - final RegionFileIOThread thread = RegionFileIOThread.selectThread(world, chunkX, chunkZ, type); - thread.scheduleSaveInternal(world, chunkX, chunkZ, data, type, priority); - } - - void scheduleSaveInternal(final ServerLevel world, final int chunkX, final int chunkZ, final CompoundTag data, - final RegionFileType type, final Priority priority) { - final ChunkDataController taskController = getControllerFor(world, type); + final RegionDataController taskController = getControllerFor(world, type); final boolean[] created = new boolean[1]; final long key = CoordinateUtils.getChunkKey(chunkX, chunkZ); - final ChunkDataTask task = taskController.tasks.compute(key, (final long keyInMap, final ChunkDataTask taskRunning) -> { + final ChunkIOTask task = taskController.chunkTasks.compute(key, (final long keyInMap, final ChunkIOTask taskRunning) -> { if (taskRunning == null || taskRunning.failedWrite) { // no task is scheduled or the previous write failed - meaning we need to overwrite it // create task - final ChunkDataTask newTask = new ChunkDataTask(world, chunkX, chunkZ, taskController, RegionFileIOThread.this, priority); - newTask.inProgressWrite = data; + final ChunkIOTask newTask = new ChunkIOTask( + world, taskController, chunkX, chunkZ, priority, new ChunkIOTask.InProgressRead(), data + ); created[0] = true; return newTask; @@ -588,9 +467,10 @@ void scheduleSaveInternal(final ServerLevel world, final int chunkX, final int c }); if (created[0]) { - task.prioritisedTask.queue(); + taskController.startTask(task); + task.scheduleWriteCompress(); } else { - task.prioritisedTask.raisePriority(priority); + task.raisePriority(priority); } } @@ -623,7 +503,7 @@ void scheduleSaveInternal(final ServerLevel world, final int chunkX, final int c */ public static Cancellable loadAllChunkData(final ServerLevel world, final int chunkX, final int chunkZ, final Consumer onComplete, final boolean intendingToBlock) { - return RegionFileIOThread.loadAllChunkData(world, chunkX, chunkZ, onComplete, intendingToBlock, Priority.NORMAL); + return MoonriseRegionFileIO.loadAllChunkData(world, chunkX, chunkZ, onComplete, intendingToBlock, Priority.NORMAL); } /** @@ -657,7 +537,7 @@ public static Cancellable loadAllChunkData(final ServerLevel world, final int ch public static Cancellable loadAllChunkData(final ServerLevel world, final int chunkX, final int chunkZ, final Consumer onComplete, final boolean intendingToBlock, final Priority priority) { - return RegionFileIOThread.loadChunkData(world, chunkX, chunkZ, onComplete, intendingToBlock, priority, CACHED_REGIONFILE_TYPES); + return MoonriseRegionFileIO.loadChunkData(world, chunkX, chunkZ, onComplete, intendingToBlock, priority, CACHED_REGIONFILE_TYPES); } /** @@ -691,7 +571,7 @@ public static Cancellable loadAllChunkData(final ServerLevel world, final int ch public static Cancellable loadChunkData(final ServerLevel world, final int chunkX, final int chunkZ, final Consumer onComplete, final boolean intendingToBlock, final RegionFileType... types) { - return RegionFileIOThread.loadChunkData(world, chunkX, chunkZ, onComplete, intendingToBlock, Priority.NORMAL, types); + return MoonriseRegionFileIO.loadChunkData(world, chunkX, chunkZ, onComplete, intendingToBlock, Priority.NORMAL, types); } /** @@ -741,7 +621,7 @@ public static Cancellable loadChunkData(final ServerLevel world, final int chunk for (int i = 0; i < expectedCompletions; ++i) { final RegionFileType type = types[i]; - reads[i] = RegionFileIOThread.loadDataAsync(world, chunkX, chunkZ, type, + reads[i] = MoonriseRegionFileIO.loadDataAsync(world, chunkX, chunkZ, type, (final CompoundTag data, final Throwable throwable) -> { if (throwable != null) { ret.setThrowable(type, throwable); @@ -787,7 +667,7 @@ public static Cancellable loadChunkData(final ServerLevel world, final int chunk public static Cancellable loadDataAsync(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, final BiConsumer onComplete, final boolean intendingToBlock) { - return RegionFileIOThread.loadDataAsync(world, chunkX, chunkZ, type, onComplete, intendingToBlock, Priority.NORMAL); + return MoonriseRegionFileIO.loadDataAsync(world, chunkX, chunkZ, type, onComplete, intendingToBlock, Priority.NORMAL); } /** @@ -820,36 +700,28 @@ public static Cancellable loadDataAsync(final ServerLevel world, final int chunk public static Cancellable loadDataAsync(final ServerLevel world, final int chunkX, final int chunkZ, final RegionFileType type, final BiConsumer onComplete, final boolean intendingToBlock, final Priority priority) { - final RegionFileIOThread thread = RegionFileIOThread.selectThread(world, chunkX, chunkZ, type); - return thread.loadDataAsyncInternal(world, chunkX, chunkZ, type, onComplete, intendingToBlock, priority); - } - - Cancellable loadDataAsyncInternal(final ServerLevel world, final int chunkX, final int chunkZ, - final RegionFileType type, final BiConsumer onComplete, - final boolean intendingToBlock, final Priority priority) { - final ChunkDataController taskController = getControllerFor(world, type); + final RegionDataController taskController = getControllerFor(world, type); final ImmediateCallbackCompletion callbackInfo = new ImmediateCallbackCompletion(); final long key = CoordinateUtils.getChunkKey(chunkX, chunkZ); - final BiLong1Function compute = (final long keyInMap, final ChunkDataTask running) -> { + final BiLong1Function compute = (final long keyInMap, final ChunkIOTask running) -> { if (running == null) { // not scheduled // set up task - final ChunkDataTask newTask = new ChunkDataTask( - world, chunkX, chunkZ, taskController, RegionFileIOThread.this, priority + final ChunkIOTask newTask = new ChunkIOTask( + world, taskController, chunkX, chunkZ, priority, new ChunkIOTask.InProgressRead(), ChunkIOTask.NOTHING_TO_WRITE ); - newTask.inProgressRead = new InProgressRead(); newTask.inProgressRead.addToAsyncWaiters(onComplete); - callbackInfo.tasksNeedsScheduling = true; + callbackInfo.tasksNeedReadScheduling = true; return newTask; } final CompoundTag pendingWrite = running.inProgressWrite; - if (pendingWrite == ChunkDataTask.NOTHING_TO_WRITE) { + if (pendingWrite == ChunkIOTask.NOTHING_TO_WRITE) { // need to add to waiters here, because the regionfile thread will use compute() to lock and check for cancellations if (!running.inProgressRead.addToAsyncWaiters(onComplete)) { callbackInfo.data = running.inProgressRead.value; @@ -866,11 +738,12 @@ Cancellable loadDataAsyncInternal(final ServerLevel world, final int chunkX, fin return running; }; - final ChunkDataTask ret = taskController.tasks.compute(key, compute); + final ChunkIOTask ret = taskController.chunkTasks.compute(key, compute); // needs to be scheduled - if (callbackInfo.tasksNeedsScheduling) { - ret.prioritisedTask.queue(); + if (callbackInfo.tasksNeedReadScheduling) { + taskController.startTask(ret); + ret.scheduleReadIO(); } else if (callbackInfo.completeNow) { try { onComplete.accept(callbackInfo.data == null ? null : callbackInfo.data.copy(), callbackInfo.throwable); @@ -879,12 +752,21 @@ Cancellable loadDataAsyncInternal(final ServerLevel world, final int chunkX, fin } } else { // we're waiting on a task we didn't schedule, so raise its priority to what we want - ret.prioritisedTask.raisePriority(priority); + ret.raisePriority(priority); } return new CancellableRead(onComplete, ret); } + private static final class ImmediateCallbackCompletion { + + private CompoundTag data; + private Throwable throwable; + private boolean completeNow; + private boolean tasksNeedReadScheduling; + + } + /** * Schedules a load task to be executed asynchronously, and blocks on that task. * @@ -902,7 +784,7 @@ public static CompoundTag loadData(final ServerLevel world, final int chunkX, fi final Priority priority) throws IOException { final CompletableFuture ret = new CompletableFuture<>(); - RegionFileIOThread.loadDataAsync(world, chunkX, chunkZ, type, (final CompoundTag compound, final Throwable thr) -> { + MoonriseRegionFileIO.loadDataAsync(world, chunkX, chunkZ, type, (final CompoundTag compound, final Throwable thr) -> { if (thr != null) { ret.completeExceptionally(thr); } else { @@ -916,22 +798,13 @@ public static CompoundTag loadData(final ServerLevel world, final int chunkX, fi throw new IOException(ex); } } - - private static final class ImmediateCallbackCompletion { - - public CompoundTag data; - public Throwable throwable; - public boolean completeNow; - public boolean tasksNeedsScheduling; - - } - + private static final class CancellableRead implements Cancellable { private BiConsumer callback; - private ChunkDataTask task; + private ChunkIOTask task; - CancellableRead(final BiConsumer callback, final ChunkDataTask task) { + private CancellableRead(final BiConsumer callback, final ChunkIOTask task) { this.callback = callback; this.task = task; } @@ -939,7 +812,7 @@ private static final class CancellableRead implements Cancellable { @Override public boolean cancel() { final BiConsumer callback = this.callback; - final ChunkDataTask task = this.task; + final ChunkIOTask task = this.task; if (callback == null || task == null) { return false; @@ -947,21 +820,17 @@ public boolean cancel() { this.callback = null; this.task = null; - - final InProgressRead read = task.inProgressRead; - - // read can be null if no read was scheduled (i.e no regionfile existed or chunk in regionfile didn't) - return read != null && read.cancel(callback); + + return task.inProgressRead.cancel(callback); } } private static final class CancellableReads implements Cancellable { private Cancellable[] reads; - private static final VarHandle READS_HANDLE = ConcurrentUtil.getVarHandle(CancellableReads.class, "reads", Cancellable[].class); - CancellableReads(final Cancellable[] reads) { + private CancellableReads(final Cancellable[] reads) { this.reads = reads; } @@ -983,258 +852,694 @@ public boolean cancel() { } } - private static final class InProgressRead { + private static final class ChunkIOTask { - private static final Logger LOGGER = LoggerFactory.getLogger(InProgressRead.class); + private static final CompoundTag NOTHING_TO_WRITE = new CompoundTag(); - private CompoundTag value; - private Throwable throwable; - private final MultiThreadedQueue> callbacks = new MultiThreadedQueue<>(); + private final ServerLevel world; + private final RegionDataController regionDataController; + private final int chunkX; + private final int chunkZ; + private Priority priority; + private PrioritisedExecutor.PrioritisedTask currentTask; + + private final InProgressRead inProgressRead; + private volatile CompoundTag inProgressWrite; + + private RegionDataController.ReadData readData; + private RegionDataController.WriteData writeData; + private boolean failedWrite; + + public ChunkIOTask(final ServerLevel world, final RegionDataController regionDataController, + final int chunkX, final int chunkZ, final Priority priority, final InProgressRead inProgressRead, + final CompoundTag inProgressWrite) { + this.world = world; + this.regionDataController = regionDataController; + this.chunkX = chunkX; + this.chunkZ = chunkZ; + this.priority = priority; + this.inProgressRead = inProgressRead; + this.inProgressWrite = inProgressWrite; + } + + public Priority getPriority() { + synchronized (this) { + return this.priority; + } + } + + public boolean setPriority(final Priority priority) { + synchronized (this) { + if (this.priority == priority) { + return false; + } + + this.priority = priority; + if (this.currentTask != null) { + this.currentTask.setPriority(priority); + } + + return true; + } + } - public boolean hasNoWaiters() { - return this.callbacks.isEmpty(); + public boolean raisePriority(final Priority priority) { + synchronized (this) { + if (this.priority.isHigherOrEqualPriority(priority)) { + return false; + } + + this.priority = priority; + if (this.currentTask != null) { + this.currentTask.setPriority(priority); + } + + return true; + } } - public boolean addToAsyncWaiters(final BiConsumer callback) { - return this.callbacks.add(callback); + public boolean lowerPriority(final Priority priority) { + synchronized (this) { + if (this.priority.isLowerOrEqualPriority(priority)) { + return false; + } + + this.priority = priority; + if (this.currentTask != null) { + this.currentTask.setPriority(priority); + } + + return true; + } } - public boolean cancel(final BiConsumer callback) { - return this.callbacks.remove(callback); + public void scheduleReadIO() { + final PrioritisedExecutor.PrioritisedTask task; + synchronized (this) { + task = this.regionDataController.ioScheduler.createTask(this.chunkX, this.chunkZ, this::performReadIO, this.priority); + this.currentTask = task; + } + task.queue(); } - public void complete(final ChunkDataTask task, final CompoundTag value, final Throwable throwable) { - this.value = value; - this.throwable = throwable; + private void performReadIO() { + final InProgressRead read = this.inProgressRead; + final long chunkKey = CoordinateUtils.getChunkKey(this.chunkX, this.chunkZ); + + final boolean[] canRead = new boolean[] { true }; + + if (read.hasNoWaiters()) { + // cancelled read? go to task controller to confirm + final ChunkIOTask inMap = this.regionDataController.chunkTasks.compute(chunkKey, (final long keyInMap, final ChunkIOTask valueInMap) -> { + if (valueInMap == null) { + throw new IllegalStateException("Write completed concurrently, expected this task: " + ChunkIOTask.this.toString() + ", report this!"); + } + if (valueInMap != ChunkIOTask.this) { + throw new IllegalStateException("Chunk task mismatch, expected this task: " + ChunkIOTask.this.toString() + ", got: " + valueInMap.toString() + ", report this!"); + } + + if (!read.hasNoWaiters()) { + return valueInMap; + } else { + canRead[0] = false; + } + + if (valueInMap.inProgressWrite != NOTHING_TO_WRITE) { + return valueInMap; + } + + return null; + }); + + if (inMap == null) { + this.regionDataController.endTask(this); + // read is cancelled - and no write pending, so we're done + return; + } + // if there is a write in progress, we don't actually have to worry about waiters gaining new entries - + // the readers will just use the in progress write, so the value in canRead is good to use without + // further synchronisation. + } + + if (canRead[0]) { + RegionDataController.ReadData readData = null; + Throwable throwable = null; - BiConsumer consumer; - while ((consumer = this.callbacks.pollOrBlockAdds()) != null) { try { - consumer.accept(value == null ? null : value.copy(), throwable); + readData = this.regionDataController.readData(this.chunkX, this.chunkZ); } catch (final Throwable thr) { - LOGGER.error("Callback " + ConcurrentUtil.genericToString(consumer) + " failed to handle chunk data for task " + task.toString(), thr); + throwable = thr; + LOGGER.error("Failed to read chunk data for task: " + this.toString(), thr); } + + if (throwable != null) { + this.finishRead(null, throwable); + } else { + switch (readData.result()) { + case NO_DATA: + case SYNC_READ: { + this.finishRead(readData.syncRead(), null); + break; + } + case HAS_DATA: { + this.readData = readData; + this.scheduleReadDecompress(); + // read will handle write scheduling + return; + } + default: { + throw new IllegalStateException("Unknown state: " + readData.result()); + } + } + } + } + + if (!this.tryAbortWrite()) { + this.scheduleWriteCompress(); } } - } - public static abstract class ChunkDataController { + private void scheduleReadDecompress() { + final PrioritisedExecutor.PrioritisedTask task; + synchronized (this) { + task = this.regionDataController.compressionExecutor.createTask(this::performReadDecompress, this.priority); + this.currentTask = task; + } + task.queue(); + } - // ConcurrentHashMap synchronizes per chain, so reduce the chance of task's hashes colliding. - private final ConcurrentLong2ReferenceChainedHashTable tasks = ConcurrentLong2ReferenceChainedHashTable.createWithCapacity(8192, 0.5f); + private void performReadDecompress() { + final RegionDataController.ReadData readData = this.readData; + this.readData = null; - public final RegionFileType type; + CompoundTag compoundTag = null; + Throwable throwable = null; - public ChunkDataController(final RegionFileType type) { - this.type = type; - } + try { + compoundTag = this.regionDataController.finishRead(this.chunkX, this.chunkZ, readData); + } catch (final Throwable thr) { + throwable = thr; + LOGGER.error("Failed to decompress chunk data for task: " + this.toString(), thr); + } - public abstract RegionFileStorage getCache(); + if (compoundTag == null) { + // need to re-try from the start + this.scheduleReadIO(); + return; + } - public abstract void writeData(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException; + this.finishRead(compoundTag, throwable); + if (!this.tryAbortWrite()) { + // we are already on the compression executor, don't bother scheduling + this.performWriteCompress(); + } + } - public abstract CompoundTag readData(final int chunkX, final int chunkZ) throws IOException; + private void finishRead(final CompoundTag compoundTag, final Throwable throwable) { + this.inProgressRead.complete(this, compoundTag, throwable); + } - public boolean hasTasks() { - return !this.tasks.isEmpty(); + public void scheduleWriteCompress() { + final PrioritisedExecutor.PrioritisedTask task; + synchronized (this) { + task = this.regionDataController.compressionExecutor.createTask(this::performWriteCompress, this.priority); + this.currentTask = task; + } + task.queue(); } - public boolean doesRegionFileNotExist(final int chunkX, final int chunkZ) { - return ((ChunkSystemRegionFileStorage)(Object)this.getCache()).moonrise$doesRegionFileNotExistNoIO(chunkX, chunkZ); + private boolean tryAbortWrite() { + final long chunkKey = CoordinateUtils.getChunkKey(this.chunkX, this.chunkZ); + if (this.inProgressWrite == NOTHING_TO_WRITE) { + final ChunkIOTask inMap = this.regionDataController.chunkTasks.compute(chunkKey, (final long keyInMap, final ChunkIOTask valueInMap) -> { + if (valueInMap == null) { + throw new IllegalStateException("Write completed concurrently, expected this task: " + ChunkIOTask.this.toString() + ", report this!"); + } + if (valueInMap != ChunkIOTask.this) { + throw new IllegalStateException("Chunk task mismatch, expected this task: " + ChunkIOTask.this.toString() + ", got: " + valueInMap.toString() + ", report this!"); + } + + if (valueInMap.inProgressWrite != NOTHING_TO_WRITE) { + return valueInMap; + } + + return null; + }); + + if (inMap == null) { + this.regionDataController.endTask(this); + return true; // set the task value to null, indicating we're done + } // else: inProgressWrite changed, so now we have something to write + } + + return false; } - public T computeForRegionFile(final int chunkX, final int chunkZ, final boolean existingOnly, final Function function) { - final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; - synchronized (cache) { + private void performWriteCompress() { + for (;;) { + final CompoundTag write = this.inProgressWrite; + if (write == NOTHING_TO_WRITE) { + throw new IllegalStateException("Should be writable"); + } + + RegionDataController.WriteData writeData = null; + boolean failedWrite = false; + try { - if (existingOnly) { - regionFile = ((ChunkSystemRegionFileStorage)(Object)cache).moonrise$getRegionFileIfExists(chunkX, chunkZ); + writeData = this.regionDataController.startWrite(this.chunkX, this.chunkZ, write); + } catch (final Throwable thr) { + // TODO implement this? + /*if (thr instanceof RegionFileStorage.RegionFileSizeException) { + final int maxSize = RegionFile.MAX_CHUNK_SIZE / (1024 * 1024); + LOGGER.error("Chunk at (" + this.chunkX + "," + this.chunkZ + ") in '" + WorldUtil.getWorldName(this.world) + "' exceeds max size of " + maxSize + "MiB, it has been deleted from disk."); + } else */ + { + failedWrite = thr instanceof IOException; + LOGGER.error("Failed to write chunk data for task: " + this.toString(), thr); + } + } + + if (writeData == null) { + // null if a throwable was encountered + + // we cannot continue to the I/O stage here, so try to complete + + if (this.tryCompleteWrite(write, failedWrite)) { + return; } else { - regionFile = cache.getRegionFile(new ChunkPos(chunkX, chunkZ)); + // fetch new data and try again + continue; } - } catch (final IOException ex) { - throw new RuntimeException(ex); + } else { + // writeData != null && !failedWrite + // we can continue to I/O stage + this.writeData = writeData; + this.scheduleWriteIO(); + return; } + } + } + + private void scheduleWriteIO() { + final PrioritisedExecutor.PrioritisedTask task; + synchronized (this) { + task = this.regionDataController.ioScheduler.createTask(this.chunkX, this.chunkZ, this::runWriteIO, this.priority); + this.currentTask = task; + } + task.queue(); + } + + private void runWriteIO() { + RegionDataController.WriteData writeData = this.writeData; + this.writeData = null; + + boolean failedWrite = false; + + try { + this.regionDataController.finishWrite(this.chunkX, this.chunkZ, writeData); + } catch (final Throwable thr) { + failedWrite = thr instanceof IOException; + LOGGER.error("Failed to write chunk data for task: " + this.toString(), thr); + } - return function.apply(regionFile); + if (!this.tryCompleteWrite(writeData.input(), failedWrite)) { + // fetch new data and try again + this.scheduleWriteCompress(); } + return; } - public T computeForRegionFileIfLoaded(final int chunkX, final int chunkZ, final Function function) { - final RegionFileStorage cache = this.getCache(); - final RegionFile regionFile; + private boolean tryCompleteWrite(final CompoundTag written, final boolean failedWrite) { + final long chunkKey = CoordinateUtils.getChunkKey(this.chunkX, this.chunkZ); + + final boolean[] done = new boolean[] { false }; - synchronized (cache) { - regionFile = ((ChunkSystemRegionFileStorage)(Object)cache).moonrise$getRegionFileIfLoaded(chunkX, chunkZ); + this.regionDataController.chunkTasks.compute(chunkKey, (final long keyInMap, final ChunkIOTask valueInMap) -> { + if (valueInMap == null) { + throw new IllegalStateException("Write completed concurrently, expected this task: " + ChunkIOTask.this.toString() + ", report this!"); + } + if (valueInMap != ChunkIOTask.this) { + throw new IllegalStateException("Chunk task mismatch, expected this task: " + ChunkIOTask.this.toString() + ", got: " + valueInMap.toString() + ", report this!"); + } + if (valueInMap.inProgressWrite == written) { + valueInMap.failedWrite = failedWrite; + done[0] = true; + // keep the data in map if we failed the write so we can try to prevent data loss + return failedWrite ? valueInMap : null; + } + // different data than expected, means we need to retry write + return valueInMap; + }); - return function.apply(regionFile); + if (done[0]) { + this.regionDataController.endTask(this); + return true; + } + return false; + } + + @Override + public String toString() { + return "Task for world: '" + WorldUtil.getWorldName(this.world) + "' at (" + this.chunkX + "," + + this.chunkZ + ") type: " + this.regionDataController.type.name() + ", hash: " + this.hashCode(); + } + + private static final class InProgressRead { + + private static final Logger LOGGER = LoggerFactory.getLogger(InProgressRead.class); + + private CompoundTag value; + private Throwable throwable; + private final MultiThreadedQueue> callbacks = new MultiThreadedQueue<>(); + + public boolean hasNoWaiters() { + return this.callbacks.isEmpty(); + } + + public CompoundTag getValue() { + return this.value; + } + + public Throwable getThrowable() { + return this.throwable; + } + + public boolean addToAsyncWaiters(final BiConsumer callback) { + return this.callbacks.add(callback); + } + + public boolean cancel(final BiConsumer callback) { + return this.callbacks.remove(callback); + } + + public void complete(final ChunkIOTask task, final CompoundTag value, final Throwable throwable) { + this.value = value; + this.throwable = throwable; + + BiConsumer consumer; + while ((consumer = this.callbacks.pollOrBlockAdds()) != null) { + try { + consumer.accept(value == null ? null : value.copy(), throwable); + } catch (final Throwable thr) { + LOGGER.error("Callback " + ConcurrentUtil.genericToString(consumer) + " failed to handle chunk data for task " + task.toString(), thr); + } + } } } } - private static final class ChunkDataTask implements Runnable { + public static abstract class RegionDataController { - private static final CompoundTag NOTHING_TO_WRITE = new CompoundTag(); + public final RegionFileType type; + private final PrioritisedExecutor compressionExecutor; + private final IOScheduler ioScheduler; + private final ConcurrentLong2ReferenceChainedHashTable chunkTasks = new ConcurrentLong2ReferenceChainedHashTable<>(); - private static final Logger LOGGER = LoggerFactory.getLogger(ChunkDataTask.class); + private final AtomicLong inProgressTasks = new AtomicLong(); - private InProgressRead inProgressRead; - private volatile CompoundTag inProgressWrite = NOTHING_TO_WRITE; // only needs to be acquire/release + public RegionDataController(final RegionFileType type, final PrioritisedExecutor ioExecutor, + final PrioritisedExecutor compressionExecutor) { + this.type = type; + this.compressionExecutor = compressionExecutor; + this.ioScheduler = new IOScheduler(ioExecutor); + } - private boolean failedWrite; + final void startTask(final ChunkIOTask task) { + this.inProgressTasks.getAndIncrement(); + } - private final ServerLevel world; - private final int chunkX; - private final int chunkZ; - private final ChunkDataController taskController; + final void endTask(final ChunkIOTask task) { + this.inProgressTasks.getAndDecrement(); + } - private final PrioritisedTask prioritisedTask; + public boolean hasTasks() { + return this.inProgressTasks.get() != 0L; + } - /* - * IO thread will perform reads before writes for a given chunk x and z - * - * How reads/writes are scheduled: - * - * If read is scheduled while scheduling write, take no special action and just schedule write - * If read is scheduled while scheduling read and no write is scheduled, chain the read task - * - * - * If write is scheduled while scheduling read, use the pending write data and ret immediately (so no read is scheduled) - * If write is scheduled while scheduling write (ignore read in progress), overwrite the write in progress data - * - * This allows the reads and writes to act as if they occur synchronously to the thread scheduling them, however - * it fails to properly propagate write failures thanks to writes overwriting each other - */ + public long getTotalWorkingTasks() { + return this.inProgressTasks.get(); + } - public ChunkDataTask(final ServerLevel world, final int chunkX, final int chunkZ, final ChunkDataController taskController, - final PrioritisedExecutor executor, final Priority priority) { - this.world = world; - this.chunkX = chunkX; - this.chunkZ = chunkZ; - this.taskController = taskController; - this.prioritisedTask = executor.createTask(this, priority); + public abstract RegionFileStorage getCache(); + + public static record WriteData(CompoundTag input, WriteResult result, DataOutputStream output, IORunnable write) { + public static enum WriteResult { + WRITE, + DELETE; + } } - @Override - public String toString() { - return "Task for world: '" + WorldUtil.getWorldName(this.world) + "' at (" + this.chunkX + "," + this.chunkZ + - ") type: " + this.taskController.type.name() + ", hash: " + this.hashCode(); + public abstract WriteData startWrite(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException; + + public abstract void finishWrite(final int chunkX, final int chunkZ, final WriteData writeData) throws IOException; + + public static record ReadData(ReadResult result, DataInputStream input, CompoundTag syncRead) { + public static enum ReadResult { + NO_DATA, + HAS_DATA, + SYNC_READ; + } } - @Override - public void run() { - final InProgressRead read = this.inProgressRead; - final long chunkKey = CoordinateUtils.getChunkKey(this.chunkX, this.chunkZ); + public abstract ReadData readData(final int chunkX, final int chunkZ) throws IOException; - if (read != null) { - final boolean[] canRead = new boolean[] { true }; + // if the return value is null, then the caller needs to re-try with a new call to readData() + public abstract CompoundTag finishRead(final int chunkX, final int chunkZ, final ReadData readData) throws IOException; - if (read.hasNoWaiters()) { - // cancelled read? go to task controller to confirm - final ChunkDataTask inMap = this.taskController.tasks.compute(chunkKey, (final long keyInMap, final ChunkDataTask valueInMap) -> { - if (valueInMap == null) { - throw new IllegalStateException("Write completed concurrently, expected this task: " + ChunkDataTask.this.toString() + ", report this!"); - } - if (valueInMap != ChunkDataTask.this) { - throw new IllegalStateException("Chunk task mismatch, expected this task: " + ChunkDataTask.this.toString() + ", got: " + valueInMap.toString() + ", report this!"); - } + public static interface IORunnable { - if (!read.hasNoWaiters()) { - return valueInMap; - } else { - canRead[0] = false; - } + public void run(final RegionFile regionFile) throws IOException; - return valueInMap.inProgressWrite == NOTHING_TO_WRITE ? null : valueInMap; - }); + } + } - if (inMap == null) { - // read is cancelled - and no write pending, so we're done - return; - } - // if there is a write in progress, we don't actually have to worry about waiters gaining new entries - - // the readers will just use the in progress write, so the value in canRead is good to use without - // further synchronisation. + private static final class IOScheduler { + + private final ConcurrentLong2ReferenceChainedHashTable regionTasks = new ConcurrentLong2ReferenceChainedHashTable<>(); + private final PrioritisedExecutor executor; + + public IOScheduler(final PrioritisedExecutor executor) { + this.executor = executor; + } + + public PrioritisedExecutor.PrioritisedTask createTask(final int chunkX, final int chunkZ, + final Runnable run, final Priority priority) { + final PrioritisedExecutor.PrioritisedTask[] ret = new PrioritisedExecutor.PrioritisedTask[1]; + final long subOrder = this.executor.generateNextSubOrder(); + this.regionTasks.compute(CoordinateUtils.getChunkKey(chunkX >> REGION_FILE_SHIFT, chunkZ >> REGION_FILE_SHIFT), + (final long regionKey, final RegionIOTasks existing) -> { + final RegionIOTasks res; + if (existing != null) { + res = existing; + } else { + res = new RegionIOTasks(regionKey, IOScheduler.this); } - if (canRead[0]) { - CompoundTag compound = null; - Throwable throwable = null; + ret[0] = res.createTask(run, priority, subOrder); - try { - compound = this.taskController.readData(this.chunkX, this.chunkZ); - } catch (final Throwable thr) { - throwable = thr; - LOGGER.error("Failed to read chunk data for task: " + this.toString(), thr); - } - read.complete(this, compound, throwable); + return res; + }); + + return ret[0]; + } + } + + private static final class RegionIOTasks implements Runnable { + + private static final Logger LOGGER = LoggerFactory.getLogger(RegionIOTasks.class); + + private final PrioritisedTaskQueue queue = new PrioritisedTaskQueue(); + private final long regionKey; + private final IOScheduler ioScheduler; + private long createdTasks; + private long executedTasks; + + private PrioritisedExecutor.PrioritisedTask task; + + public RegionIOTasks(final long regionKey, final IOScheduler ioScheduler) { + this.regionKey = regionKey; + this.ioScheduler = ioScheduler; + } + + public PrioritisedExecutor.PrioritisedTask createTask(final Runnable run, final Priority priority, + final long subOrder) { + ++this.createdTasks; + return new WrappedTask(this.queue.createTask(run, priority, subOrder)); + } + + private void adjustTaskPriority() { + final PrioritisedTaskQueue.PrioritySubOrderPair priority = this.queue.getHighestPrioritySubOrder(); + if (this.task == null) { + if (priority == null) { + return; + } + this.task = this.ioScheduler.executor.createTask(this, priority.priority(), priority.subOrder()); + this.task.queue(); + } else { + if (priority == null) { + throw new IllegalStateException(); + } else { + this.task.setPriorityAndSubOrder(priority.priority(), priority.subOrder()); } } + } - CompoundTag write = this.inProgressWrite; + @Override + public void run() { + final Runnable run; + synchronized (this) { + run = this.queue.pollTask(); + } - if (write == NOTHING_TO_WRITE) { - final ChunkDataTask inMap = this.taskController.tasks.compute(chunkKey, (final long keyInMap, final ChunkDataTask valueInMap) -> { - if (valueInMap == null) { - throw new IllegalStateException("Write completed concurrently, expected this task: " + ChunkDataTask.this.toString() + ", report this!"); + try { + run.run(); + } finally { + synchronized (this) { + this.task = null; + this.adjustTaskPriority(); + } + this.ioScheduler.regionTasks.compute(this.regionKey, (final long keyInMap, final RegionIOTasks tasks) -> { + if (tasks != RegionIOTasks.this) { + throw new IllegalStateException("Region task mismatch"); + } + ++tasks.executedTasks; + if (tasks.createdTasks != tasks.executedTasks) { + return tasks; } - if (valueInMap != ChunkDataTask.this) { - throw new IllegalStateException("Chunk task mismatch, expected this task: " + ChunkDataTask.this.toString() + ", got: " + valueInMap.toString() + ", report this!"); + + if (tasks.task != null) { + throw new IllegalStateException("Task may not be null when created==executed"); } - return valueInMap.inProgressWrite == NOTHING_TO_WRITE ? null : valueInMap; + + return null; }); + } + } - if (inMap == null) { - return; // set the task value to null, indicating we're done - } // else: inProgressWrite changed, so now we have something to write + private final class WrappedTask implements PrioritisedExecutor.PrioritisedTask { + + private final PrioritisedExecutor.PrioritisedTask wrapped; + + public WrappedTask(final PrioritisedExecutor.PrioritisedTask wrap) { + this.wrapped = wrap; } - for (;;) { - write = this.inProgressWrite; - final CompoundTag dataWritten = write; + @Override + public PrioritisedExecutor getExecutor() { + return RegionIOTasks.this.ioScheduler.executor; + } - boolean failedWrite = false; + @Override + public boolean queue() { + synchronized (RegionIOTasks.this) { + if (this.wrapped.queue()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; + } + return false; + } + } - try { - this.taskController.writeData(this.chunkX, this.chunkZ, write); - } catch (final Throwable thr) { - // TODO implement this? - /*if (thr instanceof RegionFileStorage.RegionFileSizeException) { - final int maxSize = RegionFile.MAX_CHUNK_SIZE / (1024 * 1024); - LOGGER.error("Chunk at (" + this.chunkX + "," + this.chunkZ + ") in '" + WorldUtil.getWorldName(this.world) + "' exceeds max size of " + maxSize + "MiB, it has been deleted from disk."); - } else */{ - failedWrite = thr instanceof IOException; - LOGGER.error("Failed to write chunk data for task: " + this.toString(), thr); + @Override + public boolean isQueued() { + return this.wrapped.isQueued(); + } + + @Override + public boolean cancel() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean execute() { + throw new UnsupportedOperationException(); + } + + @Override + public Priority getPriority() { + return this.wrapped.getPriority(); + } + + @Override + public boolean setPriority(final Priority priority) { + synchronized (RegionIOTasks.this) { + if (this.wrapped.setPriority(priority) && this.wrapped.isQueued()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; } + return false; } + } - final boolean finalFailWrite = failedWrite; - final boolean[] done = new boolean[] { false }; + @Override + public boolean raisePriority(final Priority priority) { + synchronized (RegionIOTasks.this) { + if (this.wrapped.raisePriority(priority) && this.wrapped.isQueued()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; + } + return false; + } + } - this.taskController.tasks.compute(chunkKey, (final long keyInMap, final ChunkDataTask valueInMap) -> { - if (valueInMap == null) { - throw new IllegalStateException("Write completed concurrently, expected this task: " + ChunkDataTask.this.toString() + ", report this!"); + @Override + public boolean lowerPriority(final Priority priority) { + synchronized (RegionIOTasks.this) { + if (this.wrapped.lowerPriority(priority) && this.wrapped.isQueued()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; } - if (valueInMap != ChunkDataTask.this) { - throw new IllegalStateException("Chunk task mismatch, expected this task: " + ChunkDataTask.this.toString() + ", got: " + valueInMap.toString() + ", report this!"); + return false; + } + } + + @Override + public long getSubOrder() { + return this.wrapped.getSubOrder(); + } + + @Override + public boolean setSubOrder(final long subOrder) { + synchronized (RegionIOTasks.this) { + if (this.wrapped.setSubOrder(subOrder) && this.wrapped.isQueued()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; } - if (valueInMap.inProgressWrite == dataWritten) { - valueInMap.failedWrite = finalFailWrite; - done[0] = true; - // keep the data in map if we failed the write so we can try to prevent data loss - return finalFailWrite ? valueInMap : null; + return false; + } + } + + @Override + public boolean raiseSubOrder(final long subOrder) { + synchronized (RegionIOTasks.this) { + if (this.wrapped.raiseSubOrder(subOrder) && this.wrapped.isQueued()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; } - // different data than expected, means we need to retry write - return valueInMap; - }); + return false; + } + } - if (done[0]) { - return; + @Override + public boolean lowerSubOrder(final long subOrder) { + synchronized (RegionIOTasks.this) { + if (this.wrapped.lowerSubOrder(subOrder) && this.wrapped.isQueued()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; + } + return false; } + } - // fetch & write new data - continue; + @Override + public boolean setPriorityAndSubOrder(final Priority priority, final long subOrder) { + synchronized (RegionIOTasks.this) { + if (this.wrapped.setPriorityAndSubOrder(priority, subOrder) && this.wrapped.isQueued()) { + RegionIOTasks.this.adjustTaskPriority(); + return true; + } + return false; + } } } } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/ChunkDataController.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/ChunkDataController.java index c35e0c29..a36ab89f 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/ChunkDataController.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/ChunkDataController.java @@ -1,22 +1,24 @@ package ca.spottedleaf.moonrise.patches.chunk_system.io.datacontroller; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.ChunkSystemRegionFileStorage; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; +import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemChunkMap; +import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import ca.spottedleaf.moonrise.patches.chunk_system.storage.ChunkSystemChunkStorage; import net.minecraft.nbt.CompoundTag; import net.minecraft.server.level.ServerLevel; import net.minecraft.world.level.ChunkPos; import net.minecraft.world.level.chunk.storage.RegionFileStorage; import java.io.IOException; -import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; -public final class ChunkDataController extends RegionFileIOThread.ChunkDataController { +public final class ChunkDataController extends MoonriseRegionFileIO.RegionDataController { private final ServerLevel world; - public ChunkDataController(final ServerLevel world) { - super(RegionFileIOThread.RegionFileType.CHUNK_DATA); + public ChunkDataController(final ServerLevel world, final ChunkTaskScheduler taskScheduler) { + super(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, taskScheduler.ioExecutor, taskScheduler.compressionExecutor); this.world = world; } @@ -26,31 +28,23 @@ public RegionFileStorage getCache() { } @Override - public void writeData(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException { - final CompletableFuture future = this.world.getChunkSource().chunkMap.write(new ChunkPos(chunkX, chunkZ), compound); - - try { - if (future != null) { - // rets non-null when sync writing (i.e. future should be completed here) - future.join(); - } - } catch (final CompletionException ex) { - if (ex.getCause() instanceof IOException ioException) { - throw ioException; - } - throw ex; - } + public WriteData startWrite(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$startWrite(chunkX, chunkZ, compound); } @Override - public CompoundTag readData(final int chunkX, final int chunkZ) throws IOException { - try { - return this.world.getChunkSource().chunkMap.read(new ChunkPos(chunkX, chunkZ)).join().orElse(null); - } catch (final CompletionException ex) { - if (ex.getCause() instanceof IOException ioException) { - throw ioException; - } - throw ex; - } + public void finishWrite(final int chunkX, final int chunkZ, final WriteData writeData) throws IOException { + ((ChunkSystemChunkMap)this.world.getChunkSource().chunkMap).moonrise$writeFinishCallback(new ChunkPos(chunkX, chunkZ)); + ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$finishWrite(chunkX, chunkZ, writeData); + } + + @Override + public ReadData readData(final int chunkX, final int chunkZ) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$readData(chunkX, chunkZ); + } + + @Override + public CompoundTag finishRead(final int chunkX, final int chunkZ, final ReadData readData) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$finishRead(chunkX, chunkZ, readData); } } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/EntityDataController.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/EntityDataController.java index fdd189ef..828c868f 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/EntityDataController.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/EntityDataController.java @@ -1,6 +1,8 @@ package ca.spottedleaf.moonrise.patches.chunk_system.io.datacontroller; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.ChunkSystemRegionFileStorage; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; +import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import net.minecraft.nbt.CompoundTag; import net.minecraft.world.level.ChunkPos; import net.minecraft.world.level.chunk.storage.EntityStorage; @@ -9,12 +11,12 @@ import java.io.IOException; import java.nio.file.Path; -public final class EntityDataController extends RegionFileIOThread.ChunkDataController { +public final class EntityDataController extends MoonriseRegionFileIO.RegionDataController { private final EntityRegionFileStorage storage; - public EntityDataController(final EntityRegionFileStorage storage) { - super(RegionFileIOThread.RegionFileType.ENTITY_DATA); + public EntityDataController(final EntityRegionFileStorage storage, final ChunkTaskScheduler taskScheduler) { + super(MoonriseRegionFileIO.RegionFileType.ENTITY_DATA, taskScheduler.ioExecutor, taskScheduler.compressionExecutor); this.storage = storage; } @@ -24,13 +26,35 @@ public RegionFileStorage getCache() { } @Override - public void writeData(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException { - this.storage.write(new ChunkPos(chunkX, chunkZ), compound); + public WriteData startWrite(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException { + checkPosition(new ChunkPos(chunkX, chunkZ), compound); + + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$startWrite(chunkX, chunkZ, compound); + } + + @Override + public void finishWrite(final int chunkX, final int chunkZ, final WriteData writeData) throws IOException { + ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$finishWrite(chunkX, chunkZ, writeData); + } + + @Override + public ReadData readData(final int chunkX, final int chunkZ) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$readData(chunkX, chunkZ); } @Override - public CompoundTag readData(final int chunkX, final int chunkZ) throws IOException { - return this.storage.read(new ChunkPos(chunkX, chunkZ)); + public CompoundTag finishRead(final int chunkX, final int chunkZ, final ReadData readData) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$finishRead(chunkX, chunkZ, readData); + } + + private static void checkPosition(final ChunkPos pos, final CompoundTag nbt) { + final ChunkPos nbtPos = nbt == null ? null : EntityStorage.readChunkPos(nbt); + if (nbtPos != null && !pos.equals(nbtPos)) { + throw new IllegalArgumentException( + "Entity chunk coordinate and serialized data do not have matching coordinates, trying to serialize coordinate " + pos.toString() + + " but compound says coordinate is " + nbtPos + ); + } } public static final class EntityRegionFileStorage extends RegionFileStorage { @@ -42,13 +66,7 @@ public EntityRegionFileStorage(final RegionStorageInfo regionStorageInfo, final @Override public void write(final ChunkPos pos, final CompoundTag nbt) throws IOException { - final ChunkPos nbtPos = nbt == null ? null : EntityStorage.readChunkPos(nbt); - if (nbtPos != null && !pos.equals(nbtPos)) { - throw new IllegalArgumentException( - "Entity chunk coordinate and serialized data do not have matching coordinates, trying to serialize coordinate " + pos.toString() - + " but compound says coordinate is " + nbtPos + " for world: " + this - ); - } + checkPosition(pos, nbt); super.write(pos, nbt); } } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/PoiDataController.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/PoiDataController.java index af867f8f..bd0d7828 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/PoiDataController.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/io/datacontroller/PoiDataController.java @@ -1,18 +1,20 @@ package ca.spottedleaf.moonrise.patches.chunk_system.io.datacontroller; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.ChunkSystemRegionFileStorage; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.level.storage.ChunkSystemSectionStorage; +import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import net.minecraft.nbt.CompoundTag; import net.minecraft.server.level.ServerLevel; import net.minecraft.world.level.chunk.storage.RegionFileStorage; import java.io.IOException; -public final class PoiDataController extends RegionFileIOThread.ChunkDataController { +public final class PoiDataController extends MoonriseRegionFileIO.RegionDataController { private final ServerLevel world; - public PoiDataController(final ServerLevel world) { - super(RegionFileIOThread.RegionFileType.POI_DATA); + public PoiDataController(final ServerLevel world, final ChunkTaskScheduler taskScheduler) { + super(MoonriseRegionFileIO.RegionFileType.POI_DATA, taskScheduler.ioExecutor, taskScheduler.compressionExecutor); this.world = world; } @@ -22,12 +24,22 @@ public RegionFileStorage getCache() { } @Override - public void writeData(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException { - ((ChunkSystemSectionStorage)this.world.getPoiManager()).moonrise$write(chunkX, chunkZ, compound); + public WriteData startWrite(final int chunkX, final int chunkZ, final CompoundTag compound) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$startWrite(chunkX, chunkZ, compound); } @Override - public CompoundTag readData(final int chunkX, final int chunkZ) throws IOException { - return ((ChunkSystemSectionStorage)this.world.getPoiManager()).moonrise$read(chunkX, chunkZ); + public void finishWrite(final int chunkX, final int chunkZ, final WriteData writeData) throws IOException { + ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$finishWrite(chunkX, chunkZ, writeData); + } + + @Override + public ReadData readData(final int chunkX, final int chunkZ) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$readData(chunkX, chunkZ); + } + + @Override + public CompoundTag finishRead(final int chunkX, final int chunkZ, final ReadData readData) throws IOException { + return ((ChunkSystemRegionFileStorage)this.getCache()).moonrise$finishRead(chunkX, chunkZ, readData); } } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemChunkMap.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemChunkMap.java new file mode 100644 index 00000000..47a4d337 --- /dev/null +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemChunkMap.java @@ -0,0 +1,10 @@ +package ca.spottedleaf.moonrise.patches.chunk_system.level; + +import net.minecraft.world.level.ChunkPos; +import java.io.IOException; + +public interface ChunkSystemChunkMap { + + public void moonrise$writeFinishCallback(final ChunkPos pos) throws IOException; + +} diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemServerLevel.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemServerLevel.java index 93583ea9..39a27550 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemServerLevel.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/level/ChunkSystemServerLevel.java @@ -1,9 +1,9 @@ package ca.spottedleaf.moonrise.patches.chunk_system.level; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.list.ReferenceList; import ca.spottedleaf.moonrise.common.misc.NearbyPlayers; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.player.RegionizedPlayerChunkLoader; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import net.minecraft.core.BlockPos; @@ -17,11 +17,11 @@ public interface ChunkSystemServerLevel extends ChunkSystemLevel { public ChunkTaskScheduler moonrise$getChunkTaskScheduler(); - public RegionFileIOThread.ChunkDataController moonrise$getChunkDataController(); + public MoonriseRegionFileIO.RegionDataController moonrise$getChunkDataController(); - public RegionFileIOThread.ChunkDataController moonrise$getPoiChunkDataController(); + public MoonriseRegionFileIO.RegionDataController moonrise$getPoiChunkDataController(); - public RegionFileIOThread.ChunkDataController moonrise$getEntityChunkDataController(); + public MoonriseRegionFileIO.RegionDataController moonrise$getEntityChunkDataController(); public int moonrise$getRegionChunkShift(); @@ -32,19 +32,19 @@ public interface ChunkSystemServerLevel extends ChunkSystemLevel { public RegionizedPlayerChunkLoader moonrise$getPlayerChunkLoader(); public void moonrise$loadChunksAsync(final BlockPos pos, final int radiusBlocks, - final PrioritisedExecutor.Priority priority, + final Priority priority, final Consumer> onLoad); public void moonrise$loadChunksAsync(final BlockPos pos, final int radiusBlocks, - final ChunkStatus chunkStatus, final PrioritisedExecutor.Priority priority, + final ChunkStatus chunkStatus, final Priority priority, final Consumer> onLoad); public void moonrise$loadChunksAsync(final int minChunkX, final int maxChunkX, final int minChunkZ, final int maxChunkZ, - final PrioritisedExecutor.Priority priority, + final Priority priority, final Consumer> onLoad); public void moonrise$loadChunksAsync(final int minChunkX, final int maxChunkX, final int minChunkZ, final int maxChunkZ, - final ChunkStatus chunkStatus, final PrioritisedExecutor.Priority priority, + final ChunkStatus chunkStatus, final Priority priority, final Consumer> onLoad); public RegionizedPlayerChunkLoader.ViewDistanceHolder moonrise$getViewDistanceHolder(); diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/player/RegionizedPlayerChunkLoader.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/player/RegionizedPlayerChunkLoader.java index 7f94a05e..f6a0b658 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/player/RegionizedPlayerChunkLoader.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/player/RegionizedPlayerChunkLoader.java @@ -1,7 +1,7 @@ package ca.spottedleaf.moonrise.patches.chunk_system.player; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.misc.AllocatingRateLimiter; import ca.spottedleaf.moonrise.common.misc.SingleUserAreaMap; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; @@ -709,7 +709,7 @@ void updateQueues(final long time) { final int queuedChunkX = CoordinateUtils.getChunkX(queuedLoadChunk); final int queuedChunkZ = CoordinateUtils.getChunkZ(queuedLoadChunk); ((ChunkSystemServerLevel)this.world).moonrise$getChunkTaskScheduler().scheduleChunkLoad( - queuedChunkX, queuedChunkZ, ChunkStatus.EMPTY, false, PrioritisedExecutor.Priority.NORMAL, null + queuedChunkX, queuedChunkZ, ChunkStatus.EMPTY, false, Priority.NORMAL, null ); if (this.removed) { return; diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkHolderManager.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkHolderManager.java index 76275935..e1d31b02 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkHolderManager.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkHolderManager.java @@ -1,14 +1,14 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.lock.ReentrantAreaLock; import ca.spottedleaf.concurrentutil.map.ConcurrentLong2ReferenceChainedHashTable; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; import ca.spottedleaf.moonrise.common.util.MoonriseCommon; import ca.spottedleaf.moonrise.common.util.TickThread; import ca.spottedleaf.moonrise.common.util.WorldUtil; import ca.spottedleaf.moonrise.common.util.ChunkSystem; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.level.entity.ChunkEntitySlices; import ca.spottedleaf.moonrise.patches.chunk_system.level.poi.PoiChunk; @@ -189,7 +189,7 @@ public void close(final boolean save, final boolean halt) { if (halt) { LOGGER.info("Waiting 60s for chunk system to halt for world '" + WorldUtil.getWorldName(this.world) + "'"); if (!this.taskScheduler.halt(true, TimeUnit.SECONDS.toNanos(60L))) { - LOGGER.warn("Failed to halt world generation/loading tasks for world '" + WorldUtil.getWorldName(this.world) + "'"); + LOGGER.warn("Failed to halt generation/loading tasks for world '" + WorldUtil.getWorldName(this.world) + "'"); } else { LOGGER.info("Halted chunk system for world '" + WorldUtil.getWorldName(this.world) + "'"); } @@ -199,21 +199,21 @@ public void close(final boolean save, final boolean halt) { this.saveAllChunks(true, true, true); } - boolean hasTasks = false; - for (final RegionFileIOThread.RegionFileType type : RegionFileIOThread.RegionFileType.values()) { - if (RegionFileIOThread.getControllerFor(this.world, type).hasTasks()) { - hasTasks = true; - break; + MoonriseRegionFileIO.flush(this.world); + + if (halt) { + LOGGER.info("Waiting 60s for chunk I/O to halt for world '" + WorldUtil.getWorldName(this.world) + "'"); + if (!this.taskScheduler.haltIO(true, TimeUnit.SECONDS.toNanos(60L))) { + LOGGER.warn("Failed to halt I/O tasks for world '" + WorldUtil.getWorldName(this.world) + "'"); + } else { + LOGGER.info("Halted I/O scheduler for world '" + WorldUtil.getWorldName(this.world) + "'"); } } - if (hasTasks) { - RegionFileIOThread.flush(); - } // kill regionfile cache - for (final RegionFileIOThread.RegionFileType type : RegionFileIOThread.RegionFileType.values()) { + for (final MoonriseRegionFileIO.RegionFileType type : MoonriseRegionFileIO.RegionFileType.values()) { try { - RegionFileIOThread.getControllerFor(this.world, type).getCache().close(); + MoonriseRegionFileIO.getControllerFor(this.world, type).getCache().close(); } catch (final IOException ex) { LOGGER.error("Failed to close '" + type.name() + "' regionfile cache for world '" + WorldUtil.getWorldName(this.world) + "'", ex); } @@ -271,8 +271,8 @@ public void saveAllChunks(final boolean flush, final boolean shutdown, final boo long start = System.nanoTime(); long lastLog = start; - boolean needsFlush = false; - final int flushInterval = 50; + final int flushInterval = 200; + int lastFlush = 0; int savedChunk = 0; int savedEntity = 0; @@ -283,43 +283,52 @@ public void saveAllChunks(final boolean flush, final boolean shutdown, final boo try { final NewChunkHolder.SaveStat saveStat = holder.save(shutdown); if (saveStat != null) { - ++saved; - needsFlush = flush; if (saveStat.savedChunk()) { ++savedChunk; + ++saved; } if (saveStat.savedEntityChunk()) { ++savedEntity; + ++saved; } if (saveStat.savedPoiChunk()) { ++savedPoi; + ++saved; } } } catch (final Throwable thr) { LOGGER.error("Failed to save chunk (" + holder.chunkX + "," + holder.chunkZ + ") in world '" + WorldUtil.getWorldName(this.world) + "'", thr); } - if (needsFlush && (saved % flushInterval) == 0) { - needsFlush = false; - RegionFileIOThread.partialFlush(flushInterval / 2); + if (flush && (saved - lastFlush) > (flushInterval / 2)) { + lastFlush = saved; + MoonriseRegionFileIO.partialFlush(this.world, flushInterval / 2); } if (logProgress) { final long currTime = System.nanoTime(); if ((currTime - lastLog) > TimeUnit.SECONDS.toNanos(10L)) { lastLog = currTime; - LOGGER.info("Saved " + saved + " chunks (" + format.format((double)(i+1)/(double)len * 100.0) + "%) in world '" + WorldUtil.getWorldName(this.world) + "'"); + LOGGER.info( + "Saved " + savedChunk + " block chunks, " + savedEntity + " entity chunks, " + savedPoi + + " poi chunks in world '" + WorldUtil.getWorldName(this.world) + "', progress: " + + format.format((double)(i+1)/(double)len * 100.0) + ); } } } if (flush) { - RegionFileIOThread.flush(); + MoonriseRegionFileIO.flush(this.world); try { - RegionFileIOThread.flushRegionStorages(this.world); + MoonriseRegionFileIO.flushRegionStorages(this.world); } catch (final IOException ex) { LOGGER.error("Exception when flushing regions in world '" + WorldUtil.getWorldName(this.world) + "'", ex); } } if (logProgress) { - LOGGER.info("Saved " + savedChunk + " block chunks, " + savedEntity + " entity chunks, " + savedPoi + " poi chunks in world '" + WorldUtil.getWorldName(this.world) + "' in " + format.format(1.0E-9 * (System.nanoTime() - start)) + "s"); + LOGGER.info( + "Saved " + savedChunk + " block chunks, " + savedEntity + " entity chunks, " + savedPoi + + " poi chunks in world '" + WorldUtil.getWorldName(this.world) + "' in " + + format.format(1.0E-9 * (System.nanoTime() - start)) + "s" + ); } } @@ -772,21 +781,21 @@ public NewChunkHolder getChunkHolder(final long position) { return this.chunkHolders.get(position); } - public void raisePriority(final int x, final int z, final PrioritisedExecutor.Priority priority) { + public void raisePriority(final int x, final int z, final Priority priority) { final NewChunkHolder chunkHolder = this.getChunkHolder(x, z); if (chunkHolder != null) { chunkHolder.raisePriority(priority); } } - public void setPriority(final int x, final int z, final PrioritisedExecutor.Priority priority) { + public void setPriority(final int x, final int z, final Priority priority) { final NewChunkHolder chunkHolder = this.getChunkHolder(x, z); if (chunkHolder != null) { chunkHolder.setPriority(priority); } } - public void lowerPriority(final int x, final int z, final PrioritisedExecutor.Priority priority) { + public void lowerPriority(final int x, final int z, final Priority priority) { final NewChunkHolder chunkHolder = this.getChunkHolder(x, z); if (chunkHolder != null) { chunkHolder.lowerPriority(priority); @@ -869,7 +878,7 @@ public ChunkEntitySlices getOrCreateEntityChunk(final int chunkX, final int chun final ChunkLoadTask.EntityDataLoadTask entityLoad = current.getEntityDataLoadTask(); if (entityLoad != null) { - entityLoad.raisePriority(PrioritisedExecutor.Priority.BLOCKING); + entityLoad.raisePriority(Priority.BLOCKING); } } } @@ -945,7 +954,7 @@ public PoiChunk loadPoiChunk(final int chunkX, final int chunkZ) { final ChunkLoadTask.PoiDataLoadTask poiLoad = current.getPoiDataLoadTask(); if (poiLoad != null) { - poiLoad.raisePriority(PrioritisedExecutor.Priority.BLOCKING); + poiLoad.raisePriority(Priority.BLOCKING); } } } finally { @@ -992,7 +1001,7 @@ void addChangedStatuses(final List changedFullStatus) { } ChunkHolderManager.this.processPendingFullUpdate(); - }, PrioritisedExecutor.Priority.HIGHEST); + }, Priority.HIGHEST); } else { final ArrayDeque pendingFullLoadUpdate = this.pendingFullLoadUpdate; for (int i = 0, len = changedFullStatus.size(); i < len; ++i) { diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkTaskScheduler.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkTaskScheduler.java index 888d6ba8..c79f29ac 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkTaskScheduler.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/ChunkTaskScheduler.java @@ -1,16 +1,17 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedThreadPool; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedThreadedTaskQueue; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.queue.PrioritisedTaskQueue; +import ca.spottedleaf.concurrentutil.executor.thread.PrioritisedThreadPool; import ca.spottedleaf.concurrentutil.lock.ReentrantAreaLock; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; +import ca.spottedleaf.moonrise.common.config.moonrise.MoonriseConfig; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; import ca.spottedleaf.moonrise.common.util.JsonUtil; import ca.spottedleaf.moonrise.common.util.MoonriseCommon; import ca.spottedleaf.moonrise.common.util.TickThread; import ca.spottedleaf.moonrise.common.util.WorldUtil; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.level.chunk.ChunkSystemChunkStatus; import ca.spottedleaf.moonrise.patches.chunk_system.player.ChunkSystemServerPlayer; @@ -54,7 +55,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Objects; @@ -66,34 +66,18 @@ public final class ChunkTaskScheduler { private static final Logger LOGGER = LoggerFactory.getLogger(ChunkTaskScheduler.class); - static int newChunkSystemIOThreads; - static int newChunkSystemGenParallelism; - static int newChunkSystemGenPopulationParallelism; - static int newChunkSystemLoadParallelism; + public static void init(final MoonriseConfig.ChunkSystem config) { + final int newChunkSystemIOThreads = Math.max(1, config.ioThreads); - private static boolean initialised = false; + final boolean useParallelGen = config.populationGenParallelism; - public static void init() { - if (initialised) { - return; - } - initialised = true; - newChunkSystemIOThreads = MoonriseCommon.getConfig().chunkSystem.ioThreads; - if (newChunkSystemIOThreads <= 0) { - newChunkSystemIOThreads = 1; - } else { - newChunkSystemIOThreads = Math.max(1, newChunkSystemIOThreads); - } + MoonriseCommon.IO_POOL.adjustThreadCount(newChunkSystemIOThreads); - boolean useParallelGen = MoonriseCommon.getConfig().chunkSystem.populationGenParallelism; - - ChunkTaskScheduler.newChunkSystemGenParallelism = MoonriseCommon.WORKER_THREADS; - ChunkTaskScheduler.newChunkSystemGenPopulationParallelism = useParallelGen ? MoonriseCommon.WORKER_THREADS : 1; - ChunkTaskScheduler.newChunkSystemLoadParallelism = MoonriseCommon.WORKER_THREADS; - - RegionFileIOThread.init(newChunkSystemIOThreads); + for (final PrioritisedThreadPool.ExecutorGroup.ThreadPoolExecutor executor : MoonriseCommon.RADIUS_AWARE_GROUP.getAllExecutors()) { + executor.setMaxParallelism(useParallelGen ? -1 : 1); + } - LOGGER.info("Chunk system is using " + newChunkSystemIOThreads + " I/O threads, " + MoonriseCommon.WORKER_THREADS + " worker threads, and population gen parallelism of " + ChunkTaskScheduler.newChunkSystemGenPopulationParallelism + " threads"); + LOGGER.info("Chunk system is using " + newChunkSystemIOThreads + " I/O threads, " + MoonriseCommon.WORKER_POOL.getCoreThreads().length + " worker threads, population gen parallelism: " + useParallelGen); } public static final TicketType CHUNK_LOAD = TicketType.create("chunk_system:chunk_load", Long::compareTo); @@ -137,13 +121,14 @@ public static int getTicketLevel(final ChunkStatus status) { } public final ServerLevel world; - public final PrioritisedThreadPool workers; public final RadiusAwarePrioritisedExecutor radiusAwareScheduler; - public final PrioritisedThreadPool.PrioritisedPoolExecutor parallelGenExecutor; - private final PrioritisedThreadPool.PrioritisedPoolExecutor radiusAwareGenExecutor; - public final PrioritisedThreadPool.PrioritisedPoolExecutor loadExecutor; + public final PrioritisedThreadPool.ExecutorGroup.ThreadPoolExecutor parallelGenExecutor; + private final PrioritisedThreadPool.ExecutorGroup.ThreadPoolExecutor radiusAwareGenExecutor; + public final PrioritisedThreadPool.ExecutorGroup.ThreadPoolExecutor loadExecutor; + public final PrioritisedThreadPool.ExecutorGroup.ThreadPoolExecutor ioExecutor; + public final PrioritisedThreadPool.ExecutorGroup.ThreadPoolExecutor compressionExecutor; - private final PrioritisedThreadedTaskQueue mainThreadExecutor = new PrioritisedThreadedTaskQueue(); + private final PrioritisedTaskQueue mainThreadExecutor = new PrioritisedTaskQueue(); public final ChunkHolderManager chunkHolderManager; @@ -292,9 +277,8 @@ public final int getChunkSystemLockShift() { return this.lockShift; } - public ChunkTaskScheduler(final ServerLevel world, final PrioritisedThreadPool workers) { + public ChunkTaskScheduler(final ServerLevel world) { this.world = world; - this.workers = workers; // must be >= region shift (in paper, doesn't exist) and must be >= ticket propagator section shift // it must be >= region shift since the regioniser assumes ticket updates do not occur in parallel for the region sections // it must be >= ticket propagator section shift so that the ticket propagator can assume that owning a position implies owning @@ -303,11 +287,17 @@ public ChunkTaskScheduler(final ServerLevel world, final PrioritisedThreadPool w this.lockShift = Math.max(((ChunkSystemServerLevel)world).moonrise$getRegionChunkShift(), ThreadedTicketLevelPropagator.SECTION_SHIFT); this.schedulingLockArea = new ReentrantAreaLock(this.getChunkSystemLockShift()); + final MoonriseConfig config = MoonriseCommon.getConfig(); + final String worldName = WorldUtil.getWorldName(world); - this.parallelGenExecutor = workers.createExecutor("Chunk parallel generation executor for world '" + worldName + "'", 1, Math.max(1, newChunkSystemGenParallelism)); - this.radiusAwareGenExecutor = workers.createExecutor("Chunk radius aware generator for world '" + worldName + "'", 1, Math.max(1, newChunkSystemGenPopulationParallelism)); - this.loadExecutor = workers.createExecutor("Chunk load executor for world '" + worldName + "'", 1, newChunkSystemLoadParallelism); - this.radiusAwareScheduler = new RadiusAwarePrioritisedExecutor(this.radiusAwareGenExecutor, Math.max(2, 1 + newChunkSystemGenPopulationParallelism)); + this.parallelGenExecutor = MoonriseCommon.PARALLEL_GEN_GROUP.createExecutor(-1, MoonriseCommon.WORKER_QUEUE_HOLD_TIME, 0); + this.radiusAwareGenExecutor = MoonriseCommon.RADIUS_AWARE_GROUP.createExecutor(config.chunkSystem.populationGenParallelism ? -1 : 1, MoonriseCommon.WORKER_QUEUE_HOLD_TIME, 0); + this.loadExecutor = MoonriseCommon.LOAD_GROUP.createExecutor(-1, MoonriseCommon.WORKER_QUEUE_HOLD_TIME, 0); + // TODO proper max to schedule + this.radiusAwareScheduler = new RadiusAwarePrioritisedExecutor(this.radiusAwareGenExecutor, Math.max(2, 1 + 1)); + this.ioExecutor = MoonriseCommon.SERVER_REGION_IO_GROUP.createExecutor(-1, MoonriseCommon.IO_QUEUE_HOLD_TIME, 0); + // we need a separate executor here so that on shutdown we can continue to process I/O tasks + this.compressionExecutor = MoonriseCommon.LOAD_GROUP.createExecutor(-1, MoonriseCommon.WORKER_QUEUE_HOLD_TIME, 0); this.chunkHolderManager = new ChunkHolderManager(world, this); } @@ -346,7 +336,7 @@ public void unrecoverableChunkSystemFailure(final int chunkX, final int chunkZ, }; // this may not be good enough, specifically thanks to stupid ass plugins swallowing exceptions - this.scheduleChunkTask(chunkX, chunkZ, crash, PrioritisedExecutor.Priority.BLOCKING); + this.scheduleChunkTask(chunkX, chunkZ, crash, Priority.BLOCKING); // so, make the main thread pick it up ((ChunkSystemMinecraftServer)this.world.getServer()).moonrise$setChunkSystemCrash(new RuntimeException("Chunk system crash propagated from unrecoverableChunkSystemFailure", reportedException)); } @@ -356,20 +346,20 @@ public boolean executeMainThreadTask() { return this.mainThreadExecutor.executeTask(); } - public void raisePriority(final int x, final int z, final PrioritisedExecutor.Priority priority) { + public void raisePriority(final int x, final int z, final Priority priority) { this.chunkHolderManager.raisePriority(x, z, priority); } - public void setPriority(final int x, final int z, final PrioritisedExecutor.Priority priority) { + public void setPriority(final int x, final int z, final Priority priority) { this.chunkHolderManager.setPriority(x, z, priority); } - public void lowerPriority(final int x, final int z, final PrioritisedExecutor.Priority priority) { + public void lowerPriority(final int x, final int z, final Priority priority) { this.chunkHolderManager.lowerPriority(x, z, priority); } public void scheduleTickingState(final int chunkX, final int chunkZ, final FullChunkStatus toStatus, - final boolean addTicket, final PrioritisedExecutor.Priority priority, + final boolean addTicket, final Priority priority, final Consumer onComplete) { final int radius = toStatus.ordinal() - 1; // 0 -> BORDER, 1 -> TICKING, 2 -> ENTITY_TICKING @@ -465,7 +455,7 @@ public void scheduleTickingState(final int chunkX, final int chunkZ, final FullC } public void scheduleChunkLoad(final int chunkX, final int chunkZ, final boolean gen, final ChunkStatus toStatus, final boolean addTicket, - final PrioritisedExecutor.Priority priority, final Consumer onComplete) { + final Priority priority, final Consumer onComplete) { if (gen) { this.scheduleChunkLoad(chunkX, chunkZ, toStatus, addTicket, priority, onComplete); return; @@ -489,7 +479,7 @@ public void scheduleChunkLoad(final int chunkX, final int chunkZ, final boolean // only appropriate to use with syncLoadNonFull public boolean beginChunkLoadForNonFullSync(final int chunkX, final int chunkZ, final ChunkStatus toStatus, - final PrioritisedExecutor.Priority priority) { + final Priority priority) { final int accessRadius = getAccessRadius(toStatus); final long chunkKey = CoordinateUtils.getChunkKey(chunkX, chunkZ); final int minLevel = ChunkTaskScheduler.getTicketLevel(toStatus); @@ -544,7 +534,7 @@ public ChunkAccess syncLoadNonFull(final int chunkX, final int chunkZ, final Chu this.chunkHolderManager.addTicketAtLevel(NON_FULL_CHUNK_LOAD, chunkX, chunkZ, ticketLevel, ticketId); this.chunkHolderManager.processTicketUpdates(); - this.beginChunkLoadForNonFullSync(chunkX, chunkZ, status, PrioritisedExecutor.Priority.BLOCKING); + this.beginChunkLoadForNonFullSync(chunkX, chunkZ, status, Priority.BLOCKING); // we could do a simple spinwait here, since we do not need to process tasks while performing this load // but we process tasks only because it's a better use of the time spent @@ -564,7 +554,7 @@ public ChunkAccess syncLoadNonFull(final int chunkX, final int chunkZ, final Chu } public void scheduleChunkLoad(final int chunkX, final int chunkZ, final ChunkStatus toStatus, final boolean addTicket, - final PrioritisedExecutor.Priority priority, final Consumer onComplete) { + final Priority priority, final Consumer onComplete) { if (!TickThread.isTickThreadFor(this.world, chunkX, chunkZ)) { this.scheduleChunkTask(chunkX, chunkZ, () -> { ChunkTaskScheduler.this.scheduleChunkLoad(chunkX, chunkZ, toStatus, addTicket, priority, onComplete); @@ -656,7 +646,7 @@ public void scheduleChunkLoad(final int chunkX, final int chunkZ, final ChunkSta private ChunkProgressionTask createTask(final int chunkX, final int chunkZ, final ChunkAccess chunk, final NewChunkHolder chunkHolder, final StaticCache2D neighbours, - final ChunkStatus toStatus, final PrioritisedExecutor.Priority initialPriority) { + final ChunkStatus toStatus, final Priority initialPriority) { if (toStatus == ChunkStatus.EMPTY) { return new ChunkLoadTask(this, this.world, chunkX, chunkZ, chunkHolder, initialPriority); } @@ -672,7 +662,7 @@ private ChunkProgressionTask createTask(final int chunkX, final int chunkZ, fina ChunkProgressionTask schedule(final int chunkX, final int chunkZ, final ChunkStatus targetStatus, final NewChunkHolder chunkHolder, final List allTasks) { - return this.schedule(chunkX, chunkZ, targetStatus, chunkHolder, allTasks, chunkHolder.getEffectivePriority(PrioritisedExecutor.Priority.NORMAL)); + return this.schedule(chunkX, chunkZ, targetStatus, chunkHolder, allTasks, chunkHolder.getEffectivePriority(Priority.NORMAL)); } // rets new task scheduled for the _specified_ chunk @@ -681,7 +671,7 @@ ChunkProgressionTask schedule(final int chunkX, final int chunkZ, final ChunkSta // schedule will ignore the generation target, so it should be checked by the caller to ensure the target is not regressed! private ChunkProgressionTask schedule(final int chunkX, final int chunkZ, final ChunkStatus targetStatus, final NewChunkHolder chunkHolder, final List allTasks, - final PrioritisedExecutor.Priority minPriority) { + final Priority minPriority) { if (!this.schedulingLockArea.isHeldByCurrentThread(chunkX, chunkZ, getAccessRadius(targetStatus))) { throw new IllegalStateException("Not holding scheduling lock"); } @@ -691,8 +681,8 @@ private ChunkProgressionTask schedule(final int chunkX, final int chunkZ, final return null; } - final PrioritisedExecutor.Priority requestedPriority = PrioritisedExecutor.Priority.max( - minPriority, chunkHolder.getEffectivePriority(PrioritisedExecutor.Priority.NORMAL) + final Priority requestedPriority = Priority.max( + minPriority, chunkHolder.getEffectivePriority(Priority.NORMAL) ); final ChunkStatus currentGenStatus = chunkHolder.getCurrentGenStatus(); final ChunkAccess chunk = chunkHolder.getCurrentChunk(); @@ -769,7 +759,7 @@ private ChunkProgressionTask schedule(final int chunkX, final int chunkZ, final final ChunkProgressionTask task = this.createTask( chunkX, chunkZ, chunk, chunkHolder, neighbours, toStatus, - chunkHolder.getEffectivePriority(PrioritisedExecutor.Priority.NORMAL) + chunkHolder.getEffectivePriority(Priority.NORMAL) ); allTasks.add(task); @@ -780,7 +770,7 @@ private ChunkProgressionTask schedule(final int chunkX, final int chunkZ, final // rets true if the neighbour is not at the required status, false otherwise private boolean checkNeighbour(final int chunkX, final int chunkZ, final ChunkStatus requiredStatus, final NewChunkHolder center, - final List tasks, final PrioritisedExecutor.Priority minPriority) { + final List tasks, final Priority minPriority) { final NewChunkHolder chunkHolder = this.chunkHolderManager.getChunkHolder(chunkX, chunkZ); if (chunkHolder == null) { @@ -816,41 +806,41 @@ private boolean checkNeighbour(final int chunkX, final int chunkZ, final ChunkSt */ @Deprecated public PrioritisedExecutor.PrioritisedTask scheduleChunkTask(final Runnable run) { - return this.scheduleChunkTask(run, PrioritisedExecutor.Priority.NORMAL); + return this.scheduleChunkTask(run, Priority.NORMAL); } /** * @deprecated Chunk tasks must be tied to coordinates in the future */ @Deprecated - public PrioritisedExecutor.PrioritisedTask scheduleChunkTask(final Runnable run, final PrioritisedExecutor.Priority priority) { - return this.mainThreadExecutor.queueRunnable(run, priority); + public PrioritisedExecutor.PrioritisedTask scheduleChunkTask(final Runnable run, final Priority priority) { + return this.mainThreadExecutor.queueTask(run, priority); } public PrioritisedExecutor.PrioritisedTask createChunkTask(final int chunkX, final int chunkZ, final Runnable run) { - return this.createChunkTask(chunkX, chunkZ, run, PrioritisedExecutor.Priority.NORMAL); + return this.createChunkTask(chunkX, chunkZ, run, Priority.NORMAL); } public PrioritisedExecutor.PrioritisedTask createChunkTask(final int chunkX, final int chunkZ, final Runnable run, - final PrioritisedExecutor.Priority priority) { + final Priority priority) { return this.mainThreadExecutor.createTask(run, priority); } public PrioritisedExecutor.PrioritisedTask scheduleChunkTask(final int chunkX, final int chunkZ, final Runnable run) { - return this.scheduleChunkTask(chunkX, chunkZ, run, PrioritisedExecutor.Priority.NORMAL); + return this.scheduleChunkTask(chunkX, chunkZ, run, Priority.NORMAL); } public PrioritisedExecutor.PrioritisedTask scheduleChunkTask(final int chunkX, final int chunkZ, final Runnable run, - final PrioritisedExecutor.Priority priority) { - return this.mainThreadExecutor.queueRunnable(run, priority); + final Priority priority) { + return this.mainThreadExecutor.queueTask(run, priority); } public boolean halt(final boolean sync, final long maxWaitNS) { this.radiusAwareGenExecutor.halt(); this.parallelGenExecutor.halt(); this.loadExecutor.halt(); - final long time = System.nanoTime(); if (sync) { + final long time = System.nanoTime(); for (long failures = 9L;; failures = ConcurrentUtil.linearLongBackoff(failures, 500_000L, 50_000_000L)) { if ( !this.radiusAwareGenExecutor.isActive() && @@ -868,6 +858,27 @@ public boolean halt(final boolean sync, final long maxWaitNS) { return true; } + public boolean haltIO(final boolean sync, final long maxWaitNS) { + this.ioExecutor.halt(); + this.compressionExecutor.halt(); + if (sync) { + final long time = System.nanoTime(); + for (long failures = 9L;; failures = ConcurrentUtil.linearLongBackoff(failures, 500_000L, 50_000_000L)) { + if ( + !this.ioExecutor.isActive() && + !this.compressionExecutor.isActive() + ) { + return true; + } + if ((System.nanoTime() - time) >= maxWaitNS) { + return false; + } + } + } + + return true; + } + public static final ArrayDeque WAITING_CHUNKS = new ArrayDeque<>(); // stack public static final class ChunkInfo { diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/NewChunkHolder.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/NewChunkHolder.java index fcfc8493..b82038d1 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/NewChunkHolder.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/NewChunkHolder.java @@ -1,18 +1,19 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling; -import ca.spottedleaf.concurrentutil.completable.Completable; +import ca.spottedleaf.concurrentutil.completable.CallbackCompletable; import ca.spottedleaf.concurrentutil.executor.Cancellable; -import ca.spottedleaf.concurrentutil.executor.standard.DelayedPrioritisedTask; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.lock.ReentrantAreaLock; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; +import ca.spottedleaf.moonrise.common.misc.LazyRunnable; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; import ca.spottedleaf.moonrise.common.util.TickThread; import ca.spottedleaf.moonrise.common.util.WorldUtil; import ca.spottedleaf.moonrise.common.util.ChunkSystem; import ca.spottedleaf.moonrise.patches.chunk_system.ChunkSystemFeatures; import ca.spottedleaf.moonrise.patches.chunk_system.async_save.AsyncChunkSaveData; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.level.chunk.ChunkSystemChunkHolder; import ca.spottedleaf.moonrise.patches.chunk_system.level.chunk.ChunkSystemChunkStatus; @@ -173,7 +174,7 @@ private void completeEntityLoad(final GenericDataLoadTask.TaskResult(); @@ -294,7 +295,7 @@ private void completePoiLoad(final GenericDataLoadTask.TaskResult(); @@ -519,15 +520,15 @@ public void addWaitingNeighbour(final NewChunkHolder neighbour, final ChunkStatu // priority state // the target priority for this chunk to generate at - private PrioritisedExecutor.Priority priority = null; + private Priority priority = null; private boolean priorityLocked; // the priority neighbouring chunks have requested this chunk generate at - private PrioritisedExecutor.Priority neighbourRequestedPriority = null; + private Priority neighbourRequestedPriority = null; - public PrioritisedExecutor.Priority getEffectivePriority(final PrioritisedExecutor.Priority dfl) { - final PrioritisedExecutor.Priority neighbour = this.neighbourRequestedPriority; - final PrioritisedExecutor.Priority us = this.priority; + public Priority getEffectivePriority(final Priority dfl) { + final Priority neighbour = this.neighbourRequestedPriority; + final Priority us = this.priority; if (neighbour == null) { return us == null ? dfl : us; @@ -536,7 +537,7 @@ public PrioritisedExecutor.Priority getEffectivePriority(final PrioritisedExecut return neighbour; } - return PrioritisedExecutor.Priority.max(us, neighbour); + return Priority.max(us, neighbour); } private void recalculateNeighbourRequestedPriority() { @@ -545,18 +546,18 @@ private void recalculateNeighbourRequestedPriority() { return; } - PrioritisedExecutor.Priority max = null; + Priority max = null; for (final NewChunkHolder holder : this.neighboursWaitingForUs.keySet()) { - final PrioritisedExecutor.Priority neighbourPriority = holder.getEffectivePriority(null); + final Priority neighbourPriority = holder.getEffectivePriority(null); if (neighbourPriority != null && (max == null || neighbourPriority.isHigherPriority(max))) { max = neighbourPriority; } } - final PrioritisedExecutor.Priority current = this.getEffectivePriority(PrioritisedExecutor.Priority.NORMAL); + final Priority current = this.getEffectivePriority(Priority.NORMAL); this.neighbourRequestedPriority = max; - final PrioritisedExecutor.Priority next = this.getEffectivePriority(PrioritisedExecutor.Priority.NORMAL); + final Priority next = this.getEffectivePriority(Priority.NORMAL); if (current == next) { return; @@ -578,7 +579,7 @@ public void recalculateNeighbourPriorities() { } // must hold scheduling lock - public void raisePriority(final PrioritisedExecutor.Priority priority) { + public void raisePriority(final Priority priority) { if (this.priority != null && this.priority.isHigherOrEqualPriority(priority)) { return; } @@ -591,13 +592,13 @@ private void lockPriority() { } // must hold scheduling lock - public void setPriority(final PrioritisedExecutor.Priority priority) { + public void setPriority(final Priority priority) { if (this.priorityLocked) { return; } - final PrioritisedExecutor.Priority old = this.getEffectivePriority(null); + final Priority old = this.getEffectivePriority(null); this.priority = priority; - final PrioritisedExecutor.Priority newPriority = this.getEffectivePriority(PrioritisedExecutor.Priority.NORMAL); + final Priority newPriority = this.getEffectivePriority(Priority.NORMAL); if (old != newPriority) { if (this.generationTask != null) { @@ -609,7 +610,7 @@ public void setPriority(final PrioritisedExecutor.Priority priority) { } // must hold scheduling lock - public void lowerPriority(final PrioritisedExecutor.Priority priority) { + public void lowerPriority(final Priority priority) { if (this.priority != null && this.priority.isLowerOrEqualPriority(priority)) { return; } @@ -788,9 +789,10 @@ static final record UnloadState(NewChunkHolder holder, ChunkAccess chunk, ChunkE private UnloadTask entityDataUnload; private UnloadTask poiDataUnload; - public static final record UnloadTask(Completable completable, DelayedPrioritisedTask task) {} + public static final record UnloadTask(CallbackCompletable completable, PrioritisedExecutor.PrioritisedTask task, + LazyRunnable toRun) {} - public UnloadTask getUnloadTask(final RegionFileIOThread.RegionFileType type) { + public UnloadTask getUnloadTask(final MoonriseRegionFileIO.RegionFileType type) { switch (type) { case CHUNK_DATA: return this.chunkDataUnload; @@ -803,7 +805,7 @@ public UnloadTask getUnloadTask(final RegionFileIOThread.RegionFileType type) { } } - private void removeUnloadTask(final RegionFileIOThread.RegionFileType type) { + private void removeUnloadTask(final MoonriseRegionFileIO.RegionFileType type) { switch (type) { case CHUNK_DATA: { this.chunkDataUnload = null; @@ -851,22 +853,23 @@ UnloadState unloadStage1() { this.priorityLocked = false; if (chunk != null) { - this.chunkDataUnload = new UnloadTask(new Completable<>(), new DelayedPrioritisedTask(PrioritisedExecutor.Priority.NORMAL)); + final LazyRunnable toRun = new LazyRunnable(); + this.chunkDataUnload = new UnloadTask(new CallbackCompletable<>(), this.scheduler.loadExecutor.createTask(toRun), toRun); } if (poiChunk != null) { - this.poiDataUnload = new UnloadTask(new Completable<>(), null); + this.poiDataUnload = new UnloadTask(new CallbackCompletable<>(), null, null); } if (entityChunk != null) { - this.entityDataUnload = new UnloadTask(new Completable<>(), null); + this.entityDataUnload = new UnloadTask(new CallbackCompletable<>(), null, null); } return this.unloadState = (chunk != null || entityChunk != null || poiChunk != null) ? new UnloadState(this, chunk, entityChunk, poiChunk) : null; } // data is null if failed or does not need to be saved - void completeAsyncUnloadDataSave(final RegionFileIOThread.RegionFileType type, final CompoundTag data) { + void completeAsyncUnloadDataSave(final MoonriseRegionFileIO.RegionFileType type, final CompoundTag data) { if (data != null) { - RegionFileIOThread.scheduleSave(this.world, this.chunkX, this.chunkZ, data, type); + MoonriseRegionFileIO.scheduleSave(this.world, this.chunkX, this.chunkZ, data, type); } this.getUnloadTask(type).completable().complete(data); @@ -897,7 +900,7 @@ void unloadStage2(final UnloadState state) { if (!shouldLevelChunkNotSave) { this.saveChunk(chunk, true); } else { - this.completeAsyncUnloadDataSave(RegionFileIOThread.RegionFileType.CHUNK_DATA, null); + this.completeAsyncUnloadDataSave(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, null); } if (chunk instanceof LevelChunk levelChunk) { @@ -1386,7 +1389,7 @@ private void completeStatusConsumers0(final ChunkStatus status, final ChunkAcces LOGGER.error("Failed to process chunk status callback", thr); } } - }, PrioritisedExecutor.Priority.HIGHEST); + }, Priority.HIGHEST); } private final Reference2ObjectOpenHashMap>> fullStatusWaiters = new Reference2ObjectOpenHashMap<>(); @@ -1414,7 +1417,7 @@ private void completeFullStatusConsumers(FullChunkStatus status, final LevelChun LOGGER.error("Failed to process chunk status callback", thr); } } - }, PrioritisedExecutor.Priority.HIGHEST); + }, Priority.HIGHEST); } // note: must hold scheduling lock @@ -1670,6 +1673,8 @@ public ChunkEntitySlices getEntityChunk() { public static final record SaveStat(boolean savedChunk, boolean savedEntityChunk, boolean savedPoiChunk) {} + private static final MoonriseRegionFileIO.RegionFileType[] REGION_FILE_TYPES = MoonriseRegionFileIO.RegionFileType.values(); + public SaveStat save(final boolean shutdown) { TickThread.ensureTickThread(this.world, this.chunkX, this.chunkZ, "Cannot save data off-main"); @@ -1677,6 +1682,7 @@ public SaveStat save(final boolean shutdown) { PoiChunk poi = this.getPoiChunk(); ChunkEntitySlices entities = this.getEntityChunk(); boolean executedUnloadTask = false; + final boolean[] executedUnloadTasks = new boolean[REGION_FILE_TYPES.length]; if (shutdown) { // make sure that the async unloads complete @@ -1686,12 +1692,17 @@ public SaveStat save(final boolean shutdown) { poi = this.unloadState.poiChunk(); entities = this.unloadState.entityChunk(); } - final UnloadTask chunkUnloadTask = this.chunkDataUnload; - final DelayedPrioritisedTask chunkDataUnloadTask = chunkUnloadTask == null ? null : chunkUnloadTask.task(); - if (chunkDataUnloadTask != null) { - final PrioritisedExecutor.PrioritisedTask unloadTask = chunkDataUnloadTask.getTask(); - if (unloadTask != null) { - executedUnloadTask = unloadTask.execute(); + for (final MoonriseRegionFileIO.RegionFileType regionFileType : REGION_FILE_TYPES) { + final UnloadTask unloadTask = this.getUnloadTask(regionFileType); + if (unloadTask == null) { + continue; + } + + final PrioritisedExecutor.PrioritisedTask task = unloadTask.task(); + if (task != null && task.isQueued()) { + final boolean executed = task.execute(); + executedUnloadTask |= executed; + executedUnloadTasks[regionFileType.ordinal()] = executed; } } } @@ -1717,7 +1728,13 @@ public SaveStat save(final boolean shutdown) { } } - return executedUnloadTask | canSaveChunk | canSaveEntities | canSavePOI ? new SaveStat(executedUnloadTask || canSaveChunk, canSaveEntities, canSavePOI): null; + return executedUnloadTask | canSaveChunk | canSaveEntities | canSavePOI ? + new SaveStat( + canSaveChunk | executedUnloadTasks[MoonriseRegionFileIO.RegionFileType.CHUNK_DATA.ordinal()], + canSaveEntities | executedUnloadTasks[MoonriseRegionFileIO.RegionFileType.ENTITY_DATA.ordinal()], + canSavePOI | executedUnloadTasks[MoonriseRegionFileIO.RegionFileType.POI_DATA.ordinal()] + ) + : null; } static final class AsyncChunkSerializeTask implements Runnable { @@ -1749,17 +1766,17 @@ public void run() { synchronousSave = ChunkSystemFeatures.saveChunkAsync(AsyncChunkSerializeTask.this.world, AsyncChunkSerializeTask.this.chunk, AsyncChunkSerializeTask.this.asyncSaveData); } catch (final Throwable throwable2) { LOGGER.error("Failed to synchronously save chunk " + AsyncChunkSerializeTask.this.chunk.getPos() + " for world '" + WorldUtil.getWorldName(AsyncChunkSerializeTask.this.world) + "', chunk data will be lost", throwable2); - AsyncChunkSerializeTask.this.toComplete.completeAsyncUnloadDataSave(RegionFileIOThread.RegionFileType.CHUNK_DATA, null); + AsyncChunkSerializeTask.this.toComplete.completeAsyncUnloadDataSave(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, null); return; } - AsyncChunkSerializeTask.this.toComplete.completeAsyncUnloadDataSave(RegionFileIOThread.RegionFileType.CHUNK_DATA, synchronousSave); + AsyncChunkSerializeTask.this.toComplete.completeAsyncUnloadDataSave(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, synchronousSave); LOGGER.info("Successfully serialized chunk " + AsyncChunkSerializeTask.this.chunk.getPos() + " for world '" + WorldUtil.getWorldName(AsyncChunkSerializeTask.this.world) + "' synchronously"); - }, PrioritisedExecutor.Priority.HIGHEST); + }, Priority.HIGHEST); return; } - this.toComplete.completeAsyncUnloadDataSave(RegionFileIOThread.RegionFileType.CHUNK_DATA, toSerialize); + this.toComplete.completeAsyncUnloadDataSave(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, toSerialize); } @Override @@ -1773,7 +1790,7 @@ public String toString() { private boolean saveChunk(final ChunkAccess chunk, final boolean unloading) { if (!chunk.isUnsaved()) { if (unloading) { - this.completeAsyncUnloadDataSave(RegionFileIOThread.RegionFileType.CHUNK_DATA, null); + this.completeAsyncUnloadDataSave(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, null); } return false; } @@ -1784,13 +1801,11 @@ private boolean saveChunk(final ChunkAccess chunk, final boolean unloading) { try { final AsyncChunkSaveData asyncSaveData = ChunkSystemFeatures.getAsyncSaveData(this.world, chunk); - final PrioritisedExecutor.PrioritisedTask task = this.scheduler.loadExecutor.createTask(new AsyncChunkSerializeTask(this.world, chunk, asyncSaveData, this)); - - this.chunkDataUnload.task().setTask(task); + this.chunkDataUnload.toRun().setRunnable(new AsyncChunkSerializeTask(this.world, chunk, asyncSaveData, this)); chunk.setUnsaved(false); - task.queue(); + this.chunkDataUnload.task().queue(); return true; } catch (final Throwable thr) { @@ -1804,18 +1819,18 @@ private boolean saveChunk(final ChunkAccess chunk, final boolean unloading) { if (unloading) { completing = true; - this.completeAsyncUnloadDataSave(RegionFileIOThread.RegionFileType.CHUNK_DATA, save); + this.completeAsyncUnloadDataSave(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, save); if (failedAsyncPrepare) { LOGGER.info("Successfully serialized chunk data (" + this.chunkX + "," + this.chunkZ + ") in world '" + WorldUtil.getWorldName(this.world) + "' synchronously"); } } else { - RegionFileIOThread.scheduleSave(this.world, this.chunkX, this.chunkZ, save, RegionFileIOThread.RegionFileType.CHUNK_DATA); + MoonriseRegionFileIO.scheduleSave(this.world, this.chunkX, this.chunkZ, save, MoonriseRegionFileIO.RegionFileType.CHUNK_DATA); } chunk.setUnsaved(false); } catch (final Throwable thr) { LOGGER.error("Failed to save chunk data (" + this.chunkX + "," + this.chunkZ + ") in world '" + WorldUtil.getWorldName(this.world) + "'", thr); if (unloading && !completing) { - this.completeAsyncUnloadDataSave(RegionFileIOThread.RegionFileType.CHUNK_DATA, null); + this.completeAsyncUnloadDataSave(MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, null); } } @@ -1834,7 +1849,7 @@ private boolean saveEntities(final ChunkEntitySlices entities, final boolean unl return false; } try { - mergeFrom = RegionFileIOThread.loadData(this.world, this.chunkX, this.chunkZ, RegionFileIOThread.RegionFileType.ENTITY_DATA, PrioritisedExecutor.Priority.BLOCKING); + mergeFrom = MoonriseRegionFileIO.loadData(this.world, this.chunkX, this.chunkZ, MoonriseRegionFileIO.RegionFileType.ENTITY_DATA, Priority.BLOCKING); } catch (final Exception ex) { LOGGER.error("Cannot merge transient entities for chunk (" + this.chunkX + "," + this.chunkZ + ") in world '" + WorldUtil.getWorldName(this.world) + "', data on disk will be replaced", ex); } @@ -1853,7 +1868,7 @@ private boolean saveEntities(final ChunkEntitySlices entities, final boolean unl return false; } - RegionFileIOThread.scheduleSave(this.world, this.chunkX, this.chunkZ, save, RegionFileIOThread.RegionFileType.ENTITY_DATA); + MoonriseRegionFileIO.scheduleSave(this.world, this.chunkX, this.chunkZ, save, MoonriseRegionFileIO.RegionFileType.ENTITY_DATA); this.lastEntitySaveNull = save == null; if (unloading) { this.lastEntityUnload = save; @@ -1877,7 +1892,7 @@ private boolean savePOI(final PoiChunk poi, final boolean unloading) { return false; } - RegionFileIOThread.scheduleSave(this.world, this.chunkX, this.chunkZ, save, RegionFileIOThread.RegionFileType.POI_DATA); + MoonriseRegionFileIO.scheduleSave(this.world, this.chunkX, this.chunkZ, save, MoonriseRegionFileIO.RegionFileType.POI_DATA); this.lastPoiSaveNull = save == null; if (unloading) { this.poiDataUnload.completable().complete(save); @@ -1924,7 +1939,7 @@ private static JsonElement serializeStacktraceElement(final StackTraceElement el return element == null ? JsonNull.INSTANCE : new JsonPrimitive(element.toString()); } - private static JsonObject serializeCompletable(final Completable completable) { + private static JsonObject serializeCompletable(final CallbackCompletable completable) { final JsonObject ret = new JsonObject(); if (completable == null) { @@ -2019,13 +2034,13 @@ public JsonObject getDebugJson() { ret.add("poi_unload_completable", serializeCompletable(poiDataUnload == null ? null : poiDataUnload.completable())); ret.add("chunk_unload_completable", serializeCompletable(chunkDataUnload == null ? null : chunkDataUnload.completable())); - final DelayedPrioritisedTask unloadTask = chunkDataUnload == null ? null : chunkDataUnload.task(); + final PrioritisedExecutor.PrioritisedTask unloadTask = chunkDataUnload == null ? null : chunkDataUnload.task(); if (unloadTask == null) { ret.addProperty("unload_task_priority", "null"); - ret.addProperty("unload_task_priority_raw", "null"); + ret.addProperty("unload_task_suborder", Long.valueOf(0L)); } else { ret.addProperty("unload_task_priority", Objects.toString(unloadTask.getPriority())); - ret.addProperty("unload_task_priority_raw", Integer.valueOf(unloadTask.getPriorityInternal())); + ret.addProperty("unload_task_suborder", Long.valueOf(unloadTask.getSubOrder())); } ret.addProperty("killed", Boolean.valueOf(this.unloaded)); diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/PriorityHolder.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/PriorityHolder.java index 261e0945..6b468c62 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/PriorityHolder.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/PriorityHolder.java @@ -1,7 +1,7 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import java.lang.invoke.VarHandle; public abstract class PriorityHolder { @@ -28,8 +28,8 @@ protected final void setPriorityPlain(final int val) { PRIORITY_HANDLE.set((PriorityHolder)this, (int)val); } - protected PriorityHolder(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + protected PriorityHolder(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.setPriorityPlain(priority.priority); @@ -69,7 +69,7 @@ public void schedule() { return; } - this.scheduleTask(PrioritisedExecutor.Priority.getPriority(priority)); + this.scheduleTask(Priority.getPriority(priority)); int failures = 0; for (;;) { @@ -86,7 +86,7 @@ public void schedule() { return; } - this.setPriorityScheduled(PrioritisedExecutor.Priority.getPriority(priority)); + this.setPriorityScheduled(Priority.getPriority(priority)); ++failures; for (int i = 0; i < failures; ++i) { @@ -95,19 +95,19 @@ public void schedule() { } } - public final PrioritisedExecutor.Priority getPriority() { + public final Priority getPriority() { final int ret = this.getPriorityVolatile(); if ((ret & PRIORITY_EXECUTED) != 0) { - return PrioritisedExecutor.Priority.COMPLETING; + return Priority.COMPLETING; } if ((ret & PRIORITY_SCHEDULED) != 0) { return this.getScheduledPriority(); } - return PrioritisedExecutor.Priority.getPriority(ret); + return Priority.getPriority(ret); } - public final void lowerPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public final void lowerPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -139,8 +139,8 @@ public final void lowerPriority(final PrioritisedExecutor.Priority priority) { } } - public final void setPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public final void setPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -168,8 +168,8 @@ public final void setPriority(final PrioritisedExecutor.Priority priority) { } } - public final void raisePriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public final void raisePriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -203,13 +203,13 @@ public final void raisePriority(final PrioritisedExecutor.Priority priority) { protected abstract void cancelScheduled(); - protected abstract PrioritisedExecutor.Priority getScheduledPriority(); + protected abstract Priority getScheduledPriority(); - protected abstract void scheduleTask(final PrioritisedExecutor.Priority priority); + protected abstract void scheduleTask(final Priority priority); - protected abstract void lowerPriorityScheduled(final PrioritisedExecutor.Priority priority); + protected abstract void lowerPriorityScheduled(final Priority priority); - protected abstract void setPriorityScheduled(final PrioritisedExecutor.Priority priority); + protected abstract void setPriorityScheduled(final Priority priority); - protected abstract void raisePriorityScheduled(final PrioritisedExecutor.Priority priority); + protected abstract void raisePriorityScheduled(final Priority priority); } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/executor/RadiusAwarePrioritisedExecutor.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/executor/RadiusAwarePrioritisedExecutor.java index e0b26ccb..c91fdb8f 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/executor/RadiusAwarePrioritisedExecutor.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/executor/RadiusAwarePrioritisedExecutor.java @@ -1,10 +1,10 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling.executor; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.CoordinateUtils; import it.unimi.dsi.fastutil.longs.Long2ReferenceOpenHashMap; import it.unimi.dsi.fastutil.objects.ReferenceOpenHashSet; - import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -16,12 +16,15 @@ public class RadiusAwarePrioritisedExecutor { return Long.compare(t1.id, t2.id); }; - private final DependencyTree[] queues = new DependencyTree[PrioritisedExecutor.Priority.TOTAL_SCHEDULABLE_PRIORITIES]; + private final PrioritisedExecutor executor; + private final DependencyTree[] queues = new DependencyTree[Priority.TOTAL_SCHEDULABLE_PRIORITIES]; private static final int NO_TASKS_QUEUED = -1; private int selectedQueue = NO_TASKS_QUEUED; private boolean canQueueTasks = true; public RadiusAwarePrioritisedExecutor(final PrioritisedExecutor executor, final int maxToSchedule) { + this.executor = executor; + for (int i = 0; i < this.queues.length; ++i) { this.queues[i] = new DependencyTree(this, executor, maxToSchedule, i); } @@ -56,7 +59,7 @@ private List treeFinished() { return null; } - private List queue(final Task task, final PrioritisedExecutor.Priority priority) { + private List queue(final Task task, final Priority priority) { final int priorityId = priority.priority; final DependencyTree queue = this.queues[priorityId]; @@ -79,7 +82,7 @@ private List queue(final Task task, final P return null; } - if (PrioritisedExecutor.Priority.isHigherPriority(priorityId, this.selectedQueue)) { + if (Priority.isHigherPriority(priorityId, this.selectedQueue)) { // prevent the lower priority tree from queueing more tasks this.canQueueTasks = false; return null; @@ -90,7 +93,7 @@ private List queue(final Task task, final P } public PrioritisedExecutor.PrioritisedTask createTask(final int chunkX, final int chunkZ, final int radius, - final Runnable run, final PrioritisedExecutor.Priority priority) { + final Runnable run, final Priority priority) { if (radius < 0) { throw new IllegalArgumentException("Radius must be > 0: " + radius); } @@ -99,11 +102,11 @@ public PrioritisedExecutor.PrioritisedTask createTask(final int chunkX, final in public PrioritisedExecutor.PrioritisedTask createTask(final int chunkX, final int chunkZ, final int radius, final Runnable run) { - return this.createTask(chunkX, chunkZ, radius, run, PrioritisedExecutor.Priority.NORMAL); + return this.createTask(chunkX, chunkZ, radius, run, Priority.NORMAL); } public PrioritisedExecutor.PrioritisedTask queueTask(final int chunkX, final int chunkZ, final int radius, - final Runnable run, final PrioritisedExecutor.Priority priority) { + final Runnable run, final Priority priority) { final PrioritisedExecutor.PrioritisedTask ret = this.createTask(chunkX, chunkZ, radius, run, priority); ret.queue(); @@ -120,15 +123,15 @@ public PrioritisedExecutor.PrioritisedTask queueTask(final int chunkX, final int return ret; } - public PrioritisedExecutor.PrioritisedTask createInfiniteRadiusTask(final Runnable run, final PrioritisedExecutor.Priority priority) { + public PrioritisedExecutor.PrioritisedTask createInfiniteRadiusTask(final Runnable run, final Priority priority) { return new Task(this, 0, 0, -1, run, priority); } public PrioritisedExecutor.PrioritisedTask createInfiniteRadiusTask(final Runnable run) { - return this.createInfiniteRadiusTask(run, PrioritisedExecutor.Priority.NORMAL); + return this.createInfiniteRadiusTask(run, Priority.NORMAL); } - public PrioritisedExecutor.PrioritisedTask queueInfiniteRadiusTask(final Runnable run, final PrioritisedExecutor.Priority priority) { + public PrioritisedExecutor.PrioritisedTask queueInfiniteRadiusTask(final Runnable run, final Priority priority) { final PrioritisedExecutor.PrioritisedTask ret = this.createInfiniteRadiusTask(run, priority); ret.queue(); @@ -137,7 +140,7 @@ public PrioritisedExecutor.PrioritisedTask queueInfiniteRadiusTask(final Runnabl } public PrioritisedExecutor.PrioritisedTask queueInfiniteRadiusTask(final Runnable run) { - final PrioritisedExecutor.PrioritisedTask ret = this.createInfiniteRadiusTask(run, PrioritisedExecutor.Priority.NORMAL); + final PrioritisedExecutor.PrioritisedTask ret = this.createInfiniteRadiusTask(run, Priority.NORMAL); ret.queue(); @@ -412,13 +415,13 @@ private static final class Task implements PrioritisedExecutor.PrioritisedTask, private final int chunkZ; private final int radius; private Runnable run; - private PrioritisedExecutor.Priority priority; + private Priority priority; private DependencyNode dependencyNode; private PrioritisedExecutor.PrioritisedTask queuedTask; private Task(final RadiusAwarePrioritisedExecutor scheduler, final int chunkX, final int chunkZ, final int radius, - final Runnable run, final PrioritisedExecutor.Priority priority) { + final Runnable run, final Priority priority) { this.scheduler = scheduler; this.chunkX = chunkX; this.chunkZ = chunkZ; @@ -460,6 +463,11 @@ private void returnNode() { scheduleTasks(toSchedule); } + @Override + public PrioritisedExecutor getExecutor() { + return this.scheduler.executor; + } + @Override public void run() { final Runnable run = this.run; @@ -475,7 +483,7 @@ public void run() { public boolean queue() { final List toSchedule; synchronized (this.scheduler) { - if (this.queuedTask != null || this.dependencyNode != null || this.priority == PrioritisedExecutor.Priority.COMPLETING) { + if (this.queuedTask != null || this.dependencyNode != null || this.priority == Priority.COMPLETING) { return false; } @@ -486,16 +494,23 @@ public boolean queue() { return true; } + @Override + public boolean isQueued() { + synchronized (this.scheduler) { + return (this.queuedTask != null || this.dependencyNode != null) && this.priority != Priority.COMPLETING; + } + } + @Override public boolean cancel() { final PrioritisedExecutor.PrioritisedTask task; synchronized (this.scheduler) { if ((task = this.queuedTask) == null) { - if (this.priority == PrioritisedExecutor.Priority.COMPLETING) { + if (this.priority == Priority.COMPLETING) { return false; } - this.priority = PrioritisedExecutor.Priority.COMPLETING; + this.priority = Priority.COMPLETING; if (this.dependencyNode != null) { this.dependencyNode.purged = true; this.dependencyNode = null; @@ -519,11 +534,11 @@ public boolean execute() { final PrioritisedExecutor.PrioritisedTask task; synchronized (this.scheduler) { if ((task = this.queuedTask) == null) { - if (this.priority == PrioritisedExecutor.Priority.COMPLETING) { + if (this.priority == Priority.COMPLETING) { return false; } - this.priority = PrioritisedExecutor.Priority.COMPLETING; + this.priority = Priority.COMPLETING; if (this.dependencyNode != null) { this.dependencyNode.purged = true; this.dependencyNode = null; @@ -543,7 +558,7 @@ public boolean execute() { } @Override - public PrioritisedExecutor.Priority getPriority() { + public Priority getPriority() { final PrioritisedExecutor.PrioritisedTask task; synchronized (this.scheduler) { if ((task = this.queuedTask) == null) { @@ -555,8 +570,8 @@ public PrioritisedExecutor.Priority getPriority() { } @Override - public boolean setPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public boolean setPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -564,7 +579,7 @@ public boolean setPriority(final PrioritisedExecutor.Priority priority) { List toSchedule = null; synchronized (this.scheduler) { if ((task = this.queuedTask) == null) { - if (this.priority == PrioritisedExecutor.Priority.COMPLETING) { + if (this.priority == Priority.COMPLETING) { return false; } @@ -592,8 +607,8 @@ public boolean setPriority(final PrioritisedExecutor.Priority priority) { } @Override - public boolean raisePriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public boolean raisePriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -601,7 +616,7 @@ public boolean raisePriority(final PrioritisedExecutor.Priority priority) { List toSchedule = null; synchronized (this.scheduler) { if ((task = this.queuedTask) == null) { - if (this.priority == PrioritisedExecutor.Priority.COMPLETING) { + if (this.priority == Priority.COMPLETING) { return false; } @@ -629,8 +644,8 @@ public boolean raisePriority(final PrioritisedExecutor.Priority priority) { } @Override - public boolean lowerPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public boolean lowerPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -638,7 +653,7 @@ public boolean lowerPriority(final PrioritisedExecutor.Priority priority) { List toSchedule = null; synchronized (this.scheduler) { if ((task = this.queuedTask) == null) { - if (this.priority == PrioritisedExecutor.Priority.COMPLETING) { + if (this.priority == Priority.COMPLETING) { return false; } @@ -664,5 +679,35 @@ public boolean lowerPriority(final PrioritisedExecutor.Priority priority) { return true; } + + @Override + public long getSubOrder() { + // TODO implement + return 0; + } + + @Override + public boolean setSubOrder(final long subOrder) { + // TODO implement + return false; + } + + @Override + public boolean raiseSubOrder(final long subOrder) { + // TODO implement + return false; + } + + @Override + public boolean lowerSubOrder(final long subOrder) { + // TODO implement + return false; + } + + @Override + public boolean setPriorityAndSubOrder(final Priority priority, final long subOrder) { + // TODO implement + return this.setPriority(priority); + } } } \ No newline at end of file diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkFullTask.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkFullTask.java index 00729ce5..78d2afbb 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkFullTask.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkFullTask.java @@ -1,7 +1,8 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling.task; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.level.chunk.ChunkSystemLevelChunk; import ca.spottedleaf.moonrise.patches.chunk_system.level.poi.ChunkSystemPoiManager; @@ -29,7 +30,7 @@ public final class ChunkFullTask extends ChunkProgressionTask implements Runnabl private final PrioritisedExecutor.PrioritisedTask convertToFullTask; public ChunkFullTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, final int chunkZ, - final NewChunkHolder chunkHolder, final ChunkAccess fromChunk, final PrioritisedExecutor.Priority priority) { + final NewChunkHolder chunkHolder, final ChunkAccess fromChunk, final Priority priority) { super(scheduler, world, chunkX, chunkZ); this.chunkHolder = chunkHolder; this.fromChunk = fromChunk; @@ -112,29 +113,29 @@ public void cancel() { } @Override - public PrioritisedExecutor.Priority getPriority() { + public Priority getPriority() { return this.convertToFullTask.getPriority(); } @Override - public void lowerPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void lowerPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.convertToFullTask.lowerPriority(priority); } @Override - public void setPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void setPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.convertToFullTask.setPriority(priority); } @Override - public void raisePriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void raisePriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.convertToFullTask.raisePriority(priority); diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLightTask.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLightTask.java index 7c2e6752..4538ccfa 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLightTask.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLightTask.java @@ -1,6 +1,6 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling.task; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.WorldUtil; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.PriorityHolder; @@ -25,9 +25,9 @@ public final class ChunkLightTask extends ChunkProgressionTask { private final LightTaskPriorityHolder priorityHolder; public ChunkLightTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, final int chunkZ, - final ChunkAccess chunk, final PrioritisedExecutor.Priority priority) { + final ChunkAccess chunk, final Priority priority) { super(scheduler, world, chunkX, chunkZ); - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.priorityHolder = new LightTaskPriorityHolder(priority, this); @@ -55,22 +55,22 @@ public void cancel() { } @Override - public PrioritisedExecutor.Priority getPriority() { + public Priority getPriority() { return this.priorityHolder.getPriority(); } @Override - public void lowerPriority(final PrioritisedExecutor.Priority priority) { + public void lowerPriority(final Priority priority) { this.priorityHolder.raisePriority(priority); } @Override - public void setPriority(final PrioritisedExecutor.Priority priority) { + public void setPriority(final Priority priority) { this.priorityHolder.setPriority(priority); } @Override - public void raisePriority(final PrioritisedExecutor.Priority priority) { + public void raisePriority(final Priority priority) { this.priorityHolder.raisePriority(priority); } @@ -78,7 +78,7 @@ private static final class LightTaskPriorityHolder extends PriorityHolder { private final ChunkLightTask task; - private LightTaskPriorityHolder(final PrioritisedExecutor.Priority priority, final ChunkLightTask task) { + private LightTaskPriorityHolder(final Priority priority, final ChunkLightTask task) { super(priority); this.task = task; } @@ -90,13 +90,13 @@ protected void cancelScheduled() { } @Override - protected PrioritisedExecutor.Priority getScheduledPriority() { + protected Priority getScheduledPriority() { final ChunkLightTask task = this.task; return ((StarLightLightingProvider)task.world.getChunkSource().getLightEngine()).starlight$getLightEngine().getServerLightQueue().getPriority(task.chunkX, task.chunkZ); } @Override - protected void scheduleTask(final PrioritisedExecutor.Priority priority) { + protected void scheduleTask(final Priority priority) { final ChunkLightTask task = this.task; final StarLightInterface starLightInterface = ((StarLightLightingProvider)task.world.getChunkSource().getLightEngine()).starlight$getLightEngine(); final StarLightInterface.ServerLightQueue lightQueue = starLightInterface.getServerLightQueue(); @@ -105,7 +105,7 @@ protected void scheduleTask(final PrioritisedExecutor.Priority priority) { } @Override - protected void lowerPriorityScheduled(final PrioritisedExecutor.Priority priority) { + protected void lowerPriorityScheduled(final Priority priority) { final ChunkLightTask task = this.task; final StarLightInterface starLightInterface = ((StarLightLightingProvider)task.world.getChunkSource().getLightEngine()).starlight$getLightEngine(); final StarLightInterface.ServerLightQueue lightQueue = starLightInterface.getServerLightQueue(); @@ -113,7 +113,7 @@ protected void lowerPriorityScheduled(final PrioritisedExecutor.Priority priorit } @Override - protected void setPriorityScheduled(final PrioritisedExecutor.Priority priority) { + protected void setPriorityScheduled(final Priority priority) { final ChunkLightTask task = this.task; final StarLightInterface starLightInterface = ((StarLightLightingProvider)task.world.getChunkSource().getLightEngine()).starlight$getLightEngine(); final StarLightInterface.ServerLightQueue lightQueue = starLightInterface.getServerLightQueue(); @@ -121,7 +121,7 @@ protected void setPriorityScheduled(final PrioritisedExecutor.Priority priority) } @Override - protected void raisePriorityScheduled(final PrioritisedExecutor.Priority priority) { + protected void raisePriorityScheduled(final Priority priority) { final ChunkLightTask task = this.task; final StarLightInterface starLightInterface = ((StarLightLightingProvider)task.world.getChunkSource().getLightEngine()).starlight$getLightEngine(); final StarLightInterface.ServerLightQueue lightQueue = starLightInterface.getServerLightQueue(); diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLoadTask.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLoadTask.java index 1636ffb7..66545e13 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLoadTask.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkLoadTask.java @@ -1,12 +1,13 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling.task; import ca.spottedleaf.concurrentutil.collection.MultiThreadedQueue; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.lock.ReentrantAreaLock; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.patches.chunk_system.ChunkSystemConverters; import ca.spottedleaf.moonrise.patches.chunk_system.ChunkSystemFeatures; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.level.poi.PoiChunk; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.NewChunkHolder; @@ -41,7 +42,7 @@ public final class ChunkLoadTask extends ChunkProgressionTask { private final AtomicInteger taskCountToComplete = new AtomicInteger(3); // one for poi, one for entity, and one for chunk data public ChunkLoadTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, final int chunkZ, - final NewChunkHolder chunkHolder, final PrioritisedExecutor.Priority priority) { + final NewChunkHolder chunkHolder, final Priority priority) { super(scheduler, world, chunkX, chunkZ); this.chunkHolder = chunkHolder; this.loadTask = new ChunkDataLoadTask(scheduler, world, chunkX, chunkZ, priority); @@ -170,12 +171,12 @@ public void cancel() { } @Override - public PrioritisedExecutor.Priority getPriority() { + public Priority getPriority() { return this.loadTask.getPriority(); } @Override - public void lowerPriority(final PrioritisedExecutor.Priority priority) { + public void lowerPriority(final Priority priority) { final EntityDataLoadTask entityLoad = this.chunkHolder.getEntityDataLoadTask(); if (entityLoad != null) { entityLoad.lowerPriority(priority); @@ -191,7 +192,7 @@ public void lowerPriority(final PrioritisedExecutor.Priority priority) { } @Override - public void setPriority(final PrioritisedExecutor.Priority priority) { + public void setPriority(final Priority priority) { final EntityDataLoadTask entityLoad = this.chunkHolder.getEntityDataLoadTask(); if (entityLoad != null) { entityLoad.setPriority(priority); @@ -207,7 +208,7 @@ public void setPriority(final PrioritisedExecutor.Priority priority) { } @Override - public void raisePriority(final PrioritisedExecutor.Priority priority) { + public void raisePriority(final Priority priority) { final EntityDataLoadTask entityLoad = this.chunkHolder.getEntityDataLoadTask(); if (entityLoad != null) { entityLoad.raisePriority(priority); @@ -231,8 +232,8 @@ protected static abstract class CallbackDataLoadTask ext protected static final VarHandle COMPLETED_HANDLE = ConcurrentUtil.getVarHandle(CallbackDataLoadTask.class, "completed", boolean.class); protected CallbackDataLoadTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, - final int chunkZ, final RegionFileIOThread.RegionFileType type, - final PrioritisedExecutor.Priority priority) { + final int chunkZ, final MoonriseRegionFileIO.RegionFileType type, + final Priority priority) { super(scheduler, world, chunkX, chunkZ, type, priority); } @@ -274,8 +275,8 @@ protected void onComplete(final TaskResult result) { private static final class ChunkDataLoadTask extends CallbackDataLoadTask { private ChunkDataLoadTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, - final int chunkZ, final PrioritisedExecutor.Priority priority) { - super(scheduler, world, chunkX, chunkZ, RegionFileIOThread.RegionFileType.CHUNK_DATA, priority); + final int chunkZ, final Priority priority) { + super(scheduler, world, chunkX, chunkZ, MoonriseRegionFileIO.RegionFileType.CHUNK_DATA, priority); } @Override @@ -289,12 +290,12 @@ protected boolean hasOnMain() { } @Override - protected PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final PrioritisedExecutor.Priority priority) { + protected PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final Priority priority) { return this.scheduler.loadExecutor.createTask(run, priority); } @Override - protected PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final PrioritisedExecutor.Priority priority) { + protected PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final Priority priority) { return this.scheduler.createChunkTask(this.chunkX, this.chunkZ, run, priority); } @@ -368,8 +369,8 @@ protected TaskResult runOnMain(final CompoundTag data, f public static final class PoiDataLoadTask extends CallbackDataLoadTask { public PoiDataLoadTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, - final int chunkZ, final PrioritisedExecutor.Priority priority) { - super(scheduler, world, chunkX, chunkZ, RegionFileIOThread.RegionFileType.POI_DATA, priority); + final int chunkZ, final Priority priority) { + super(scheduler, world, chunkX, chunkZ, MoonriseRegionFileIO.RegionFileType.POI_DATA, priority); } @Override @@ -383,12 +384,12 @@ protected boolean hasOnMain() { } @Override - protected PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final PrioritisedExecutor.Priority priority) { + protected PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final Priority priority) { return this.scheduler.loadExecutor.createTask(run, priority); } @Override - protected PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final PrioritisedExecutor.Priority priority) { + protected PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final Priority priority) { throw new UnsupportedOperationException(); } @@ -430,8 +431,8 @@ protected TaskResult runOnMain(final PoiChunk data, final T public static final class EntityDataLoadTask extends CallbackDataLoadTask { public EntityDataLoadTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, - final int chunkZ, final PrioritisedExecutor.Priority priority) { - super(scheduler, world, chunkX, chunkZ, RegionFileIOThread.RegionFileType.ENTITY_DATA, priority); + final int chunkZ, final Priority priority) { + super(scheduler, world, chunkX, chunkZ, MoonriseRegionFileIO.RegionFileType.ENTITY_DATA, priority); } @Override @@ -445,12 +446,12 @@ protected boolean hasOnMain() { } @Override - protected PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final PrioritisedExecutor.Priority priority) { + protected PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final Priority priority) { return this.scheduler.loadExecutor.createTask(run, priority); } @Override - protected PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final PrioritisedExecutor.Priority priority) { + protected PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final Priority priority) { throw new UnsupportedOperationException(); } diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkProgressionTask.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkProgressionTask.java index 70e900b0..002ee365 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkProgressionTask.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkProgressionTask.java @@ -1,8 +1,8 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling.task; import ca.spottedleaf.concurrentutil.collection.MultiThreadedQueue; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.WorldUtil; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import net.minecraft.server.level.ServerLevel; @@ -46,15 +46,15 @@ protected ChunkProgressionTask(final ChunkTaskScheduler scheduler, final ServerL /* May be called multiple times */ public abstract void cancel(); - public abstract PrioritisedExecutor.Priority getPriority(); + public abstract Priority getPriority(); /* Schedule lock is always held for the priority update calls */ - public abstract void lowerPriority(final PrioritisedExecutor.Priority priority); + public abstract void lowerPriority(final Priority priority); - public abstract void setPriority(final PrioritisedExecutor.Priority priority); + public abstract void setPriority(final Priority priority); - public abstract void raisePriority(final PrioritisedExecutor.Priority priority); + public abstract void raisePriority(final Priority priority); public final void onComplete(final BiConsumer onComplete) { if (!this.waiters.add(onComplete)) { diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkUpgradeGenericStatusTask.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkUpgradeGenericStatusTask.java index 2c17d558..25d8da47 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkUpgradeGenericStatusTask.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/ChunkUpgradeGenericStatusTask.java @@ -1,7 +1,8 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling.task; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.WorldUtil; import ca.spottedleaf.moonrise.patches.chunk_system.level.chunk.ChunkSystemChunkStatus; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; @@ -36,9 +37,9 @@ public final class ChunkUpgradeGenericStatusTask extends ChunkProgressionTask im public ChunkUpgradeGenericStatusTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, final int chunkZ, final ChunkAccess chunk, final StaticCache2D neighbours, - final ChunkStatus toStatus, final PrioritisedExecutor.Priority priority) { + final ChunkStatus toStatus, final Priority priority) { super(scheduler, world, chunkX, chunkZ); - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.fromChunk = chunk; @@ -187,29 +188,29 @@ public void cancel() { } @Override - public PrioritisedExecutor.Priority getPriority() { + public Priority getPriority() { return this.generateTask.getPriority(); } @Override - public void lowerPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void lowerPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.generateTask.lowerPriority(priority); } @Override - public void setPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void setPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.generateTask.setPriority(priority); } @Override - public void raisePriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void raisePriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.generateTask.raisePriority(priority); diff --git a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/GenericDataLoadTask.java b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/GenericDataLoadTask.java index 7a65d351..bdcd1879 100644 --- a/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/GenericDataLoadTask.java +++ b/src/main/java/ca/spottedleaf/moonrise/patches/chunk_system/scheduling/task/GenericDataLoadTask.java @@ -1,12 +1,13 @@ package ca.spottedleaf.moonrise.patches.chunk_system.scheduling.task; +import ca.spottedleaf.concurrentutil.completable.CallbackCompletable; import ca.spottedleaf.concurrentutil.completable.Completable; import ca.spottedleaf.concurrentutil.executor.Cancellable; -import ca.spottedleaf.concurrentutil.executor.standard.DelayedPrioritisedTask; -import ca.spottedleaf.concurrentutil.executor.standard.PrioritisedExecutor; +import ca.spottedleaf.concurrentutil.executor.PrioritisedExecutor; import ca.spottedleaf.concurrentutil.util.ConcurrentUtil; +import ca.spottedleaf.concurrentutil.util.Priority; import ca.spottedleaf.moonrise.common.util.WorldUtil; -import ca.spottedleaf.moonrise.patches.chunk_system.io.RegionFileIOThread; +import ca.spottedleaf.moonrise.patches.chunk_system.io.MoonriseRegionFileIO; import ca.spottedleaf.moonrise.patches.chunk_system.level.ChunkSystemServerLevel; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.ChunkTaskScheduler; import ca.spottedleaf.moonrise.patches.chunk_system.scheduling.NewChunkHolder; @@ -47,11 +48,11 @@ public abstract class GenericDataLoadTask { protected final ServerLevel world; protected final int chunkX; protected final int chunkZ; - protected final RegionFileIOThread.RegionFileType type; + protected final MoonriseRegionFileIO.RegionFileType type; public GenericDataLoadTask(final ChunkTaskScheduler scheduler, final ServerLevel world, final int chunkX, - final int chunkZ, final RegionFileIOThread.RegionFileType type, - final PrioritisedExecutor.Priority priority) { + final int chunkZ, final MoonriseRegionFileIO.RegionFileType type, + final Priority priority) { this.scheduler = scheduler; this.world = world; this.chunkX = chunkX; @@ -89,9 +90,9 @@ public static final record TaskResult(L left, R right) {} protected abstract boolean hasOnMain(); - protected abstract PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final PrioritisedExecutor.Priority priority); + protected abstract PrioritisedExecutor.PrioritisedTask createOffMain(final Runnable run, final Priority priority); - protected abstract PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final PrioritisedExecutor.Priority priority); + protected abstract PrioritisedExecutor.PrioritisedTask createOnMain(final Runnable run, final Priority priority); protected abstract TaskResult runOffMain(final CompoundTag data, final Throwable throwable); @@ -108,7 +109,7 @@ public String toString() { ", type: " + this.type.toString() + "}"; } - public PrioritisedExecutor.Priority getPriority() { + public Priority getPriority() { if (this.processOnMain != null) { return this.processOnMain.getPriority(); } else { @@ -116,7 +117,7 @@ public PrioritisedExecutor.Priority getPriority() { } } - public void lowerPriority(final PrioritisedExecutor.Priority priority) { + public void lowerPriority(final Priority priority) { // can't lower I/O tasks, we don't know what they affect if (this.processOffMain != null) { this.processOffMain.lowerPriority(priority); @@ -126,7 +127,7 @@ public void lowerPriority(final PrioritisedExecutor.Priority priority) { } } - public void setPriority(final PrioritisedExecutor.Priority priority) { + public void setPriority(final Priority priority) { // can't lower I/O tasks, we don't know what they affect this.loadDataFromDiskTask.raisePriority(priority); if (this.processOffMain != null) { @@ -137,7 +138,7 @@ public void setPriority(final PrioritisedExecutor.Priority priority) { } } - public void raisePriority(final PrioritisedExecutor.Priority priority) { + public void raisePriority(final Priority priority) { // can't lower I/O tasks, we don't know what they affect this.loadDataFromDiskTask.raisePriority(priority); if (this.processOffMain != null) { @@ -382,10 +383,10 @@ private final void setPriorityPlain(final int val) { private final int chunkX; private final int chunkZ; - private final RegionFileIOThread.RegionFileType type; + private final MoonriseRegionFileIO.RegionFileType type; private Cancellable dataLoadTask; private Cancellable dataUnloadCancellable; - private DelayedPrioritisedTask dataUnloadTask; + private PrioritisedExecutor.PrioritisedTask dataUnloadTask; private final BiConsumer onComplete; private final AtomicBoolean scheduled = new AtomicBoolean(); @@ -393,10 +394,10 @@ private final void setPriorityPlain(final int val) { // onComplete should be caller sensitive, it may complete synchronously with schedule() - which does // hold a priority lock. public LoadDataFromDiskTask(final ServerLevel world, final int chunkX, final int chunkZ, - final RegionFileIOThread.RegionFileType type, + final MoonriseRegionFileIO.RegionFileType type, final BiConsumer onComplete, - final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } this.world = world; @@ -426,8 +427,8 @@ private boolean isMarkedExecuted() { return (this.getPriorityVolatile() & PRIORITY_EXECUTED) != 0; } - public void lowerPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void lowerPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -439,7 +440,7 @@ public void lowerPriority(final PrioritisedExecutor.Priority priority) { } if ((curr & PRIORITY_LOAD_SCHEDULED) != 0) { - RegionFileIOThread.lowerPriority(this.world, this.chunkX, this.chunkZ, this.type, priority); + MoonriseRegionFileIO.lowerPriority(this.world, this.chunkX, this.chunkZ, this.type, priority); return; } @@ -467,8 +468,8 @@ public void lowerPriority(final PrioritisedExecutor.Priority priority) { } } - public void setPriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void setPriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -480,7 +481,7 @@ public void setPriority(final PrioritisedExecutor.Priority priority) { } if ((curr & PRIORITY_LOAD_SCHEDULED) != 0) { - RegionFileIOThread.setPriority(this.world, this.chunkX, this.chunkZ, this.type, priority); + MoonriseRegionFileIO.setPriority(this.world, this.chunkX, this.chunkZ, this.type, priority); return; } @@ -504,8 +505,8 @@ public void setPriority(final PrioritisedExecutor.Priority priority) { } } - public void raisePriority(final PrioritisedExecutor.Priority priority) { - if (!PrioritisedExecutor.Priority.isValidPriority(priority)) { + public void raisePriority(final Priority priority) { + if (!Priority.isValidPriority(priority)) { throw new IllegalArgumentException("Invalid priority " + priority); } @@ -517,7 +518,7 @@ public void raisePriority(final PrioritisedExecutor.Priority priority) { } if ((curr & PRIORITY_LOAD_SCHEDULED) != 0) { - RegionFileIOThread.raisePriority(this.world, this.chunkX, this.chunkZ, this.type, priority); + MoonriseRegionFileIO.raisePriority(this.world, this.chunkX, this.chunkZ, this.type, priority); return; } @@ -583,7 +584,7 @@ public void schedule() { } // else: cancelled }; - final PrioritisedExecutor.Priority initialPriority = PrioritisedExecutor.Priority.getPriority(priority); + final Priority initialPriority = Priority.getPriority(priority); boolean scheduledUnload = false; final NewChunkHolder holder = ((ChunkSystemServerLevel)this.world).moonrise$getChunkTaskScheduler().chunkHolderManager.getChunkHolder(this.chunkX, this.chunkZ); @@ -593,13 +594,13 @@ public void schedule() { consumer.accept(data, null); } else { // need to schedule task - LoadDataFromDiskTask.this.schedule(false, consumer, PrioritisedExecutor.Priority.getPriority(LoadDataFromDiskTask.this.getPriorityVolatile() & ~PRIORITY_FLAGS)); + LoadDataFromDiskTask.this.schedule(false, consumer, Priority.getPriority(LoadDataFromDiskTask.this.getPriorityVolatile() & ~PRIORITY_FLAGS)); } }; Cancellable unloadCancellable = null; CompoundTag syncComplete = null; final NewChunkHolder.UnloadTask unloadTask = holder.getUnloadTask(this.type); // can be null if no task exists - final Completable unloadCompletable = unloadTask == null ? null : unloadTask.completable(); + final CallbackCompletable unloadCompletable = unloadTask == null ? null : unloadTask.completable(); if (unloadCompletable != null) { unloadCancellable = unloadCompletable.addAsynchronousWaiter(unloadConsumer); if (unloadCancellable == null) { @@ -622,7 +623,7 @@ public void schedule() { this.schedule(scheduledUnload, consumer, initialPriority); } - private void schedule(final boolean scheduledUnload, final BiConsumer consumer, final PrioritisedExecutor.Priority initialPriority) { + private void schedule(final boolean scheduledUnload, final BiConsumer consumer, final Priority initialPriority) { int priority = this.getPriorityVolatile(); if ((priority & PRIORITY_EXECUTED) != 0) { @@ -631,9 +632,9 @@ private void schedule(final boolean scheduledUnload, final BiConsumer { if (valueInMap == null) { valueInMap = new ServerChunkTasks( @@ -879,11 +880,11 @@ public static final class ServerChunkTasks extends ChunkTasks { public ServerChunkTasks(final long chunkCoordinate, final StarLightInterface lightEngine, final ServerLightQueue queue) { - this(chunkCoordinate, lightEngine, queue, PrioritisedExecutor.Priority.NORMAL); + this(chunkCoordinate, lightEngine, queue, Priority.NORMAL); } public ServerChunkTasks(final long chunkCoordinate, final StarLightInterface lightEngine, - final ServerLightQueue queue, final PrioritisedExecutor.Priority priority) { + final ServerLightQueue queue, final Priority priority) { super(chunkCoordinate, lightEngine, queue); this.task = ((ChunkSystemServerLevel)(ServerLevel)lightEngine.getWorld()).moonrise$getChunkTaskScheduler().radiusAwareScheduler.createTask( CoordinateUtils.getChunkX(chunkCoordinate), CoordinateUtils.getChunkZ(chunkCoordinate), @@ -903,19 +904,19 @@ public boolean cancel() { return this.task.cancel(); } - public PrioritisedExecutor.Priority getPriority() { + public Priority getPriority() { return this.task.getPriority(); } - public void lowerPriority(final PrioritisedExecutor.Priority priority) { + public void lowerPriority(final Priority priority) { this.task.lowerPriority(priority); } - public void setPriority(final PrioritisedExecutor.Priority priority) { + public void setPriority(final Priority priority) { this.task.setPriority(priority); } - public void raisePriority(final PrioritisedExecutor.Priority priority) { + public void raisePriority(final Priority priority) { this.task.raisePriority(priority); } diff --git a/src/main/resources/moonrise.accesswidener b/src/main/resources/moonrise.accesswidener index 661d1420..b4f53fba 100644 --- a/src/main/resources/moonrise.accesswidener +++ b/src/main/resources/moonrise.accesswidener @@ -278,4 +278,12 @@ accessible method net/minecraft/server/level/ServerChunkCache$ChunkAndHolder