From 9ac7b4caf4af4dc94d9f39a2b641654e6c4e8255 Mon Sep 17 00:00:00 2001 From: Jozufozu Date: Mon, 27 May 2024 18:06:17 -0500 Subject: [PATCH] Little light lut - Commit entire chunks of light at a time - Share all light data between embeddings --- .../api/visualization/VisualEmbedding.java | 27 +-- .../flywheel/backend/Backends.java | 4 +- .../flywheel/backend/engine/EngineImpl.java | 11 +- .../backend/engine/EnvironmentStorage.java | 17 -- .../embed/AbstractEmbeddedEnvironment.java | 15 +- .../flywheel/backend/engine/embed/Arena.java | 50 +++++ .../engine/embed/EmbeddedLightTexture.java | 87 -------- .../engine/embed/EmbeddedLightVolume.java | 187 ----------------- .../backend/engine/embed/LightLut.java | 139 +++++++++++++ .../backend/engine/embed/LightStorage.java | 195 ++++++++++++++++++ .../embed/NestedEmbeddedEnvironment.java | 11 +- .../embed/TopLevelEmbeddedEnvironment.java | 52 +---- .../backend/engine/indirect/LightBuffers.java | 29 +++ .../flywheel/flywheel/internal/common.vert | 9 +- .../flywheel/internal/indirect/main.vert | 71 +++++++ .../flywheel/internal/instancing/main.vert | 6 + 16 files changed, 524 insertions(+), 386 deletions(-) create mode 100644 common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/Arena.java delete mode 100644 common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightTexture.java delete mode 100644 common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightVolume.java create mode 100644 common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightLut.java create mode 100644 common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightStorage.java create mode 100644 common/src/backend/java/dev/engine_room/flywheel/backend/engine/indirect/LightBuffers.java diff --git a/common/src/api/java/dev/engine_room/flywheel/api/visualization/VisualEmbedding.java b/common/src/api/java/dev/engine_room/flywheel/api/visualization/VisualEmbedding.java index f935ab006..e0b6d4f81 100644 --- a/common/src/api/java/dev/engine_room/flywheel/api/visualization/VisualEmbedding.java +++ b/common/src/api/java/dev/engine_room/flywheel/api/visualization/VisualEmbedding.java @@ -4,7 +4,7 @@ import org.joml.Matrix4fc; import dev.engine_room.flywheel.api.BackendImplemented; -import net.minecraft.world.level.BlockAndTintGetter; +import it.unimi.dsi.fastutil.longs.LongSet; @BackendImplemented public interface VisualEmbedding extends VisualizationContext { @@ -16,30 +16,7 @@ public interface VisualEmbedding extends VisualizationContext { */ void transforms(Matrix4fc pose, Matrix3fc normal); - /** - * Collect light information from the given level for the given box. - * - *

Call this method on as many or as few boxes as you need to - * encompass all child visuals of this embedding.

- * - *

After this method is called, instances rendered from this - * embedding within the given box will be lit as if they were in - * the given level.

- * - * @param level The level to collect light information from. - * @param minX The minimum x coordinate of the box. - * @param minY The minimum y coordinate of the box. - * @param minZ The minimum z coordinate of the box. - * @param sizeX The size of the box in the x direction. - * @param sizeY The size of the box in the y direction. - * @param sizeZ The size of the box in the z direction. - */ - void collectLight(BlockAndTintGetter level, int minX, int minY, int minZ, int sizeX, int sizeY, int sizeZ); - - /** - * Reset any collected lighting information. - */ - void invalidateLight(); + void lightChunks(LongSet chunks); /** * Delete this embedding. diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/Backends.java b/common/src/backend/java/dev/engine_room/flywheel/backend/Backends.java index 001f8f39b..59e255cd5 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/Backends.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/Backends.java @@ -16,7 +16,7 @@ public final class Backends { * Use GPU instancing to render everything. */ public static final Backend INSTANCING = SimpleBackend.builder() - .engineFactory(level -> new EngineImpl(new InstancedDrawManager(InstancingPrograms.get()), 256)) + .engineFactory(level -> new EngineImpl(level, new InstancedDrawManager(InstancingPrograms.get()), 256)) .supported(() -> GlCompat.SUPPORTS_INSTANCING && InstancingPrograms.allLoaded() && !ShadersModHandler.isShaderPackInUse()) .register(Flywheel.rl("instancing")); @@ -24,7 +24,7 @@ public final class Backends { * Use Compute shaders to cull instances. */ public static final Backend INDIRECT = SimpleBackend.builder() - .engineFactory(level -> new EngineImpl(new IndirectDrawManager(IndirectPrograms.get()), 256)) + .engineFactory(level -> new EngineImpl(level, new IndirectDrawManager(IndirectPrograms.get()), 256)) .fallback(() -> Backends.INSTANCING) .supported(() -> GlCompat.SUPPORTS_INDIRECT && IndirectPrograms.allLoaded() && !ShadersModHandler.isShaderPackInUse()) .register(Flywheel.rl("indirect")); diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EngineImpl.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EngineImpl.java index 3e7dd9d61..87349d721 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EngineImpl.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EngineImpl.java @@ -15,6 +15,7 @@ import dev.engine_room.flywheel.api.visualization.VisualEmbedding; import dev.engine_room.flywheel.api.visualization.VisualizationContext; import dev.engine_room.flywheel.backend.engine.embed.Environment; +import dev.engine_room.flywheel.backend.engine.embed.LightStorage; import dev.engine_room.flywheel.backend.engine.embed.TopLevelEmbeddedEnvironment; import dev.engine_room.flywheel.backend.engine.uniform.Uniforms; import dev.engine_room.flywheel.backend.gl.GlStateTracker; @@ -24,19 +25,22 @@ import net.minecraft.client.Camera; import net.minecraft.core.BlockPos; import net.minecraft.core.Vec3i; +import net.minecraft.world.level.LevelAccessor; import net.minecraft.world.phys.Vec3; public class EngineImpl implements Engine { private final int sqrMaxOriginDistance; private final DrawManager> drawManager; private final EnvironmentStorage environmentStorage = new EnvironmentStorage(); + private final LightStorage lightStorage; private final Flag flushFlag = new NamedFlag("flushed"); private BlockPos renderOrigin = BlockPos.ZERO; - public EngineImpl(DrawManager> drawManager, int maxOriginDistance) { + public EngineImpl(LevelAccessor level, DrawManager> drawManager, int maxOriginDistance) { this.drawManager = drawManager; sqrMaxOriginDistance = maxOriginDistance * maxOriginDistance; + lightStorage = new LightStorage(level); } @Override @@ -93,6 +97,7 @@ public Vec3i renderOrigin() { public void delete() { drawManager.delete(); environmentStorage.delete(); + lightStorage.delete(); } public Instancer instancer(Environment environment, InstanceType type, Model model, RenderStage stage) { @@ -113,6 +118,10 @@ public EnvironmentStorage environmentStorage() { return environmentStorage; } + public LightStorage lightStorage() { + return lightStorage; + } + private class VisualizationContextImpl implements VisualizationContext { private final InstancerProviderImpl instancerProvider; private final RenderStage stage; diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EnvironmentStorage.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EnvironmentStorage.java index c6500069f..5a260eb04 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EnvironmentStorage.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/EnvironmentStorage.java @@ -1,8 +1,5 @@ package dev.engine_room.flywheel.backend.engine; -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; - import dev.engine_room.flywheel.backend.engine.embed.AbstractEmbeddedEnvironment; import it.unimi.dsi.fastutil.objects.ReferenceLinkedOpenHashSet; import it.unimi.dsi.fastutil.objects.ReferenceSet; @@ -10,30 +7,16 @@ public class EnvironmentStorage { protected final ReferenceSet environments = ReferenceSets.synchronize(new ReferenceLinkedOpenHashSet<>()); - private final Queue forDeletion = new ConcurrentLinkedQueue<>(); public void track(AbstractEmbeddedEnvironment environment) { environments.add(environment); } - public void enqueueDeletion(AbstractEmbeddedEnvironment environment) { - environments.remove(environment); - - forDeletion.add(environment); - } - public void flush() { - AbstractEmbeddedEnvironment env; - - while ((env = forDeletion.poll()) != null) { - env.actuallyDelete(); - } - environments.forEach(AbstractEmbeddedEnvironment::flush); } public void delete() { - environments.forEach(AbstractEmbeddedEnvironment::actuallyDelete); environments.clear(); } } diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/AbstractEmbeddedEnvironment.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/AbstractEmbeddedEnvironment.java index 07adb4805..f2a7912be 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/AbstractEmbeddedEnvironment.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/AbstractEmbeddedEnvironment.java @@ -24,7 +24,7 @@ public abstract class AbstractEmbeddedEnvironment extends AtomicReferenceCounted private final Matrix4f poseComposed = new Matrix4f(); private final Matrix3f normalComposed = new Matrix3f(); private final InstancerProvider instancerProvider; - private final EngineImpl engine; + protected final EngineImpl engine; private final RenderStage renderStage; public AbstractEmbeddedEnvironment(EngineImpl engine, RenderStage renderStage) { @@ -105,20 +105,7 @@ public void delete() { release(); } - /** - * Called when referenceCount goes to 0 - */ - @Override - public void _delete() { - engine.environmentStorage().enqueueDeletion(this); - } - public abstract void setupLight(GlProgram program); public abstract void composeMatrices(Matrix4f pose, Matrix3f normal); - - /** - * Called in EnvironmentStorage#flush - */ - public abstract void actuallyDelete(); } diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/Arena.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/Arena.java new file mode 100644 index 000000000..31dbc77cd --- /dev/null +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/Arena.java @@ -0,0 +1,50 @@ +package dev.engine_room.flywheel.backend.engine.embed; + +import dev.engine_room.flywheel.lib.memory.MemoryBlock; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntList; + +public class Arena { + private final long elementSizeBytes; + + private MemoryBlock memoryBlock; + + // Monotonic index, generally represents the size of the arena. + private int top = 0; + // List of free indices. + private final IntList freeStack = new IntArrayList(); + + public Arena(long elementSizeBytes, int initialCapacity) { + this.elementSizeBytes = elementSizeBytes; + + memoryBlock = MemoryBlock.malloc(elementSizeBytes * initialCapacity); + } + + public int alloc() { + // First re-use freed elements. + if (!freeStack.isEmpty()) { + return freeStack.removeInt(freeStack.size() - 1); + } + + // Make sure there's room to increment top. + if (top * elementSizeBytes >= memoryBlock.size()) { + memoryBlock = memoryBlock.realloc(memoryBlock.size() * 2); + } + + // Return the top index and increment. + return top++; + } + + public void free(int i) { + // That's it! Now pls don't try to use it. + freeStack.add(i); + } + + public long indexToPointer(int i) { + return memoryBlock.ptr() + i * elementSizeBytes; + } + + public void delete() { + memoryBlock.free(); + } +} diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightTexture.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightTexture.java deleted file mode 100644 index 7e4325fb9..000000000 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightTexture.java +++ /dev/null @@ -1,87 +0,0 @@ -package dev.engine_room.flywheel.backend.engine.embed; - -import static org.lwjgl.opengl.GL11.GL_LINEAR; -import static org.lwjgl.opengl.GL11.GL_TEXTURE_MAG_FILTER; -import static org.lwjgl.opengl.GL11.GL_TEXTURE_MIN_FILTER; -import static org.lwjgl.opengl.GL11.GL_TEXTURE_WRAP_S; -import static org.lwjgl.opengl.GL11.GL_TEXTURE_WRAP_T; -import static org.lwjgl.opengl.GL11.GL_UNPACK_ALIGNMENT; -import static org.lwjgl.opengl.GL11.GL_UNPACK_ROW_LENGTH; -import static org.lwjgl.opengl.GL11.GL_UNPACK_SKIP_PIXELS; -import static org.lwjgl.opengl.GL11.GL_UNPACK_SKIP_ROWS; -import static org.lwjgl.opengl.GL11.GL_UNSIGNED_BYTE; -import static org.lwjgl.opengl.GL11.glPixelStorei; -import static org.lwjgl.opengl.GL11.glTexParameteri; -import static org.lwjgl.opengl.GL12.GL_TEXTURE_3D; -import static org.lwjgl.opengl.GL12.GL_TEXTURE_WRAP_R; -import static org.lwjgl.opengl.GL12.GL_UNPACK_IMAGE_HEIGHT; -import static org.lwjgl.opengl.GL12.GL_UNPACK_SKIP_IMAGES; -import static org.lwjgl.opengl.GL12.glTexImage3D; -import static org.lwjgl.opengl.GL12.glTexSubImage3D; -import static org.lwjgl.opengl.GL14.GL_MIRRORED_REPEAT; - -import org.jetbrains.annotations.Nullable; -import org.lwjgl.opengl.GL30; - -import dev.engine_room.flywheel.backend.gl.GlTexture; -import net.minecraft.util.Mth; - -public class EmbeddedLightTexture { - @Nullable - private GlTexture texture; - - public int sizeX; - public int sizeY; - public int sizeZ; - - public void bind() { - - texture().bind(); - } - - private GlTexture texture() { - if (texture == null) { - texture = new GlTexture(GL_TEXTURE_3D); - } - return texture; - } - - public void ensureCapacity(int sizeX, int sizeY, int sizeZ) { - sizeX = Mth.smallestEncompassingPowerOfTwo(sizeX); - sizeY = Mth.smallestEncompassingPowerOfTwo(sizeY); - sizeZ = Mth.smallestEncompassingPowerOfTwo(sizeZ); - - if (sizeX > this.sizeX || sizeY > this.sizeY || sizeZ > this.sizeZ) { - this.sizeX = sizeX; - this.sizeY = sizeY; - this.sizeZ = sizeZ; - - glTexImage3D(GL_TEXTURE_3D, 0, GL30.GL_RG8, sizeX, sizeY, sizeZ, 0, GL30.GL_RG, GL_UNSIGNED_BYTE, 0); - - glTexParameteri(GL_TEXTURE_3D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_3D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); - glTexParameteri(GL_TEXTURE_3D, GL_TEXTURE_WRAP_S, GL_MIRRORED_REPEAT); - glTexParameteri(GL_TEXTURE_3D, GL_TEXTURE_WRAP_R, GL_MIRRORED_REPEAT); - glTexParameteri(GL_TEXTURE_3D, GL_TEXTURE_WRAP_T, GL_MIRRORED_REPEAT); - } - } - - public void upload(long ptr, int sizeX, int sizeY, int sizeZ) { - glPixelStorei(GL_UNPACK_ROW_LENGTH, 0); - glPixelStorei(GL_UNPACK_SKIP_PIXELS, 0); - glPixelStorei(GL_UNPACK_SKIP_ROWS, 0); - glPixelStorei(GL_UNPACK_SKIP_IMAGES, 0); - glPixelStorei(GL_UNPACK_IMAGE_HEIGHT, 0); - glPixelStorei(GL_UNPACK_ALIGNMENT, (int) EmbeddedLightVolume.STRIDE); - - glTexSubImage3D(GL_TEXTURE_3D, 0, 0, 0, 0, sizeX, sizeY, sizeZ, GL30.GL_RG, GL_UNSIGNED_BYTE, ptr); - - glPixelStorei(GL_UNPACK_ALIGNMENT, 4); // 4 is the default - } - - public void delete() { - if (texture != null) { - texture.delete(); - } - } -} diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightVolume.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightVolume.java deleted file mode 100644 index c815ae3ea..000000000 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/EmbeddedLightVolume.java +++ /dev/null @@ -1,187 +0,0 @@ -package dev.engine_room.flywheel.backend.engine.embed; - -import org.jetbrains.annotations.Nullable; -import org.lwjgl.system.MemoryUtil; - -import dev.engine_room.flywheel.lib.memory.MemoryBlock; -import net.minecraft.core.BlockPos; -import net.minecraft.world.level.BlockAndTintGetter; -import net.minecraft.world.level.LightLayer; - -public class EmbeddedLightVolume { - public static final long STRIDE = Short.BYTES; - - private int minX; - private int minY; - private int minZ; - private int maxX; - private int maxY; - private int maxZ; - - private final BlockPos.MutableBlockPos scratchPos = new BlockPos.MutableBlockPos(); - - @Nullable - protected MemoryBlock memoryBlock; - protected boolean empty = true; - - public boolean empty() { - return empty; - } - - public void collect(BlockAndTintGetter level, int minX, int minY, int minZ, int sizeX, int sizeY, int sizeZ) { - maybeExpandForBox(minX, minY, minZ, sizeX, sizeY, sizeZ); - - empty = false; - - for (int z = minZ; z < minZ + sizeZ; z++) { - for (int y = minY; y < minY + sizeY; y++) { - for (int x = minX; x < minX + sizeX; x++) { - paintLight(level, x, y, z); - } - } - } - } - - private void paintLight(BlockAndTintGetter level, int x, int y, int z) { - scratchPos.set(x, y, z); - - int block = level.getBrightness(LightLayer.BLOCK, scratchPos); - int sky = level.getBrightness(LightLayer.SKY, scratchPos); - - long ptr = this.memoryBlock.ptr() + offset(x - x(), y - y(), z - z(), sizeX(), sizeY()); - MemoryUtil.memPutShort(ptr, (short) ((block << 4) | sky << 12)); - } - - private void maybeExpandForBox(int x, int y, int z, int sizeX, int sizeY, int sizeZ) { - if (empty || memoryBlock == null) { - // We're either brand new or recently #clear'd, - // so none of the previous min/max values have any meaning. - this.minX = x; - this.minY = y; - this.minZ = z; - this.maxX = x + sizeX; - this.maxY = y + sizeY; - this.maxZ = z + sizeZ; - - int volume = sizeX * sizeY * sizeZ; - long neededSize = volume * STRIDE; - - if (memoryBlock == null) { - memoryBlock = MemoryBlock.malloc(neededSize); - } else if (memoryBlock.size() < neededSize) { - // There's some memory left over from before the last #clear, - // but not enough to hold this initial box. Need to grow the block. - memoryBlock.realloc(neededSize); - } - // else: we have enough memory left over to hold this box, nothing to do! - return; - } - - int oldMinX = this.minX; - int oldMinY = this.minY; - int oldMinZ = this.minZ; - int oldSizeX = this.sizeX(); - int oldSizeY = this.sizeY(); - int oldSizeZ = this.sizeZ(); - boolean changed = false; - - if (x < this.minX) { - this.minX = x; - changed = true; - } - if (y < this.minY) { - this.minY = y; - changed = true; - } - if (z < this.minZ) { - this.minZ = z; - changed = true; - } - if (x + sizeX > this.maxX) { - this.maxX = x + sizeX; - changed = true; - } - if (y + sizeY > this.maxY) { - this.maxY = y + sizeY; - changed = true; - } - if (z + sizeZ > this.maxZ) { - this.maxZ = z + sizeZ; - changed = true; - } - - if (!changed) { - return; - } - - int volume = volume(); - - memoryBlock = memoryBlock.realloc(volume * STRIDE); - - int xOff = oldMinX - minX; - int yOff = oldMinY - minY; - int zOff = oldMinZ - minZ; - - blit(memoryBlock.ptr(), 0, 0, 0, oldSizeX, oldSizeY, memoryBlock.ptr(), xOff, yOff, zOff, sizeX(), sizeY(), oldSizeX, oldSizeY, oldSizeZ); - } - - public static void blit(long src, int srcX, int srcY, int srcZ, int srcSizeX, int srcSizeY, long dst, int dstX, int dstY, int dstZ, int dstSizeX, int dstSizeY, int sizeX, int sizeY, int sizeZ) { - for (int z = 0; z < sizeZ; z++) { - for (int y = 0; y < sizeY; y++) { - for (int x = 0; x < sizeX; x++) { - long srcPtr = src + offset(x + srcX, y + srcY, z + srcZ, srcSizeX, srcSizeY); - long dstPtr = dst + offset(x + dstX, y + dstY, z + dstZ, dstSizeX, dstSizeY); - - MemoryUtil.memPutShort(dstPtr, MemoryUtil.memGetShort(srcPtr)); - } - } - } - } - - public static long offset(int x, int y, int z, int sizeX, int sizeY) { - return (x + sizeX * (y + sizeY * z)) * STRIDE; - } - - public void clear() { - empty = true; - } - - public void delete() { - if (memoryBlock != null) { - memoryBlock.free(); - memoryBlock = null; - } - } - - public long ptr() { - return memoryBlock.ptr(); - } - - public int x() { - return minX; - } - - public int y() { - return minY; - } - - public int z() { - return minZ; - } - - public int sizeX() { - return maxX - minX; - } - - public int sizeY() { - return maxY - minY; - } - - public int sizeZ() { - return maxZ - minZ; - } - - public int volume() { - return sizeX() * sizeY() * sizeZ(); - } -} diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightLut.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightLut.java new file mode 100644 index 000000000..cfe4ced2f --- /dev/null +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightLut.java @@ -0,0 +1,139 @@ +package dev.engine_room.flywheel.backend.engine.embed; + +import org.jetbrains.annotations.NotNull; + +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.ints.IntObjectImmutablePair; +import it.unimi.dsi.fastutil.ints.IntObjectPair; +import it.unimi.dsi.fastutil.longs.Long2IntMap; +import it.unimi.dsi.fastutil.longs.LongArrayList; +import it.unimi.dsi.fastutil.longs.LongComparator; +import it.unimi.dsi.fastutil.objects.ReferenceArrayList; +import net.minecraft.core.SectionPos; + +public class LightLut { + public static final LongComparator SECTION_X_THEN_Y_THEN_Z = (long a, long b) -> { + final var xComp = Integer.compare(SectionPos.x(a), SectionPos.x(b)); + if (xComp != 0) { + return xComp; + } + var yComp = Integer.compare(SectionPos.y(a), SectionPos.y(b)); + if (yComp != 0) { + return yComp; + } + return Integer.compare(SectionPos.z(a), SectionPos.z(b)); + }; + + + // Massive kudos to RogueLogix for figuring out this LUT scheme. + // TODO: switch to y x z or x z y ordering + // DATA LAYOUT + // [0] : base chunk X, X index count, followed by linear indices of y blocks + // [yBlockIndex] : baseChunk Y, Y index count, followed by linear indices of z blocks for this x + // [zBlockIndex] : baseChunk Z, Z index count, followed by linear indices of lighting chunks + // this data layout allows a single buffer to represent the lighting volume, without requiring the entire 3d lookup volume to be allocated + public static IntArrayList buildLut(Long2IntMap sectionIndicesMaps) { + if (sectionIndicesMaps.isEmpty()) { + return new IntArrayList(); + } + final var positions = sortedKeys(sectionIndicesMaps); + final var baseX = SectionPos.x(positions.getLong(0)); + + return buildLut(baseX, buildIndices(sectionIndicesMaps, positions, baseX)); + } + + private static ReferenceArrayList>> buildIndices(Long2IntMap sectionIndicesMaps, LongArrayList positions, int baseX) { + final var indices = new ReferenceArrayList>>(); + for (long position : positions) { + final var x = SectionPos.x(position); + final var y = SectionPos.y(position); + final var z = SectionPos.z(position); + + final var xIndex = x - baseX; + if (indices.size() <= xIndex) { + indices.ensureCapacity(xIndex + 1); + indices.size(xIndex + 1); + } + var yLookup = indices.get(xIndex); + if (yLookup == null) { + //noinspection SuspiciousNameCombination + yLookup = new IntObjectImmutablePair<>(y, new ReferenceArrayList<>()); + indices.set(xIndex, yLookup); + } + + final var yIndices = yLookup.right(); + final var yIndex = y - yLookup.leftInt(); + if (yIndices.size() <= yIndex) { + yIndices.ensureCapacity(yIndex + 1); + yIndices.size(yIndex + 1); + } + var zLookup = yIndices.get(yIndex); + if (zLookup == null) { + zLookup = new IntArrayList(); + zLookup.add(z); + zLookup.add(0); // this value will be filled in later + yIndices.set(yIndex, zLookup); + } + + final var zIndex = z - zLookup.getInt(0); + if ((zLookup.size() - 2) <= zIndex) { + zLookup.ensureCapacity(zIndex + 3); + zLookup.size(zIndex + 3); + } + zLookup.set(zIndex + 2, sectionIndicesMaps.get(position)); + } + return indices; + } + + private static @NotNull LongArrayList sortedKeys(Long2IntMap sectionIndicesMaps) { + final var out = new LongArrayList(sectionIndicesMaps.keySet()); + out.unstableSort(SECTION_X_THEN_Y_THEN_Z); + return out; + } + + private static IntArrayList buildLut(int baseX, ReferenceArrayList>> indices) { + final var out = new IntArrayList(); + out.add(baseX); + out.add(indices.size()); + for (int i = 0; i < indices.size(); i++) { + out.add(0); + } + for (int x = 0; x < indices.size(); x++) { + final var yLookup = indices.get(x); + if (yLookup == null) { + out.set(x + 2, 0); + continue; + } + // ensure that the base position and size dont cross a (64 byte) cache line + if ((out.size() & 0xF) == 0xF) { + out.add(0); + } + + final var baseYIndex = out.size(); + out.set(x + 2, baseYIndex); + + final var yIndices = yLookup.right(); + out.add(yLookup.leftInt()); + out.add(yIndices.size()); + for (int i = 0; i < indices.size(); i++) { + out.add(0); + } + + for (int y = 0; y < yIndices.size(); y++) { + final var zLookup = yIndices.get(y); + if (zLookup == null) { + out.set(baseYIndex + y + 2, 0); + continue; + } + // ensure that the base position and size dont cross a (64 byte) cache line + if ((out.size() & 0xF) == 0xF) { + out.add(0); + } + out.set(baseYIndex + y + 2, out.size()); + zLookup.set(1, zLookup.size() - 2); + out.addAll(zLookup); + } + } + return out; + } +} diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightStorage.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightStorage.java new file mode 100644 index 000000000..c1edb2fcf --- /dev/null +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/LightStorage.java @@ -0,0 +1,195 @@ +package dev.engine_room.flywheel.backend.engine.embed; + +import java.util.BitSet; + +import org.lwjgl.system.MemoryUtil; + +import dev.engine_room.flywheel.backend.engine.indirect.StagingBuffer; +import it.unimi.dsi.fastutil.ints.IntArrayList; +import it.unimi.dsi.fastutil.longs.Long2IntMap; +import it.unimi.dsi.fastutil.longs.Long2IntOpenHashMap; +import net.minecraft.core.BlockPos; +import net.minecraft.core.SectionPos; +import net.minecraft.world.level.LevelAccessor; +import net.minecraft.world.level.LightLayer; + +/** + * TODO: AO data + * A managed arena of light sections for uploading to the GPU. + * + *

Each section represents an 18x18x18 block volume of light data. + * The "edges" are taken from the neighboring sections, so that each + * shader invocation only needs to access a single section of data. + * Even still, neighboring shader invocations may need to access other sections. + * + *

Sections are logically stored as a 9x9x9 array of longs, + * where each long holds a 2x2x2 array of light data. + *
Both the greater array and the longs are packed in x, z, y order. + * + *

Thus, each section occupies 5832 bytes. + */ +public class LightStorage { + private final long SECTION_SIZE_BYTES = 9 * 9 * 9 * 8; + private final int DEFAULT_ARENA_CAPACITY_SECTIONS = 64; + private final int INVALID_SECTION = -1; + + private final LevelAccessor level; + + private final Arena arena; + private final Long2IntMap section2ArenaIndex = new Long2IntOpenHashMap(); + { + section2ArenaIndex.defaultReturnValue(INVALID_SECTION); + } + + private final BitSet changed = new BitSet(); + private boolean newSections = false; + + public LightStorage(LevelAccessor level) { + this.level = level; + + arena = new Arena(SECTION_SIZE_BYTES, DEFAULT_ARENA_CAPACITY_SECTIONS); + } + + public void addSection(long section) { + var lightEngine = level.getLightEngine(); + + var blockLight = lightEngine.getLayerListener(LightLayer.BLOCK); + var skyLight = lightEngine.getLayerListener(LightLayer.SKY); + + var blockPos = new BlockPos.MutableBlockPos(); + + int xMin = SectionPos.sectionToBlockCoord(SectionPos.x(section)); + int yMin = SectionPos.sectionToBlockCoord(SectionPos.y(section)); + int zMin = SectionPos.sectionToBlockCoord(SectionPos.z(section)); + + int index = indexForSection(section); + + changed.set(index); + + long ptr = arena.indexToPointer(index); + + for (int y = -1; y < 17; y++) { + for (int z = -1; z < 17; z++) { + for (int x = -1; x < 17; x++) { + blockPos.set(xMin + x, yMin + y, zMin + z); + var block = blockLight.getLightValue(blockPos); + var sky = skyLight.getLightValue(blockPos); + + write(ptr, x, y, z, block, sky); + } + } + } + } + + void addSectionFast(long section) { + // TODO: get this to work. it should be MUCH faster to read directly from the data layer + // though it's more complicated to manage which section datas we fetch + var lightEngine = level.getLightEngine(); + + var blockLight = lightEngine.getLayerListener(LightLayer.BLOCK); + var skyLight = lightEngine.getLayerListener(LightLayer.SKY); + + var sectionPos = SectionPos.of(section); + var blockData = blockLight.getDataLayerData(sectionPos); + var skyData = skyLight.getDataLayerData(sectionPos); + + if (blockData == null || skyData == null) { + return; + } + + long ptr = ptrForSection(section); + + for (int y = 0; y < 16; y++) { + for (int z = 0; z < 16; z++) { + for (int x = 0; x < 16; x++) { + var block = blockData.get(x, y, z); + var sky = skyData.get(x, y, z); + + write(ptr, x, y, z, block, sky); + } + } + } + } + + /** + * Write to the given section. + * @param ptr Pointer to the base of a section's data. + * @param x X coordinate in the section, from [-1, 16]. + * @param y Y coordinate in the section, from [-1, 16]. + * @param z Z coordinate in the section, from [-1, 16]. + * @param block The block light level, from [0, 15]. + * @param sky The sky light level, from [0, 15]. + */ + private void write(long ptr, int x, int y, int z, int block, int sky) { + int x1 = x + 1; + int y1 = y + 1; + int z1 = z + 1; + + int offset = x1 + z1 * 18 + y1 * 18 * 18; + + long packedByte = (block & 0xF) | ((sky & 0xF) << 4); + + MemoryUtil.memPutByte(ptr + offset, (byte) packedByte); + } + + private void writeFor2Cubed(long ptr, int x, int y, int z, int block, int sky) { + int x1 = x + 1; + int y1 = y + 1; + int z1 = z + 1; + + int longIndex = (x1 >> 1) + (z1 >> 1) * 9 + (y1 >> 1) * 9 * 9; + int byteIndexInLong = (x1 & 1) + ((z1 & 1) << 1) + ((y1 & 1) << 2); + + long packedByte = (block & 0xF) | ((sky & 0xF) << 4); + + MemoryUtil.memPutByte(ptr + longIndex * 8L + byteIndexInLong, (byte) packedByte); + } + + /** + * Get a pointer to the base of the given section. + *

If the section is not yet reserved, allocate a chunk in the arena. + * @param section The section to write to. + * @return A raw pointer to the base of the section. + */ + private long ptrForSection(long section) { + return arena.indexToPointer(indexForSection(section)); + } + + private int indexForSection(long section) { + int out = section2ArenaIndex.get(section); + + // Need to allocate. + if (out == INVALID_SECTION) { + out = arena.alloc(); + section2ArenaIndex.put(section, out); + newSections = true; + } + return out; + } + + public void removeSection(long section) { + + } + + public void delete() { + arena.delete(); + } + + public boolean hasChanges() { + return !changed.isEmpty(); + } + + public boolean hasNewSections() { + return newSections; + } + + public void uploadChangedSections(StagingBuffer staging, int dstVbo) { + for (int i = changed.nextSetBit(0); i >= 0; i = changed.nextSetBit(i + 1)) { + staging.enqueueCopy(arena.indexToPointer(i), SECTION_SIZE_BYTES, dstVbo, i * SECTION_SIZE_BYTES); + } + } + + public IntArrayList createLut() { + return LightLut.buildLut(section2ArenaIndex); + } +} diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/NestedEmbeddedEnvironment.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/NestedEmbeddedEnvironment.java index f957a7e43..82afd22fa 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/NestedEmbeddedEnvironment.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/NestedEmbeddedEnvironment.java @@ -6,7 +6,7 @@ import dev.engine_room.flywheel.api.event.RenderStage; import dev.engine_room.flywheel.backend.engine.EngineImpl; import dev.engine_room.flywheel.backend.gl.shader.GlProgram; -import net.minecraft.world.level.BlockAndTintGetter; +import it.unimi.dsi.fastutil.longs.LongSet; public class NestedEmbeddedEnvironment extends AbstractEmbeddedEnvironment { private final AbstractEmbeddedEnvironment parent; @@ -18,11 +18,8 @@ public NestedEmbeddedEnvironment(AbstractEmbeddedEnvironment parent, EngineImpl } @Override - public void collectLight(BlockAndTintGetter level, int minX, int minY, int minZ, int sizeX, int sizeY, int sizeZ) { - } - - @Override - public void invalidateLight() { + public void lightChunks(LongSet chunks) { + // noop } @Override @@ -38,7 +35,7 @@ public void composeMatrices(Matrix4f pose, Matrix3f normal) { } @Override - public void actuallyDelete() { + public void _delete() { parent.release(); } } diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/TopLevelEmbeddedEnvironment.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/TopLevelEmbeddedEnvironment.java index d5a8aa9f9..926d246b9 100644 --- a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/TopLevelEmbeddedEnvironment.java +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/embed/TopLevelEmbeddedEnvironment.java @@ -4,14 +4,13 @@ import org.joml.Matrix4f; import dev.engine_room.flywheel.api.event.RenderStage; -import dev.engine_room.flywheel.backend.Samplers; import dev.engine_room.flywheel.backend.engine.EngineImpl; import dev.engine_room.flywheel.backend.gl.shader.GlProgram; -import net.minecraft.world.level.BlockAndTintGetter; +import it.unimi.dsi.fastutil.longs.LongArraySet; +import it.unimi.dsi.fastutil.longs.LongSet; public class TopLevelEmbeddedEnvironment extends AbstractEmbeddedEnvironment { - private final EmbeddedLightVolume lightVolume = new EmbeddedLightVolume(); - private final EmbeddedLightTexture lightTexture = new EmbeddedLightTexture(); + private final LongSet lightSections = new LongArraySet(); public TopLevelEmbeddedEnvironment(EngineImpl engine, RenderStage renderStage) { super(engine, renderStage); @@ -21,45 +20,18 @@ public TopLevelEmbeddedEnvironment(EngineImpl engine, RenderStage renderStage) { public void flush() { super.flush(); - if (lightVolume.empty()) { - return; - } - Samplers.EMBEDDED_LIGHT.makeActive(); - - lightTexture.bind(); - - lightTexture.ensureCapacity(lightVolume.sizeX(), lightVolume.sizeY(), lightVolume.sizeZ()); - - lightTexture.upload(lightVolume.ptr(), lightVolume.sizeX(), lightVolume.sizeY(), lightVolume.sizeZ()); - } - - @Override - public void collectLight(BlockAndTintGetter level, int minX, int minY, int minZ, int sizeX, int sizeY, int sizeZ) { - lightVolume.collect(level, minX, minY, minZ, sizeX, sizeY, sizeZ); + lightSections.forEach(engine.lightStorage()::addSection); } @Override - public void invalidateLight() { - lightVolume.clear(); + public void lightChunks(LongSet chunks) { + lightSections.clear(); + lightSections.addAll(chunks); } @Override public void setupLight(GlProgram program) { - if (!lightVolume.empty()) { - Samplers.EMBEDDED_LIGHT.makeActive(); - - lightTexture.bind(); - - float oneOverSizeX = 1f / (float) lightTexture.sizeX; - float oneOverSizeY = 1f / (float) lightTexture.sizeY; - float oneOverSizeZ = 1f / (float) lightTexture.sizeZ; - - program.setVec3(EmbeddingUniforms.ONE_OVER_LIGHT_BOX_SIZE, oneOverSizeX, oneOverSizeY, oneOverSizeZ); - program.setVec3(EmbeddingUniforms.LIGHT_VOLUME_MIN, lightVolume.x(), lightVolume.y(), lightVolume.z()); - program.setBool(EmbeddingUniforms.USE_LIGHT_VOLUME, true); - } else { - program.setBool(EmbeddingUniforms.USE_LIGHT_VOLUME, false); - } + program.setBool(EmbeddingUniforms.USE_LIGHT_VOLUME, !lightSections.isEmpty()); } @Override @@ -69,11 +41,7 @@ public void composeMatrices(Matrix4f pose, Matrix3f normal) { } @Override - public void actuallyDelete() { - // We could technically free the light volume right away in _delete, but - // the control flow here is so convoluted that it's probably best to do - // everything in one place. - lightVolume.delete(); - lightTexture.delete(); + protected void _delete() { + } } diff --git a/common/src/backend/java/dev/engine_room/flywheel/backend/engine/indirect/LightBuffers.java b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/indirect/LightBuffers.java new file mode 100644 index 000000000..9a0a37a57 --- /dev/null +++ b/common/src/backend/java/dev/engine_room/flywheel/backend/engine/indirect/LightBuffers.java @@ -0,0 +1,29 @@ +package dev.engine_room.flywheel.backend.engine.indirect; + +import org.lwjgl.system.MemoryUtil; + +import dev.engine_room.flywheel.backend.engine.embed.LightStorage; + +public class LightBuffers { + private final ResizableStorageBuffer lightArena = new ResizableStorageBuffer(); + private final ResizableStorageArray lut = new ResizableStorageArray(4); + + public LightBuffers() { + } + + public void flush(StagingBuffer staging, LightStorage light) { + light.uploadChangedSections(staging, lightArena.handle()); + + if (light.hasNewSections()) { + var lut = light.createLut(); + + this.lut.ensureCapacity(lut.size()); + + staging.enqueueCopy((long) lut.size() * Integer.BYTES, this.lut.handle(), 0, ptr -> { + for (int i = 0; i < lut.size(); i++) { + MemoryUtil.memPutInt(ptr + (long) i * Integer.BYTES, lut.getInt(i)); + } + }); + } + } +} diff --git a/common/src/backend/resources/assets/flywheel/flywheel/internal/common.vert b/common/src/backend/resources/assets/flywheel/flywheel/internal/common.vert index ec009e414..fae9fd91c 100644 --- a/common/src/backend/resources/assets/flywheel/flywheel/internal/common.vert +++ b/common/src/backend/resources/assets/flywheel/flywheel/internal/common.vert @@ -67,12 +67,10 @@ vec2 getCrumblingTexCoord() { #endif #ifdef _FLW_EMBEDDED -uniform vec3 _flw_oneOverLightBoxSize; -uniform vec3 _flw_lightVolumeMin; uniform mat4 _flw_modelMatrix; uniform mat3 _flw_normalMatrix; -out vec3 _flw_lightVolumeCoord; +bool _flw_embeddedLight(vec3 worldPos, vec3 normal, out vec2 lightCoord); #endif flat out uint _flw_instanceID; @@ -90,7 +88,10 @@ void _flw_main(in FlwInstance instance, in uint stableInstanceID) { flw_vertexPos = _flw_modelMatrix * flw_vertexPos; flw_vertexNormal = _flw_normalMatrix * flw_vertexNormal; - _flw_lightVolumeCoord = (flw_vertexPos.xyz - _flw_lightVolumeMin) * _flw_oneOverLightBoxSize; + vec2 embeddedLight; + if (_flw_embeddedLight(flw_vertexPos, flw_vertexNormal, embeddedLight)) { + flw_vertexLight = max(flw_vertexLight, embeddedLight); + } #endif flw_vertexNormal = normalize(flw_vertexNormal); diff --git a/common/src/backend/resources/assets/flywheel/flywheel/internal/indirect/main.vert b/common/src/backend/resources/assets/flywheel/flywheel/internal/indirect/main.vert index 55bd8fc0f..de5493d3c 100644 --- a/common/src/backend/resources/assets/flywheel/flywheel/internal/indirect/main.vert +++ b/common/src/backend/resources/assets/flywheel/flywheel/internal/indirect/main.vert @@ -11,6 +11,77 @@ layout(std430, binding = _FLW_DRAW_BUFFER_BINDING) restrict readonly buffer Draw MeshDrawCommand _flw_drawCommands[]; }; +#ifdef _FLW_EMBEDDED + +layout(std430, binding = 8) restrict readonly buffer EmbeddingLut { + uint _flw_embeddingLut[]; +}; + +const uint _FLW_LIGHT_SECTION_SIZE_BYTES = 18 * 18 * 18; +const uint _FLW_LIGHT_SECTION_SIZE_INTS = _FLW_LIGHT_SECTION_SIZE_BYTES / 4; + +layout(std430, binding = 9) restrict readonly buffer LightSections { + uint _flw_lightSections[]; +}; + +bool _flw_nextLut(uint base, int coord, out uint next) { + int start = int(_flw_embeddingLut[base]); + uint size = _flw_embeddingLut[base + 1]; + + int i = coord - start; + + if (i < 0 || i >= size) { + return true; + } + + next = _flw_embeddingLut[base + 2 + i]; + + return false; +} + +bool _flw_chunkCoordToSectionIndex(ivec3 sectionPos, out uint index) { + uint y; + if (_flw_nextLut(0, sectionPos.x, y)) { + return true; + } + + uint z; + if (_flw_nextLut(y, sectionPos.y, z)) { + return true; + } + return _flw_nextLut(z, sectionPos.z, index); +} + +vec2 _flw_lightAt(uint sectionOffset, ivec3 blockInSectionPos) { + uint byteOffset = blockInSectionPos.x + blockInSectionPos.z * 18u + blockInSectionPos.y * 18u * 18u; + + uint uintOffset = byteOffset >> 2u; + uint bitOffset = (byteOffset & 3u) << 3; + + uint packed = _flw_lightSections[sectionOffset + uintOffset]; + uint block = (packed >> bitOffset) & 0xFu; + uint sky = (packed >> (bitOffset + 4u)) & 0xFu; + + return vec2(block, sky); +} + +bool _flw_embeddedLight(vec3 worldPos, vec3 normal, out vec2 lightCoord) { + ivec3 sectionPos = blockPos >> 4; + uvec3 blockInSectionPos = (blockPos & 0xF) + 1; + + uint lightSectionIndex; + if (_flw_chunkCoordToSectionIndex(sectionPos, lightSectionIndex)) { + return false; + } + + uint sectionOffset = lightSectionIndex * _FLW_LIGHT_SECTION_SIZE_INTS; + + lightCoord = _flw_lightAt(sectionOffset, blockInSectionPos); + return true; +} + +#endif + uniform uint _flw_baseDraw; flat out uvec3 _flw_packedMaterial; diff --git a/common/src/backend/resources/assets/flywheel/flywheel/internal/instancing/main.vert b/common/src/backend/resources/assets/flywheel/flywheel/internal/instancing/main.vert index 8c4b9d6a3..ad3e9f9e3 100644 --- a/common/src/backend/resources/assets/flywheel/flywheel/internal/instancing/main.vert +++ b/common/src/backend/resources/assets/flywheel/flywheel/internal/instancing/main.vert @@ -4,6 +4,12 @@ uniform uvec4 _flw_packedMaterial; uniform int _flw_baseInstance = 0; +#ifdef _FLW_EMBEDDED +bool _flw_embeddedLight(vec3 worldPos, vec3 normal, out vec2 lightCoord) { + return true; +} +#endif + void main() { _flw_uberMaterialVertexIndex = _flw_packedMaterial.x; _flw_unpackMaterialProperties(_flw_packedMaterial.w, flw_material);