feat: Simulator blockstream manager

.gitignore

@@ -56,3 +56,5 @@ server/docker/.env
 # manual test files
 server/src/test/resources/test_output/
 
+# simulator files, this files are un-tarred before build
+/simulator/src/main/resources/block-0.0.3/

simulator/build.gradle.kts

@@ -38,3 +38,23 @@ testModuleInfo {
     requires("org.mockito.junit.jupiter")
     requiresStatic("com.github.spotbugs.annotations")
 }
+
+tasks.register<Copy>("untarTestBlockStream") {
+    description = "Untar the test block stream data"
+    group = "build"
+
+    val targetDir = file("src/main/resources")
+
+    from(tarTree(resources.gzip(file("src/main/resources/block-0.0.3.tar.gz"))))
+    into(targetDir)
+
+    // Mark task as not up-to-date if the directory is empty
+    outputs.upToDateWhen { targetDir.listFiles()?.isNotEmpty() ?: false }
+
+    // Adding a simple logging to verify
+    doLast { println("Untar task completed. Files should be in: ${targetDir.absolutePath}") }
+}
+
+tasks.named("processResources") { dependsOn(tasks.named("untarTestBlockStream")) }
+
+tasks.named("sourcesJar") { dependsOn(tasks.named("untarTestBlockStream")) }

simulator/src/main/java/com/hedera/block/simulator/config/data/BlockStreamConfig.java

@@ -19,12 +19,46 @@
 import com.hedera.block.simulator.config.types.GenerationMode;
 import com.swirlds.config.api.ConfigData;
 import com.swirlds.config.api.ConfigProperty;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 
 /**
  * The BlockStreamConfig class defines the configuration data for the block stream.
  *
  * @param generationMode the mode of generation for the block stream
+ * @param folderRootPath the root path of the folder containing the block stream
  */
 @ConfigData("blockStream")
 public record BlockStreamConfig(
-        @ConfigProperty(defaultValue = "DIR") GenerationMode generationMode) {}
+        @ConfigProperty(defaultValue = "DIR") GenerationMode generationMode,
+        @ConfigProperty(defaultValue = "") String folderRootPath) {
+
+    /**
+     * Constructor to set the default root path if not provided, it will be set to the data
+     * directory in the current working directory
+     */
+    public BlockStreamConfig {
+        // verify rootPath prop
+        Path path = Path.of(folderRootPath);
+
+        // if rootPath is empty, set it to the default data directory
+        // default data directory is the "src/main/resources/blocks" directory
+        if (folderRootPath.isEmpty()) {
+            path =
+                    Paths.get(folderRootPath)
+                            .toAbsolutePath()
+                            .resolve("src/main/resources/block-0.0.3");
+        }
+        // Check if absolute
+        if (!path.isAbsolute()) {
+            throw new IllegalArgumentException(folderRootPath + " Root path must be absolute");
+        }
+        // Check if the folder exists
+        if (Files.notExists(path) && generationMode == GenerationMode.DIR) {
+            throw new IllegalArgumentException("Folder does not exist: " + path);
+        }
+
+        folderRootPath = path.toString();
+    }
+}

simulator/src/main/java/com/hedera/block/simulator/generator/BlockAsFileBlockStreamManager.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hedera.block.simulator.generator;
+
+import com.hedera.block.simulator.config.data.BlockStreamConfig;
+import com.hedera.block.simulator.config.types.GenerationMode;
+import com.hedera.hapi.block.stream.Block;
+import com.hedera.hapi.block.stream.BlockItem;
+import com.hedera.pbj.runtime.ParseException;
+import com.hedera.pbj.runtime.io.buffer.Bytes;
+import edu.umd.cs.findbugs.annotations.NonNull;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.stream.Stream;
+import javax.inject.Inject;
+
+/** The block as file block stream manager. */
+public class BlockAsFileBlockStreamManager implements BlockStreamManager {
+
+    private static final System.Logger LOGGER =
+            System.getLogger(BlockAsFileBlockStreamManager.class.getName());
+
+    final String rootFolder;
+
+    final List<Block> blockList = new ArrayList<>();
+
+    int currentBlockIndex = 0;
+    int currentBlockItemIndex = 0;
+    int lastGivenBlockNumber = 0;
+
+    /**
+     * Constructor for the block as file block stream manager.
+     *
+     * @param blockStreamConfig the block stream config
+     */
+    @Inject
+    public BlockAsFileBlockStreamManager(@NonNull BlockStreamConfig blockStreamConfig) {
+        this.rootFolder = blockStreamConfig.folderRootPath();
+        try {
+            this.loadBlocks();
+        } catch (IOException | ParseException | IllegalArgumentException e) {
+            LOGGER.log(System.Logger.Level.ERROR, "Error loading blocks", e);
+            throw new RuntimeException(e);
+        }
+
+        LOGGER.log(System.Logger.Level.INFO, "Loaded " + blockList.size() + " blocks into memory");
+    }
+
+    @Override
+    public GenerationMode getGenerationMode() {
+        return GenerationMode.DIR;
+    }
+
+    @Override
+    public BlockItem getNextBlockItem() {
+        BlockItem nextBlockItem =
+                blockList.get(currentBlockIndex).items().get(currentBlockItemIndex);
+        currentBlockItemIndex++;
+        if (currentBlockItemIndex >= blockList.get(currentBlockIndex).items().size()) {
+            currentBlockItemIndex = 0;
+            currentBlockIndex++;
+        }
+        return nextBlockItem;
+    }
+
+    @Override
+    public Block getNextBlock() {
+        Block nextBlock = blockList.get(currentBlockIndex);
+        currentBlockIndex++;
+        lastGivenBlockNumber++;
+        if (currentBlockIndex >= blockList.size()) {
+            currentBlockIndex = 0;
+        }
+        return nextBlock;
+    }
+
+    private void loadBlocks() throws IOException, ParseException {
+
+        Path rootPath = Path.of(rootFolder);
+
+        try (Stream<Path> blockFiles = Files.list(rootPath)) {
+
+            List<Path> sortedBlockFiles =
+                    blockFiles.sorted(Comparator.comparing(Path::getFileName)).toList();
+
+            for (Path blockPath : sortedBlockFiles) {
+
+                byte[] blockBytes;
+                if (blockPath.toString().endsWith(".gz")) {
+                    blockBytes = Utils.readGzFile(blockPath);
+                } else if (blockPath.toString().endsWith(".blk")) {
+                    blockBytes = Files.readAllBytes(blockPath);
+                } else {
+                    throw new IllegalArgumentException("Invalid file format: " + blockPath);
+                }
+
+                Block block = Block.PROTOBUF.parse(Bytes.wrap(blockBytes));
+                blockList.add(block);
+                LOGGER.log(System.Logger.Level.DEBUG, "Loaded block: " + blockPath);
+            }
+        }
+    }
+}

simulator/src/main/java/com/hedera/block/simulator/generator/BlockStreamManager.java

@@ -16,15 +16,31 @@
 
 package com.hedera.block.simulator.generator;
 
+import com.hedera.block.simulator.config.types.GenerationMode;
 import com.hedera.hapi.block.stream.Block;
+import com.hedera.hapi.block.stream.BlockItem;
 
-/** this interface defines the contract for managing the block stream. */
+/** The block stream manager interface. */
 public interface BlockStreamManager {
 
     /**
-     * Gets the next block in the block stream.
+     * Get the generation mode.
      *
-     * @return the next block in the block stream
+     * @return the generation mode
+     */
+    GenerationMode getGenerationMode();
+
+    /**
+     * Get the next block item.
+     *
+     * @return the next block item
+     */
+    BlockItem getNextBlockItem();
+
+    /**
+     * Get the next block.
+     *
+     * @return the next block
      */
     Block getNextBlock();
 }

simulator/src/main/java/com/hedera/block/simulator/generator/GeneratorInjectionModule.java

@@ -27,10 +27,11 @@ public interface GeneratorInjectionModule {
     /**
      * Provides the block stream manager.
      *
-     * @param blockStreamManager the block stream manager to be used
+     * @param blockAsFileBlockStreamManager the block as file block stream manager
      * @return the block stream manager
      */
     @Singleton
     @Binds
-    BlockStreamManager bindBlockStreamManager(MockBlockStreamManagerImpl blockStreamManager);
+    BlockStreamManager provideBlockStreamManager(
+            BlockAsFileBlockStreamManager blockAsFileBlockStreamManager);
 }

simulator/src/main/java/com/hedera/block/simulator/generator/Utils.java

@@ -0,0 +1,43 @@
+/*
+ * Copyright (C) 2024 Hedera Hashgraph, LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hedera.block.simulator.generator;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.zip.GZIPInputStream;
+
+/** Utility class for the simulator. */
+public class Utils {
+
+    private Utils() {}
+
+    /**
+     * Read a GZIP file and return the content as a byte array.
+     *
+     * @param filePath Path to the GZIP file
+     * @return byte array of the content of the GZIP file
+     * @throws IOException if an I/O error occurs
+     */
+    public static byte[] readGzFile(Path filePath) throws IOException {
+        try (InputStream fileInputStream = Files.newInputStream(filePath);
+                GZIPInputStream gzipInputStream = new GZIPInputStream(fileInputStream)) {
+            return gzipInputStream.readAllBytes();
+        }
+    }
+}

simulator/src/main/java/module-info.java

@@ -7,7 +7,7 @@
     requires static com.github.spotbugs.annotations;
     requires static com.google.auto.service;
     requires com.hedera.block.stream;
-    // requires com.hedera.pbj.runtime; // leaving it here since it will be needed soon.
+    requires com.hedera.pbj.runtime;
     requires com.swirlds.config.api;
     requires com.swirlds.config.extensions;
     requires dagger;

simulator/src/main/resources/app.properties

@@ -1 +1 @@
-blockStream.folderRootPath=/Users/user/Projects/hedera-block-node/simulator/src/main/resources/block-0.0.3/
+