From 5fe56b2c79b62c59a3fb24705f4279bb3c3b1d89 Mon Sep 17 00:00:00 2001 From: Jasper Potts <1466205+jasperpotts@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:00:36 -0700 Subject: [PATCH] feat: 231 add block stream tools (#300) Signed-off-by: jasperpotts Co-authored-by: jasperpotts --- .../kotlin/com.hedera.block.tools.gradle.kts | 21 + settings.gradle.kts | 2 + stream/src/main/java/module-info.java | 59 +++ tools/README.md | 45 +++ tools/build.gradle.kts | 35 ++ .../hedera/block/tools/BlockStreamTool.java | 48 +++ .../block/tools/commands/BlockInfo.java | 376 ++++++++++++++++++ .../block/tools/commands/ConvertToJson.java | 197 +++++++++ tools/src/main/java/module-info.java | 15 + 9 files changed, 798 insertions(+) create mode 100644 buildSrc/src/main/kotlin/com.hedera.block.tools.gradle.kts create mode 100644 tools/README.md create mode 100644 tools/build.gradle.kts create mode 100644 tools/src/main/java/com/hedera/block/tools/BlockStreamTool.java create mode 100644 tools/src/main/java/com/hedera/block/tools/commands/BlockInfo.java create mode 100644 tools/src/main/java/com/hedera/block/tools/commands/ConvertToJson.java create mode 100644 tools/src/main/java/module-info.java diff --git a/buildSrc/src/main/kotlin/com.hedera.block.tools.gradle.kts b/buildSrc/src/main/kotlin/com.hedera.block.tools.gradle.kts new file mode 100644 index 000000000..0779a3619 --- /dev/null +++ b/buildSrc/src/main/kotlin/com.hedera.block.tools.gradle.kts @@ -0,0 +1,21 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("application") + id("com.hedera.block.conventions") + id("me.champeau.jmh") +} diff --git a/settings.gradle.kts b/settings.gradle.kts index e1441bb3a..9e2361aa9 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -24,6 +24,7 @@ include(":suites") include(":stream") include(":server") include(":simulator") +include(":tools") includeBuild(".") // https://github.com/gradle/gradle/issues/21490#issuecomment-1458887481 @@ -62,6 +63,7 @@ dependencyResolutionManagement { version("com.google.auto.service.processor", "1.1.1") version("com.google.auto.service", "1.1.1") version("org.hyperledger.besu.nativelib.secp256k1", "0.8.2") + version("info.picocli", "4.7.6") // gRPC dependencies version("io.grpc", "1.65.1") diff --git a/stream/src/main/java/module-info.java b/stream/src/main/java/module-info.java index f557cc459..71b3bed38 100644 --- a/stream/src/main/java/module-info.java +++ b/stream/src/main/java/module-info.java @@ -5,6 +5,65 @@ exports com.hedera.hapi.block.stream; exports com.hedera.hapi.block.stream.input; exports com.hedera.hapi.block.stream.output; + exports com.hedera.hapi.node.base; + exports com.hedera.hapi.node.base.codec; + exports com.hedera.hapi.node.base.schema; + exports com.hedera.hapi.node.consensus; + exports com.hedera.hapi.node.consensus.codec; + exports com.hedera.hapi.node.consensus.schema; + exports com.hedera.hapi.node.contract; + exports com.hedera.hapi.node.contract.codec; + exports com.hedera.hapi.node.contract.schema; + exports com.hedera.hapi.node.file; + exports com.hedera.hapi.node.file.codec; + exports com.hedera.hapi.node.file.schema; + exports com.hedera.hapi.node.freeze; + exports com.hedera.hapi.node.freeze.codec; + exports com.hedera.hapi.node.freeze.schema; + exports com.hedera.hapi.node.network; + exports com.hedera.hapi.node.network.codec; + exports com.hedera.hapi.node.network.schema; + exports com.hedera.hapi.node.scheduled; + exports com.hedera.hapi.node.scheduled.codec; + exports com.hedera.hapi.node.scheduled.schema; + exports com.hedera.hapi.node.token; + exports com.hedera.hapi.node.token.codec; + exports com.hedera.hapi.node.token.schema; + exports com.hedera.hapi.node.transaction; + exports com.hedera.hapi.node.transaction.codec; + exports com.hedera.hapi.node.transaction.schema; + exports com.hedera.hapi.node.util; + exports com.hedera.hapi.node.util.codec; + exports com.hedera.hapi.node.util.schema; + exports com.hedera.hapi.streams; + exports com.hedera.hapi.streams.codec; + exports com.hedera.hapi.streams.schema; + exports com.hedera.hapi.node.addressbook; + exports com.hedera.hapi.node.state.addressbook.codec; + exports com.hedera.hapi.node.state.addressbook; + exports com.hedera.hapi.node.state.consensus.codec; + exports com.hedera.hapi.node.state.consensus; + exports com.hedera.hapi.node.state.token; + exports com.hedera.hapi.node.state.common; + exports com.hedera.hapi.node.state.contract; + exports com.hedera.hapi.node.state.file; + exports com.hedera.hapi.node.state.recordcache; + exports com.hedera.hapi.node.state.recordcache.codec; + exports com.hedera.hapi.node.state.blockrecords; + exports com.hedera.hapi.node.state.blockrecords.codec; + exports com.hedera.hapi.node.state.blockrecords.schema; + exports com.hedera.hapi.node.state.blockstream; + exports com.hedera.hapi.node.state.schedule; + exports com.hedera.hapi.node.state.primitives; + exports com.hedera.hapi.node.state.throttles; + exports com.hedera.hapi.node.state.congestion; + exports com.hedera.hapi.platform.event; + exports com.hedera.services.stream.proto; + exports com.hederahashgraph.api.proto.java; + exports com.hederahashgraph.service.proto.java; + exports com.hedera.hapi.platform.state; + exports com.hedera.hapi.node.state.roster; + exports com.hedera.hapi.block.stream.schema; requires transitive com.google.common; requires transitive com.google.protobuf; diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 000000000..5def135d9 --- /dev/null +++ b/tools/README.md @@ -0,0 +1,45 @@ +# Command Line Tools for Block Nodes & Streams + +This subproject provides command line tools for working with block stream files and maybe other things in the future. It +uses [picocli](https://picocli.info) to provide a command line interface which makes it easy to extend and add new +subcommands or options. + +## Running from command line +You can run through gradle with the `tools:run` task. For example, to see the help for the `info` subcommand, you can +run: + +`./gradlew -q tools:run --args="info --help"` + +## Subcommands +The following subcommands are available: +- **json** Converts a binary block stream to JSON +- **info** Prints info for block files + +# JSON Subcommand +Converts a binary block stream to JSON + +`Usage: subcommands json [-t] [-ms=] [...]` + +**Options:** +- `-ms ` or `--min-size=` + - Filter to only files bigger than this minimum file size in megabytes +- `-t` or `--transactions` + - expand transactions, this is no longer pure json conversion but is very useful making the +transactions human-readable. +- `...` + - The block files or directories of block files to convert to JSON + +# Info Subcommand +Prints info for block files + +`Usage: subcommands info [-c] [-ms=] [-o=] [...]` + +**Options:** +- `-c` or `--csv` + - Enable CSV output mode (default: false) +- `-ms ` or `--min-size=` + - Filter to only files bigger than this minimum file size in megabytes +- `-o ` or `--output-file=` + - Output to file rather than stdout +- `...` + - The block files or directories of block files to print info for diff --git a/tools/build.gradle.kts b/tools/build.gradle.kts new file mode 100644 index 000000000..26a09c31b --- /dev/null +++ b/tools/build.gradle.kts @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2022-2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id("application") + id("com.hedera.block.tools") +} + +description = "Hedera Block Stream Tools" + +application { + mainModule = "com.hedera.block.tools" + mainClass = "com.hedera.block.tools.BlockStreamTool" +} + +mainModuleInfo { + runtimeOnly("com.swirlds.config.impl") + runtimeOnly("org.apache.logging.log4j.slf4j2.impl") + runtimeOnly("io.grpc.netty.shaded") +} + +testModuleInfo { requiresStatic("com.github.spotbugs.annotations") } diff --git a/tools/src/main/java/com/hedera/block/tools/BlockStreamTool.java b/tools/src/main/java/com/hedera/block/tools/BlockStreamTool.java new file mode 100644 index 000000000..01d275ad6 --- /dev/null +++ b/tools/src/main/java/com/hedera/block/tools/BlockStreamTool.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.tools; + +import com.hedera.block.tools.commands.BlockInfo; +import com.hedera.block.tools.commands.ConvertToJson; +import picocli.CommandLine; +import picocli.CommandLine.Command; + +/** + * Command line tool for working with Hedera block stream files + */ +@SuppressWarnings("InstantiationOfUtilityClass") +@Command( + name = "subcommands", + mixinStandardHelpOptions = true, + version = "BlockStreamTool 0.1", + subcommands = {ConvertToJson.class, BlockInfo.class}) +public final class BlockStreamTool { + + /** + * Empty Default constructor to remove JavaDoc warning + */ + public BlockStreamTool() {} + + /** + * Main entry point for the app + * @param args command line arguments + */ + public static void main(String... args) { + int exitCode = new CommandLine(new BlockStreamTool()).execute(args); + System.exit(exitCode); + } +} diff --git a/tools/src/main/java/com/hedera/block/tools/commands/BlockInfo.java b/tools/src/main/java/com/hedera/block/tools/commands/BlockInfo.java new file mode 100644 index 000000000..446a99246 --- /dev/null +++ b/tools/src/main/java/com/hedera/block/tools/commands/BlockInfo.java @@ -0,0 +1,376 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.tools.commands; + +import com.hedera.hapi.block.stream.Block; +import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.node.base.Transaction; +import com.hedera.hapi.node.transaction.SignedTransaction; +import com.hedera.hapi.node.transaction.TransactionBody; +import com.hedera.hapi.node.transaction.TransactionBody.DataOneOfType; +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.zip.GZIPInputStream; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +/** + * Command line command that prints info for block files + */ +@SuppressWarnings({ + "DataFlowIssue", + "unused", + "StringConcatenationInsideStringBufferAppend", + "DuplicatedCode", + "FieldMayBeFinal" +}) +@Command(name = "info", description = "Prints info for block files") +public class BlockInfo implements Runnable { + + @Parameters(index = "0..*") + private File[] files; + + @Option( + names = {"-ms", "--min-size"}, + description = "Filter to only files bigger than this minimum file size in megabytes") + private double minSizeMb = Double.MAX_VALUE; + + @Option( + names = {"-c", "--csv"}, + description = "Enable CSV output mode (default: ${DEFAULT-VALUE})") + private boolean csvMode = false; + + @Option( + names = {"-o", "--output-file"}, + description = "Output to file rather than stdout") + private File outputFile; + + // atomic counters for total blocks, transactions, items, compressed bytes, and uncompressed bytes + private final AtomicLong totalBlocks = new AtomicLong(0); + private final AtomicLong totalTransactions = new AtomicLong(0); + private final AtomicLong totalItems = new AtomicLong(0); + private final AtomicLong totalBytesCompressed = new AtomicLong(0); + private final AtomicLong totalBytesUncompressed = new AtomicLong(0); + + /** + * Empty Default constructor to remove JavaDoc warning + */ + public BlockInfo() {} + + /** + * Main method to run the command + */ + @Override + public void run() { + System.out.println("csvMode = " + csvMode); + System.out.println("outputFile = " + outputFile.getAbsoluteFile()); + if (files == null || files.length == 0) { + System.err.println("No files to convert"); + } else { + totalTransactions.set(0); + totalItems.set(0); + totalBytesCompressed.set(0); + totalBytesUncompressed.set(0); + // if none of the files exist then print error message + if (Arrays.stream(files).noneMatch(File::exists)) { + System.err.println("No files found"); + System.exit(1); + } + // collect all the block file paths sorted by file name + final List blockFiles = Arrays.stream(files) + .filter( + f -> { // filter out non existent files + if (!f.exists()) { + System.err.println("File not found : " + f); + return false; + } else { + return true; + } + }) + .map(File::toPath) + .flatMap(path -> { + try { + return Files.walk(path); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .filter(Files::isRegularFile) + .filter(file -> file.getFileName().toString().endsWith(".blk") + || file.getFileName().toString().endsWith(".blk.gz")) + .filter( + file -> { // handle min file size + try { + return minSizeMb == Double.MAX_VALUE + || Files.size(file) / 1024.0 / 1024.0 >= minSizeMb; + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .sorted(Comparator.comparing(file -> file.getFileName().toString())) + .toList(); + // create stream of block info strings + final var blockInfoStream = blockFiles.stream().parallel().map(this::blockInfo); + // create CSV header line + final String csvHeader = "\"Block\",\"Items\",\"Transactions\",\"Java Objects\"," + + "\"Original Size (MB)\",\"Uncompressed Size(MB)\",\"Compression\""; + if (outputFile != null) { + // check if file exists and throw error + if (outputFile.exists()) { + System.err.println("Output file already exists : " + outputFile); + System.exit(1); + } + AtomicInteger completedFileCount = new AtomicInteger(0); + try (var writer = Files.newBufferedWriter(outputFile.toPath())) { + if (csvMode) { + writer.write(csvHeader); + writer.newLine(); + } + printProgress(0, blockFiles.size(), 0); + blockInfoStream.forEachOrdered(line -> { + printProgress( + (double) completedFileCount.incrementAndGet() / blockFiles.size(), + blockFiles.size(), + completedFileCount.get()); + try { + writer.write(line); + writer.newLine(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } catch (IOException e) { + throw new RuntimeException(e); + } + } else { + if (csvMode) { + // print CSV column headers + System.out.println(csvHeader); + } + blockInfoStream.forEachOrdered(System.out::println); + } + // print output file complete + if (outputFile != null) { + System.out.println("\nOutput written to CSV file: " + outputFile.getAbsoluteFile()); + } + // print summary + if (!(csvMode && outputFile != null)) { + System.out.println("\n========================================================="); + System.out.println("Summary : "); + System.out.printf(" Total Blocks = %,d \n", totalBlocks.get()); + System.out.printf(" Total Transactions = %,d \n", totalTransactions.get()); + System.out.printf(" Total Items = %,d \n", totalItems.get()); + System.out.printf( + " Total Bytes Compressed = %,.2f MB\n", + totalBytesCompressed.get() / 1024.0 / 1024.0); + System.out.printf( + " Total Bytes Uncompressed = %,.2f MB\n", + totalBytesUncompressed.get() / 1024.0 / 1024.0); + System.out.printf( + " Average transactions per block = %,.2f \n", + totalTransactions.get() / (double) totalBlocks.get()); + System.out.printf( + " Average items per transaction = %,.2f \n", + totalItems.get() / (double) totalTransactions.get()); + System.out.printf( + " Average uncompressed bytes per transaction = %,d \n", + totalTransactions.get() == 0 ? 0 : (totalBytesUncompressed.get() / totalTransactions.get())); + System.out.printf( + " Average compressed bytes per transaction = %,d \n", + totalTransactions.get() == 0 ? 0 : totalBytesCompressed.get() / totalTransactions.get()); + System.out.printf( + " Average uncompressed bytes per item = %,d \n", + totalItems.get() == 0 ? 0 : totalBytesUncompressed.get() / totalItems.get()); + System.out.printf( + " Average compressed bytes per item = %,d \n", + totalItems.get() == 0 ? 0 : totalBytesCompressed.get() / totalItems.get()); + System.out.println("========================================================="); + } + } + } + + /** + * Print progress bar to console + * + * @param progress the progress percentage between 0 and 1 + * @param totalBlockFiles the total number of block files + * @param completedBlockFiles the number of block files completed + */ + public void printProgress(double progress, int totalBlockFiles, int completedBlockFiles) { + final int width = 50; + System.out.print("\r["); + int i = 0; + for (; i <= (int) (progress * width); i++) { + System.out.print("="); + } + for (; i < width; i++) { + System.out.print(" "); + } + System.out.printf( + "] %.0f%% completed %,d of %,d block files", progress * 100, completedBlockFiles, totalBlockFiles); + } + + /** + * Collect info for a block file + * + * @param blockProtoFile the block file to produce info for + * @return the info string + */ + public String blockInfo(Path blockProtoFile) { + try (InputStream fIn = Files.newInputStream(blockProtoFile)) { + byte[] uncompressedData; + if (blockProtoFile.getFileName().toString().endsWith(".gz")) { + uncompressedData = new GZIPInputStream(fIn).readAllBytes(); + } else { + uncompressedData = fIn.readAllBytes(); + } + long start = System.currentTimeMillis(); + final Block block = Block.PROTOBUF.parse(Bytes.wrap(uncompressedData)); + long end = System.currentTimeMillis(); + return blockInfo(block, end - start, Files.size(blockProtoFile), uncompressedData.length); + } catch (Exception e) { + StringWriter sw = new StringWriter(); + sw.append("Error processing file : " + blockProtoFile + "\n"); + e.printStackTrace(new java.io.PrintWriter(sw)); + return sw.toString(); + } + } + + /** + * Collect info for a block + * + * @param block the block to produce info for + * @param parseTimeMs the time taken to parse the block in milliseconds + * @param originalFileSizeBytes the original file size in bytes + * @param uncompressedFileSizeBytes the uncompressed file size in bytes + * @return the info string + */ + public String blockInfo(Block block, long parseTimeMs, long originalFileSizeBytes, long uncompressedFileSizeBytes) { + final StringBuffer output = new StringBuffer(); + long numOfTransactions = + block.items().stream().filter(BlockItem::hasEventTransaction).count(); + totalBlocks.incrementAndGet(); + totalTransactions.addAndGet(numOfTransactions); + totalItems.addAndGet(block.items().size()); + totalBytesCompressed.addAndGet(originalFileSizeBytes); + totalBytesUncompressed.addAndGet(uncompressedFileSizeBytes); + String json = ConvertToJson.toJson(block, false); + // count number of '{' chars in json string to get number of objects + final long numberOfObjectsInBlock = json.chars().filter(c -> c == '{').count(); + if (!csvMode) { + output.append(String.format( + "Block [%d] contains = %d items, %d transactions, %d java objects : parse time = %d ms\n", + block.items().getFirst().blockHeader().number(), + block.items().size(), + numOfTransactions, + numberOfObjectsInBlock, + parseTimeMs)); + } + + final double originalFileSizeMb = originalFileSizeBytes / 1024.0 / 1024.0; + final double uncompressedFileSizeMb = uncompressedFileSizeBytes / 1024.0 / 1024.0; + final double compressionPercent = 100.0 - (originalFileSizeMb / uncompressedFileSizeMb * 100.0); + if (!csvMode) { + output.append(String.format( + " Original File Size = %,.2f MB, Uncompressed File Size = %,.2f MB, Compression = %.2f%%\n", + originalFileSizeMb, uncompressedFileSizeMb, compressionPercent)); + } + Map transactionTypeCounts = new HashMap<>(); + List unknownTransactionInfo = new ArrayList<>(); + long numOfSystemTransactions = block.items().stream() + .filter(BlockItem::hasEventTransaction) + .filter(item -> item.eventTransaction().hasStateSignatureTransaction()) + .count(); + if (numOfSystemTransactions > 0) { + transactionTypeCounts.put("SystemSignature", numOfTransactions); + } + block.items().stream() + .filter(BlockItem::hasEventTransaction) + .map(item -> { + if (item.eventTransaction().hasStateSignatureTransaction()) { + return "SystemSignature"; + } else if (item.eventTransaction().hasApplicationTransaction()) { + try { + final Transaction transaction = Transaction.PROTOBUF.parse( + item.eventTransaction().applicationTransaction()); + final TransactionBody transactionBody; + if (transaction.signedTransactionBytes().length() > 0) { + transactionBody = TransactionBody.PROTOBUF.parse(SignedTransaction.PROTOBUF + .parse(transaction.signedTransactionBytes()) + .bodyBytes()); + } else { + transactionBody = TransactionBody.PROTOBUF.parse(transaction.bodyBytes()); + } + final DataOneOfType kind = transactionBody.data().kind(); + if (kind == DataOneOfType.UNSET) { // should never happen, unless there is a bug somewhere + unknownTransactionInfo.add(" " + TransactionBody.JSON.toJSON(transactionBody)); + unknownTransactionInfo.add(" " + + Transaction.JSON.toJSON(Transaction.PROTOBUF.parse( + item.eventTransaction().applicationTransaction()))); + unknownTransactionInfo.add(" " + BlockItem.JSON.toJSON(item)); + } + return kind.toString(); + } catch (ParseException e) { + System.err.println("Error parsing transaction body : " + e.getMessage()); + throw new RuntimeException(e); + } + } else { + unknownTransactionInfo.add(" " + BlockItem.JSON.toJSON(item)); + return "Unknown"; + } + }) + .forEach(kind -> transactionTypeCounts.put(kind, transactionTypeCounts.getOrDefault(kind, 0L) + 1)); + if (!csvMode) { + transactionTypeCounts.forEach((k, v) -> output.append(String.format(" %s = %,d transactions\n", k, v))); + if (!unknownTransactionInfo.isEmpty()) { + output.append("------------------------------------------\n"); + output.append(" Unknown Transactions : \n"); + unknownTransactionInfo.forEach( + info -> output.append(" " + info).append("\n")); + output.append("------------------------------------------\n"); + } + } else { + + // print CSV column headers + output.append(String.format( + "\"%d\",\"%d\",\"%d\",\"%d\",\"%.2f\",\"%.2f\",\"%.2f\"", + block.items().getFirst().blockHeader().number(), + block.items().size(), + numOfTransactions, + numberOfObjectsInBlock, + originalFileSizeMb, + uncompressedFileSizeMb, + compressionPercent)); + } + return output.toString(); + } +} diff --git a/tools/src/main/java/com/hedera/block/tools/commands/ConvertToJson.java b/tools/src/main/java/com/hedera/block/tools/commands/ConvertToJson.java new file mode 100644 index 000000000..70a2565ad --- /dev/null +++ b/tools/src/main/java/com/hedera/block/tools/commands/ConvertToJson.java @@ -0,0 +1,197 @@ +/* + * Copyright (C) 2024 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.block.tools.commands; + +import com.hedera.hapi.block.stream.Block; +import com.hedera.hapi.block.stream.BlockItem; +import com.hedera.hapi.node.base.Transaction; +import com.hedera.hapi.node.transaction.SignedTransaction; +import com.hedera.hapi.node.transaction.TransactionBody; +import com.hedera.pbj.runtime.ParseException; +import com.hedera.pbj.runtime.io.buffer.Bytes; +import com.hedera.pbj.runtime.io.stream.WritableStreamingData; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Objects; +import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.Parameters; + +/** + * Command line command that converts binary block stream files to JSON + */ +@SuppressWarnings({"DataFlowIssue", "unused", "DuplicatedCode", "ConstantValue"}) +@Command(name = "json", description = "Converts a binary block stream to JSON") +public class ConvertToJson implements Runnable { + + @Parameters(index = "0..*") + private File[] files; + + @SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"}) + @Option( + names = {"-t", "--transactions"}, + description = "expand transactions") + private boolean expandTransactions = false; + + @SuppressWarnings({"FieldMayBeFinal", "FieldCanBeLocal"}) + @Option( + names = {"-ms", "--min-size"}, + description = "minimum file size in megabytes") + private double minSizeMb = Double.MAX_VALUE; + + /** + * Empty Default constructor to remove JavaDoc warning + */ + public ConvertToJson() {} + + /** + * Main method to run the command + */ + @Override + public void run() { + if (files == null || files.length == 0) { + System.err.println("No files to convert"); + } else { + Arrays.stream(files) + .map(File::toPath) + .flatMap(path -> { + try { + return Files.walk(path); + } catch (Exception e) { + throw new RuntimeException(e); + } + }) + .filter(Files::isRegularFile) + .filter(file -> file.getFileName().toString().endsWith(".blk") + || file.getFileName().toString().endsWith(".blk.gz")) + .filter( + file -> { // handle min file size + try { + return minSizeMb == Double.MAX_VALUE + || Files.size(file) / 1024.0 / 1024.0 >= minSizeMb; + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .sorted(Comparator.comparing(file -> file.getFileName().toString())) + .parallel() + .forEach(this::convert); + } + } + + /** + * Converts a binary block stream to JSON + * + * @param blockProtoFile the binary block stream file + */ + private void convert(Path blockProtoFile) { + final String fileName = blockProtoFile.getFileName().toString(); + final String fileNameNoExt = fileName.substring(0, fileName.lastIndexOf('.')); + final Path outputFile = blockProtoFile.resolveSibling(fileNameNoExt + ".json"); + try (InputStream fIn = Files.newInputStream(blockProtoFile)) { + byte[] uncompressedData; + if (blockProtoFile.getFileName().toString().endsWith(".gz")) { + uncompressedData = new GZIPInputStream(fIn).readAllBytes(); + } else { + uncompressedData = fIn.readAllBytes(); + } + final Block block = Block.PROTOBUF.parse(Bytes.wrap(uncompressedData)); + writeJsonBlock(block, outputFile); + final long numOfTransactions = block.items().stream() + .filter(BlockItem::hasEventTransaction) + .count(); + final String blockNumber = + block.items().size() > 1 && block.items().getFirst().hasBlockHeader() + ? String.valueOf(Objects.requireNonNull( + block.items().getFirst().blockHeader()) + .number()) + : "unknown"; + System.out.println("Converted \"" + blockProtoFile.getFileName() + "\" " + + "Block [" + blockNumber + "] " + + "contains = " + block.items().size() + " items, " + numOfTransactions + " transactions"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Converts a parsed block to JSON + * + * @param block the block to convert + * @param expandTransactions whether to expand transactions + * @return the JSON representation of the block + */ + public static String toJson(Block block, boolean expandTransactions) { + if (expandTransactions) { + String blockJson = Block.JSON.toJSON(block); + // get iterator over all transactions + final Iterator transactionBodyJsonIterator = block.items().stream() + .filter(BlockItem::hasEventTransaction) + .filter(item -> item.eventTransaction().hasApplicationTransaction()) + .map(item -> { + try { + return " " + + TransactionBody.JSON + .toJSON(TransactionBody.PROTOBUF.parse(SignedTransaction.PROTOBUF + .parse(Transaction.PROTOBUF + .parse(item.eventTransaction() + .applicationTransaction()) + .signedTransactionBytes()) + .bodyBytes())) + .replaceAll("\n", "\n "); + } catch (ParseException e) { + System.err.println("Error parsing transaction body : " + e.getMessage()); + throw new RuntimeException(e); + } + }) + .iterator(); + // find all "applicationTransaction" fields and expand them, replacing with json from iterator + return Pattern.compile("(\"applicationTransaction\": )\"([^\"]+)\"") + .matcher(blockJson) + .replaceAll(matchResult -> matchResult.group(1) + transactionBodyJsonIterator.next()); + } else { + return Block.JSON.toJSON(block); + } + } + + /** + * Writes a block to a JSON file + * + * @param block the block to write + * @param outputFile the file to write to + * @throws IOException if an I/O error occurs + */ + private void writeJsonBlock(Block block, Path outputFile) throws IOException { + if (expandTransactions) { + Files.writeString(outputFile, toJson(block, expandTransactions)); + } else { + try (OutputStream fOut = new BufferedOutputStream(Files.newOutputStream(outputFile), 1024 * 1024)) { + Block.JSON.write(block, new WritableStreamingData(fOut)); + } + } + } +} diff --git a/tools/src/main/java/module-info.java b/tools/src/main/java/module-info.java new file mode 100644 index 000000000..098aaa36e --- /dev/null +++ b/tools/src/main/java/module-info.java @@ -0,0 +1,15 @@ +/** Runtime module of block stream tools. */ +module com.hedera.block.tools { + exports com.hedera.block.tools; + + opens com.hedera.block.tools to + info.picocli; + opens com.hedera.block.tools.commands to + info.picocli; + + requires static com.github.spotbugs.annotations; + requires static com.google.auto.service; + requires com.hedera.block.stream; + requires com.hedera.pbj.runtime; + requires info.picocli; +}