From a25bcd58778302448e48713ad7a0b61c05a95775 Mon Sep 17 00:00:00 2001 From: Ned Loynd Date: Sun, 12 Nov 2023 11:23:59 +1100 Subject: [PATCH] Add option to disable merging blocks --- .../com/github/NeRdTheNed/deft4j/Deft.java | 7 ++++++- .../deft4j/deflate/DeflateStream.java | 13 +++++------- .../github/NeRdTheNed/deft4j/cmd/CMDUtil.java | 14 +++++++------ .../NeRdTheNed/deft4j/cmd/GZipCompress.java | 9 ++++++--- .../NeRdTheNed/deft4j/cmd/Optimise.java | 5 ++++- .../NeRdTheNed/deft4j/cmd/OptimiseFolder.java | 5 ++++- .../util/compression/CompressionUtil.java | 20 ++++++++++--------- .../util/compression/CompressorTask.java | 6 ++++-- .../container/DeflateFilesContainer.java | 11 +++++++--- runTestOpt.sh | 4 ++-- 10 files changed, 58 insertions(+), 36 deletions(-) diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/Deft.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/Deft.java index 78b2999..b884604 100644 --- a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/Deft.java +++ b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/Deft.java @@ -14,10 +14,15 @@ public class Deft { /** Optimise a raw deflate stream */ public static byte[] optimiseDeflateStream(byte[] original) { + return optimiseDeflateStream(original, true); + } + + /** Optimise a raw deflate stream */ + public static byte[] optimiseDeflateStream(byte[] original, boolean mergeBlocks) { final DeflateStream stream = new DeflateStream(); try { - if (stream.parse(original) && (stream.optimise() > 0)) { + if (stream.parse(original) && (stream.optimise(mergeBlocks) > 0)) { return stream.asBytes(); } } catch (final IOException e) { diff --git a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java index 1b6224a..c193ab7 100644 --- a/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java +++ b/deft4j-base/src/main/java/com/github/NeRdTheNed/deft4j/deflate/DeflateStream.java @@ -490,6 +490,10 @@ private static DeflateBlock optimiseBlock(DeflateBlock toOptimise, long position } public long optimise() { + return optimise(true); + } + + public long optimise(boolean mergeBlocks) { int block = 0; int pass = 0; long pos = 0; @@ -558,14 +562,7 @@ public long optimise() { // TODO Try other types of blocks // TODO Try merging more block types // TODO Try merging blocks at different passes - final long mergeSaved = mergeBlocks(); - saved += mergeSaved; - - if (mergeSaved > 0) { - return saved + optimise(); - } - - return saved; + return mergeBlocks ? saved + mergeBlocks() : saved; } public long mergeBlocks() { diff --git a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/CMDUtil.java b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/CMDUtil.java index 1e0e18c..e6c29bf 100644 --- a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/CMDUtil.java +++ b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/CMDUtil.java @@ -37,18 +37,20 @@ public enum RecompressMode { /** Recompress each deflate stream, and optimise. If the result is smaller, use it. */ private final boolean recompress; + private final boolean mergeBlocks; private final CompressionUtil compUtil; - CMDUtil(RecompressMode recompressMode, int passes) { + CMDUtil(RecompressMode recompressMode, boolean mergeBlocks, int passes) { recompress = recompressMode.ordinal() > RecompressMode.NONE.ordinal(); final boolean zopfli = recompressMode.ordinal() >= RecompressMode.ZOPFLI.ordinal(); final Strategy strat = recompressMode.ordinal() >= RecompressMode.ZOPFLI_EXTENSIVE.ordinal() ? Strategy.EXTENSIVE : Strategy.MULTI_CHEAP; - compUtil = recompress ? new CompressionUtil(true, true, recompressMode.ordinal() >= RecompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, passes, strat, true, true) : null; + compUtil = recompress ? new CompressionUtil(true, true, recompressMode.ordinal() >= RecompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, passes, strat, true, true, mergeBlocks) : null; + this.mergeBlocks = mergeBlocks; } - CMDUtil(RecompressMode recompressMode) { - this(recompressMode, 20); + CMDUtil(RecompressMode recompressMode, boolean mergeBlocks) { + this(recompressMode, mergeBlocks, 20); } /** Read from the given input stream into the container, optimise, and write to the output stream */ @@ -65,7 +67,7 @@ private boolean optimise(InputStream is, OutputStream os, DeflateFilesContainer System.out.println(container.getStreamInfo()); } - final long saved = NO_OPT ? 0 : container.optimise(); + final long saved = NO_OPT ? 0 : container.optimise(mergeBlocks); if (saved != 0) { System.out.println("Saved " + saved + " bits with optimisation"); @@ -84,7 +86,7 @@ private boolean optimise(InputStream is, OutputStream os, DeflateFilesContainer final ByteArrayInputStream bais = new ByteArrayInputStream(recompresed); if (recompStream.parse(bais)) { - recompStream.optimise(); + recompStream.optimise(mergeBlocks); final long recompSize = recompStream.getSizeBits(); final long originalSize = stream.getSizeBits(); final long streamSaved = originalSize - recompSize; diff --git a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/GZipCompress.java b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/GZipCompress.java index e9703a9..b96ef58 100644 --- a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/GZipCompress.java +++ b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/GZipCompress.java @@ -40,6 +40,9 @@ public enum CompressMode { @Option(names = { "--zopfli-iter", "--iter", "-z" }, defaultValue = "20", description = "Zopfli iterations") int iter = 20; + @Option(names = { "--merge-blocks", "-b" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Try merging deflate blocks. May majorly increase time spent optimising files.") + boolean mergeBlocks = true; + @Option(names = { "--keep-name", "-n" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Write filename to GZip header") boolean name = true; @@ -55,10 +58,10 @@ public enum CompressMode { @Option(names = { "--text", "-T" }, defaultValue = "false", description = "File is ASCII text") boolean text; - private static CompressionUtil getComp(CompressMode mode, int iter) { + private static CompressionUtil getComp(CompressMode mode, boolean mergeBlocks, int iter) { final boolean zopfli = mode.ordinal() >= CompressMode.ZOPFLI.ordinal(); final Strategy strat = mode.ordinal() >= CompressMode.ZOPFLI_EXTENSIVE.ordinal() ? Strategy.EXTENSIVE : Strategy.MULTI_CHEAP; - return new CompressionUtil(true, true, mode.ordinal() >= CompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, iter, strat, true, true); + return new CompressionUtil(true, true, mode.ordinal() >= CompressMode.ZOPFLI_VERY_EXTENSIVE.ordinal(), zopfli, iter, strat, true, true, mergeBlocks); } private static int getOS(int fallback) { @@ -107,7 +110,7 @@ public Integer call() throws Exception { return 1; } - final CompressionUtil compUtil = getComp(recompressMode, iter); + final CompressionUtil compUtil = getComp(recompressMode, mergeBlocks, iter); final byte[] uncompressed; try diff --git a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/Optimise.java b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/Optimise.java index 209c5fe..4f7b1bd 100644 --- a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/Optimise.java +++ b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/Optimise.java @@ -30,9 +30,12 @@ class Optimise implements Callable { @Option(names = { "--zopfli-iter", "--iter", "-I" }, defaultValue = "20", description = "Zopfli iterations. More iterations increases time spent optimising files.") int recompressZopfliPasses = 20; + @Option(names = { "--merge-blocks", "-b" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Try merging deflate blocks. May majorly increase time spent optimising files.") + boolean mergeBlocks = true; + @Override public Integer call() throws Exception { - final CMDUtil deft = new CMDUtil(recompressMode, recompressZopfliPasses); + final CMDUtil deft = new CMDUtil(recompressMode, mergeBlocks, recompressZopfliPasses); final boolean didOpt; try { diff --git a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/OptimiseFolder.java b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/OptimiseFolder.java index 2fa23a7..007bb31 100644 --- a/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/OptimiseFolder.java +++ b/deft4j-cmd/src/main/java/com/github/NeRdTheNed/deft4j/cmd/OptimiseFolder.java @@ -27,9 +27,12 @@ class OptimiseFolder implements Callable { @Option(names = { "--zopfli-iter", "--iter", "-I" }, defaultValue = "20", description = "Zopfli iterations. More iterations increases time spent optimising files.") int recompressZopfliPasses = 20; + @Option(names = { "--merge-blocks", "-b" }, negatable = true, defaultValue = "true", fallbackValue = "true", description = "Try merging deflate blocks. May majorly increase time spent optimising files.") + boolean mergeBlocks = true; + @Override public Integer call() throws Exception { - final CMDUtil deft = new CMDUtil(recompressMode, recompressZopfliPasses); + final CMDUtil deft = new CMDUtil(recompressMode, mergeBlocks, recompressZopfliPasses); boolean didOpt = true; try diff --git a/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressionUtil.java b/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressionUtil.java index 2f5c279..142367d 100644 --- a/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressionUtil.java +++ b/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressionUtil.java @@ -38,6 +38,7 @@ public enum Strategy { private final boolean useDeft; private final boolean compareDeft; + private final boolean mergeBlocks; /** Construct the list of compressors for the given settings */ private static Compressor[] getCompressors(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, int defaultSplit) { @@ -76,22 +77,23 @@ private static Compressor[] getCompressors(boolean java, boolean jzlib, boolean return compressorsList.toArray(new Compressor[0]); } - private CompressionUtil(Compressor[] compressors, boolean useDeft, boolean compareDeft) { + private CompressionUtil(Compressor[] compressors, boolean useDeft, boolean compareDeft, boolean mergeBlocks) { this.compressors = compressors; this.useDeft = useDeft; this.compareDeft = compareDeft; + this.mergeBlocks = mergeBlocks; } - public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, int defaultSplit, boolean useDeft, boolean compareDeft) { - this(getCompressors(java, jzlib, jzopfli, cafeundzopfli, iter, mode, defaultSplit), useDeft, compareDeft); + public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, int defaultSplit, boolean useDeft, boolean compareDeft, boolean mergeBlocks) { + this(getCompressors(java, jzlib, jzopfli, cafeundzopfli, iter, mode, defaultSplit), useDeft, compareDeft, mergeBlocks); } - public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, boolean useDeft, boolean compareDeft) { - this(java, jzlib, jzopfli, cafeundzopfli, iter, mode, JZOPFLI_DEFAULT_SPLIT, useDeft, compareDeft); + public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, int iter, Strategy mode, boolean useDeft, boolean compareDeft, boolean mergeBlocks) { + this(java, jzlib, jzopfli, cafeundzopfli, iter, mode, JZOPFLI_DEFAULT_SPLIT, useDeft, compareDeft, mergeBlocks); } - public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, Strategy mode, boolean useDeft, boolean compareDeft) { - this(java, jzlib, jzopfli, cafeundzopfli, ZOPFLI_ITER, mode, useDeft, compareDeft); + public CompressionUtil(boolean java, boolean jzlib, boolean jzopfli, boolean cafeundzopfli, Strategy mode, boolean useDeft, boolean compareDeft, boolean mergeBlocks) { + this(java, jzlib, jzopfli, cafeundzopfli, ZOPFLI_ITER, mode, useDeft, compareDeft, mergeBlocks); } private Supplier execService = () -> { @@ -110,7 +112,7 @@ public byte[] compress(byte[] uncompressedData, boolean threaded) throws IOExcep int tasks = 0; for (final Compressor compressor : compressors) { - compService.submit(new CompressorTask(compressor, uncompressedData, useDeft)); + compService.submit(new CompressorTask(compressor, uncompressedData, useDeft, mergeBlocks)); tasks++; } @@ -149,7 +151,7 @@ public byte[] compress(byte[] uncompressedData, boolean threaded) throws IOExcep for (byte[] currentResult : currentResultList) { if (useDeft) { - currentResult = Deft.optimiseDeflateStream(currentResult); + currentResult = Deft.optimiseDeflateStream(currentResult, mergeBlocks); } if ((compressedData == null) || (compareDeft ? Deft.getSizeBitsFallback(currentResult) < currentSizeBits : currentResult.length < compressedData.length)) { diff --git a/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressorTask.java b/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressorTask.java index ca7c7bf..34f8a45 100644 --- a/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressorTask.java +++ b/deft4j-compress/src/main/java/com/github/NeRdTheNed/deft4j/util/compression/CompressorTask.java @@ -9,11 +9,13 @@ class CompressorTask implements Callable { private final Compressor compressor; private final byte[] uncompressedData; private final boolean optimiseDeft; + private final boolean mergeBlocks; - CompressorTask(Compressor comp, byte[] uncompressedData, boolean optimiseDeft) { + CompressorTask(Compressor comp, byte[] uncompressedData, boolean optimiseDeft, boolean mergeBlocks) { compressor = comp; this.uncompressedData = uncompressedData; this.optimiseDeft = optimiseDeft; + this.mergeBlocks = mergeBlocks; } @Override @@ -28,7 +30,7 @@ public byte[][] call() throws Exception { final int length = compressed.length; for (int i = 0; i < length; i++) { - compressed[i] = Deft.optimiseDeflateStream(compressed[i]); + compressed[i] = Deft.optimiseDeflateStream(compressed[i], mergeBlocks); } } diff --git a/deft4j-container/src/main/java/com/github/NeRdTheNed/deft4j/container/DeflateFilesContainer.java b/deft4j-container/src/main/java/com/github/NeRdTheNed/deft4j/container/DeflateFilesContainer.java index abde745..14a17ff 100644 --- a/deft4j-container/src/main/java/com/github/NeRdTheNed/deft4j/container/DeflateFilesContainer.java +++ b/deft4j-container/src/main/java/com/github/NeRdTheNed/deft4j/container/DeflateFilesContainer.java @@ -15,7 +15,7 @@ public interface DeflateFilesContainer extends Closeable { boolean RECALC = true; /** Optimise all given streams. Returns the total amount of bits saved. */ - static long optimise(List streams) { + static long optimise(List streams, boolean mergeBlocks) { long savedTotal = 0; final int size = streams.size(); @@ -26,7 +26,7 @@ static long optimise(List streams) { System.out.println("Stream " + defStream + " (" + stream.getName() + ")"); } - final long saved = stream.optimise(); + final long saved = stream.optimise(mergeBlocks); if (Deft.PRINT_OPT && (saved > 0)) { System.out.println(saved + " bits saved in stream " + defStream + " (" + stream.getName() + ")"); @@ -64,7 +64,12 @@ default byte[] write() throws IOException { /** Optimise all streams in this container. Returns the total amount of bits saved. */ default long optimise() { - return optimise(getDeflateStreams()); + return optimise(true); + } + + /** Optimise all streams in this container. Returns the total amount of bits saved. */ + default long optimise(boolean mergeBlocks) { + return optimise(getDeflateStreams(), mergeBlocks); } /** Returns debug information for the given stream */ diff --git a/runTestOpt.sh b/runTestOpt.sh index 57e59a6..d6ecc9a 100755 --- a/runTestOpt.sh +++ b/runTestOpt.sh @@ -7,5 +7,5 @@ java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/asyouli java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/text.png ./test/text-opt.png | tee ./test/text-opt.png.txt java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/apng/ball.png ./test/apng/ball-opt.png | tee ./test/apng/ball-opt.png.txt java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/284-edge-case/284.png ./test/284-edge-case/284-opt.png | tee ./test/284-edge-case/284-opt.png.txt -java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/nerd/nerd.png ./test/nerd/nerd-opt.png | tee ./test/nerd/nerd-opt.png.txt -java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise ./test/nerd/nerd-extopt.png ./test/nerd/nerd-fullopt.png | tee ./test/nerd/nerd-fullopt.png.txt +java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise --no-merge-blocks ./test/nerd/nerd.png ./test/nerd/nerd-opt.png | tee ./test/nerd/nerd-opt.png.txt +java -ea -jar ./deft4j-cmd/build/libs/deft4j-cmd-all.jar optimise --no-merge-blocks ./test/nerd/nerd-extopt.png ./test/nerd/nerd-fullopt.png | tee ./test/nerd/nerd-fullopt.png.txt