From ae94b502caee4229c8500351336624ad03f79cb6 Mon Sep 17 00:00:00 2001 From: xdark Date: Sat, 27 Apr 2024 17:58:20 +0300 Subject: [PATCH 1/2] Add method to stream decompressed data --- .../compression/MemorySegmentInputStream.java | 157 ++++++++++++++++++ .../format/compression/ZipCompressions.java | 25 +++ 2 files changed, 182 insertions(+) create mode 100644 src/main/java/software/coley/lljzip/format/compression/MemorySegmentInputStream.java diff --git a/src/main/java/software/coley/lljzip/format/compression/MemorySegmentInputStream.java b/src/main/java/software/coley/lljzip/format/compression/MemorySegmentInputStream.java new file mode 100644 index 0000000..9cff222 --- /dev/null +++ b/src/main/java/software/coley/lljzip/format/compression/MemorySegmentInputStream.java @@ -0,0 +1,157 @@ +package software.coley.lljzip.format.compression; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +final class MemorySegmentInputStream extends InputStream { + private final MemorySegment data; + private long read; + private long markedOffset = -1; + private long markedLimit; + private volatile boolean closed; + + MemorySegmentInputStream(MemorySegment data) { + this.data = data; + } + + private void checkMarkLimit() { + if (markedOffset > -1) { + // Discard if we passed the read limit for our mark + long diff = read - markedOffset; + if (diff > markedLimit) { + markedOffset = -1; + } + } + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public synchronized void mark(int limit) { + // Record current position and read-limit + markedOffset = read; + markedLimit = limit; + } + + @Override + public synchronized void reset() { + // Revert read to marked position. + read = markedOffset; + } + + @Override + public int read() throws IOException { + ensureOpen(); + MemorySegment data = this.data; + if (read >= data.byteSize()) { + return -1; + } + byte b = data.get(ValueLayout.JAVA_BYTE, read++); + checkMarkLimit(); + return b & 0xff; + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long read = this.read; + long length = data.byteSize(); + if (read >= length) { + return -1; + } + long remaining = length - read; + len = (int) Math.min(remaining, len); + MemorySegment.copy(data, read, MemorySegment.ofArray(b), off, len); + this.read += len; + checkMarkLimit(); + return len; + } + + @Override + public byte[] readNBytes(int len) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long read = this.read; + long length = data.byteSize(); + if (read >= length) { + return new byte[0]; + } + long remaining = length - read; + len = (int) Math.min(remaining, len); + byte[] buf = new byte[len]; + MemorySegment.copy(data, read, MemorySegment.ofArray(buf), 0, len); + this.read += len; + checkMarkLimit(); + return buf; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long read = this.read; + long length = data.byteSize(); + if (read >= length) { + return 0; + } + n = Math.min(n, length - read); + this.read += n; + checkMarkLimit(); + return n; + } + + @Override + public int available() throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long length = data.byteSize(); + long read = this.read; + if (read >= length) { + return 0; + } + long remaining = length - read; + if (remaining > Integer.MAX_VALUE) + return Integer.MAX_VALUE; + return (int) remaining; + } + + @Override + public void close() throws IOException { + closed = true; + } + + @Override + public long transferTo(OutputStream out) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long length = data.byteSize(); + long read = this.read; + if (read >= length) { + return 0L; + } + long remaining = length - read; + byte[] buffer = new byte[(int) Math.min(16384, remaining)]; + MemorySegment bufferSegment = MemorySegment.ofArray(buffer); + while (read < length) { + int copyable = (int) Math.min(buffer.length, length - read); + MemorySegment.copy(data, read, bufferSegment, 0, copyable); + out.write(buffer, 0, copyable); + read += copyable; + } + this.read = length; + checkMarkLimit(); + return remaining; + } + + private void ensureOpen() throws IOException { + if (closed) + throw new IOException("Stream closed"); + } +} \ No newline at end of file diff --git a/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java b/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java index b4c7b7f..1b5d0b8 100644 --- a/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java +++ b/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java @@ -3,7 +3,9 @@ import software.coley.lljzip.format.model.LocalFileHeader; import java.io.IOException; +import java.io.InputStream; import java.lang.foreign.MemorySegment; +import java.util.zip.DeflaterInputStream; /** * Constants for {@link LocalFileHeader#getCompressionMethod()}. @@ -185,4 +187,27 @@ static MemorySegment decompress(LocalFileHeader header) throws IOException { } }; } + + /** + * @param header + * Header with {@link LocalFileHeader#getFileData()} to decompress. + * + * @return Stream with decompressed data. + * + * @throws IOException + * When the decompression failed. + */ + static InputStream decompressStream(LocalFileHeader header) throws IOException { + int method = header.getCompressionMethod(); + InputStream in = new MemorySegmentInputStream(header.getFileData()); + return switch (method) { + case STORED -> in; + case DEFLATED -> new DeflaterInputStream(in); + default -> { + // TODO: Support other decompressing techniques + String methodName = getName(method); + throw new IOException("Unsupported compression method: " + methodName); + } + }; + } } From 4f42427d432d10e939eb3a4467f13786d634e64c Mon Sep 17 00:00:00 2001 From: Matt Date: Sat, 27 Apr 2024 23:29:38 -0400 Subject: [PATCH 2/2] Move MemorySegmentInputStream and make it public --- .../format/compression/ZipCompressions.java | 8 +- .../MemorySegmentInputStream.java | 339 ++++++++++-------- 2 files changed, 189 insertions(+), 158 deletions(-) rename src/main/java/software/coley/lljzip/{format/compression => util}/MemorySegmentInputStream.java (83%) diff --git a/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java b/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java index 1b5d0b8..7de2e8b 100644 --- a/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java +++ b/src/main/java/software/coley/lljzip/format/compression/ZipCompressions.java @@ -1,7 +1,9 @@ package software.coley.lljzip.format.compression; import software.coley.lljzip.format.model.LocalFileHeader; +import software.coley.lljzip.util.MemorySegmentInputStream; +import javax.annotation.Nonnull; import java.io.IOException; import java.io.InputStream; import java.lang.foreign.MemorySegment; @@ -175,7 +177,8 @@ static String getName(int method) { * @throws IOException * When the decompression failed. */ - static MemorySegment decompress(LocalFileHeader header) throws IOException { + @Nonnull + static MemorySegment decompress(@Nonnull LocalFileHeader header) throws IOException { int method = header.getCompressionMethod(); return switch (method) { case STORED -> header.getFileData(); @@ -197,7 +200,8 @@ static MemorySegment decompress(LocalFileHeader header) throws IOException { * @throws IOException * When the decompression failed. */ - static InputStream decompressStream(LocalFileHeader header) throws IOException { + @Nonnull + static InputStream decompressStream(@Nonnull LocalFileHeader header) throws IOException { int method = header.getCompressionMethod(); InputStream in = new MemorySegmentInputStream(header.getFileData()); return switch (method) { diff --git a/src/main/java/software/coley/lljzip/format/compression/MemorySegmentInputStream.java b/src/main/java/software/coley/lljzip/util/MemorySegmentInputStream.java similarity index 83% rename from src/main/java/software/coley/lljzip/format/compression/MemorySegmentInputStream.java rename to src/main/java/software/coley/lljzip/util/MemorySegmentInputStream.java index 9cff222..626c357 100644 --- a/src/main/java/software/coley/lljzip/format/compression/MemorySegmentInputStream.java +++ b/src/main/java/software/coley/lljzip/util/MemorySegmentInputStream.java @@ -1,157 +1,184 @@ -package software.coley.lljzip.format.compression; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.lang.foreign.MemorySegment; -import java.lang.foreign.ValueLayout; - -final class MemorySegmentInputStream extends InputStream { - private final MemorySegment data; - private long read; - private long markedOffset = -1; - private long markedLimit; - private volatile boolean closed; - - MemorySegmentInputStream(MemorySegment data) { - this.data = data; - } - - private void checkMarkLimit() { - if (markedOffset > -1) { - // Discard if we passed the read limit for our mark - long diff = read - markedOffset; - if (diff > markedLimit) { - markedOffset = -1; - } - } - } - - @Override - public boolean markSupported() { - return true; - } - - @Override - public synchronized void mark(int limit) { - // Record current position and read-limit - markedOffset = read; - markedLimit = limit; - } - - @Override - public synchronized void reset() { - // Revert read to marked position. - read = markedOffset; - } - - @Override - public int read() throws IOException { - ensureOpen(); - MemorySegment data = this.data; - if (read >= data.byteSize()) { - return -1; - } - byte b = data.get(ValueLayout.JAVA_BYTE, read++); - checkMarkLimit(); - return b & 0xff; - } - - @Override - public int read(byte[] b, int off, int len) throws IOException { - ensureOpen(); - MemorySegment data = this.data; - long read = this.read; - long length = data.byteSize(); - if (read >= length) { - return -1; - } - long remaining = length - read; - len = (int) Math.min(remaining, len); - MemorySegment.copy(data, read, MemorySegment.ofArray(b), off, len); - this.read += len; - checkMarkLimit(); - return len; - } - - @Override - public byte[] readNBytes(int len) throws IOException { - ensureOpen(); - MemorySegment data = this.data; - long read = this.read; - long length = data.byteSize(); - if (read >= length) { - return new byte[0]; - } - long remaining = length - read; - len = (int) Math.min(remaining, len); - byte[] buf = new byte[len]; - MemorySegment.copy(data, read, MemorySegment.ofArray(buf), 0, len); - this.read += len; - checkMarkLimit(); - return buf; - } - - @Override - public long skip(long n) throws IOException { - ensureOpen(); - MemorySegment data = this.data; - long read = this.read; - long length = data.byteSize(); - if (read >= length) { - return 0; - } - n = Math.min(n, length - read); - this.read += n; - checkMarkLimit(); - return n; - } - - @Override - public int available() throws IOException { - ensureOpen(); - MemorySegment data = this.data; - long length = data.byteSize(); - long read = this.read; - if (read >= length) { - return 0; - } - long remaining = length - read; - if (remaining > Integer.MAX_VALUE) - return Integer.MAX_VALUE; - return (int) remaining; - } - - @Override - public void close() throws IOException { - closed = true; - } - - @Override - public long transferTo(OutputStream out) throws IOException { - ensureOpen(); - MemorySegment data = this.data; - long length = data.byteSize(); - long read = this.read; - if (read >= length) { - return 0L; - } - long remaining = length - read; - byte[] buffer = new byte[(int) Math.min(16384, remaining)]; - MemorySegment bufferSegment = MemorySegment.ofArray(buffer); - while (read < length) { - int copyable = (int) Math.min(buffer.length, length - read); - MemorySegment.copy(data, read, bufferSegment, 0, copyable); - out.write(buffer, 0, copyable); - read += copyable; - } - this.read = length; - checkMarkLimit(); - return remaining; - } - - private void ensureOpen() throws IOException { - if (closed) - throw new IOException("Stream closed"); - } +package software.coley.lljzip.util; + +import javax.annotation.Nonnull; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; + +/** + * Input stream implementation backed by {@link MemorySegment}. + * + * @author xDark + */ +public class MemorySegmentInputStream extends InputStream { + private final MemorySegment data; + private long read; + private long markedOffset = -1; + private long markedLimit; + private volatile boolean closed; + + public MemorySegmentInputStream(@Nonnull MemorySegment data) { + this.data = data; + } + + @Nonnull + public MemorySegment getData() { + return data; + } + + public long getRead() { + return read; + } + + public long getMarkedOffset() { + return markedOffset; + } + + public long getMarkedLimit() { + return markedLimit; + } + + public boolean isClosed() { + return closed; + } + + private void checkMarkLimit() { + if (markedOffset > -1) { + // Discard if we passed the read limit for our mark + long diff = read - markedOffset; + if (diff > markedLimit) { + markedOffset = -1; + } + } + } + + @Override + public boolean markSupported() { + return true; + } + + @Override + public synchronized void mark(int limit) { + // Record current position and read-limit + markedOffset = read; + markedLimit = limit; + } + + @Override + public synchronized void reset() { + // Revert read to marked position. + read = markedOffset; + } + + @Override + public int read() throws IOException { + ensureOpen(); + MemorySegment data = this.data; + if (read >= data.byteSize()) { + return -1; + } + byte b = data.get(ValueLayout.JAVA_BYTE, read++); + checkMarkLimit(); + return b & 0xff; + } + + @Override + public int read(@Nonnull byte[] b, int off, int len) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long read = this.read; + long length = data.byteSize(); + if (read >= length) { + return -1; + } + long remaining = length - read; + len = (int) Math.min(remaining, len); + MemorySegment.copy(data, read, MemorySegment.ofArray(b), off, len); + this.read += len; + checkMarkLimit(); + return len; + } + + @Override + public byte[] readNBytes(int len) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long read = this.read; + long length = data.byteSize(); + if (read >= length) { + return new byte[0]; + } + long remaining = length - read; + len = (int) Math.min(remaining, len); + byte[] buf = new byte[len]; + MemorySegment.copy(data, read, MemorySegment.ofArray(buf), 0, len); + this.read += len; + checkMarkLimit(); + return buf; + } + + @Override + public long skip(long n) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long read = this.read; + long length = data.byteSize(); + if (read >= length) { + return 0; + } + n = Math.min(n, length - read); + this.read += n; + checkMarkLimit(); + return n; + } + + @Override + public int available() throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long length = data.byteSize(); + long read = this.read; + if (read >= length) { + return 0; + } + long remaining = length - read; + if (remaining > Integer.MAX_VALUE) + return Integer.MAX_VALUE; + return (int) remaining; + } + + @Override + public void close() { + closed = true; + } + + @Override + public long transferTo(OutputStream out) throws IOException { + ensureOpen(); + MemorySegment data = this.data; + long length = data.byteSize(); + long read = this.read; + if (read >= length) { + return 0L; + } + long remaining = length - read; + byte[] buffer = new byte[(int) Math.min(16384, remaining)]; + MemorySegment bufferSegment = MemorySegment.ofArray(buffer); + while (read < length) { + int copyable = (int) Math.min(buffer.length, length - read); + MemorySegment.copy(data, read, bufferSegment, 0, copyable); + out.write(buffer, 0, copyable); + read += copyable; + } + this.read = length; + checkMarkLimit(); + return remaining; + } + + private void ensureOpen() throws IOException { + if (closed) + throw new IOException("Stream closed"); + } } \ No newline at end of file