Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adds testing for macro-aware transcoding of Ion 1.1 streams from text and to binary. #67

Merged
merged 3 commits into from
Dec 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions src/com/amazon/ion/benchmark/Format.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ Path convert(Path input, Path output, OptionsCombinationBase options) throws IOE
|| options.ionInlineSymbols != null
|| options.preallocation != null
|| options.floatWidth != null
|| (options.ionMinorVersion != null && !IonUtilities.minorVersionsEqual(options.ionMinorVersion, input.toFile()))
|| (options.ionMinorVersion != null && !IonUtilities.minorVersionsEqual(Format.ION_BINARY, options.ionMinorVersion, input.toFile()))
|| (options.importsForBenchmarkFile != null
&& !IonUtilities.importsEqual(options.importsForBenchmarkFile, input.toFile()))
|| !IonUtilities.importsFilesEqual(options.importsForInputFile, options.importsForBenchmarkFile);
Expand All @@ -34,6 +34,12 @@ Path convert(Path input, Path output, OptionsCombinationBase options) throws IOE
} else if (options.limit == Integer.MAX_VALUE) {
// There are no settings that require mutating the original input.
return input;
} else if (options.ionMinorVersion != null && options.ionMinorVersion > 0) {
// TODO this case is eligible for simple truncation (the 'else' below), but currently it does
// not work because the SpanProvider facet, which is used by our truncation implementation,
// can't handle macro invocations or delimited containers. If that changes, this branch will
// no longer be necessary. See https://github.com/amazon-ion/ion-java/issues/1011
IonUtilities.rewriteIonFile(ION_BINARY, input, output, options, IonUtilities::newBinaryWriterSupplier);
} else {
// This combination of settings requires simple truncation.
return IonUtilities.truncateBinaryIonFile(input, output, options.limit);
Expand Down Expand Up @@ -86,8 +92,13 @@ Path convert(Path input, Path output, OptionsCombinationBase options) throws IOE
Format sourceFormat = classify(input);
switch (sourceFormat) {
case ION_TEXT:
if (options.limit == Integer.MAX_VALUE && (getMinorVersion(ION_TEXT, input.toFile()) == options.ionMinorVersion)) {
// The input is already text in the requested minor version, and it is not being limited.
if (
options.flushPeriod == null &&
options.limit == Integer.MAX_VALUE &&
(getMinorVersion(ION_TEXT, input.toFile()) == options.ionMinorVersion)
) {
// The input is already text in the requested minor version, and it is not being limited or
// written with a specific flush period.
return input;
}
IonUtilities.rewriteIonFile(ION_TEXT, input, output, options, IonUtilities::newTextWriterSupplier);
Expand Down
53 changes: 47 additions & 6 deletions src/com/amazon/ion/benchmark/IonUtilities.java
Original file line number Diff line number Diff line change
Expand Up @@ -175,13 +175,11 @@ static boolean isFormatHeaderPresent(byte[] formatHeader, File file) throws IOEx
* @return true if the file contains binary Ion data of the given minor version; otherwise, false.
* @throws IOException if thrown while reading the file.
*/
static boolean minorVersionsEqual(int minorVersion, File input) throws IOException {
if (minorVersion == 1) {
return isFormatHeaderPresent(_Private_IonConstants.BINARY_VERSION_MARKER_1_1, input);
} else if (minorVersion == 0) {
return isFormatHeaderPresent(_Private_IonConstants.BINARY_VERSION_MARKER_1_0, input);
static boolean minorVersionsEqual(Format format, int minorVersion, File input) throws IOException {
if (!format.isIon()) {
return false;
}
throw new IllegalStateException("Unknown Ion minor version: " + minorVersion);
return minorVersion == getMinorVersion(format, input);
}

/**
Expand Down Expand Up @@ -210,6 +208,49 @@ static int getMinorVersion(Format inputFormat, File input) throws IOException {
throw new IllegalStateException("File is either not Ion or has unknown minor version: " + input);
}

/**
* Determine whether the given data starts with the given format header.
* @param formatHeader the format header to match.
* @param input the data.
* @return true if the first bytes in the data match the given format header; otherwise, false.
*/
private static boolean isFormatHeaderPresent(byte[] formatHeader, byte[] input) {
if (input.length < formatHeader.length) {
return false;
}
for (int i = 0; i < formatHeader.length; i++) {
if (input[i] != formatHeader[i]) {
return false;
}
}
return true;
}

/**
* Get the minor version for the given text or binary Ion data.
* @param inputFormat the format of 'input'; must be ION_BINARY, ION_TEXT, or JSON.
* @param input the data to examine.
* @return the Ion minor version of the data.
*/
static int getMinorVersion(Format inputFormat, byte[] input) {
if (inputFormat == Format.ION_BINARY) {
if (isFormatHeaderPresent(_Private_IonConstants.BINARY_VERSION_MARKER_1_0, input)) {
return 0;
}
if (isFormatHeaderPresent(_Private_IonConstants.BINARY_VERSION_MARKER_1_1, input)) {
return 1;
}
} else if (inputFormat == Format.ION_TEXT) {
if (isFormatHeaderPresent(ION_1_1_TEXT_IVM, input)) {
return 1;
}
return 0;
} else if (inputFormat == Format.JSON) {
return 0;
}
throw new IllegalStateException("Input is either not Ion or has unknown minor version.");
}

/**
* Supplies IonWriter instances.
*/
Expand Down
2 changes: 1 addition & 1 deletion src/com/amazon/ion/benchmark/MeasurableWriteTask.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ interface WriteInstruction<T> {
*/
MeasurableWriteTask(Path inputPath, WriteOptionsCombination options) throws IOException {
Format inputFormat = Format.classify(inputPath);
if (options.format.canParse(inputFormat)) {
if (options.format.canParse(inputFormat) && IonUtilities.minorVersionsEqual(inputFormat, options.ionMinorVersion, inputPath.toFile())) {
this.inputFile = inputPath.toFile();
} else {
this.inputFile = options.format.convert(
Expand Down
Loading