From a23beae6dc6600d6d9d78d925c66e1e0ae11b5eb Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Thu, 14 Mar 2024 23:48:43 -0400 Subject: [PATCH 01/11] Add QAT accelerated compression. Signed-off-by: Mulugeta Mammo --- build.gradle | 1 + licenses/qat-java-1.1.0.jar.sha1 | 1 + licenses/qat-java-LICENSE.txt | 36 ++++ licenses/qat-java-NOTICE.txt | 1 + .../codec/customcodecs/CustomCodecPlugin.java | 39 +++- .../customcodecs/CustomCodecService.java | 34 +++- .../CustomCodecServiceFactory.java | 4 +- .../customcodecs/Lucene99CustomCodec.java | 88 ++++++--- .../Lucene99CustomStoredFieldsFormat.java | 124 +++++++++--- .../codec/customcodecs/QatDeflate99Codec.java | 91 +++++++++ .../QatDeflateCompressionMode.java | 183 +++++++++++++++++ .../codec/customcodecs/QatLz499Codec.java | 91 +++++++++ .../customcodecs/QatLz4CompressionMode.java | 178 +++++++++++++++++ .../codec/customcodecs/QatZipperFactory.java | 185 ++++++++++++++++++ .../plugin-metadata/plugin-security.policy | 5 + .../services/org.apache.lucene.codecs.Codec | 4 +- 16 files changed, 992 insertions(+), 73 deletions(-) create mode 100644 licenses/qat-java-1.1.0.jar.sha1 create mode 100644 licenses/qat-java-LICENSE.txt create mode 100644 licenses/qat-java-NOTICE.txt create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java diff --git a/build.gradle b/build.gradle index 00de61c..8a5d123 100644 --- a/build.gradle +++ b/build.gradle @@ -72,6 +72,7 @@ opensearchplugin { dependencies { api "com.github.luben:zstd-jni:1.5.5-5" + api "com.intel.qat:qat-java:1.1.1" } allprojects { diff --git a/licenses/qat-java-1.1.0.jar.sha1 b/licenses/qat-java-1.1.0.jar.sha1 new file mode 100644 index 0000000..7a10220 --- /dev/null +++ b/licenses/qat-java-1.1.0.jar.sha1 @@ -0,0 +1 @@ +fbcaabdbf9d2a72d4b8222e7cfb2043a68b7860e diff --git a/licenses/qat-java-LICENSE.txt b/licenses/qat-java-LICENSE.txt new file mode 100644 index 0000000..d935639 --- /dev/null +++ b/licenses/qat-java-LICENSE.txt @@ -0,0 +1,36 @@ +----------------------------------------------------------------------------- +** Beginning of "BSD License" text. ** + +Qat-Java: Qat-Java is a compression library that uses Intel® QAT to accelerate +compression and decompression. + +Copyright(c) 2007-2023 Intel Corporation. All rights reserved. +All rights reserved. + +BSD License + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/licenses/qat-java-NOTICE.txt b/licenses/qat-java-NOTICE.txt new file mode 100644 index 0000000..9e422fd --- /dev/null +++ b/licenses/qat-java-NOTICE.txt @@ -0,0 +1 @@ +Qat-Java is a compression library that uses Intel® QAT to accelerate compression and decompression. diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 91a13a1..72cf6f1 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -8,28 +8,47 @@ package org.opensearch.index.codec.customcodecs; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Setting.Property; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineConfig; import org.opensearch.plugins.EnginePlugin; import org.opensearch.plugins.Plugin; +import java.util.Arrays; +import java.util.List; import java.util.Optional; +import com.intel.qat.QatZipper; + /** * A plugin that implements custom codecs. Supports these codecs: + * * * * @opensearch.internal */ public final class CustomCodecPlugin extends Plugin implements EnginePlugin { - /** - * Creates a new instance - */ + /** A setting to specifiy the QAT acceleration mode. */ + public static final Setting INDEX_CODEC_QAT_MODE_SETTING = new Setting<>("index.codec.qatmode", "hardware", s -> { + switch (s) { + case "auto": + return QatZipper.Mode.AUTO; + case "hardware": + return QatZipper.Mode.HARDWARE; + default: + throw new IllegalArgumentException("Unknown value for [index.codec.qatmode] must be one of [auto, hardware] but was: " + s); + } + }, Property.IndexScope, Property.Dynamic); + + /** Creates a new instance */ public CustomCodecPlugin() {} /** @@ -39,9 +58,17 @@ public CustomCodecPlugin() {} @Override public Optional getCustomCodecServiceFactory(final IndexSettings indexSettings) { String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); - if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC)) { + if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) + || codecName.equals(CustomCodecService.ZSTD_CODEC) + || codecName.equals(CustomCodecService.QAT_DEFLATE_CODEC) + || codecName.equals(CustomCodecService.QAT_LZ4_CODEC)) { return Optional.of(new CustomCodecServiceFactory()); } return Optional.empty(); } + + @Override + public List> getSettings() { + return Arrays.asList(INDEX_CODEC_QAT_MODE_SETTING); + } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index 90519d2..71a62b5 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -21,26 +21,28 @@ import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; -/** - * CustomCodecService provides ZSTD and ZSTD_NO_DICT compression codecs. - */ +/** CustomCodecService provides ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, and QAT_LZ4 compression codecs. */ public class CustomCodecService extends CodecService { private final Map codecs; - /** - * ZStandard codec - */ + + /** ZStandard codec */ public static final String ZSTD_CODEC = "zstd"; - /** - * ZStandard without dictionary codec - */ + + /** ZStandard without dictionary codec */ public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; + /** Hardware accelerated (Intel QAT) compression codec for DEFLATE. */ + public static final String QAT_DEFLATE_CODEC = "qat_deflate"; + + /** Hardware accelerated (Intel QAT) compression codec for LZ4. */ + public static final String QAT_LZ4_CODEC = "qat_lz4"; + /** * Creates a new CustomCodecService. * * @param mapperService The mapper service. * @param indexSettings The index settings. - * @param logger The logger. + * @param logger The logger. */ public CustomCodecService(MapperService mapperService, IndexSettings indexSettings, Logger logger) { super(mapperService, indexSettings, logger); @@ -49,9 +51,21 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin if (mapperService == null) { codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel)); codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel)); + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> { + return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + })); + codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> { + return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + })); } else { codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel)); codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel)); + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> { + return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + })); + codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> { + return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + })); } this.codecs = codecs.immutableMap(); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java index d634616..39c9d67 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java @@ -12,9 +12,7 @@ import org.opensearch.index.codec.CodecServiceConfig; import org.opensearch.index.codec.CodecServiceFactory; -/** - * A factory for creating new {@link CodecService} instance - */ +/** A factory for creating new {@link CodecService} instance */ public class CustomCodecServiceFactory implements CodecServiceFactory { /** Creates a new instance. */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java index e0b3c2b..bf72330 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java @@ -16,12 +16,13 @@ import org.opensearch.index.mapper.MapperService; import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; /** - * - * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. - * Supports two modes zstd and zstd_no_dict. - * Uses Lucene99 as the delegate codec + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. Supports two modes zstd + * and zstd_no_dict. Uses Lucene99 as the delegate codec * * @opensearch.internal */ @@ -30,16 +31,19 @@ public abstract class Lucene99CustomCodec extends FilterCodec { /** Default compression level used for compression */ public static final int DEFAULT_COMPRESSION_LEVEL = 3; + /** The default QAT mode */ + public static final QatZipper.Mode DEFAULT_QAT_MODE = QatZipper.Mode.HARDWARE; + /** Each mode represents a compression algorithm. */ public enum Mode { - /** - * ZStandard mode with dictionary - */ + /** ZStandard mode with dictionary */ ZSTD("ZSTD99", Set.of("zstd")), - /** - * ZStandard mode without dictionary - */ - ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")); + /** ZStandard mode without dictionary */ + ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")), + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE99", Set.of("qat_deflate")), + /** QAT lz4 mode. */ + QAT_LZ4("QATLZ499", Set.of("qat_lz4")); private final String codec; private final Set aliases; @@ -49,16 +53,12 @@ public enum Mode { this.aliases = aliases; } - /** - * Returns the Codec that is registered with Lucene - */ + /** Returns the Codec that is registered with Lucene */ public String getCodec() { return codec; } - /** - * Returns the aliases of the Codec - */ + /** Returns the aliases of the Codec */ public Set getAliases() { return aliases; } @@ -69,18 +69,18 @@ public Set getAliases() { /** * Creates a new compression codec with the default compression level. * - * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). */ public Lucene99CustomCodec(Mode mode) { this(mode, DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. * - * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). * @param compressionLevel The compression level. */ public Lucene99CustomCodec(Mode mode, int compressionLevel) { @@ -89,11 +89,25 @@ public Lucene99CustomCodec(Mode mode, int compressionLevel) { } /** - * Creates a new compression codec with the given compression level. We use - * lowercase letters when registering the codec so that we remain consistent with - * the other compression codecs: default, lucene_default, and best_compression. + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + */ + public Lucene99CustomCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene99Codec()); + this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel, supplier); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. * - * @param mode The compression codec (ZSTD or ZSTDNODICT). + * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). * @param compressionLevel The compression level. * @param mapperService The mapper service. * @param logger The logger. @@ -103,6 +117,28 @@ public Lucene99CustomCodec(Mode mode, int compressionLevel, MapperService mapper this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); } + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + * @param supplier supplier for QAT mode. + */ + public Lucene99CustomCodec( + Mode mode, + int compressionLevel, + MapperService mapperService, + Logger logger, + Supplier supplier + ) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel, supplier); + } + @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java index ff32c11..f310913 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java @@ -17,10 +17,12 @@ import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.opensearch.index.codec.customcodecs.backward_codecs.Lucene95CustomCodec; import java.io.IOException; import java.util.Objects; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; /** Stored field format used by pluggable codec */ public class Lucene99CustomStoredFieldsFormat extends StoredFieldsFormat { @@ -32,8 +34,18 @@ public class Lucene99CustomStoredFieldsFormat extends StoredFieldsFormat { protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; protected static final int ZSTD_BLOCK_SHIFT = 10; + private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; + + private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; + private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; + private static final int QATLZ4_BLOCK_SHIFT = 10; + private final CompressionMode zstdCompressionMode; private final CompressionMode zstdNoDictCompressionMode; + private final CompressionMode qatDeflateCompressionMode; + private final CompressionMode qatLz4CompressionMode; private final Lucene99CustomCodec.Mode mode; private final int compressionLevel; @@ -46,35 +58,59 @@ public Lucene99CustomStoredFieldsFormat() { /** * Creates a new instance. * - * @param mode The mode represents ZSTD or ZSTDNODICT + * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 */ public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode) { this(mode, Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL); } + /** + * Creates a new instance. + * + * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, Supplier supplier) { + this(mode, Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, supplier); + } + /** * Creates a new instance with the specified mode and compression level. * - * @param mode The mode represents ZSTD or ZSTDNODICT + * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 * @param compressionLevel The compression level for the mode. */ public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return Lucene99CustomCodec.DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 + * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compressionLevel, Supplier supplier) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; zstdCompressionMode = new ZstdCompressionMode(compressionLevel); zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); + qatDeflateCompressionMode = new QatDeflateCompressionMode(compressionLevel, supplier); + qatLz4CompressionMode = new QatLz4CompressionMode(compressionLevel, supplier); } /** - * Returns a {@link StoredFieldsReader} to load stored fields. - * @param directory The index directory. - * @param si The SegmentInfo that stores segment information. - * @param fn The fieldInfos. - * @param context The IOContext that holds additional details on the merge/search context. - */ + * Returns a {@link StoredFieldsReader} to load stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ @Override public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { - if (si.getAttribute(MODE_KEY) !=null){ + if (si.getAttribute(MODE_KEY) != null) { String value = si.getAttribute(MODE_KEY); Lucene99CustomCodec.Mode mode = Lucene99CustomCodec.Mode.valueOf(value); return impl(mode).fieldsReader(directory, si, fn, context); @@ -84,11 +120,12 @@ public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, Fiel } /** - * Returns a {@link StoredFieldsReader} to write stored fields. - * @param directory The index directory. - * @param si The SegmentInfo that stores segment information. - * @param context The IOContext that holds additional details on the merge/search context. - */ + * Returns a {@link StoredFieldsReader} to write stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ @Override public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { String previous = si.putAttribute(MODE_KEY, mode.name()); @@ -106,35 +143,68 @@ StoredFieldsFormat impl(Lucene99CustomCodec.Mode mode) { return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); case ZSTD_NO_DICT: return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); + case QAT_DEFLATE: + return getCustomCompressingStoredFieldsFormat( + "CustomStoredFieldsQatDeflate", + this.qatDeflateCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + case QAT_LZ4: + return getCustomCompressingStoredFieldsFormat( + "CustomStoredFieldsQatLz4", + this.qatLz4CompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); default: throw new AssertionError(); } } - private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { - return new Lucene90CompressingStoredFieldsFormat( - formatName, - compressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT + return getCustomCompressingStoredFieldsFormat( + formatName, + compressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT ); } + private StoredFieldsFormat getCustomCompressingStoredFieldsFormat( + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift + ) { + return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); + } + public Lucene99CustomCodec.Mode getMode() { return mode; } - /** - * Returns the compression level. - */ + /** Returns the compression level. */ public int getCompressionLevel() { return compressionLevel; } public CompressionMode getCompressionMode() { - return mode == Lucene99CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; + switch (mode) { + case ZSTD: + return zstdCompressionMode; + case ZSTD_NO_DICT: + return zstdNoDictCompressionMode; + case QAT_DEFLATE: + return qatDeflateCompressionMode; + case QAT_LZ4: + return qatLz4CompressionMode; + default: + throw new AssertionError(); + } } - } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java new file mode 100644 index 0000000..3048f60 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** + * QatDeflate99Codec provides a DEFLATE compressor using the qat-java library. + */ +public class QatDeflate99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatDeflate99Codec instance with the default compression level. */ + public QatDeflate99Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatDeflate99Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatDeflate99Codec(int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel); + } + + /** + * Creates a new QatDeflate99Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate99Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, supplier); + } + + /** + * Creates a new QatDeflate99Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public QatDeflate99Codec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger); + } + + /** + * Creates a new QatDeflate99Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatDeflate99Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + super(Mode.QAT_DEFLATE, compressionLevel, mapperService, logger, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.QAT_DEFLATE.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java new file mode 100644 index 0000000..3cfbb58 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java @@ -0,0 +1,183 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.store.ByteBuffersDataInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** QAT_DEFLATE ompression Mode */ +public class QatDeflateCompressionMode extends CompressionMode { + + private static final int NUM_SUB_BLOCKS = 10; + private static final int DEFAULT_COMPRESSION_LEVEL = 6; + + private final int compressionLevel; + private final Supplier supplier; + + /** default constructor */ + protected QatDeflateCompressionMode() { + this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; + this.supplier = () -> { return Lucene99CustomCodec.DEFAULT_QAT_MODE; }; + } + + /** + * Creates a new instance. + * + * @param compressionLevel The compression level to use. + * @param supplier a supplier for QAT acceleration mode. + */ + protected QatDeflateCompressionMode(int compressionLevel, Supplier supplier) { + this.compressionLevel = compressionLevel; + this.supplier = supplier; + } + + @Override + public Compressor newCompressor() { + return new QatCompressor(compressionLevel, supplier.get()); + } + + @Override + public Decompressor newDecompressor() { + return new QatDecompressor(supplier.get()); + } + + /** QAT_DEFLATE compressor */ + private static final class QatCompressor extends Compressor { + + private byte[] compressedBuffer; + + private QatZipper qatZipper; + + /** compressor with a given compresion level */ + public QatCompressor(int compressionLevel, QatZipper.Mode mode) { + compressedBuffer = BytesRef.EMPTY_BYTES; + qatZipper = QatZipperFactory.createInstance( + QatZipper.Algorithm.DEFLATE, + compressionLevel, + mode, + QatZipper.PollingMode.PERIODICAL + ); + } + + private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { + assert offset >= 0 : "Offset value must be greater than 0."; + + int blockLength = (length + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS; + out.writeVInt(blockLength); + + final int end = offset + length; + assert end >= 0 : "Buffer read size must be greater than 0."; + + for (int start = offset; start < end; start += blockLength) { + int l = Math.min(blockLength, end - start); + + if (l == 0) { + out.writeVInt(0); + return; + } + + final int maxCompressedLength = qatZipper.maxCompressedLength(l); + compressedBuffer = ArrayUtil.grow(compressedBuffer, maxCompressedLength); + + int compressedSize = qatZipper.compress(bytes, start, l, compressedBuffer, 0, compressedBuffer.length); + out.writeVInt(compressedSize); + out.writeBytes(compressedBuffer, compressedSize); + } + } + + @Override + public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { + final int length = (int) buffersInput.size(); + byte[] bytes = new byte[length]; + buffersInput.readBytes(bytes, 0, length); + compress(bytes, 0, length, out); + } + + @Override + public void close() throws IOException {} + } + + /** QAT_DEFLATE decompressor */ + private static final class QatDecompressor extends Decompressor { + + private byte[] compressed; + private QatZipper qatZipper; + final QatZipper.Mode qatMode; + + /** default decompressor */ + public QatDecompressor(QatZipper.Mode mode) { + compressed = BytesRef.EMPTY_BYTES; + qatZipper = QatZipperFactory.createInstance(QatZipper.Algorithm.DEFLATE, mode, QatZipper.PollingMode.PERIODICAL); + this.qatMode = mode; + } + + /*resuable decompress function*/ + @Override + public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { + assert offset + length <= originalLength : "Buffer read size must be within limit."; + + if (length == 0) { + bytes.length = 0; + return; + } + + final int blockLength = in.readVInt(); + bytes.offset = bytes.length = 0; + int offsetInBlock = 0; + int offsetInBytesRef = offset; + + // Skip unneeded blocks + while (offsetInBlock + blockLength < offset) { + final int compressedLength = in.readVInt(); + in.skipBytes(compressedLength); + offsetInBlock += blockLength; + offsetInBytesRef -= blockLength; + } + + // Read blocks that intersect with the interval we need + while (offsetInBlock < offset + length) { + final int compressedLength = in.readVInt(); + if (compressedLength == 0) { + return; + } + compressed = ArrayUtil.grow(compressed, compressedLength); + in.readBytes(compressed, 0, compressedLength); + + int l = Math.min(blockLength, originalLength - offsetInBlock); + bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + l); + + final int uncompressed = qatZipper.decompress(compressed, 0, compressedLength, bytes.bytes, bytes.length, l); + + bytes.length += uncompressed; + offsetInBlock += blockLength; + } + + bytes.offset = offsetInBytesRef; + bytes.length = length; + + assert bytes.isValid() : "Decompression output is corrupted."; + } + + @Override + public Decompressor clone() { + return new QatDecompressor(qatMode); + } + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java new file mode 100644 index 0000000..9da6e12 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java @@ -0,0 +1,91 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.opensearch.common.settings.Setting; +import org.opensearch.index.codec.CodecAliases; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.engine.EngineConfig; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** + * QatLz499Codec provides an LZ4 compressor using the qat-java library. + */ +public class QatLz499Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { + + /** Creates a new QatLz499Codec instance with the default compression level. */ + public QatLz499Codec() { + this(DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new QatLz499Codec instance. + * + * @param compressionLevel The compression level. + */ + public QatLz499Codec(int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel); + } + + /** + * Creates a new QatLz499Codec instance with the default compression level. + * + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz499Codec(int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, supplier); + } + + /** + * Creates a new QatLz499Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + */ + public QatLz499Codec(MapperService mapperService, Logger logger, int compressionLevel) { + super(Mode.QAT_LZ4, compressionLevel, mapperService, logger); + } + + /** + * Creates a new QatLz499Codec instance. + * + * @param mapperService The mapper service. + * @param logger The logger. + * @param compressionLevel The compression level. + * @param supplier supplier for QAT acceleration mode. + */ + public QatLz499Codec(MapperService mapperService, Logger logger, int compressionLevel, Supplier supplier) { + super(Mode.QAT_LZ4, compressionLevel, mapperService, logger, supplier); + } + + /** The name for this codec. */ + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean supports(Setting setting) { + return setting.equals(EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING); + } + + @Override + public Set aliases() { + return Mode.QAT_LZ4.getAliases(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java new file mode 100644 index 0000000..51ffb7a --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java @@ -0,0 +1,178 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.store.ByteBuffersDataInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** QAT_LZ4 Compression Mode */ +public class QatLz4CompressionMode extends CompressionMode { + + private static final int NUM_SUB_BLOCKS = 10; + private static final int DEFAULT_COMPRESSION_LEVEL = 6; + + private final int compressionLevel; + private final Supplier supplier; + + /** default constructor */ + protected QatLz4CompressionMode() { + this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; + this.supplier = () -> { return Lucene99CustomCodec.DEFAULT_QAT_MODE; }; + } + + /** + * Creates a new instance. + * + * @param compressionLevel The compression level to use. + * @param supplier a supplier for QAT acceleration mode. + */ + protected QatLz4CompressionMode(int compressionLevel, Supplier supplier) { + this.compressionLevel = compressionLevel; + this.supplier = supplier; + } + + @Override + public Compressor newCompressor() { + return new QatCompressor(compressionLevel, supplier.get()); + } + + @Override + public Decompressor newDecompressor() { + return new QatDecompressor(supplier.get()); + } + + /** QAT_LZ4 compressor */ + private static final class QatCompressor extends Compressor { + + private byte[] compressedBuffer; + + private QatZipper qatZipper; + + /** compressor with a given compresion level */ + public QatCompressor(int compressionLevel, QatZipper.Mode mode) { + compressedBuffer = BytesRef.EMPTY_BYTES; + qatZipper = QatZipperFactory.createInstance(QatZipper.Algorithm.LZ4, compressionLevel, mode, QatZipper.PollingMode.PERIODICAL); + } + + private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { + assert offset >= 0 : "Offset value must be greater than 0."; + + int blockLength = (length + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS; + out.writeVInt(blockLength); + + final int end = offset + length; + assert end >= 0 : "Buffer read size must be greater than 0."; + + for (int start = offset; start < end; start += blockLength) { + int l = Math.min(blockLength, end - start); + + if (l == 0) { + out.writeVInt(0); + return; + } + + final int maxCompressedLength = qatZipper.maxCompressedLength(l); + compressedBuffer = ArrayUtil.grow(compressedBuffer, maxCompressedLength); + + int compressedSize = qatZipper.compress(bytes, start, l, compressedBuffer, 0, compressedBuffer.length); + out.writeVInt(compressedSize); + out.writeBytes(compressedBuffer, compressedSize); + } + } + + @Override + public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { + final int length = (int) buffersInput.size(); + byte[] bytes = new byte[length]; + buffersInput.readBytes(bytes, 0, length); + compress(bytes, 0, length, out); + } + + @Override + public void close() throws IOException {} + } + + /** QAT_LZ4 decompressor */ + private static final class QatDecompressor extends Decompressor { + + private byte[] compressed; + private QatZipper qatZipper; + private final QatZipper.Mode qatMode; + + /** default decompressor */ + public QatDecompressor(QatZipper.Mode mode) { + compressed = BytesRef.EMPTY_BYTES; + qatZipper = QatZipperFactory.createInstance(QatZipper.Algorithm.LZ4, mode, QatZipper.PollingMode.PERIODICAL); + qatMode = mode; + } + + /*resuable decompress function*/ + @Override + public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { + assert offset + length <= originalLength : "Buffer read size must be within limit."; + + if (length == 0) { + bytes.length = 0; + return; + } + + final int blockLength = in.readVInt(); + bytes.offset = bytes.length = 0; + int offsetInBlock = 0; + int offsetInBytesRef = offset; + + // Skip unneeded blocks + while (offsetInBlock + blockLength < offset) { + final int compressedLength = in.readVInt(); + in.skipBytes(compressedLength); + offsetInBlock += blockLength; + offsetInBytesRef -= blockLength; + } + + // Read blocks that intersect with the interval we need + while (offsetInBlock < offset + length) { + final int compressedLength = in.readVInt(); + if (compressedLength == 0) { + return; + } + compressed = ArrayUtil.grow(compressed, compressedLength); + in.readBytes(compressed, 0, compressedLength); + + int l = Math.min(blockLength, originalLength - offsetInBlock); + bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + l); + + final int uncompressed = qatZipper.decompress(compressed, 0, compressedLength, bytes.bytes, bytes.length, l); + + bytes.length += uncompressed; + offsetInBlock += blockLength; + } + + bytes.offset = offsetInBytesRef; + bytes.length = length; + + assert bytes.isValid() : "Decompression output is corrupted."; + } + + @Override + public Decompressor clone() { + return new QatDecompressor(qatMode); + } + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java new file mode 100644 index 0000000..39cbcac --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java @@ -0,0 +1,185 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import com.intel.qat.QatZipper; + +import static com.intel.qat.QatZipper.Algorithm; +import static com.intel.qat.QatZipper.DEFAULT_COMPRESS_LEVEL; +import static com.intel.qat.QatZipper.DEFAULT_MODE; +import static com.intel.qat.QatZipper.DEFAULT_POLLING_MODE; +import static com.intel.qat.QatZipper.DEFAULT_RETRY_COUNT; +import static com.intel.qat.QatZipper.Mode; +import static com.intel.qat.QatZipper.PollingMode; + +/** A factory class to create instances of QatZipper */ +public class QatZipperFactory + +{ + + /** + * Creates a new QatZipper with the specified parameters. + * + * @param algorithm the compression algorithm + * @param level the compression level. + * @param mode the mode of QAT execution + * @param retryCount the number of attempts to acquire hardware resources + * @param pmode polling mode. + */ + public static QatZipper createInstance(Algorithm algorithm, int level, Mode mode, int retryCount, PollingMode pmode) { + return new QatZipper(algorithm, level, mode, retryCount, pmode); + } + + /** + * Creates a new QatZipper that uses the DEFLATE algorithm and the default compression level, + * mode, retry count, and polling mode. + */ + public static QatZipper createInstance() { + return createInstance(Algorithm.DEFLATE, DEFAULT_COMPRESS_LEVEL, DEFAULT_MODE, DEFAULT_RETRY_COUNT, DEFAULT_POLLING_MODE); + } + + /** + * Creates a new QatZipper with the specified compression algorithm. Uses the default compression + * level, mode, retry count, and polling mode. + * + * @param algorithm the compression algorithm + */ + public static QatZipper createInstance(Algorithm algorithm) { + return createInstance(algorithm, DEFAULT_COMPRESS_LEVEL, DEFAULT_MODE, DEFAULT_RETRY_COUNT, DEFAULT_POLLING_MODE); + } + + /** + * Creates a new QatZipper with the specified execution mode. Uses the DEFLATE algorithm with the + * default compression level, retry count, and polling mode. + * + * @param mode the mode of QAT execution + */ + public static QatZipper createInstance(Mode mode) { + return createInstance(Algorithm.DEFLATE, DEFAULT_COMPRESS_LEVEL, mode, DEFAULT_RETRY_COUNT, DEFAULT_POLLING_MODE); + } + + /** + * Creates a new QatZipper with the specified polling polling mode. Uses the DEFLATE algorithm + * with the default compression level, mode, and retry count. + * + * @param pmode the polling mode. + */ + public static QatZipper createInstance(PollingMode pmode) { + return createInstance(Algorithm.DEFLATE, DEFAULT_COMPRESS_LEVEL, DEFAULT_MODE, DEFAULT_RETRY_COUNT, pmode); + } + + /** + * Creates a new QatZipper with the specified algorithm and compression level. Uses the default + * mode, retry count, and polling mode. + * + * @param algorithm the compression algorithm (deflate or LZ4). + * @param level the compression level. + */ + public static QatZipper createInstance(Algorithm algorithm, int level) { + return createInstance(algorithm, level, DEFAULT_MODE, DEFAULT_RETRY_COUNT, DEFAULT_POLLING_MODE); + } + + /** + * Creates a new QatZipper with the specified algorithm and mode of execution. Uses the default + * compression level, retry count, and polling mode. + * + * @param algorithm the compression algorithm + * @param mode the mode of QAT execution + */ + public static QatZipper createInstance(Algorithm algorithm, Mode mode) { + return createInstance(algorithm, DEFAULT_COMPRESS_LEVEL, mode, DEFAULT_RETRY_COUNT, DEFAULT_POLLING_MODE); + } + + /** + * Creates a new QatZipper with the specified algorithm and polling mode of execution. Uses the + * default compression level, mode, and retry count. + * + * @param algorithm the compression algorithm + * @param pmode the polling mode. + */ + public static QatZipper createInstance(Algorithm algorithm, PollingMode pmode) { + return createInstance(algorithm, DEFAULT_COMPRESS_LEVEL, DEFAULT_MODE, DEFAULT_RETRY_COUNT, pmode); + } + + /** + * Creates a new QatZipper with the specified algorithm and mode of execution. Uses compression + * level and retry count. + * + * @param algorithm the compression algorithm + * @param mode the mode of QAT execution + * @param pmode the polling mode. + */ + public static QatZipper createInstance(Algorithm algorithm, Mode mode, PollingMode pmode) { + return createInstance(algorithm, DEFAULT_COMPRESS_LEVEL, mode, DEFAULT_RETRY_COUNT, pmode); + } + + /** + * Creates a new QatZipper with the specified algorithm, compression level, and mode . Uses the + * default retry count and polling mode. + * + * @param algorithm the compression algorithm (deflate or LZ4). + * @param level the compression level. + * @param mode the mode of operation (HARDWARE - only hardware, AUTO - hardware with a software + * failover.) + */ + public static QatZipper createInstance(Algorithm algorithm, int level, Mode mode) { + return createInstance(algorithm, level, mode, DEFAULT_RETRY_COUNT, DEFAULT_POLLING_MODE); + } + + /** + * Creates a new QatZipper with the specified algorithm, compression level, and polling mode . + * Uses the default mode and retry count. + * + * @param algorithm the compression algorithm (deflate or LZ4). + * @param level the compression level. + * @param pmode the polling mode. + */ + public static QatZipper createInstance(Algorithm algorithm, int level, PollingMode pmode) { + return createInstance(algorithm, level, DEFAULT_MODE, DEFAULT_RETRY_COUNT, pmode); + } + + /** + * Creates a new QatZipper with the specified parameters and polling mode. + * + * @param algorithm the compression algorithm + * @param level the compression level. + * @param mode the mode of QAT execution + * @param retryCount the number of attempts to acquire hardware resources + */ + public static QatZipper createInstance(Algorithm algorithm, int level, Mode mode, int retryCount) { + return createInstance(algorithm, level, mode, retryCount, DEFAULT_POLLING_MODE); + } + + /** + * Creates a new QatZipper with the specified parameters and retry count. + * + * @param algorithm the compression algorithm + * @param level the compression level. + * @param mode the mode of QAT execution + * @param pmode the polling mode. + */ + public static QatZipper createInstance(Algorithm algorithm, int level, Mode mode, PollingMode pmode) { + return createInstance(algorithm, level, mode, DEFAULT_RETRY_COUNT, pmode); + } + + /** + * Checks if QAT hardware is available. + * + * @return true if QAT hardware is available, false otherwise. + */ + public static boolean isQatAvailable() { + try { + QatZipper qzip = QatZipperFactory.createInstance(); + qzip.end(); + return true; + } catch (UnsatisfiedLinkError | ExceptionInInitializerError | NoClassDefFoundError e) { + return false; + } + } +} diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 8161010..615ea9d 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -9,3 +9,8 @@ grant codeBase "${codebase.zstd-jni}" { permission java.lang.RuntimePermission "loadLibrary.*"; }; + +grant codeBase "${codebase.qat-java}" { + permission java.lang.RuntimePermission "loadLibrary.*"; + permission org.opensearch.secure_sm.ThreadPermission "modifyArbitraryThread"; +}; diff --git a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec index 5912ede..4e25917 100644 --- a/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec +++ b/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -2,4 +2,6 @@ org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95Codec org.opensearch.index.codec.customcodecs.backward_codecs.ZstdNoDict95Codec org.opensearch.index.codec.customcodecs.backward_codecs.Zstd95DeprecatedCodec org.opensearch.index.codec.customcodecs.Zstd99Codec -org.opensearch.index.codec.customcodecs.ZstdNoDict99Codec \ No newline at end of file +org.opensearch.index.codec.customcodecs.ZstdNoDict99Codec +org.opensearch.index.codec.customcodecs.QatDeflate99Codec +org.opensearch.index.codec.customcodecs.QatLz499Codec From 32e6ab78a2a5c7d25348e570032e9c657ff98b74 Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Wed, 27 Mar 2024 13:05:28 -0400 Subject: [PATCH 02/11] Use own classes for QAT codec. Apply SpotlessJavaCheck. Signed-off-by: Mulugeta Mammo --- licenses/qat-java-1.1.0.jar.sha1 | 1 - licenses/qat-java-1.1.1.jar.sha1 | 1 + .../customcodecs/Lucene99CustomCodec.java | 88 +++----- .../Lucene99CustomStoredFieldsFormat.java | 110 ++-------- .../codec/customcodecs/Lucene99QatCodec.java | 147 +++++++++++++ .../Lucene99QatStoredFieldsFormat.java | 184 ++++++++++++++++ .../codec/customcodecs/QatDeflate99Codec.java | 2 +- .../QatDeflateCompressionMode.java | 2 +- .../codec/customcodecs/QatLz499Codec.java | 2 +- .../customcodecs/QatLz4CompressionMode.java | 2 +- .../index/codec/customcodecs/Zstd99Codec.java | 7 +- .../customcodecs/ZstdCompressionMode.java | 3 +- .../codec/customcodecs/ZstdNoDict99Codec.java | 8 +- .../ZstdNoDictCompressionMode.java | 10 +- .../codec/customcodecs/package-info.java | 4 +- .../customcodecs/AbstractCompressorTests.java | 3 +- ...Lucene95CustomStoredFieldsFormatTests.java | 21 +- ...Lucene99CustomStoredFieldsFormatTests.java | 8 +- .../Lucene99QatStoredFieldsFormatTests.java | 60 ++++++ .../codec/customcodecs/QatCodecTests.java | 201 ++++++++++++++++++ .../QatDeflateCompressorTests.java | 83 ++++++++ .../customcodecs/QatLz4CompressorTests.java | 83 ++++++++ .../ZstdNoDictCompressorTests.java | 4 +- 23 files changed, 836 insertions(+), 198 deletions(-) delete mode 100644 licenses/qat-java-1.1.0.jar.sha1 create mode 100644 licenses/qat-java-1.1.1.jar.sha1 create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java create mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java create mode 100644 src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java diff --git a/licenses/qat-java-1.1.0.jar.sha1 b/licenses/qat-java-1.1.0.jar.sha1 deleted file mode 100644 index 7a10220..0000000 --- a/licenses/qat-java-1.1.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fbcaabdbf9d2a72d4b8222e7cfb2043a68b7860e diff --git a/licenses/qat-java-1.1.1.jar.sha1 b/licenses/qat-java-1.1.1.jar.sha1 new file mode 100644 index 0000000..ca4633f --- /dev/null +++ b/licenses/qat-java-1.1.1.jar.sha1 @@ -0,0 +1 @@ +3333601cdedf6a711d445118d5bc44ec6a9c65f9 diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java index bf72330..e0b3c2b 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java @@ -16,13 +16,12 @@ import org.opensearch.index.mapper.MapperService; import java.util.Set; -import java.util.function.Supplier; - -import com.intel.qat.QatZipper; /** - * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. Supports two modes zstd - * and zstd_no_dict. Uses Lucene99 as the delegate codec + * + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. + * Supports two modes zstd and zstd_no_dict. + * Uses Lucene99 as the delegate codec * * @opensearch.internal */ @@ -31,19 +30,16 @@ public abstract class Lucene99CustomCodec extends FilterCodec { /** Default compression level used for compression */ public static final int DEFAULT_COMPRESSION_LEVEL = 3; - /** The default QAT mode */ - public static final QatZipper.Mode DEFAULT_QAT_MODE = QatZipper.Mode.HARDWARE; - /** Each mode represents a compression algorithm. */ public enum Mode { - /** ZStandard mode with dictionary */ + /** + * ZStandard mode with dictionary + */ ZSTD("ZSTD99", Set.of("zstd")), - /** ZStandard mode without dictionary */ - ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")), - /** QAT deflate mode. */ - QAT_DEFLATE("QATDEFLATE99", Set.of("qat_deflate")), - /** QAT lz4 mode. */ - QAT_LZ4("QATLZ499", Set.of("qat_lz4")); + /** + * ZStandard mode without dictionary + */ + ZSTD_NO_DICT("ZSTDNODICT99", Set.of("zstd_no_dict")); private final String codec; private final Set aliases; @@ -53,12 +49,16 @@ public enum Mode { this.aliases = aliases; } - /** Returns the Codec that is registered with Lucene */ + /** + * Returns the Codec that is registered with Lucene + */ public String getCodec() { return codec; } - /** Returns the aliases of the Codec */ + /** + * Returns the aliases of the Codec + */ public Set getAliases() { return aliases; } @@ -69,18 +69,18 @@ public Set getAliases() { /** * Creates a new compression codec with the default compression level. * - * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). + * @param mode The compression codec (ZSTD or ZSTDNODICT). */ public Lucene99CustomCodec(Mode mode) { this(mode, DEFAULT_COMPRESSION_LEVEL); } /** - * Creates a new compression codec with the given compression level. We use lowercase letters when - * registering the codec so that we remain consistent with the other compression codecs: default, - * lucene_default, and best_compression. + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. * - * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). + * @param mode The compression codec (ZSTD or ZSTDNODICT). * @param compressionLevel The compression level. */ public Lucene99CustomCodec(Mode mode, int compressionLevel) { @@ -89,25 +89,11 @@ public Lucene99CustomCodec(Mode mode, int compressionLevel) { } /** - * Creates a new compression codec with the given compression level. We use lowercase letters when - * registering the codec so that we remain consistent with the other compression codecs: default, - * lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). - * @param compressionLevel The compression level. - * @param supplier supplier for QAT mode. - */ - public Lucene99CustomCodec(Mode mode, int compressionLevel, Supplier supplier) { - super(mode.getCodec(), new Lucene99Codec()); - this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel, supplier); - } - - /** - * Creates a new compression codec with the given compression level. We use lowercase letters when - * registering the codec so that we remain consistent with the other compression codecs: default, - * lucene_default, and best_compression. + * Creates a new compression codec with the given compression level. We use + * lowercase letters when registering the codec so that we remain consistent with + * the other compression codecs: default, lucene_default, and best_compression. * - * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). + * @param mode The compression codec (ZSTD or ZSTDNODICT). * @param compressionLevel The compression level. * @param mapperService The mapper service. * @param logger The logger. @@ -117,28 +103,6 @@ public Lucene99CustomCodec(Mode mode, int compressionLevel, MapperService mapper this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel); } - /** - * Creates a new compression codec with the given compression level. We use lowercase letters when - * registering the codec so that we remain consistent with the other compression codecs: default, - * lucene_default, and best_compression. - * - * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). - * @param compressionLevel The compression level. - * @param mapperService The mapper service. - * @param logger The logger. - * @param supplier supplier for QAT mode. - */ - public Lucene99CustomCodec( - Mode mode, - int compressionLevel, - MapperService mapperService, - Logger logger, - Supplier supplier - ) { - super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); - this.storedFieldsFormat = new Lucene99CustomStoredFieldsFormat(mode, compressionLevel, supplier); - } - @Override public StoredFieldsFormat storedFieldsFormat() { return storedFieldsFormat; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java index f310913..235c801 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java @@ -20,9 +20,6 @@ import java.io.IOException; import java.util.Objects; -import java.util.function.Supplier; - -import com.intel.qat.QatZipper; /** Stored field format used by pluggable codec */ public class Lucene99CustomStoredFieldsFormat extends StoredFieldsFormat { @@ -34,18 +31,8 @@ public class Lucene99CustomStoredFieldsFormat extends StoredFieldsFormat { protected static final int ZSTD_MAX_DOCS_PER_BLOCK = 4096; protected static final int ZSTD_BLOCK_SHIFT = 10; - private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; - private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; - private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; - - private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; - private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; - private static final int QATLZ4_BLOCK_SHIFT = 10; - private final CompressionMode zstdCompressionMode; private final CompressionMode zstdNoDictCompressionMode; - private final CompressionMode qatDeflateCompressionMode; - private final CompressionMode qatLz4CompressionMode; private final Lucene99CustomCodec.Mode mode; private final int compressionLevel; @@ -58,56 +45,32 @@ public Lucene99CustomStoredFieldsFormat() { /** * Creates a new instance. * - * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 + * @param mode The mode represents ZSTD or ZSTDNODICT */ public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode) { this(mode, Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL); } - /** - * Creates a new instance. - * - * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 - * @param supplier a supplier for QAT acceleration mode. - */ - public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, Supplier supplier) { - this(mode, Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, supplier); - } - /** * Creates a new instance with the specified mode and compression level. * - * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 + * @param mode The mode represents ZSTD or ZSTDNODICT * @param compressionLevel The compression level for the mode. */ public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compressionLevel) { - this(mode, compressionLevel, () -> { return Lucene99CustomCodec.DEFAULT_QAT_MODE; }); - } - - /** - * Creates a new instance with the specified mode and compression level. - * - * @param mode The mode represents ZSTD, ZSTDNODICT, QAT_DEFLATE, or QAT_LZ4 - * @param compressionLevel The compression level for the mode. - * @param supplier a supplier for QAT acceleration mode. - */ - public Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode mode, int compressionLevel, Supplier supplier) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; zstdCompressionMode = new ZstdCompressionMode(compressionLevel); zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel); - qatDeflateCompressionMode = new QatDeflateCompressionMode(compressionLevel, supplier); - qatLz4CompressionMode = new QatLz4CompressionMode(compressionLevel, supplier); } /** - * Returns a {@link StoredFieldsReader} to load stored fields. - * - * @param directory The index directory. - * @param si The SegmentInfo that stores segment information. - * @param fn The fieldInfos. - * @param context The IOContext that holds additional details on the merge/search context. - */ + * Returns a {@link StoredFieldsReader} to load stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ @Override public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { if (si.getAttribute(MODE_KEY) != null) { @@ -120,12 +83,11 @@ public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, Fiel } /** - * Returns a {@link StoredFieldsReader} to write stored fields. - * - * @param directory The index directory. - * @param si The SegmentInfo that stores segment information. - * @param context The IOContext that holds additional details on the merge/search context. - */ + * Returns a {@link StoredFieldsReader} to write stored fields. + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ @Override public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { String previous = si.putAttribute(MODE_KEY, mode.name()); @@ -143,29 +105,13 @@ StoredFieldsFormat impl(Lucene99CustomCodec.Mode mode) { return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstd", this.zstdCompressionMode); case ZSTD_NO_DICT: return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); - case QAT_DEFLATE: - return getCustomCompressingStoredFieldsFormat( - "CustomStoredFieldsQatDeflate", - this.qatDeflateCompressionMode, - QAT_DEFLATE_BLOCK_LENGTH, - QAT_DEFLATE_MAX_DOCS_PER_BLOCK, - QAT_DEFLATE_BLOCK_SHIFT - ); - case QAT_LZ4: - return getCustomCompressingStoredFieldsFormat( - "CustomStoredFieldsQatLz4", - this.qatLz4CompressionMode, - QAT_LZ4_BLOCK_LENGTH, - QAT_DEFLATE_MAX_DOCS_PER_BLOCK, - QAT_DEFLATE_BLOCK_SHIFT - ); default: throw new AssertionError(); } } private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatName, CompressionMode compressionMode) { - return getCustomCompressingStoredFieldsFormat( + return new Lucene90CompressingStoredFieldsFormat( formatName, compressionMode, ZSTD_BLOCK_LENGTH, @@ -174,37 +120,19 @@ private StoredFieldsFormat getCustomCompressingStoredFieldsFormat(String formatN ); } - private StoredFieldsFormat getCustomCompressingStoredFieldsFormat( - String formatName, - CompressionMode compressionMode, - int blockSize, - int maxDocs, - int blockShift - ) { - return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); - } - public Lucene99CustomCodec.Mode getMode() { return mode; } - /** Returns the compression level. */ + /** + * Returns the compression level. + */ public int getCompressionLevel() { return compressionLevel; } public CompressionMode getCompressionMode() { - switch (mode) { - case ZSTD: - return zstdCompressionMode; - case ZSTD_NO_DICT: - return zstdNoDictCompressionMode; - case QAT_DEFLATE: - return qatDeflateCompressionMode; - case QAT_LZ4: - return qatLz4CompressionMode; - default: - throw new AssertionError(); - } + return mode == Lucene99CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; } + } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java new file mode 100644 index 0000000..54d96c5 --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java @@ -0,0 +1,147 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.Logger; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; +import org.opensearch.index.mapper.MapperService; + +import java.util.Set; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. Supports two modes zstd + * and zstd_no_dict. Uses Lucene99 as the delegate codec + * + * @opensearch.internal + */ +public abstract class Lucene99QatCodec extends FilterCodec { + + /** Default compression level used for compression */ + public static final int DEFAULT_COMPRESSION_LEVEL = 3; + + /** The default QAT mode */ + public static final QatZipper.Mode DEFAULT_QAT_MODE = QatZipper.Mode.HARDWARE; + + /** Each mode represents a compression algorithm. */ + public enum Mode { + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE99", Set.of("qat_deflate")), + /** QAT lz4 mode. */ + QAT_LZ4("QATLZ499", Set.of("qat_lz4")); + + private final String codec; + private final Set aliases; + + Mode(String codec, Set aliases) { + this.codec = codec; + this.aliases = aliases; + } + + /** Returns the Codec that is registered with Lucene */ + public String getCodec() { + return codec; + } + + /** Returns the aliases of the Codec */ + public Set getAliases() { + return aliases; + } + } + + private final StoredFieldsFormat storedFieldsFormat; + + /** + * Creates a new compression codec with the default compression level. + * + * @param mode The compression codec (QAT_DEFLATE or QAT_LZ4). + */ + public Lucene99QatCodec(Mode mode) { + this(mode, DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_DEFLATE or QAT_LZ4). + * @param compressionLevel The compression level. + */ + public Lucene99QatCodec(Mode mode, int compressionLevel) { + super(mode.getCodec(), new Lucene99Codec()); + this.storedFieldsFormat = new Lucene99QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_DEFLATE, or QAT_LZ4). + * @param compressionLevel The compression level. + * @param supplier supplier for QAT mode. + */ + public Lucene99QatCodec(Mode mode, int compressionLevel, Supplier supplier) { + super(mode.getCodec(), new Lucene99Codec()); + this.storedFieldsFormat = new Lucene99QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (QAT_DEFLATE or QAT_LZ4). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + */ + public Lucene99QatCodec(Mode mode, int compressionLevel, MapperService mapperService, Logger logger) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene99QatStoredFieldsFormat(mode, compressionLevel); + } + + /** + * Creates a new compression codec with the given compression level. We use lowercase letters when + * registering the codec so that we remain consistent with the other compression codecs: default, + * lucene_default, and best_compression. + * + * @param mode The compression codec (ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, or QAT_LZ4). + * @param compressionLevel The compression level. + * @param mapperService The mapper service. + * @param logger The logger. + * @param supplier supplier for QAT mode. + */ + public Lucene99QatCodec( + Mode mode, + int compressionLevel, + MapperService mapperService, + Logger logger, + Supplier supplier + ) { + super(mode.getCodec(), new PerFieldMappingPostingFormatCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, logger)); + this.storedFieldsFormat = new Lucene99QatStoredFieldsFormat(mode, compressionLevel, supplier); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java new file mode 100644 index 0000000..dbafa7c --- /dev/null +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java @@ -0,0 +1,184 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.StoredFieldsReader; +import org.apache.lucene.codecs.StoredFieldsWriter; +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +import java.io.IOException; +import java.util.Objects; +import java.util.function.Supplier; + +import com.intel.qat.QatZipper; + +/** Stored field format used by pluggable codec */ +public class Lucene99QatStoredFieldsFormat extends StoredFieldsFormat { + + /** A key that we use to map to a mode */ + public static final String MODE_KEY = Lucene99QatStoredFieldsFormat.class.getSimpleName() + ".mode"; + + private static final int QAT_DEFLATE_BLOCK_LENGTH = 10 * 48 * 1024; + private static final int QAT_DEFLATE_MAX_DOCS_PER_BLOCK = 4096; + private static final int QAT_DEFLATE_BLOCK_SHIFT = 10; + + private static final int QAT_LZ4_BLOCK_LENGTH = 10 * 8 * 1024; + private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; + private static final int QATLZ4_BLOCK_SHIFT = 10; + + private final CompressionMode qatDeflateCompressionMode; + private final CompressionMode qatLz4CompressionMode; + + private final Lucene99QatCodec.Mode mode; + private final int compressionLevel; + + /** default constructor */ + public Lucene99QatStoredFieldsFormat() { + this(Lucene99QatCodec.Mode.QAT_DEFLATE, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_DEFLATE or QAT_LZ4 + */ + public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode) { + this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL); + } + + /** + * Creates a new instance. + * + * @param mode The mode represents QAT_DEFLATE or QAT_LZ4 + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, Supplier supplier) { + this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, supplier); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_DEFLATE or QAT_LZ4 + * @param compressionLevel The compression level for the mode. + */ + public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance with the specified mode and compression level. + * + * @param mode The mode represents QAT_DEFLATE or QAT_LZ4 + * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. + */ + public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.mode = Objects.requireNonNull(mode); + this.compressionLevel = compressionLevel; + qatDeflateCompressionMode = new QatDeflateCompressionMode(compressionLevel, supplier); + qatLz4CompressionMode = new QatLz4CompressionMode(compressionLevel, supplier); + } + + /** + * Returns a {@link StoredFieldsReader} to load stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param fn The fieldInfos. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException { + if (si.getAttribute(MODE_KEY) != null) { + String value = si.getAttribute(MODE_KEY); + Lucene99QatCodec.Mode mode = Lucene99QatCodec.Mode.valueOf(value); + return impl(mode).fieldsReader(directory, si, fn, context); + } else { + throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name); + } + } + + /** + * Returns a {@link StoredFieldsReader} to write stored fields. + * + * @param directory The index directory. + * @param si The SegmentInfo that stores segment information. + * @param context The IOContext that holds additional details on the merge/search context. + */ + @Override + public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException { + String previous = si.putAttribute(MODE_KEY, mode.name()); + if (previous != null && previous.equals(mode.name()) == false) { + throw new IllegalStateException( + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + ); + } + return impl(mode).fieldsWriter(directory, si, context); + } + + StoredFieldsFormat impl(Lucene99QatCodec.Mode mode) { + switch (mode) { + case QAT_DEFLATE: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsDeflate", + this.qatDeflateCompressionMode, + QAT_DEFLATE_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + case QAT_LZ4: + return getQatCompressingStoredFieldsFormat( + "QatStoredFieldsLz4", + this.qatLz4CompressionMode, + QAT_LZ4_BLOCK_LENGTH, + QAT_DEFLATE_MAX_DOCS_PER_BLOCK, + QAT_DEFLATE_BLOCK_SHIFT + ); + default: + throw new AssertionError(); + } + } + + private StoredFieldsFormat getQatCompressingStoredFieldsFormat( + String formatName, + CompressionMode compressionMode, + int blockSize, + int maxDocs, + int blockShift + ) { + return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); + } + + public Lucene99QatCodec.Mode getMode() { + return mode; + } + + /** Returns the compression level. */ + public int getCompressionLevel() { + return compressionLevel; + } + + public CompressionMode getCompressionMode() { + switch (mode) { + case QAT_DEFLATE: + return qatDeflateCompressionMode; + case QAT_LZ4: + return qatLz4CompressionMode; + default: + throw new AssertionError(); + } + } +} diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java index 3048f60..b24ead0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflate99Codec.java @@ -24,7 +24,7 @@ * QatDeflate99Codec provides a DEFLATE compressor using the qat-java library. */ -public class QatDeflate99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { +public class QatDeflate99Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { /** Creates a new QatDeflate99Codec instance with the default compression level. */ public QatDeflate99Codec() { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java index 3cfbb58..b1b787c 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java @@ -34,7 +34,7 @@ public class QatDeflateCompressionMode extends CompressionMode { /** default constructor */ protected QatDeflateCompressionMode() { this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; - this.supplier = () -> { return Lucene99CustomCodec.DEFAULT_QAT_MODE; }; + this.supplier = () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java index 9da6e12..9e242b0 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz499Codec.java @@ -24,7 +24,7 @@ * QatLz499Codec provides an LZ4 compressor using the qat-java library. */ -public class QatLz499Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { +public class QatLz499Codec extends Lucene99QatCodec implements CodecSettings, CodecAliases { /** Creates a new QatLz499Codec instance with the default compression level. */ public QatLz499Codec() { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java index 51ffb7a..3f39702 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java @@ -34,7 +34,7 @@ public class QatLz4CompressionMode extends CompressionMode { /** default constructor */ protected QatLz4CompressionMode() { this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; - this.supplier = () -> { return Lucene99CustomCodec.DEFAULT_QAT_MODE; }; + this.supplier = () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java index df86806..29b5f4b 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Zstd99Codec.java @@ -18,13 +18,12 @@ import java.util.Set; /** - * ZstdCodec provides ZSTD compressor using the zstd-jni library. + * ZstdCodec provides ZSTD compressor using the zstd-jni library. */ public class Zstd99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { - /** - * Creates a new ZstdCodec instance with the default compression level. - */ + /** Creates a new ZstdCodec instance with the default compression level. */ public Zstd99Codec() { this(DEFAULT_COMPRESSION_LEVEL); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index 05ff725..d88ef1f 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -48,7 +48,7 @@ protected ZstdCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } - /** Creates a new compressor instance.*/ + /** Creates a new compressor instance. */ @Override public Compressor newCompressor() { return new ZstdCompressor(compressionLevel); @@ -188,7 +188,6 @@ public void decompress(DataInput in, int originalLength, int offset, int length, // Read blocks that intersect with the interval we need while (offsetInBlock < offset + length) { - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength); int l = Math.min(blockLength, originalLength - offsetInBlock); doDecompress(in, dctx, bytes, l); offsetInBlock += blockLength; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java index 1d65aee..76c3ce6 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDict99Codec.java @@ -17,14 +17,10 @@ import java.util.Set; -/** - * ZstdNoDictCodec provides ZSTD compressor without a dictionary support. - */ +/** ZstdNoDictCodec provides ZSTD compressor without a dictionary support. */ public class ZstdNoDict99Codec extends Lucene99CustomCodec implements CodecSettings, CodecAliases { - /** - * Creates a new ZstdNoDictCodec instance with the default compression level. - */ + /** Creates a new ZstdNoDictCodec instance with the default compression level. */ public ZstdNoDict99Codec() { this(DEFAULT_COMPRESSION_LEVEL); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index af4e92b..36e8d82 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -43,7 +43,7 @@ protected ZstdNoDictCompressionMode(int compressionLevel) { this.compressionLevel = compressionLevel; } - /** Creates a new compressor instance.*/ + /** Creates a new compressor instance. */ @Override public Compressor newCompressor() { return new ZstdCompressor(compressionLevel); @@ -148,7 +148,6 @@ public void decompress(DataInput in, int originalLength, int offset, int length, // Read blocks that intersect with the interval we need while (offsetInBlock < offset + length) { - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + blockLength); final int compressedLength = in.readVInt(); if (compressedLength == 0) { return; @@ -156,13 +155,10 @@ public void decompress(DataInput in, int originalLength, int offset, int length, compressed = ArrayUtil.growNoCopy(compressed, compressedLength); in.readBytes(compressed, 0, compressedLength); - int l = Math.min(blockLength, originalLength - offsetInBlock); + final int l = Math.min(blockLength, originalLength - offsetInBlock); bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + l); - byte[] output = new byte[l]; - - final int uncompressed = (int) Zstd.decompressByteArray(output, 0, l, compressed, 0, compressedLength); - System.arraycopy(output, 0, bytes.bytes, bytes.length, uncompressed); + final int uncompressed = (int) Zstd.decompressByteArray(bytes.bytes, bytes.length, l, compressed, 0, compressedLength); bytes.length += uncompressed; offsetInBlock += blockLength; diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java b/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java index e996873..2b197c2 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/package-info.java @@ -6,7 +6,5 @@ * compatible open source license. */ -/** - * A plugin that implements compression codecs with native implementation. - */ +/** A plugin that implements compression codecs with native implementation. */ package org.opensearch.index.codec.customcodecs; diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java index cc794eb..573898c 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/AbstractCompressorTests.java @@ -193,7 +193,7 @@ private void addBytes(Random r, ByteArrayOutputStream bos) throws IOException { bos.write(bytes); } - private void doTest(byte[] bytes) throws IOException { + protected void doTest(byte[] bytes) throws IOException { final int length = bytes.length; ByteBuffersDataInput in = new ByteBuffersDataInput(List.of(ByteBuffer.wrap(bytes))); @@ -215,5 +215,4 @@ private void doTest(byte[] bytes) throws IOException { assertArrayEquals(bytes, restored); } - } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java index f03f164..4643246 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene95CustomStoredFieldsFormatTests.java @@ -21,7 +21,7 @@ public void testDefaultLucene95CustomCodecMode() { public void testZstdNoDictLucene95CustomCodecMode() { Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT + Lucene95CustomCodec.Mode.ZSTD_NO_DICT ); assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); } @@ -29,8 +29,8 @@ public void testZstdNoDictLucene95CustomCodecMode() { public void testZstdModeWithCompressionLevel() { int randomCompressionLevel = randomIntBetween(1, 6); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD, - randomCompressionLevel + Lucene95CustomCodec.Mode.ZSTD, + randomCompressionLevel ); assertEquals(Lucene95CustomCodec.Mode.ZSTD, lucene95CustomStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); @@ -39,21 +39,22 @@ public void testZstdModeWithCompressionLevel() { public void testZstdNoDictLucene95CustomCodecModeWithCompressionLevel() { int randomCompressionLevel = randomIntBetween(1, 6); Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( - Lucene95CustomCodec.Mode.ZSTD_NO_DICT, - randomCompressionLevel + Lucene95CustomCodec.Mode.ZSTD_NO_DICT, + randomCompressionLevel ); assertEquals(Lucene95CustomCodec.Mode.ZSTD_NO_DICT, lucene95CustomStoredFieldsFormat.getMode()); assertEquals(randomCompressionLevel, lucene95CustomStoredFieldsFormat.getCompressionLevel()); } - public void testCompressionModes(){ + public void testCompressionModes() { Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(); assertTrue(lucene95CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); } - public void testZstdNoDictCompressionModes(){ - Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode.ZSTD_NO_DICT); + public void testZstdNoDictCompressionModes() { + Lucene95CustomStoredFieldsFormat lucene95CustomStoredFieldsFormat = new Lucene95CustomStoredFieldsFormat( + Lucene95CustomCodec.Mode.ZSTD_NO_DICT + ); assertTrue(lucene95CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); } - -} \ No newline at end of file +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java index c966901..d0ab2ff 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormatTests.java @@ -44,13 +44,15 @@ public void testZstdNoDictLucene99CustomCodecModeWithCompressionLevel() { assertEquals(randomCompressionLevel, lucene99CustomStoredFieldsFormat.getCompressionLevel()); } - public void testCompressionModes(){ + public void testCompressionModes() { Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat(); assertTrue(lucene99CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdCompressionMode); } - public void testZstdNoDictCompressionModes(){ - Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat(Lucene99CustomCodec.Mode.ZSTD_NO_DICT); + public void testZstdNoDictCompressionModes() { + Lucene99CustomStoredFieldsFormat lucene99CustomStoredFieldsFormat = new Lucene99CustomStoredFieldsFormat( + Lucene99CustomCodec.Mode.ZSTD_NO_DICT + ); assertTrue(lucene99CustomStoredFieldsFormat.getCompressionMode() instanceof ZstdNoDictCompressionMode); } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java new file mode 100644 index 0000000..91f8550 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java @@ -0,0 +1,60 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.opensearch.test.OpenSearchTestCase; + +public class Lucene99QatStoredFieldsFormatTests extends OpenSearchTestCase { + + public void testDeflateLucene99QatCodecMode() { + if (!QatZipperFactory.isQatAvailable()) return; + Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_DEFLATE); + assertEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, lucene99QatStoredFieldsFormat.getMode()); + } + + public void testLz4Lucene99QatCodecMode() { + if (!QatZipperFactory.isQatAvailable()) return; + Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_LZ4); + assertEquals(Lucene99QatCodec.Mode.QAT_LZ4, lucene99QatStoredFieldsFormat.getMode()); + } + + public void testDeflateLucene99QatCodecModeWithCompressionLevel() { + if (!QatZipperFactory.isQatAvailable()) return; + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat( + Lucene99QatCodec.Mode.QAT_DEFLATE, + randomCompressionLevel + ); + assertEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, lucene99QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene99QatStoredFieldsFormat.getCompressionLevel()); + } + + public void testLz4Lucene99QatCodecModeWithCompressionLevel() { + if (!QatZipperFactory.isQatAvailable()) return; + int randomCompressionLevel = randomIntBetween(1, 6); + Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat( + Lucene99QatCodec.Mode.QAT_LZ4, + randomCompressionLevel + ); + assertEquals(Lucene99QatCodec.Mode.QAT_LZ4, lucene99QatStoredFieldsFormat.getMode()); + assertEquals(randomCompressionLevel, lucene99QatStoredFieldsFormat.getCompressionLevel()); + } + + public void testDeflateCompressionModes() { + if (!QatZipperFactory.isQatAvailable()) return; + Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_DEFLATE); + assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatDeflateCompressionMode); + } + + public void testLz4CompressionModes() { + if (!QatZipperFactory.isQatAvailable()) return; + Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_LZ4); + assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatLz4CompressionMode); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java new file mode 100644 index 0000000..30bb241 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -0,0 +1,201 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.index.codec.customcodecs; + +import org.apache.logging.log4j.LogManager; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.SegmentReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; +import org.opensearch.common.settings.Settings; +import org.opensearch.env.Environment; +import org.opensearch.index.IndexSettings; +import org.opensearch.index.analysis.IndexAnalyzers; +import org.opensearch.index.codec.CodecService; +import org.opensearch.index.codec.CodecServiceConfig; +import org.opensearch.index.codec.CodecServiceFactory; +import org.opensearch.index.codec.CodecSettings; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.indices.mapper.MapperRegistry; +import org.opensearch.plugins.MapperPlugin; +import org.opensearch.test.IndexSettingsModule; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Collections; +import java.util.Optional; + +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + +@SuppressCodecs("*") // we test against default codec so never get a random one here! +public class QatCodecTests extends OpenSearchTestCase { + + private CustomCodecPlugin plugin; + + @Before + public void setup() { + plugin = new CustomCodecPlugin(); + } + + public void testQatDeflate() throws Exception { + if (!QatZipperFactory.isQatAvailable()) return; + Codec codec = createCodecService(false).codec("qat_deflate"); + assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); + Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + public void testQatLz4() throws Exception { + if (!QatZipperFactory.isQatAvailable()) return; + Codec codec = createCodecService(false).codec("qat_lz4"); + assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); + Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + public void testQatDeflateWithCompressionLevel() throws Exception { + if (!QatZipperFactory.isQatAvailable()) return; + int randomCompressionLevel = randomIntBetween(1, 6); + Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); + assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); + Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); + } + + public void testQatLz4WithCompressionLevel() throws Exception { + if (!QatZipperFactory.isQatAvailable()) return; + int randomCompressionLevel = randomIntBetween(1, 6); + Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); + assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); + Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); + } + + public void testQatCompressionLevelSupport() throws Exception { + CodecService codecService = createCodecService(false); + CodecSettings qatDeflateCodec = (CodecSettings) codecService.codec("qat_deflate"); + CodecSettings qatLz4Codec = (CodecSettings) codecService.codec("qat_lz4"); + assertTrue(qatDeflateCodec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); + assertTrue(qatLz4Codec.supports(INDEX_CODEC_COMPRESSION_LEVEL_SETTING)); + } + + public void testQatDeflateMapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("qat_deflate"); + assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); + Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + public void testQatLz4MapperServiceNull() throws Exception { + Codec codec = createCodecService(true).codec("qat_lz4"); + assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); + Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); + assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + } + + private void assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode expected, Codec actual) throws Exception { + SegmentReader sr = getSegmentReader(actual); + String v = sr.getSegmentInfo().info.getAttribute(Lucene99QatStoredFieldsFormat.MODE_KEY); + assertNotNull(v); + assertEquals(expected, Lucene99QatCodec.Mode.valueOf(v)); + } + + private CodecService createCodecService(boolean isMapperServiceNull) throws IOException { + Settings nodeSettings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()).build(); + if (isMapperServiceNull) { + return new CustomCodecService( + null, + IndexSettingsModule.newIndexSettings("_na", nodeSettings, CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING), + LogManager.getLogger("test") + ); + } + return buildCodecService(nodeSettings); + } + + private CodecService createCodecService(int randomCompressionLevel, String codec) throws IOException { + Settings nodeSettings = Settings.builder() + .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir()) + .put("index.codec", codec) + .put("index.codec.compression_level", randomCompressionLevel) + .build(); + return buildCodecService(nodeSettings); + } + + private CodecService buildCodecService(Settings nodeSettings) throws IOException { + IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( + "_na", + nodeSettings, + CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING + ); + SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); + IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; + MapperRegistry mapperRegistry = new MapperRegistry(Collections.emptyMap(), Collections.emptyMap(), MapperPlugin.NOOP_FIELD_FILTER); + MapperService service = new MapperService( + indexSettings, + indexAnalyzers, + xContentRegistry(), + similarityService, + mapperRegistry, + () -> null, + () -> false, + null + ); + + Optional customCodecServiceFactory = plugin.getCustomCodecServiceFactory(indexSettings); + if (customCodecServiceFactory.isPresent()) { + return customCodecServiceFactory.get().createCodecService(new CodecServiceConfig(indexSettings, service, logger)); + } + return new CustomCodecService(service, indexSettings, LogManager.getLogger("test")); + } + + private SegmentReader getSegmentReader(Codec codec) throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(null); + iwc.setCodec(codec); + IndexWriter iw = new IndexWriter(dir, iwc); + iw.addDocument(new Document()); + iw.commit(); + iw.close(); + DirectoryReader ir = DirectoryReader.open(dir); + SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); + ir.close(); + dir.close(); + return sr; + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java new file mode 100644 index 0000000..cc9c092 --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; + +import java.io.IOException; + +/** Test QAT DEFLATE compression */ +public class QatDeflateCompressorTests extends AbstractCompressorTests { + + @Override + Compressor compressor() { + if (!QatZipperFactory.isQatAvailable()) return null; + return new QatDeflateCompressionMode().newCompressor(); + } + + @Override + Decompressor decompressor() { + if (!QatZipperFactory.isQatAvailable()) return null; + return new QatDeflateCompressionMode().newDecompressor(); + } + + @Override + public void testEmpty() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testEmpty(); + } + + @Override + public void testShortLiterals() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testShortLiterals(); + } + + @Override + public void testRandom() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRandom(); + } + + @Override + public void testLineDocs() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testLineDocs(); + } + + @Override + public void testRepetitionsL() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRepetitionsL(); + } + + @Override + public void testRepetitionsI() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRepetitionsI(); + } + + @Override + public void testRepetitionsS() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRepetitionsS(); + } + + @Override + public void testMixed() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testMixed(); + } + + @Override + protected void doTest(byte[] bytes) throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.doTest(bytes); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java new file mode 100644 index 0000000..d0e783e --- /dev/null +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.index.codec.customcodecs; + +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; + +import java.io.IOException; + +/** Test QAT LZ4 */ +public class QatLz4CompressorTests extends AbstractCompressorTests { + + @Override + Compressor compressor() { + if (!QatZipperFactory.isQatAvailable()) return null; + return new QatLz4CompressionMode().newCompressor(); + } + + @Override + Decompressor decompressor() { + if (!QatZipperFactory.isQatAvailable()) return null; + return new QatLz4CompressionMode().newDecompressor(); + } + + @Override + public void testEmpty() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testEmpty(); + } + + @Override + public void testShortLiterals() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testShortLiterals(); + } + + @Override + public void testRandom() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRandom(); + } + + @Override + public void testLineDocs() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testLineDocs(); + } + + @Override + public void testRepetitionsL() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRepetitionsL(); + } + + @Override + public void testRepetitionsI() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRepetitionsI(); + } + + @Override + public void testRepetitionsS() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testRepetitionsS(); + } + + @Override + public void testMixed() throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.testMixed(); + } + + @Override + protected void doTest(byte[] bytes) throws IOException { + if (!QatZipperFactory.isQatAvailable()) return; + super.doTest(bytes); + } +} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java index 2eda81a..caa272e 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressorTests.java @@ -10,9 +10,7 @@ import org.apache.lucene.codecs.compressing.Compressor; import org.apache.lucene.codecs.compressing.Decompressor; -/** - * Test ZSTD compression (with no dictionary). - */ +/** Test ZSTD compression (with no dictionary). */ public class ZstdNoDictCompressorTests extends AbstractCompressorTests { private final Compressor compressor = new ZstdNoDictCompressionMode().newCompressor(); From 69a8dc38a335345644eb27804cf59deb56ae0dbe Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Wed, 27 Mar 2024 18:22:29 -0400 Subject: [PATCH 03/11] Declare fields final, unless required not to. Throw a valid type of exception. Signed-off-by: Mulugeta Mammo --- .../Lucene99CustomStoredFieldsFormat.java | 2 +- .../Lucene99QatStoredFieldsFormat.java | 11 +----- .../QatDeflateCompressionMode.java | 5 +-- .../customcodecs/QatLz4CompressionMode.java | 5 +-- .../Lucene95CustomStoredFieldsFormat.java | 39 ++++++++++--------- 5 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java index 235c801..271827b 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomStoredFieldsFormat.java @@ -106,7 +106,7 @@ StoredFieldsFormat impl(Lucene99CustomCodec.Mode mode) { case ZSTD_NO_DICT: return getCustomCompressingStoredFieldsFormat("CustomStoredFieldsZstdNoDict", this.zstdNoDictCompressionMode); default: - throw new AssertionError(); + throw new IllegalStateException("Unsupported compression mode: " + mode); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java index dbafa7c..f9dc42a 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java @@ -148,7 +148,7 @@ StoredFieldsFormat impl(Lucene99QatCodec.Mode mode) { QAT_DEFLATE_BLOCK_SHIFT ); default: - throw new AssertionError(); + throw new IllegalStateException("Unsupported compression mode: " + mode); } } @@ -172,13 +172,6 @@ public int getCompressionLevel() { } public CompressionMode getCompressionMode() { - switch (mode) { - case QAT_DEFLATE: - return qatDeflateCompressionMode; - case QAT_LZ4: - return qatLz4CompressionMode; - default: - throw new AssertionError(); - } + return mode == Lucene99QatCodec.Mode.QAT_DEFLATE ? qatDeflateCompressionMode : qatLz4CompressionMode; } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java index b1b787c..3afceef 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java @@ -62,8 +62,7 @@ public Decompressor newDecompressor() { private static final class QatCompressor extends Compressor { private byte[] compressedBuffer; - - private QatZipper qatZipper; + private final QatZipper qatZipper; /** compressor with a given compresion level */ public QatCompressor(int compressionLevel, QatZipper.Mode mode) { @@ -118,7 +117,7 @@ public void close() throws IOException {} private static final class QatDecompressor extends Decompressor { private byte[] compressed; - private QatZipper qatZipper; + private final QatZipper qatZipper; final QatZipper.Mode qatMode; /** default decompressor */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java index 3f39702..d77de4f 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java @@ -62,8 +62,7 @@ public Decompressor newDecompressor() { private static final class QatCompressor extends Compressor { private byte[] compressedBuffer; - - private QatZipper qatZipper; + private final QatZipper qatZipper; /** compressor with a given compresion level */ public QatCompressor(int compressionLevel, QatZipper.Mode mode) { @@ -113,7 +112,7 @@ public void close() throws IOException {} private static final class QatDecompressor extends Decompressor { private byte[] compressed; - private QatZipper qatZipper; + private final QatZipper qatZipper; private final QatZipper.Mode qatMode; /** default decompressor */ diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java index be92032..aee7aff 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/backward_codecs/Lucene95CustomStoredFieldsFormat.java @@ -62,12 +62,15 @@ public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode) { public Lucene95CustomStoredFieldsFormat(Lucene95CustomCodec.Mode mode, int compressionLevel) { this.mode = Objects.requireNonNull(mode); this.compressionLevel = compressionLevel; - zstdCompressionMode = new ZstdCompressionMode(compressionLevel){}; - zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel){}; + zstdCompressionMode = new ZstdCompressionMode(compressionLevel) { + }; + zstdNoDictCompressionMode = new ZstdNoDictCompressionMode(compressionLevel) { + }; } /** * Returns a {@link StoredFieldsReader} to load stored fields. + * * @param directory The index directory. * @param si The SegmentInfo that stores segment information. * @param fn The fieldInfos. @@ -85,6 +88,7 @@ public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, Fiel /** * Returns a {@link StoredFieldsReader} to write stored fields. + * * @param directory The index directory. * @param si The SegmentInfo that stores segment information. * @param context The IOContext that holds additional details on the merge/search context. @@ -94,7 +98,7 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo String previous = si.putAttribute(MODE_KEY, mode.name()); if (previous != null && previous.equals(mode.name()) == false) { throw new IllegalStateException( - "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() + "found existing value for " + MODE_KEY + " for segment: " + si.name + " old = " + previous + ", new = " + mode.name() ); } return impl(mode).fieldsWriter(directory, si, context); @@ -105,22 +109,22 @@ StoredFieldsFormat impl(Lucene95CustomCodec.Mode mode) { case ZSTD: case ZSTD_DEPRECATED: return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstd", - zstdCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT + "CustomStoredFieldsZstd", + zstdCompressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT ); case ZSTD_NO_DICT: return new Lucene90CompressingStoredFieldsFormat( - "CustomStoredFieldsZstdNoDict", - zstdNoDictCompressionMode, - ZSTD_BLOCK_LENGTH, - ZSTD_MAX_DOCS_PER_BLOCK, - ZSTD_BLOCK_SHIFT + "CustomStoredFieldsZstdNoDict", + zstdNoDictCompressionMode, + ZSTD_BLOCK_LENGTH, + ZSTD_MAX_DOCS_PER_BLOCK, + ZSTD_BLOCK_SHIFT ); default: - throw new AssertionError(); + throw new IllegalStateException("Unsupported compression mode: " + mode); } } @@ -128,9 +132,7 @@ public Lucene95CustomCodec.Mode getMode() { return mode; } - /** - * Returns the compression level. - */ + /** Returns the compression level. */ public int getCompressionLevel() { return compressionLevel; } @@ -138,5 +140,4 @@ public int getCompressionLevel() { public CompressionMode getCompressionMode() { return mode == Lucene95CustomCodec.Mode.ZSTD_NO_DICT ? zstdNoDictCompressionMode : zstdCompressionMode; } - -} \ No newline at end of file +} From 6178b693ae6524e67118cc61ab1c5982850d91b6 Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Wed, 27 Mar 2024 19:12:24 -0400 Subject: [PATCH 04/11] Use assumeThat in the Qat test classes. Signed-off-by: Mulugeta Mammo --- .../codec/rest/CreateIndexWithCodecIT.java | 77 +++++----- .../index/codec/CodecCompressionLevelIT.java | 131 ++++++++++++++++++ .../index/codec/MultiCodecMergeIT.java | 6 + .../Lucene99QatStoredFieldsFormatTests.java | 15 +- .../codec/customcodecs/QatCodecTests.java | 10 +- .../QatDeflateCompressorTests.java | 25 ++-- .../customcodecs/QatLz4CompressorTests.java | 25 ++-- .../customcodecs/ZstdCompressorTests.java | 4 +- 8 files changed, 227 insertions(+), 66 deletions(-) diff --git a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java index ba9ca1f..713b96e 100644 --- a/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java +++ b/src/integrationTest/java/org/opensearch/index/codec/rest/CreateIndexWithCodecIT.java @@ -16,38 +16,34 @@ import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.function.Factory; -import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpHost; -import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; import org.apache.hc.core5.reactor.ssl.TlsDetails; import org.apache.hc.core5.ssl.SSLContextBuilder; -import org.apache.hc.core5.util.Timeout; -import org.junit.After; -import org.junit.BeforeClass; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.concurrent.ThreadContext; import org.opensearch.core.common.Strings; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.test.rest.OpenSearchRestTestCase; +import javax.net.ssl.SSLEngine; + import java.io.IOException; import java.security.KeyManagementException; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; -import java.util.Map; import java.util.Objects; -import java.util.Optional; - -import javax.net.ssl.SSLEngine; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_PER_ROUTE; import static org.opensearch.client.RestClientBuilder.DEFAULT_MAX_CONN_TOTAL; +import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_DEFLATE_CODEC; +import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; public class CreateIndexWithCodecIT extends OpenSearchRestTestCase { public void testCreateIndexWithZstdCodec() throws IOException { @@ -71,6 +67,29 @@ public void testCreateIndexWithZstdCodec() throws IOException { } } + public void testCreateIndexWithQatCodec() throws IOException { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + + final String index = "custom-codecs-test-index"; + + // creating index + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC)) + .put("index.codec.compression_level", randomIntBetween(1, 6)) + .build() + ); + + try { + ensureGreen(index); + } finally { + deleteIndex(index); + } + } + @Override protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOException { RestClientBuilder builder = RestClient.builder(hosts); @@ -81,7 +100,7 @@ protected RestClient buildClient(Settings settings, HttpHost[] hosts) throws IOE protected void configureHttpOrHttpsClient(RestClientBuilder builder, Settings settings) throws IOException { configureClient(builder, settings); - + if (getProtocol().equalsIgnoreCase("https")) { final String username = System.getProperty("user"); if (Strings.isNullOrEmpty(username)) { @@ -98,30 +117,26 @@ protected void configureHttpOrHttpsClient(RestClientBuilder builder, Settings se credentialsProvider.setCredentials(anyScope, new UsernamePasswordCredentials(username, password.toCharArray())); try { - final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder - .create() - .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) - // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 - .setTlsDetailsFactory(new Factory() { - @Override - public TlsDetails create(final SSLEngine sslEngine) { - return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); - } - }) - .build(); + final TlsStrategy tlsStrategy = ClientTlsStrategyBuilder.create() + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .setSslContext(SSLContextBuilder.create().loadTrustMaterial(null, (chains, authType) -> true).build()) + // See https://issues.apache.org/jira/browse/HTTPCLIENT-2219 + .setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); + } + }) + .build(); builder.setHttpClientConfigCallback(httpClientBuilder -> { - final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder - .create() + final PoolingAsyncClientConnectionManager connectionManager = PoolingAsyncClientConnectionManagerBuilder.create() .setMaxConnPerRoute(DEFAULT_MAX_CONN_PER_ROUTE) .setMaxConnTotal(DEFAULT_MAX_CONN_TOTAL) .setTlsStrategy(tlsStrategy) .build(); - - return httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager(connectionManager); + + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(connectionManager); }); } catch (final NoSuchAlgorithmException | KeyManagementException | KeyStoreException ex) { throw new IOException(ex); @@ -134,7 +149,7 @@ public TlsDetails create(final SSLEngine sslEngine) { protected String getProtocol() { return Objects.equals(System.getProperty("https"), "true") ? "https" : "http"; } - + /** * wipeAllIndices won't work since it cannot delete security index */ diff --git a/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java b/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java index 7810b58..1858f3d 100644 --- a/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java +++ b/src/internalClusterTest/java/org/opensearch/index/codec/CodecCompressionLevelIT.java @@ -13,6 +13,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.customcodecs.CustomCodecPlugin; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -20,9 +21,13 @@ import java.util.Collections; import java.util.concurrent.ExecutionException; +import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_DEFLATE_CODEC; +import static org.opensearch.index.codec.customcodecs.CustomCodecService.QAT_LZ4_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_CODEC; import static org.opensearch.index.codec.customcodecs.CustomCodecService.ZSTD_NO_DICT_CODEC; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) public class CodecCompressionLevelIT extends OpenSearchIntegTestCase { @@ -80,6 +85,26 @@ public void testZStandardCodecsCreateIndexWithCompressionLevel() { ensureGreen(index); } + public void testQatCodecsCreateIndexWithCompressionLevel() { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + + internalCluster().ensureAtLeastNumDataNodes(1); + final String index = "test-index"; + + // creating index + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC)) + .put("index.codec.compression_level", randomIntBetween(1, 6)) + .build() + ); + + ensureGreen(index); + } + public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionException, InterruptedException { internalCluster().ensureAtLeastNumDataNodes(1); @@ -132,6 +157,59 @@ public void testZStandardToLuceneCodecsWithCompressionLevel() throws ExecutionEx ensureGreen(index); } + public void testQatToLuceneCodecsWithCompressionLevel() throws ExecutionException, InterruptedException { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + + internalCluster().ensureAtLeastNumDataNodes(1); + final String index = "test-index"; + + // creating index + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC)) + .put("index.codec.compression_level", randomIntBetween(1, 6)) + .build() + ); + ensureGreen(index); + + assertAcked(client().admin().indices().prepareClose(index).setWaitForActiveShards(1)); + + Throwable executionException = expectThrows( + ExecutionException.class, + () -> client().admin() + .indices() + .updateSettings( + new UpdateSettingsRequest(index).settings( + Settings.builder().put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + ) + ) + .get() + ); + + Throwable rootCause = Throwables.getRootCause(executionException); + assertEquals(IllegalArgumentException.class, rootCause.getClass()); + assertTrue(rootCause.getMessage().startsWith("Compression level cannot be set")); + + assertAcked( + client().admin() + .indices() + .updateSettings( + new UpdateSettingsRequest(index).settings( + Settings.builder() + .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put("index.codec.compression_level", (String) null) + ) + ) + .get() + ); + + assertAcked(client().admin().indices().prepareOpen(index).setWaitForActiveShards(1)); + ensureGreen(index); + } + public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionException, InterruptedException { internalCluster().ensureAtLeastNumDataNodes(1); @@ -185,4 +263,57 @@ public void testLuceneToZStandardCodecsWithCompressionLevel() throws ExecutionEx ensureGreen(index); } + public void testLuceneToQatCodecsWithCompressionLevel() throws ExecutionException, InterruptedException { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); + + internalCluster().ensureAtLeastNumDataNodes(1); + final String index = "test-index"; + + // creating index + createIndex( + index, + Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .build() + ); + ensureGreen(index); + + assertAcked(client().admin().indices().prepareClose(index).setWaitForActiveShards(1)); + + Throwable executionException = expectThrows( + ExecutionException.class, + () -> client().admin() + .indices() + .updateSettings( + new UpdateSettingsRequest(index).settings( + Settings.builder() + .put("index.codec", randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC)) + .put("index.codec.compression_level", randomIntBetween(1, 6)) + ) + ) + .get() + ); + + Throwable rootCause = Throwables.getRootCause(executionException); + assertEquals(IllegalArgumentException.class, rootCause.getClass()); + assertTrue(rootCause.getMessage().startsWith("Compression level cannot be set")); + + assertAcked( + client().admin() + .indices() + .updateSettings( + new UpdateSettingsRequest(index).settings( + Settings.builder() + .put("index.codec", randomFrom(QAT_DEFLATE_CODEC, QAT_LZ4_CODEC)) + .put("index.codec.compression_level", randomIntBetween(1, 6)) + ) + ) + .get() + ); + + assertAcked(client().admin().indices().prepareOpen(index).setWaitForActiveShards(1)); + ensureGreen(index); + } } diff --git a/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java b/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java index bb50828..7d35075 100644 --- a/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java +++ b/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java @@ -16,6 +16,7 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.customcodecs.CustomCodecPlugin; +import org.opensearch.index.codec.customcodecs.QatZipperFactory; import org.opensearch.index.engine.Segment; import org.opensearch.plugins.Plugin; import org.opensearch.test.OpenSearchIntegTestCase; @@ -66,6 +67,11 @@ public void testForceMergeMultipleCodecs() throws ExecutionException, Interrupte "BEST_SPEED" ); + if (QatZipperFactory.isQatAvailable()) { + codecMap.put("QAT_DEFLATE", "qat_deflate"); + codecMap.put("QAT_LZ4", "qat_lz4"); + } + for (Map.Entry codec : codecMap.entrySet()) { forceMergeMultipleCodecs(codec.getKey(), codec.getValue(), codecMap); } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java index 91f8550..78ba772 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java @@ -10,22 +10,25 @@ import org.opensearch.test.OpenSearchTestCase; +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + public class Lucene99QatStoredFieldsFormatTests extends OpenSearchTestCase { public void testDeflateLucene99QatCodecMode() { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_DEFLATE); assertEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, lucene99QatStoredFieldsFormat.getMode()); } public void testLz4Lucene99QatCodecMode() { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_LZ4); assertEquals(Lucene99QatCodec.Mode.QAT_LZ4, lucene99QatStoredFieldsFormat.getMode()); } public void testDeflateLucene99QatCodecModeWithCompressionLevel() { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat( Lucene99QatCodec.Mode.QAT_DEFLATE, @@ -36,7 +39,7 @@ public void testDeflateLucene99QatCodecModeWithCompressionLevel() { } public void testLz4Lucene99QatCodecModeWithCompressionLevel() { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat( Lucene99QatCodec.Mode.QAT_LZ4, @@ -47,13 +50,13 @@ public void testLz4Lucene99QatCodecModeWithCompressionLevel() { } public void testDeflateCompressionModes() { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_DEFLATE); assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatDeflateCompressionMode); } public void testLz4CompressionModes() { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_LZ4); assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatLz4CompressionMode); } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index 30bb241..9b38a9c 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -62,6 +62,8 @@ import java.util.Optional; import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; @SuppressCodecs("*") // we test against default codec so never get a random one here! public class QatCodecTests extends OpenSearchTestCase { @@ -74,7 +76,7 @@ public void setup() { } public void testQatDeflate() throws Exception { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_deflate"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); @@ -82,7 +84,7 @@ public void testQatDeflate() throws Exception { } public void testQatLz4() throws Exception { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(false).codec("qat_lz4"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); @@ -90,7 +92,7 @@ public void testQatLz4() throws Exception { } public void testQatDeflateWithCompressionLevel() throws Exception { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); @@ -99,7 +101,7 @@ public void testQatDeflateWithCompressionLevel() throws Exception { } public void testQatLz4WithCompressionLevel() throws Exception { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); int randomCompressionLevel = randomIntBetween(1, 6); Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java index cc9c092..5b9d184 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java @@ -12,72 +12,75 @@ import java.io.IOException; +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + /** Test QAT DEFLATE compression */ public class QatDeflateCompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { - if (!QatZipperFactory.isQatAvailable()) return null; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); return new QatDeflateCompressionMode().newCompressor(); } @Override Decompressor decompressor() { - if (!QatZipperFactory.isQatAvailable()) return null; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); return new QatDeflateCompressionMode().newDecompressor(); } @Override public void testEmpty() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testEmpty(); } @Override public void testShortLiterals() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testShortLiterals(); } @Override public void testRandom() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRandom(); } @Override public void testLineDocs() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testLineDocs(); } @Override public void testRepetitionsL() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRepetitionsL(); } @Override public void testRepetitionsI() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRepetitionsI(); } @Override public void testRepetitionsS() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRepetitionsS(); } @Override public void testMixed() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testMixed(); } @Override protected void doTest(byte[] bytes) throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.doTest(bytes); } } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java index d0e783e..db42fa8 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java @@ -12,72 +12,75 @@ import java.io.IOException; +import static org.hamcrest.Matchers.is; +import static org.junit.Assume.assumeThat; + /** Test QAT LZ4 */ public class QatLz4CompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { - if (!QatZipperFactory.isQatAvailable()) return null; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); return new QatLz4CompressionMode().newCompressor(); } @Override Decompressor decompressor() { - if (!QatZipperFactory.isQatAvailable()) return null; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); return new QatLz4CompressionMode().newDecompressor(); } @Override public void testEmpty() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testEmpty(); } @Override public void testShortLiterals() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testShortLiterals(); } @Override public void testRandom() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRandom(); } @Override public void testLineDocs() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testLineDocs(); } @Override public void testRepetitionsL() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRepetitionsL(); } @Override public void testRepetitionsI() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRepetitionsI(); } @Override public void testRepetitionsS() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testRepetitionsS(); } @Override public void testMixed() throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.testMixed(); } @Override protected void doTest(byte[] bytes) throws IOException { - if (!QatZipperFactory.isQatAvailable()) return; + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); super.doTest(bytes); } } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java index 78cf62c..3094884 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/ZstdCompressorTests.java @@ -10,9 +10,7 @@ import org.apache.lucene.codecs.compressing.Compressor; import org.apache.lucene.codecs.compressing.Decompressor; -/** - * Test ZSTD compression (with dictionary enabled) - */ +/** Test ZSTD compression (with dictionary enabled) */ public class ZstdCompressorTests extends AbstractCompressorTests { private final Compressor compressor = new ZstdCompressionMode().newCompressor(); From 544aa9259bc99266239503f3e445f32d989e722a Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Wed, 27 Mar 2024 20:32:09 -0400 Subject: [PATCH 05/11] Add more QAT availability check in QatCodecTests. Signed-off-by: Mulugeta Mammo --- .../opensearch/index/codec/customcodecs/QatZipperFactory.java | 4 +--- .../opensearch/index/codec/customcodecs/QatCodecTests.java | 4 ++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java index 39cbcac..1b55b5e 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatZipperFactory.java @@ -19,9 +19,7 @@ import static com.intel.qat.QatZipper.PollingMode; /** A factory class to create instances of QatZipper */ -public class QatZipperFactory - -{ +public class QatZipperFactory { /** * Creates a new QatZipper with the specified parameters. diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index 9b38a9c..14def40 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -110,6 +110,7 @@ public void testQatLz4WithCompressionLevel() throws Exception { } public void testQatCompressionLevelSupport() throws Exception { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); CodecService codecService = createCodecService(false); CodecSettings qatDeflateCodec = (CodecSettings) codecService.codec("qat_deflate"); CodecSettings qatLz4Codec = (CodecSettings) codecService.codec("qat_lz4"); @@ -118,6 +119,7 @@ public void testQatCompressionLevelSupport() throws Exception { } public void testQatDeflateMapperServiceNull() throws Exception { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_deflate"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); @@ -125,6 +127,7 @@ public void testQatDeflateMapperServiceNull() throws Exception { } public void testQatLz4MapperServiceNull() throws Exception { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Codec codec = createCodecService(true).codec("qat_lz4"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); @@ -132,6 +135,7 @@ public void testQatLz4MapperServiceNull() throws Exception { } private void assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode expected, Codec actual) throws Exception { + assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); SegmentReader sr = getSegmentReader(actual); String v = sr.getSegmentInfo().info.getAttribute(Lucene99QatStoredFieldsFormat.MODE_KEY); assertNotNull(v); From e5f3ac50304594c32f6f7a4a55854ecab57ae46b Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Thu, 28 Mar 2024 02:57:08 -0400 Subject: [PATCH 06/11] Make LZ4 the default algorithm for QAT. Signed-off-by: Mulugeta Mammo --- .../index/codec/MultiCodecMergeIT.java | 2 +- .../codec/customcodecs/CustomCodecPlugin.java | 6 ++-- .../customcodecs/CustomCodecService.java | 16 ++++----- .../codec/customcodecs/Lucene99QatCodec.java | 17 ++++----- .../Lucene99QatStoredFieldsFormat.java | 30 ++++++++-------- .../Lucene99QatStoredFieldsFormatTests.java | 32 ++++++++--------- .../codec/customcodecs/QatCodecTests.java | 36 +++++++++---------- 7 files changed, 70 insertions(+), 69 deletions(-) diff --git a/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java b/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java index 7d35075..12d763c 100644 --- a/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java +++ b/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java @@ -68,8 +68,8 @@ public void testForceMergeMultipleCodecs() throws ExecutionException, Interrupte ); if (QatZipperFactory.isQatAvailable()) { - codecMap.put("QAT_DEFLATE", "qat_deflate"); codecMap.put("QAT_LZ4", "qat_lz4"); + codecMap.put("QAT_DEFLATE", "qat_deflate"); } for (Map.Entry codec : codecMap.entrySet()) { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 72cf6f1..84cc6a6 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -28,8 +28,8 @@ *
    *
  • ZSTD_CODEC *
  • ZSTD_NO_DICT_CODEC - *
  • QAT_DEFLATE *
  • QAT_LZ4 + *
  • QAT_DEFLATE *
* * @opensearch.internal @@ -60,8 +60,8 @@ public Optional getCustomCodecServiceFactory(final IndexSet String codecName = indexSettings.getValue(EngineConfig.INDEX_CODEC_SETTING); if (codecName.equals(CustomCodecService.ZSTD_NO_DICT_CODEC) || codecName.equals(CustomCodecService.ZSTD_CODEC) - || codecName.equals(CustomCodecService.QAT_DEFLATE_CODEC) - || codecName.equals(CustomCodecService.QAT_LZ4_CODEC)) { + || codecName.equals(CustomCodecService.QAT_LZ4_CODEC) + || codecName.equals(CustomCodecService.QAT_DEFLATE_CODEC)) { return Optional.of(new CustomCodecServiceFactory()); } return Optional.empty(); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index 71a62b5..5e5d08e 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -21,7 +21,7 @@ import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; -/** CustomCodecService provides ZSTD, ZSTD_NO_DICT, QAT_DEFLATE, and QAT_LZ4 compression codecs. */ +/** CustomCodecService provides ZSTD, ZSTD_NO_DICT, QAT_LZ4, and QAT_DEFLATE compression codecs. */ public class CustomCodecService extends CodecService { private final Map codecs; @@ -31,12 +31,12 @@ public class CustomCodecService extends CodecService { /** ZStandard without dictionary codec */ public static final String ZSTD_NO_DICT_CODEC = "zstd_no_dict"; - /** Hardware accelerated (Intel QAT) compression codec for DEFLATE. */ - public static final String QAT_DEFLATE_CODEC = "qat_deflate"; - /** Hardware accelerated (Intel QAT) compression codec for LZ4. */ public static final String QAT_LZ4_CODEC = "qat_lz4"; + /** Hardware accelerated (Intel QAT) compression codec for DEFLATE. */ + public static final String QAT_DEFLATE_CODEC = "qat_deflate"; + /** * Creates a new CustomCodecService. * @@ -51,19 +51,19 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin if (mapperService == null) { codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel)); codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel)); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> { + codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> { return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); })); - codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> { return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); })); } else { codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel)); codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel)); - codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> { + codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> { return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); })); - codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> { + codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> { return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); })); } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java index 54d96c5..7cef2cd 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java @@ -36,10 +36,11 @@ public abstract class Lucene99QatCodec extends FilterCodec { /** Each mode represents a compression algorithm. */ public enum Mode { - /** QAT deflate mode. */ - QAT_DEFLATE("QATDEFLATE99", Set.of("qat_deflate")), /** QAT lz4 mode. */ - QAT_LZ4("QATLZ499", Set.of("qat_lz4")); + QAT_LZ4("QATLZ499", Set.of("qat_lz4")), + + /** QAT deflate mode. */ + QAT_DEFLATE("QATDEFLATE99", Set.of("qat_deflate")); private final String codec; private final Set aliases; @@ -65,7 +66,7 @@ public Set getAliases() { /** * Creates a new compression codec with the default compression level. * - * @param mode The compression codec (QAT_DEFLATE or QAT_LZ4). + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). */ public Lucene99QatCodec(Mode mode) { this(mode, DEFAULT_COMPRESSION_LEVEL); @@ -76,7 +77,7 @@ public Lucene99QatCodec(Mode mode) { * registering the codec so that we remain consistent with the other compression codecs: default, * lucene_default, and best_compression. * - * @param mode The compression codec (QAT_DEFLATE or QAT_LZ4). + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). * @param compressionLevel The compression level. */ public Lucene99QatCodec(Mode mode, int compressionLevel) { @@ -89,7 +90,7 @@ public Lucene99QatCodec(Mode mode, int compressionLevel) { * registering the codec so that we remain consistent with the other compression codecs: default, * lucene_default, and best_compression. * - * @param mode The compression codec (QAT_DEFLATE, or QAT_LZ4). + * @param mode The compression codec (QAT_LZ4 or QAT_DEFLATE). * @param compressionLevel The compression level. * @param supplier supplier for QAT mode. */ @@ -103,7 +104,7 @@ public Lucene99QatCodec(Mode mode, int compressionLevel, Supplier supplier) { @@ -71,7 +71,7 @@ public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, Supplier Date: Fri, 29 Mar 2024 01:09:41 -0400 Subject: [PATCH 07/11] Make 'auto' the default execution mode for QAT. Also, minor clean up work. Signed-off-by: Mulugeta Mammo --- .../codec/customcodecs/CustomCodecPlugin.java | 17 +-------------- .../customcodecs/CustomCodecService.java | 8 +++---- .../customcodecs/Lucene99CustomCodec.java | 2 +- .../codec/customcodecs/Lucene99QatCodec.java | 21 ++++++++++++++++--- .../QatDeflateCompressionMode.java | 3 +-- .../customcodecs/QatLz4CompressionMode.java | 3 +-- .../customcodecs/ZstdCompressionMode.java | 3 +-- .../ZstdNoDictCompressionMode.java | 3 +-- .../codec/customcodecs/QatCodecTests.java | 4 ++-- 9 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java index 84cc6a6..dc3a917 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java @@ -9,7 +9,6 @@ package org.opensearch.index.codec.customcodecs; import org.opensearch.common.settings.Setting; -import org.opensearch.common.settings.Setting.Property; import org.opensearch.index.IndexSettings; import org.opensearch.index.codec.CodecServiceFactory; import org.opensearch.index.engine.EngineConfig; @@ -20,8 +19,6 @@ import java.util.List; import java.util.Optional; -import com.intel.qat.QatZipper; - /** * A plugin that implements custom codecs. Supports these codecs: * @@ -36,18 +33,6 @@ */ public final class CustomCodecPlugin extends Plugin implements EnginePlugin { - /** A setting to specifiy the QAT acceleration mode. */ - public static final Setting INDEX_CODEC_QAT_MODE_SETTING = new Setting<>("index.codec.qatmode", "hardware", s -> { - switch (s) { - case "auto": - return QatZipper.Mode.AUTO; - case "hardware": - return QatZipper.Mode.HARDWARE; - default: - throw new IllegalArgumentException("Unknown value for [index.codec.qatmode] must be one of [auto, hardware] but was: " + s); - } - }, Property.IndexScope, Property.Dynamic); - /** Creates a new instance */ public CustomCodecPlugin() {} @@ -69,6 +54,6 @@ public Optional getCustomCodecServiceFactory(final IndexSet @Override public List> getSettings() { - return Arrays.asList(INDEX_CODEC_QAT_MODE_SETTING); + return Arrays.asList(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java index 5e5d08e..175613e 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java @@ -52,19 +52,19 @@ public CustomCodecService(MapperService mapperService, IndexSettings indexSettin codecs.put(ZSTD_CODEC, new Zstd99Codec(compressionLevel)); codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(compressionLevel)); codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(compressionLevel, () -> { - return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); })); codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(compressionLevel, () -> { - return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); })); } else { codecs.put(ZSTD_CODEC, new Zstd99Codec(mapperService, logger, compressionLevel)); codecs.put(ZSTD_NO_DICT_CODEC, new ZstdNoDict99Codec(mapperService, logger, compressionLevel)); codecs.put(QAT_LZ4_CODEC, new QatLz499Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); })); codecs.put(QAT_DEFLATE_CODEC, new QatDeflate99Codec(mapperService, logger, compressionLevel, () -> { - return indexSettings.getValue(CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING); + return indexSettings.getValue(Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING); })); } this.codecs = codecs.immutableMap(); diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java index e0b3c2b..05e908c 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java @@ -28,7 +28,7 @@ public abstract class Lucene99CustomCodec extends FilterCodec { /** Default compression level used for compression */ - public static final int DEFAULT_COMPRESSION_LEVEL = 3; + public static final int DEFAULT_COMPRESSION_LEVEL = 6; /** Each mode represents a compression algorithm. */ public enum Mode { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java index 7cef2cd..beb94cf 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java @@ -12,6 +12,9 @@ import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Setting.Property; +import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; @@ -29,10 +32,22 @@ public abstract class Lucene99QatCodec extends FilterCodec { /** Default compression level used for compression */ - public static final int DEFAULT_COMPRESSION_LEVEL = 3; + public static final int DEFAULT_COMPRESSION_LEVEL = 6; + + /** A setting to specifiy the QAT acceleration mode. */ + public static final Setting INDEX_CODEC_QAT_MODE_SETTING = new Setting<>("index.codec.qatmode", "auto", s -> { + switch (s) { + case "auto": + return QatZipper.Mode.AUTO; + case "hardware": + return QatZipper.Mode.HARDWARE; + default: + throw new IllegalArgumentException("Unknown value for [index.codec.qatmode] must be one of [auto, hardware] but was: " + s); + } + }, Property.IndexScope, Property.Dynamic); - /** The default QAT mode */ - public static final QatZipper.Mode DEFAULT_QAT_MODE = QatZipper.Mode.HARDWARE; + /** Just a terse way to reference the default execution mode. */ + public static final QatZipper.Mode DEFAULT_QAT_MODE = INDEX_CODEC_QAT_MODE_SETTING.getDefault(Settings.EMPTY); /** Each mode represents a compression algorithm. */ public enum Mode { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java index 3afceef..94c0b34 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java @@ -26,14 +26,13 @@ public class QatDeflateCompressionMode extends CompressionMode { private static final int NUM_SUB_BLOCKS = 10; - private static final int DEFAULT_COMPRESSION_LEVEL = 6; private final int compressionLevel; private final Supplier supplier; /** default constructor */ protected QatDeflateCompressionMode() { - this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; this.supplier = () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java index d77de4f..395174f 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java @@ -26,14 +26,13 @@ public class QatLz4CompressionMode extends CompressionMode { private static final int NUM_SUB_BLOCKS = 10; - private static final int DEFAULT_COMPRESSION_LEVEL = 6; private final int compressionLevel; private final Supplier supplier; /** default constructor */ protected QatLz4CompressionMode() { - this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; this.supplier = () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }; } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java index d88ef1f..67476ed 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdCompressionMode.java @@ -30,13 +30,12 @@ public class ZstdCompressionMode extends CompressionMode { private static final int NUM_SUB_BLOCKS = 10; private static final int DICT_SIZE_FACTOR = 6; - private static final int DEFAULT_COMPRESSION_LEVEL = 6; private final int compressionLevel; /** default constructor */ protected ZstdCompressionMode() { - this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; } /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java index 36e8d82..2a44581 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/ZstdNoDictCompressionMode.java @@ -25,13 +25,12 @@ public class ZstdNoDictCompressionMode extends CompressionMode { private static final int NUM_SUB_BLOCKS = 10; - private static final int DEFAULT_COMPRESSION_LEVEL = 6; private final int compressionLevel; /** default constructor */ protected ZstdNoDictCompressionMode() { - this.compressionLevel = DEFAULT_COMPRESSION_LEVEL; + this.compressionLevel = Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL; } /** diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index 0449189..72798a4 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -147,7 +147,7 @@ private CodecService createCodecService(boolean isMapperServiceNull) throws IOEx if (isMapperServiceNull) { return new CustomCodecService( null, - IndexSettingsModule.newIndexSettings("_na", nodeSettings, CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING), + IndexSettingsModule.newIndexSettings("_na", nodeSettings, Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING), LogManager.getLogger("test") ); } @@ -167,7 +167,7 @@ private CodecService buildCodecService(Settings nodeSettings) throws IOException IndexSettings indexSettings = IndexSettingsModule.newIndexSettings( "_na", nodeSettings, - CustomCodecPlugin.INDEX_CODEC_QAT_MODE_SETTING + Lucene99QatCodec.INDEX_CODEC_QAT_MODE_SETTING ); SimilarityService similarityService = new SimilarityService(indexSettings, null, Collections.emptyMap()); IndexAnalyzers indexAnalyzers = createTestAnalysis(indexSettings, nodeSettings).indexAnalyzers; From f020c7fe8ff3126e99e31f4f3c8482a9f24c9169 Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Fri, 29 Mar 2024 01:31:45 -0400 Subject: [PATCH 08/11] Revert compression level for ZSTD to 3. Signed-off-by: Mulugeta Mammo --- .../index/codec/customcodecs/Lucene99CustomCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java index 05e908c..e0b3c2b 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java @@ -28,7 +28,7 @@ public abstract class Lucene99CustomCodec extends FilterCodec { /** Default compression level used for compression */ - public static final int DEFAULT_COMPRESSION_LEVEL = 6; + public static final int DEFAULT_COMPRESSION_LEVEL = 3; /** Each mode represents a compression algorithm. */ public enum Mode { From 4353f6c87431b158ef72a36696f659df5aa8be6c Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Mon, 1 Apr 2024 15:31:33 -0400 Subject: [PATCH 09/11] Replace QatLz4/DeflateCompressionMode classes with QatCompressionMode. Signed-off-by: Mulugeta Mammo --- .../codec/customcodecs/Lucene99QatCodec.java | 6 +- .../Lucene99QatStoredFieldsFormat.java | 51 +++-- ...ssionMode.java => QatCompressionMode.java} | 61 ++++-- .../QatDeflateCompressionMode.java | 181 ------------------ .../Lucene99QatStoredFieldsFormatTests.java | 8 +- .../codec/customcodecs/QatCodecTests.java | 12 +- .../QatDeflateCompressorTests.java | 4 +- .../customcodecs/QatLz4CompressorTests.java | 4 +- 8 files changed, 88 insertions(+), 239 deletions(-) rename src/main/java/org/opensearch/index/codec/customcodecs/{QatLz4CompressionMode.java => QatCompressionMode.java} (70%) delete mode 100644 src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java index beb94cf..3e314c7 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java @@ -24,8 +24,7 @@ import com.intel.qat.QatZipper; /** - * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. Supports two modes zstd - * and zstd_no_dict. Uses Lucene99 as the delegate codec + * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. * * @opensearch.internal */ @@ -76,6 +75,9 @@ public Set getAliases() { } } + /** The default compression mode. */ + public static final Mode DEFAULT_COMPRESSION_MODE = Mode.QAT_LZ4; + private final StoredFieldsFormat storedFieldsFormat; /** diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java index dba44b2..9ea9dd9 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormat.java @@ -38,15 +38,12 @@ public class Lucene99QatStoredFieldsFormat extends StoredFieldsFormat { private static final int QAT_LZ4_MAX_DOCS_PER_BLOCK = 4096; private static final int QAT_LZ4_BLOCK_SHIFT = 10; - private final CompressionMode qatDeflateCompressionMode; - private final CompressionMode qatLz4CompressionMode; - + private final QatCompressionMode qatCompressionMode; private final Lucene99QatCodec.Mode mode; - private final int compressionLevel; /** default constructor */ public Lucene99QatStoredFieldsFormat() { - this(Lucene99QatCodec.Mode.QAT_LZ4, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL); + this(Lucene99QatCodec.DEFAULT_COMPRESSION_MODE, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL); } /** @@ -59,23 +56,23 @@ public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode) { } /** - * Creates a new instance. + * Creates a new instance with the specified mode and compression level. * * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE - * @param supplier a supplier for QAT acceleration mode. + * @param compressionLevel The compression level for the mode. */ - public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, Supplier supplier) { - this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, supplier); + public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); } /** - * Creates a new instance with the specified mode and compression level. + * Creates a new instance. * * @param mode The mode represents QAT_LZ4 or QAT_DEFLATE - * @param compressionLevel The compression level for the mode. + * @param supplier a supplier for QAT acceleration mode. */ - public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, int compressionLevel) { - this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, Supplier supplier) { + this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, supplier); } /** @@ -87,9 +84,7 @@ public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, int compression */ public Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode mode, int compressionLevel, Supplier supplier) { this.mode = Objects.requireNonNull(mode); - this.compressionLevel = compressionLevel; - qatDeflateCompressionMode = new QatDeflateCompressionMode(compressionLevel, supplier); - qatLz4CompressionMode = new QatLz4CompressionMode(compressionLevel, supplier); + qatCompressionMode = new QatCompressionMode(mode, compressionLevel, supplier); } /** @@ -129,12 +124,12 @@ public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOCo return impl(mode).fieldsWriter(directory, si, context); } - StoredFieldsFormat impl(Lucene99QatCodec.Mode mode) { + private StoredFieldsFormat impl(Lucene99QatCodec.Mode mode) { switch (mode) { case QAT_LZ4: return getQatCompressingStoredFieldsFormat( "QatStoredFieldsLz4", - this.qatLz4CompressionMode, + qatCompressionMode, QAT_LZ4_BLOCK_LENGTH, QAT_LZ4_MAX_DOCS_PER_BLOCK, QAT_LZ4_BLOCK_SHIFT @@ -142,7 +137,7 @@ StoredFieldsFormat impl(Lucene99QatCodec.Mode mode) { case QAT_DEFLATE: return getQatCompressingStoredFieldsFormat( "QatStoredFieldsDeflate", - this.qatDeflateCompressionMode, + qatCompressionMode, QAT_DEFLATE_BLOCK_LENGTH, QAT_DEFLATE_MAX_DOCS_PER_BLOCK, QAT_DEFLATE_BLOCK_SHIFT @@ -162,16 +157,20 @@ private StoredFieldsFormat getQatCompressingStoredFieldsFormat( return new Lucene90CompressingStoredFieldsFormat(formatName, compressionMode, blockSize, maxDocs, blockShift); } + /** + * Gets the mode of compression. + * + * @return either QAT_LZ4 or QAT_DEFLATE + */ public Lucene99QatCodec.Mode getMode() { return mode; } - /** Returns the compression level. */ - public int getCompressionLevel() { - return compressionLevel; - } - - public CompressionMode getCompressionMode() { - return mode == Lucene99QatCodec.Mode.QAT_LZ4 ? qatLz4CompressionMode : qatDeflateCompressionMode; + /** + * + * @return the CompressionMode instance. + */ + public QatCompressionMode getCompressionMode() { + return qatCompressionMode; } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java similarity index 70% rename from src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java rename to src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java index 395174f..193c825 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatLz4CompressionMode.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/QatCompressionMode.java @@ -22,51 +22,78 @@ import com.intel.qat.QatZipper; -/** QAT_LZ4 Compression Mode */ -public class QatLz4CompressionMode extends CompressionMode { +/** QatCompressionMode offers QAT_LZ4 and QAT_DEFLATE compressors. */ +public class QatCompressionMode extends CompressionMode { private static final int NUM_SUB_BLOCKS = 10; + private final QatZipper.Algorithm algorithm; private final int compressionLevel; private final Supplier supplier; /** default constructor */ - protected QatLz4CompressionMode() { - this.compressionLevel = Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; - this.supplier = () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }; + protected QatCompressionMode() { + this(Lucene99QatCodec.DEFAULT_COMPRESSION_MODE, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { + return Lucene99QatCodec.DEFAULT_QAT_MODE; + }); } /** * Creates a new instance. * + * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + */ + protected QatCompressionMode(Lucene99QatCodec.Mode mode) { + this(mode, Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) + * @param compressionLevel The compression level to use. + */ + protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel) { + this(mode, compressionLevel, () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }); + } + + /** + * Creates a new instance. + * + * @param mode The compression mode (QAT_LZ4 or QAT_DEFLATE) * @param compressionLevel The compression level to use. * @param supplier a supplier for QAT acceleration mode. */ - protected QatLz4CompressionMode(int compressionLevel, Supplier supplier) { + protected QatCompressionMode(Lucene99QatCodec.Mode mode, int compressionLevel, Supplier supplier) { + this.algorithm = mode == Lucene99QatCodec.Mode.QAT_LZ4 ? QatZipper.Algorithm.LZ4 : QatZipper.Algorithm.DEFLATE; this.compressionLevel = compressionLevel; this.supplier = supplier; } @Override public Compressor newCompressor() { - return new QatCompressor(compressionLevel, supplier.get()); + return new QatCompressor(algorithm, compressionLevel, supplier.get()); } @Override public Decompressor newDecompressor() { - return new QatDecompressor(supplier.get()); + return new QatDecompressor(algorithm, supplier.get()); + } + + public int getCompressionLevel() { + return compressionLevel; } - /** QAT_LZ4 compressor */ + /** The QatCompressor. */ private static final class QatCompressor extends Compressor { private byte[] compressedBuffer; private final QatZipper qatZipper; /** compressor with a given compresion level */ - public QatCompressor(int compressionLevel, QatZipper.Mode mode) { + public QatCompressor(QatZipper.Algorithm algorithm, int compressionLevel, QatZipper.Mode qatMode) { compressedBuffer = BytesRef.EMPTY_BYTES; - qatZipper = QatZipperFactory.createInstance(QatZipper.Algorithm.LZ4, compressionLevel, mode, QatZipper.PollingMode.PERIODICAL); + qatZipper = QatZipperFactory.createInstance(algorithm, compressionLevel, qatMode, QatZipper.PollingMode.PERIODICAL); } private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { @@ -107,18 +134,20 @@ public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws I public void close() throws IOException {} } - /** QAT_LZ4 decompressor */ + /** QAT_DEFLATE decompressor */ private static final class QatDecompressor extends Decompressor { private byte[] compressed; private final QatZipper qatZipper; private final QatZipper.Mode qatMode; + private final QatZipper.Algorithm algorithm; /** default decompressor */ - public QatDecompressor(QatZipper.Mode mode) { + public QatDecompressor(QatZipper.Algorithm algorithm, QatZipper.Mode qatMode) { + this.algorithm = algorithm; + this.qatMode = qatMode; compressed = BytesRef.EMPTY_BYTES; - qatZipper = QatZipperFactory.createInstance(QatZipper.Algorithm.LZ4, mode, QatZipper.PollingMode.PERIODICAL); - qatMode = mode; + qatZipper = QatZipperFactory.createInstance(algorithm, qatMode, QatZipper.PollingMode.PERIODICAL); } /*resuable decompress function*/ @@ -170,7 +199,7 @@ public void decompress(DataInput in, int originalLength, int offset, int length, @Override public Decompressor clone() { - return new QatDecompressor(qatMode); + return new QatDecompressor(algorithm, qatMode); } } } diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java b/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java deleted file mode 100644 index 94c0b34..0000000 --- a/src/main/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressionMode.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.codec.customcodecs; - -import org.apache.lucene.codecs.compressing.CompressionMode; -import org.apache.lucene.codecs.compressing.Compressor; -import org.apache.lucene.codecs.compressing.Decompressor; -import org.apache.lucene.store.ByteBuffersDataInput; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.DataOutput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; - -import java.io.IOException; -import java.util.function.Supplier; - -import com.intel.qat.QatZipper; - -/** QAT_DEFLATE ompression Mode */ -public class QatDeflateCompressionMode extends CompressionMode { - - private static final int NUM_SUB_BLOCKS = 10; - - private final int compressionLevel; - private final Supplier supplier; - - /** default constructor */ - protected QatDeflateCompressionMode() { - this.compressionLevel = Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL; - this.supplier = () -> { return Lucene99QatCodec.DEFAULT_QAT_MODE; }; - } - - /** - * Creates a new instance. - * - * @param compressionLevel The compression level to use. - * @param supplier a supplier for QAT acceleration mode. - */ - protected QatDeflateCompressionMode(int compressionLevel, Supplier supplier) { - this.compressionLevel = compressionLevel; - this.supplier = supplier; - } - - @Override - public Compressor newCompressor() { - return new QatCompressor(compressionLevel, supplier.get()); - } - - @Override - public Decompressor newDecompressor() { - return new QatDecompressor(supplier.get()); - } - - /** QAT_DEFLATE compressor */ - private static final class QatCompressor extends Compressor { - - private byte[] compressedBuffer; - private final QatZipper qatZipper; - - /** compressor with a given compresion level */ - public QatCompressor(int compressionLevel, QatZipper.Mode mode) { - compressedBuffer = BytesRef.EMPTY_BYTES; - qatZipper = QatZipperFactory.createInstance( - QatZipper.Algorithm.DEFLATE, - compressionLevel, - mode, - QatZipper.PollingMode.PERIODICAL - ); - } - - private void compress(byte[] bytes, int offset, int length, DataOutput out) throws IOException { - assert offset >= 0 : "Offset value must be greater than 0."; - - int blockLength = (length + NUM_SUB_BLOCKS - 1) / NUM_SUB_BLOCKS; - out.writeVInt(blockLength); - - final int end = offset + length; - assert end >= 0 : "Buffer read size must be greater than 0."; - - for (int start = offset; start < end; start += blockLength) { - int l = Math.min(blockLength, end - start); - - if (l == 0) { - out.writeVInt(0); - return; - } - - final int maxCompressedLength = qatZipper.maxCompressedLength(l); - compressedBuffer = ArrayUtil.grow(compressedBuffer, maxCompressedLength); - - int compressedSize = qatZipper.compress(bytes, start, l, compressedBuffer, 0, compressedBuffer.length); - out.writeVInt(compressedSize); - out.writeBytes(compressedBuffer, compressedSize); - } - } - - @Override - public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { - final int length = (int) buffersInput.size(); - byte[] bytes = new byte[length]; - buffersInput.readBytes(bytes, 0, length); - compress(bytes, 0, length, out); - } - - @Override - public void close() throws IOException {} - } - - /** QAT_DEFLATE decompressor */ - private static final class QatDecompressor extends Decompressor { - - private byte[] compressed; - private final QatZipper qatZipper; - final QatZipper.Mode qatMode; - - /** default decompressor */ - public QatDecompressor(QatZipper.Mode mode) { - compressed = BytesRef.EMPTY_BYTES; - qatZipper = QatZipperFactory.createInstance(QatZipper.Algorithm.DEFLATE, mode, QatZipper.PollingMode.PERIODICAL); - this.qatMode = mode; - } - - /*resuable decompress function*/ - @Override - public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { - assert offset + length <= originalLength : "Buffer read size must be within limit."; - - if (length == 0) { - bytes.length = 0; - return; - } - - final int blockLength = in.readVInt(); - bytes.offset = bytes.length = 0; - int offsetInBlock = 0; - int offsetInBytesRef = offset; - - // Skip unneeded blocks - while (offsetInBlock + blockLength < offset) { - final int compressedLength = in.readVInt(); - in.skipBytes(compressedLength); - offsetInBlock += blockLength; - offsetInBytesRef -= blockLength; - } - - // Read blocks that intersect with the interval we need - while (offsetInBlock < offset + length) { - final int compressedLength = in.readVInt(); - if (compressedLength == 0) { - return; - } - compressed = ArrayUtil.grow(compressed, compressedLength); - in.readBytes(compressed, 0, compressedLength); - - int l = Math.min(blockLength, originalLength - offsetInBlock); - bytes.bytes = ArrayUtil.grow(bytes.bytes, bytes.length + l); - - final int uncompressed = qatZipper.decompress(compressed, 0, compressedLength, bytes.bytes, bytes.length, l); - - bytes.length += uncompressed; - offsetInBlock += blockLength; - } - - bytes.offset = offsetInBytesRef; - bytes.length = length; - - assert bytes.isValid() : "Decompression output is corrupted."; - } - - @Override - public Decompressor clone() { - return new QatDecompressor(qatMode); - } - } -} diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java index 91978b0..de3722b 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/Lucene99QatStoredFieldsFormatTests.java @@ -35,7 +35,7 @@ public void testLz4Lucene99QatCodecModeWithCompressionLevel() { randomCompressionLevel ); assertEquals(Lucene99QatCodec.Mode.QAT_LZ4, lucene99QatStoredFieldsFormat.getMode()); - assertEquals(randomCompressionLevel, lucene99QatStoredFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, lucene99QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testDeflateLucene99QatCodecModeWithCompressionLevel() { @@ -46,18 +46,18 @@ public void testDeflateLucene99QatCodecModeWithCompressionLevel() { randomCompressionLevel ); assertEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, lucene99QatStoredFieldsFormat.getMode()); - assertEquals(randomCompressionLevel, lucene99QatStoredFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, lucene99QatStoredFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testLz4CompressionModes() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_LZ4); - assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatLz4CompressionMode); + assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); } public void testDeflateCompressionModes() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); Lucene99QatStoredFieldsFormat lucene99QatStoredFieldsFormat = new Lucene99QatStoredFieldsFormat(Lucene99QatCodec.Mode.QAT_DEFLATE); - assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatDeflateCompressionMode); + assertTrue(lucene99QatStoredFieldsFormat.getCompressionMode() instanceof QatCompressionMode); } } diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java index 72798a4..e4220d3 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatCodecTests.java @@ -80,7 +80,7 @@ public void testQatLz4() throws Exception { Codec codec = createCodecService(false).codec("qat_lz4"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflate() throws Exception { @@ -88,7 +88,7 @@ public void testQatDeflate() throws Exception { Codec codec = createCodecService(false).codec("qat_deflate"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertEquals(Lucene99CustomCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatLz4WithCompressionLevel() throws Exception { @@ -97,7 +97,7 @@ public void testQatLz4WithCompressionLevel() throws Exception { Codec codec = createCodecService(randomCompressionLevel, "qat_lz4").codec("qat_lz4"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflateWithCompressionLevel() throws Exception { @@ -106,7 +106,7 @@ public void testQatDeflateWithCompressionLevel() throws Exception { Codec codec = createCodecService(randomCompressionLevel, "qat_deflate").codec("qat_deflate"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionLevel()); + assertEquals(randomCompressionLevel, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatCompressionLevelSupport() throws Exception { @@ -123,7 +123,7 @@ public void testQatLz4MapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("qat_lz4"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_LZ4, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } public void testQatDeflateMapperServiceNull() throws Exception { @@ -131,7 +131,7 @@ public void testQatDeflateMapperServiceNull() throws Exception { Codec codec = createCodecService(true).codec("qat_deflate"); assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode.QAT_DEFLATE, codec); Lucene99QatStoredFieldsFormat storedFieldsFormat = (Lucene99QatStoredFieldsFormat) codec.storedFieldsFormat(); - assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionLevel()); + assertEquals(Lucene99QatCodec.DEFAULT_COMPRESSION_LEVEL, storedFieldsFormat.getCompressionMode().getCompressionLevel()); } private void assertStoredFieldsCompressionEquals(Lucene99QatCodec.Mode expected, Codec actual) throws Exception { diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java index 5b9d184..1994f69 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatDeflateCompressorTests.java @@ -21,13 +21,13 @@ public class QatDeflateCompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatDeflateCompressionMode().newCompressor(); + return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_DEFLATE).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatDeflateCompressionMode().newDecompressor(); + return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_DEFLATE).newDecompressor(); } @Override diff --git a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java index db42fa8..905773f 100644 --- a/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java +++ b/src/test/java/org/opensearch/index/codec/customcodecs/QatLz4CompressorTests.java @@ -21,13 +21,13 @@ public class QatLz4CompressorTests extends AbstractCompressorTests { @Override Compressor compressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatLz4CompressionMode().newCompressor(); + return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_LZ4).newCompressor(); } @Override Decompressor decompressor() { assumeThat("Qat library is available", QatZipperFactory.isQatAvailable(), is(true)); - return new QatLz4CompressionMode().newDecompressor(); + return new QatCompressionMode(Lucene99QatCodec.Mode.QAT_LZ4).newDecompressor(); } @Override From 98947b135c51fe9f3f0b0c85336ecfb8928a8bdb Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Mon, 1 Apr 2024 16:16:22 -0400 Subject: [PATCH 10/11] Fix a MultiCodecMergeIT test fail. Signed-off-by: Mulugeta Mammo --- .../index/codec/MultiCodecMergeIT.java | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java b/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java index 12d763c..ae16b5d 100644 --- a/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java +++ b/src/internalClusterTest/java/org/opensearch/index/codec/MultiCodecMergeIT.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; @@ -52,30 +53,24 @@ protected Collection> nodePlugins() { public void testForceMergeMultipleCodecs() throws ExecutionException, InterruptedException { - Map codecMap = Map.of( - "best_compression", - "BEST_COMPRESSION", - "zlib", - "BEST_COMPRESSION", - "zstd_no_dict", - "ZSTD_NO_DICT", - "zstd", - "ZSTD", - "default", - "BEST_SPEED", - "lz4", - "BEST_SPEED" - ); - - if (QatZipperFactory.isQatAvailable()) { - codecMap.put("QAT_LZ4", "qat_lz4"); - codecMap.put("QAT_DEFLATE", "qat_deflate"); - } + Map codecMap = new HashMap() { + { + put("best_compression", "BEST_COMPRESSION"); + put("zlib", "BEST_COMPRESSION"); + put("zstd_no_dict", "ZSTD_NO_DICT"); + put("zstd", "ZSTD"); + put("default", "BEST_SPEED"); + put("lz4", "BEST_SPEED"); + if (QatZipperFactory.isQatAvailable()) { + put("qat_lz4", "QAT_LZ4"); + put("qat_deflate", "QAT_DEFLATE"); + } + } + }; for (Map.Entry codec : codecMap.entrySet()) { forceMergeMultipleCodecs(codec.getKey(), codec.getValue(), codecMap); } - } private void forceMergeMultipleCodecs(String finalCodec, String finalCodecMode, Map codecMap) throws ExecutionException, From ef53f3b73bdc5f740497921e7f711bfae681db38 Mon Sep 17 00:00:00 2001 From: Mulugeta Mammo Date: Wed, 3 Apr 2024 22:21:03 -0700 Subject: [PATCH 11/11] Remove hard-coded values for default compression level. Signed-off-by: Mulugeta Mammo --- .../index/codec/customcodecs/Lucene99CustomCodec.java | 5 ++++- .../index/codec/customcodecs/Lucene99QatCodec.java | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java index e0b3c2b..6b31167 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99CustomCodec.java @@ -12,11 +12,14 @@ import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.StoredFieldsFormat; import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.PerFieldMappingPostingFormatCodec; import org.opensearch.index.mapper.MapperService; import java.util.Set; +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + /** * * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. @@ -28,7 +31,7 @@ public abstract class Lucene99CustomCodec extends FilterCodec { /** Default compression level used for compression */ - public static final int DEFAULT_COMPRESSION_LEVEL = 3; + public static final int DEFAULT_COMPRESSION_LEVEL = INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getDefault(Settings.EMPTY); /** Each mode represents a compression algorithm. */ public enum Mode { diff --git a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java index 3e314c7..20f265d 100644 --- a/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java +++ b/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java @@ -23,6 +23,8 @@ import com.intel.qat.QatZipper; +import static org.opensearch.index.engine.EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING; + /** * Extends {@link FilterCodec} to reuse the functionality of Lucene Codec. * @@ -30,9 +32,6 @@ */ public abstract class Lucene99QatCodec extends FilterCodec { - /** Default compression level used for compression */ - public static final int DEFAULT_COMPRESSION_LEVEL = 6; - /** A setting to specifiy the QAT acceleration mode. */ public static final Setting INDEX_CODEC_QAT_MODE_SETTING = new Setting<>("index.codec.qatmode", "auto", s -> { switch (s) { @@ -45,9 +44,12 @@ public abstract class Lucene99QatCodec extends FilterCodec { } }, Property.IndexScope, Property.Dynamic); - /** Just a terse way to reference the default execution mode. */ + /** A terse way to reference the default QAT execution mode. */ public static final QatZipper.Mode DEFAULT_QAT_MODE = INDEX_CODEC_QAT_MODE_SETTING.getDefault(Settings.EMPTY); + /** Default compression level used for compression */ + public static final int DEFAULT_COMPRESSION_LEVEL = INDEX_CODEC_COMPRESSION_LEVEL_SETTING.getDefault(Settings.EMPTY); + /** Each mode represents a compression algorithm. */ public enum Mode { /** QAT lz4 mode. */