Add a plugin for hardware-accelerated compression.

sandbox/modules/custom-codecs/build.gradle

@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+apply plugin: 'opensearch.opensearchplugin'
+apply plugin: 'opensearch.yaml-rest-test'
+
+opensearchplugin {
+  name 'custom-codecs'
+  description 'A plugin that implements custom compression codecs.'
+  classname 'org.opensearch.index.codec.customcodecs.CustomCodecPlugin'
+  licenseFile rootProject.file('licenses/APACHE-LICENSE-2.0.txt')
+  noticeFile rootProject.file('NOTICE.txt')
+}
+
+dependencies {
+  api "com.intel.qat:qat-java:1.1.0"
+  testImplementation project(':server').sourceSets.test.output
+}
+
+yamlRestTest.enabled = false;
+testingConventions.enabled = false;

sandbox/modules/custom-codecs/licenses/qat-java-1.1.0.jar.sha1

@@ -0,0 +1 @@
+fbcaabdbf9d2a72d4b8222e7cfb2043a68b7860e

sandbox/modules/custom-codecs/licenses/qat-java-LICENSE.txt

@@ -0,0 +1,36 @@
+-----------------------------------------------------------------------------
+** Beginning of "BSD License" text. **
+
+Qat-Java: Qat-Java is a compression library that uses Intel® QAT to accelerate
+compression and decompression.
+
+Copyright(c) 2007-2023 Intel Corporation. All rights reserved.
+All rights reserved.
+
+BSD License
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+  * Redistributions of source code must retain the above copyright
+    notice, this list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright
+    notice, this list of conditions and the following disclaimer in
+    the documentation and/or other materials provided with the
+    distribution.
+  * Neither the name of Intel Corporation nor the names of its
+    contributors may be used to endorse or promote products derived
+    from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

sandbox/modules/custom-codecs/licenses/qat-java-NOTICE.txt

@@ -0,0 +1 @@
+Qat-Java is a compression library that uses Intel® QAT to accelerate compression and decompression.

sandbox/modules/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecPlugin.java

@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.opensearch.common.settings.Setting;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.codec.CodecServiceFactory;
+import org.opensearch.plugins.EnginePlugin;
+import org.opensearch.plugins.Plugin;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * A plugin that implements custom codecs. Supports these codecs:
+ * <ul>
+ * <li>QDEFLATE
+ * <li>QLZ4
+ * </ul>
+ *
+ * @opensearch.internal
+ */
+public final class CustomCodecPlugin extends Plugin implements EnginePlugin {
+
+    /** Creates a new instance */
+    public CustomCodecPlugin() {}
+
+    /**
+     * @param indexSettings is the default indexSettings
+     * @return the engine factory
+     */
+    @Override
+    public Optional<CodecServiceFactory> getCustomCodecServiceFactory(final IndexSettings indexSettings) {
+        return Optional.of(new CustomCodecServiceFactory());
+    }
+
+    @Override
+    public List<Setting<?>> getSettings() {
+        return Arrays.asList(Lucene99QatCodec.INDEX_CODEC_MODE_SETTING);
+    }
+}

sandbox/modules/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecService.java

@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.apache.lucene.codecs.Codec;
+import org.opensearch.common.collect.MapBuilder;
+import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.codec.CodecServiceConfig;
+import org.opensearch.index.mapper.MapperService;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Map;
+
+/**
+ * CustomCodecService provides QDEFLATE and QLZ4 compression codecs.
+ */
+public class CustomCodecService extends CodecService {
+    private final Map<String, Codec> codecs;
+
+    /**
+     * Parameterized ctor for CustomCodecService
+     * @param codecServiceConfig Generic codec service config
+     */
+    public CustomCodecService(CodecServiceConfig codecServiceConfig) {
+        super(codecServiceConfig.getMapperService(), codecServiceConfig.getIndexSettings(), codecServiceConfig.getLogger());
+        MapperService mapperService = codecServiceConfig.getMapperService();
+
+        final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
+        String accelerationMode = codecServiceConfig.getIndexSettings().getValue(Lucene99QatCodec.INDEX_CODEC_MODE_SETTING);
+        if (mapperService == null) {
+            codecs.put(Lucene99QatCodec.Mode.QDEFLATE.name(), new QatDeflateCodec(accelerationMode));
+            codecs.put(Lucene99QatCodec.Mode.QLZ4.name(), new QatLz4Codec(accelerationMode));
+        } else {
+            codecs.put(
+                Lucene99QatCodec.Mode.QDEFLATE.name(),
+                new PerFieldMappingPostingFormatCodec(Lucene99QatCodec.Mode.QDEFLATE, accelerationMode, mapperService)
+            );
+            codecs.put(
+                Lucene99QatCodec.Mode.QLZ4.name(),
+                new PerFieldMappingPostingFormatCodec(Lucene99QatCodec.Mode.QLZ4, accelerationMode, mapperService)
+            );
+        }
+        this.codecs = codecs.immutableMap();
+    }
+
+    @Override
+    public Codec codec(String name) {
+        Codec codec = super.codec(name);
+        if (codec == null) {
+            codec = codecs.get(name);
+        }
+        return codec;
+    }
+
+    @Override
+    public String[] availableCodecs() {
+        ArrayList<String> ac = new ArrayList<String>(Arrays.asList(super.availableCodecs()));
+        ac.addAll(codecs.keySet());
+        return ac.toArray(new String[0]);
+    }
+}

sandbox/modules/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/CustomCodecServiceFactory.java

@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.opensearch.index.codec.CodecService;
+import org.opensearch.index.codec.CodecServiceConfig;
+import org.opensearch.index.codec.CodecServiceFactory;
+
+/**
+ * A factory for creating new {@link CodecService} instance
+ */
+public class CustomCodecServiceFactory implements CodecServiceFactory {
+
+    /** Creates a new instance. */
+    public CustomCodecServiceFactory() {}
+
+    @Override
+    public CodecService createCodecService(CodecServiceConfig config) {
+        return new CustomCodecService(config);
+    }
+}

sandbox/modules/custom-codecs/src/main/java/org/opensearch/index/codec/customcodecs/Lucene99QatCodec.java

@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.codec.customcodecs;
+
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Setting.Property;
+
+import com.intel.qat.QatZipper;
+
+import static com.intel.qat.QatZipper.Algorithm;
+import static com.intel.qat.QatZipper.PollingMode;
+
+abstract class Lucene99QatCodec extends FilterCodec {
+
+    public static QatZipper getCompressor(Algorithm algorithm, int level, QatZipper.Mode mode, PollingMode pmode) {
+        return new QatZipper(algorithm, level, mode, pmode);
+    }
+
+    public static QatZipper getCompressor(Algorithm algorithm, QatZipper.Mode mode, PollingMode pmode) {
+        return new QatZipper(algorithm, mode, pmode);
+    }
+
+    public static final int DEFAULT_COMPRESSION_LEVEL = 6;
+
+    /** Each mode represents a compression algorithm. */
+    public enum Mode {
+        QDEFLATE,
+        QLZ4
+    }
+
+    public static final Setting<String> INDEX_CODEC_MODE_SETTING = new Setting<>("index.codec.mode", "hardware", s -> {
+        switch (s) {
+            case "auto":
+            case "hardware":
+                return s;
+            default:
+                throw new IllegalArgumentException("unknown value for [index.codec.mode] must be one of [auto, hardware] but was: " + s);
+        }
+    }, Property.IndexScope, Property.NodeScope);
+
+    private final StoredFieldsFormat storedFieldsFormat;
+
+    /**
+     * new codec for a given compression algorithm and default compression level
+     */
+    public Lucene99QatCodec(Mode mode, String accelerationMode) {
+        this(mode, DEFAULT_COMPRESSION_LEVEL, accelerationMode);
+    }
+
+    public Lucene99QatCodec(Mode mode, int compressionLevel, String accelerationMode) {
+        super(mode.name(), new Lucene99Codec());
+        this.storedFieldsFormat = new Lucene99QatStoredFieldsFormat(mode, compressionLevel, accelerationMode);
+    }
+
+    @Override
+    public StoredFieldsFormat storedFieldsFormat() {
+        return storedFieldsFormat;
+    }
+
+    @Override
+    public String toString() {
+        return getClass().getSimpleName();
+    }
+}