From ec5619ec36c413ae2e0d6fd683e14f670f6c4034 Mon Sep 17 00:00:00 2001 From: xinyual Date: Wed, 27 Sep 2023 09:27:12 +0800 Subject: [PATCH] fix license problem Signed-off-by: xinyual --- .../algorithms/SentenceTransformerTranslator.java | 14 +++++--------- .../ml/engine/algorithms/TextEmbeddingModel.java | 8 +++++--- ...SentenceTransformerTextEmbeddingTranslator.java | 9 --------- .../text_embedding/TextEmbeddingDenseModel.java | 5 ----- .../TextEmbeddingSparseEncodingModelTest.java | 5 +++++ .../tokenize/SparseTokenizerModelTest.java | 13 +++++++++++-- 6 files changed, 26 insertions(+), 28 deletions(-) diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java index b02a8c9092..302dc641c2 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java @@ -1,26 +1,22 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.ml.engine.algorithms; import ai.djl.huggingface.tokenizers.Encoding; import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer; import ai.djl.modality.Input; -import ai.djl.modality.Output; import ai.djl.ndarray.NDArray; import ai.djl.ndarray.NDList; import ai.djl.ndarray.NDManager; -import ai.djl.ndarray.types.DataType; import ai.djl.translate.Batchifier; import ai.djl.translate.ServingTranslator; import ai.djl.translate.TranslatorContext; -import org.opensearch.ml.common.output.model.MLResultDataType; -import org.opensearch.ml.common.output.model.ModelTensor; -import org.opensearch.ml.common.output.model.ModelTensors; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; import java.util.Map; public abstract class SentenceTransformerTranslator implements ServingTranslator { diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/TextEmbeddingModel.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/TextEmbeddingModel.java index 5164b1e951..b531c9e5a9 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/TextEmbeddingModel.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/TextEmbeddingModel.java @@ -1,11 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.ml.engine.algorithms; import ai.djl.inference.Predictor; import ai.djl.modality.Input; import ai.djl.modality.Output; import ai.djl.translate.TranslateException; -import ai.djl.translate.Translator; -import ai.djl.translate.TranslatorFactory; import org.opensearch.ml.common.dataset.MLInputDataset; import org.opensearch.ml.common.dataset.TextDocsInputDataSet; import org.opensearch.ml.common.input.MLInput; @@ -14,7 +17,6 @@ import org.opensearch.ml.common.output.model.ModelResultFilter; import org.opensearch.ml.common.output.model.ModelTensorOutput; import org.opensearch.ml.common.output.model.ModelTensors; -import org.opensearch.ml.engine.algorithms.DLModel; import java.util.ArrayList; import java.util.HashMap; diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/SentenceTransformerTextEmbeddingTranslator.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/SentenceTransformerTextEmbeddingTranslator.java index 898c619fce..caa932ae6c 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/SentenceTransformerTextEmbeddingTranslator.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/SentenceTransformerTextEmbeddingTranslator.java @@ -5,29 +5,20 @@ package org.opensearch.ml.engine.algorithms.text_embedding; -import ai.djl.huggingface.tokenizers.Encoding; -import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer; -import ai.djl.modality.Input; import ai.djl.modality.Output; import ai.djl.ndarray.NDArray; import ai.djl.ndarray.NDList; -import ai.djl.ndarray.NDManager; import ai.djl.ndarray.types.DataType; -import ai.djl.translate.Batchifier; -import ai.djl.translate.ServingTranslator; import ai.djl.translate.TranslatorContext; import org.opensearch.ml.common.output.model.MLResultDataType; import org.opensearch.ml.common.output.model.ModelTensor; import org.opensearch.ml.common.output.model.ModelTensors; import org.opensearch.ml.engine.algorithms.SentenceTransformerTranslator; -import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map; public class SentenceTransformerTextEmbeddingTranslator extends SentenceTransformerTranslator { @Override diff --git a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/TextEmbeddingDenseModel.java b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/TextEmbeddingDenseModel.java index 145ae7c1ec..cec8c79642 100644 --- a/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/TextEmbeddingDenseModel.java +++ b/ml-algorithms/src/main/java/org/opensearch/ml/engine/algorithms/text_embedding/TextEmbeddingDenseModel.java @@ -5,10 +5,8 @@ package org.opensearch.ml.engine.algorithms.text_embedding; -import ai.djl.inference.Predictor; import ai.djl.modality.Input; import ai.djl.modality.Output; -import ai.djl.translate.TranslateException; import ai.djl.translate.Translator; import ai.djl.translate.TranslatorFactory; import lombok.extern.log4j.Log4j2; @@ -18,9 +16,6 @@ import org.opensearch.ml.engine.algorithms.TextEmbeddingModel; import org.opensearch.ml.engine.annotation.Function; -import java.util.HashMap; -import java.util.Map; - import static org.opensearch.ml.common.model.TextEmbeddingModelConfig.FrameworkType.SENTENCE_TRANSFORMERS; import static org.opensearch.ml.engine.ModelHelper.ONNX_ENGINE; import static org.opensearch.ml.engine.ModelHelper.PYTORCH_ENGINE; diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/sparse_encoding/TextEmbeddingSparseEncodingModelTest.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/sparse_encoding/TextEmbeddingSparseEncodingModelTest.java index 92d72fd7d4..ad92822b15 100644 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/sparse_encoding/TextEmbeddingSparseEncodingModelTest.java +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/sparse_encoding/TextEmbeddingSparseEncodingModelTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.ml.engine.algorithms.sparse_encoding; import ai.djl.Model; diff --git a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/tokenize/SparseTokenizerModelTest.java b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/tokenize/SparseTokenizerModelTest.java index 5134900f23..e8ca48418f 100644 --- a/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/tokenize/SparseTokenizerModelTest.java +++ b/ml-algorithms/src/test/java/org/opensearch/ml/engine/algorithms/tokenize/SparseTokenizerModelTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.ml.engine.algorithms.tokenize; import ai.djl.MalformedModelException; @@ -35,7 +40,12 @@ import java.net.URISyntaxException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -43,7 +53,6 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import static org.opensearch.ml.engine.algorithms.DLModel.*; -import static org.opensearch.ml.engine.algorithms.DLModel.ML_ENGINE; public class SparseTokenizerModelTest { @Rule