use truncate flag in huggingface tokenizer

Signed-off-by: HenryL27 <[email protected]>
HenryL27 · Dec 7, 2023 · 9024a4c · 9024a4c
1 parent 2761d7d
commit 9024a4c
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/...thms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java b/...thms/src/main/java/org/opensearch/ml/engine/algorithms/SentenceTransformerTranslator.java
@@ -25,7 +25,12 @@ public Batchifier getBatchifier() {
  @Override
  public void prepare(TranslatorContext ctx) throws IOException {
  Path path = ctx.getModel().getModelPath();
- tokenizer = HuggingFaceTokenizer.builder().optPadding(true).optTokenizerPath(path.resolve("tokenizer.json")).build();
+ tokenizer = HuggingFaceTokenizer
+ .builder()
+ .optPadding(true)
+ .optTruncation(true)
+ .optTokenizerPath(path.resolve("tokenizer.json"))
+ .build();
  }
 
  @Override