[hotfix] convert_HF eos_tokens slicing

eole-nlp · Sep 20, 2024 · 4b44fdc · 4b44fdc
1 parent 4c3adf1
commit 4b44fdc
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/eole/bin/convert/convert_HF.py b/eole/bin/convert/convert_HF.py
@@ -944,7 +944,7 @@ def get_weight(checkpoint, tensor_name):
  if "added_tokens_decoder" in data.keys():
  eos_tokens = [
  data["added_tokens_decoder"][str(index)]["content"]
- for index in eos_token_id[1:]
+ for index in eos_token_id
  ]
  optional_eos = eos_tokens[1:]
  eos_token = eos_tokens[0]