get_transforms_cls after update_config_with_checkpoint

eole/models/model_saver.py

@@ -42,13 +42,14 @@ def load_checkpoint(model_path):
  if not os.path.exists(model_path):
  raise FileNotFoundError(f"{model_path} does not seem to exist.")
  elif os.path.isdir(model_path):
+ os.environ["MODEL_PATH"] = model_path
  logger.info("Loading checkpoint from %s" % model_path)
  # checkpoint = torch.load(ckpt_path, map_location=torch.device("cpu"))
  checkpoint = {}
  config_path = os.path.join(model_path, "config.json")
  if os.path.exists(config_path):
  with open(config_path) as f:
- config_dict = json.load(f)
+ config_dict = json.loads(os.path.expandvars(f.read()))
  # drop data to prevent validation issues
  config_dict["data"] = {}
  # drop inference to prevent validation issues

eole/train_single.py

@@ -61,8 +61,6 @@ def _init_train(config):
  - resume training but transforms have changed
  - resume training but vocab file has been modified
  """
- transforms_cls = get_transforms_cls(config._all_transform)
-
  if config.training.train_from:
  checkpoint = load_checkpoint(config.training.train_from)
 
@@ -93,6 +91,7 @@ def _init_train(config):
  checkpoint = None
 
  config = update_config_with_checkpoint(config, checkpoint=checkpoint)
+ transforms_cls = get_transforms_cls(config._all_transform)
  vocabs, transforms = prepare_transforms_vocabs(config, transforms_cls)
  if config.training.train_from and not config.training.update_vocab:
  logger.info("Keeping checkpoint vocabulary")

recipes/llama2/llama-finetune.yaml

@@ -11,14 +11,14 @@ overwrite: true
 report_every: 10
 
 # transforms config
-transforms: [sentencepiece, filtertoolong]
-transforms_configs:
- sentencepiece:
- src_subword_model: "${EOLE_MODEL_DIR}/llama2-7b-chat-hf/tokenizer.model"
- tgt_subword_model: "${EOLE_MODEL_DIR}/llama2-7b-chat-hf/tokenizer.model"
- filtertoolong:
- src_seq_length: 896
- tgt_seq_length: 896
+# transforms: [sentencepiece, filtertoolong]
+# transforms_configs:
+#  sentencepiece:
+#  src_subword_model: "${EOLE_MODEL_DIR}/llama2-7b-chat-hf/tokenizer.model"
+#  tgt_subword_model: "${EOLE_MODEL_DIR}/llama2-7b-chat-hf/tokenizer.model"
+#  filtertoolong:
+#  src_seq_length: 896
+#  tgt_seq_length: 896
 
 # datasets
 data: