update data config explicitly after config update from checkpoint (#111)

eole/config/data.py

@@ -1,6 +1,6 @@
 import os
 from typing import Dict, List, Literal
-from pydantic import Field, field_validator, model_validator
+from pydantic import Field, field_validator # , model_validator
 from pydantic import create_model
 
 from eole import constants
@@ -320,7 +320,10 @@ def _validate_data(self):
  # self.data = corpora
  self.__dict__["data"] = corpora # skip validation to avoid recursion error
 
- @model_validator(mode="after")
+ # @model_validator(mode="after")
+ # This needs to be called only after the config is fully up to date,
+ # to prevent setting and keeping some unwanted values
+ # (e.g. transforms [] for datasets)
  def _validate_data_config(self, build_vocab_only=False):
  if self.data is not None: # patch to allow None data
  self._validate_data()

eole/tests/test_data_prepare.py

@@ -27,6 +27,7 @@ def get_default_opts():
  from eole.config.data import DataConfig
 
  opt = DataConfig(**default_opts)
+ opt._validate_data_config()
  # Inject some dummy training options that may needed when build fields
  return opt
 

eole/train_single.py

@@ -91,6 +91,8 @@ def _init_train(config):
  checkpoint = None
 
  config = update_config_with_checkpoint(config, checkpoint=checkpoint)
+ # explicitly validate data_config after checkpoint update
+ config._validate_data_config()
  transforms_cls = get_transforms_cls(config._all_transform)
  vocabs, transforms = prepare_transforms_vocabs(config, transforms_cls)
  if config.training.train_from and not config.training.update_vocab: