make path to small model training modules and config.json dynamic

src/fabricator/dataset_generator.py

@@ -303,7 +303,7 @@ def _inner_generate_loop(
  if train_small_model_every_x_generations is not None and train_small_model_every_x_generations > 0:
  if prompt_call_idx % train_small_model_every_x_generations == 0:
  logger.info("Commencing small model training.")
- small_model = import_module(small_model_training,"src.small_model_training")
+ small_model = import_module("src.small_model_training." + small_model_training, __package__)
  inf_subset = small_model.get_influential_subset(generated_dataset)
  fewshot_dataset = inf_subset
  logger.info("Continuing generation.")

src/small_model_training/text_classification.py

@@ -1,6 +1,7 @@
 import math
 import torch.nn
 import numpy as np
+import os
 from dataclasses import dataclass, field
 from transformers import AutoTokenizer
 from transformers import DataCollatorWithPadding
@@ -31,14 +32,27 @@ def get_influential_subset(dataset):
 
  # trace labels
  for entry in dataset:
- if entry['label'] not in label2id:
- label2id[entry['label']] = counter
- id2label[counter] = entry['label']
+ if entry["label"] not in label2id:
+ label2id[entry["label"]] = counter
+ id2label[counter] = entry["label"]
  counter += 1
 
+ #get path to config
+ def get_project_root() -> str:
+ current_dir = os.path.abspath(os.path.dirname(__file__))
+ while not os.path.isfile(os.path.join(current_dir, "README.md")):
+ current_dir = os.path.dirname(current_dir)
+ return current_dir
+
+ def get_path_to_json_file(relative_path_to_json_file) -> str:
+ project_root = get_project_root()
+ return os.path.join(project_root, relative_path_to_json_file)
+
+ path_to_json_file = get_path_to_json_file("src/small_model_training/config.json")
+
  # get parameters from config
  parser = HfArgumentParser((ModelArguments, TrainingArguments, FewshotArguments))
- model_args, training_args, fewshot_args = parser.parse_json_file('config.json')
+ model_args, training_args, fewshot_args = parser.parse_json_file(path_to_json_file)
 
  # setup preprocessing
  tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path)
@@ -100,7 +114,7 @@ def compute_metrics(eval_pred):
 
  # Iterate through the dataset and filter out duplicates
  for example in inf_subset:
- text = example['text']
+ text = example["text"]
  if text not in unique_samples:
  unique_samples.add(text)
  deduplicated_inf_subset.append(example)