prepare experiments

src/small_model_training/project_experiments/cola/cola_baseline_downstream.py

@@ -0,0 +1,90 @@
+import numpy as np
+import datetime
+import os
+from transformers import AutoTokenizer, TrainingArguments
+from transformers import DataCollatorWithPadding
+from transformers import AutoModelForSequenceClassification, Trainer
+from datasets import load_dataset
+import evaluate
+
+generated_dataset = load_dataset("stolzenp/500-cola-sentences-baseline", split="train")
+test_split = load_dataset("glue", "cola", split="test")
+
+model_name = "bert-base-uncased"
+
+label2id = {"correct": 1, "wrong": 0, "acceptable": 1, "unacceptable": 0}
+id2label = {1: "acceptable", 0: "unacceptable"}
+
+# setup preprocessing
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+
+def preprocess_text(batch):
+ preprocessed_tokens = tokenizer(batch["text"], truncation=True, padding=True)
+ return preprocessed_tokens
+
+
+def preprocess_text_and_labels(batch):
+ preprocessed_tokens = tokenizer(batch["text"], truncation=True, padding=True)
+ preprocessed_tokens["label"] = [label2id[label] for label in batch["label"]]
+ return preprocessed_tokens
+
+
+# setup compute_metrics
+accuracy = evaluate.load("accuracy")
+
+
+def compute_metrics(eval_pred):
+ predictions, labels = eval_pred
+ predictions = np.argmax(predictions, axis=1)
+ return accuracy.compute(predictions=predictions, references=labels)
+
+
+generated_dataset = generated_dataset.train_test_split(test_size=0.1)
+train_split = generated_dataset["train"]
+val_split = generated_dataset["test"]
+
+tokenized_train = train_split.map(preprocess_text_and_labels, batched=True)
+tokenized_val = val_split.map(preprocess_text_and_labels, batched=True)
+tokenized_test = test_split.map(preprocess_text, batched=True)
+
+data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+
+model = AutoModelForSequenceClassification.from_pretrained(
+ model_name, num_labels=2, id2label=id2label, label2id=label2id
+)
+
+# initialize training
+trainer = Trainer(
+ model=model,
+ args=TrainingArguments(
+ output_dir="baseline_downstream_model",
+ learning_rate=2e-5,
+ per_device_train_batch_size=16,
+ per_device_eval_batch_size=16,
+ num_train_epochs=2,
+ weight_decay=0.01,
+ evaluation_strategy="epoch",
+ save_strategy="epoch",
+ load_best_model_at_end=True,
+ ),
+ train_dataset=tokenized_train,
+ eval_dataset=tokenized_val,
+ tokenizer=tokenizer,
+ data_collator=data_collator,
+ compute_metrics=compute_metrics,
+)
+
+trainer.train()
+outputs = trainer.predict(tokenized_test)
+test_accuracy = outputs[2]["test_accuracy"]
+
+results_file = "results.txt"
+experiments_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+results_path = f"{experiments_directory}/{results_file}"
+
+results_timestamp = datetime.datetime.now()
+
+with open(results_path, "a") as file:
+ file.write(f"{results_timestamp} - cola_baseline\n")
+ file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/cola/cola_baseline_generation.py

@@ -0,0 +1,28 @@
+from haystack.nodes import PromptNode
+from fabricator import DatasetGenerator
+from fabricator.prompts import BasePrompt
+
+label_options = ["correct", "wrong"]
+
+prompt = BasePrompt(
+ task_description="Generate a grammatically {} sentence similar to this:",
+ label_options=label_options,
+ target_formatting_template="",
+)
+
+prompt_node = PromptNode(
+ model_name_or_path="mistralai/Mistral-7B-Instruct-v0.1",
+ max_length=50,
+ model_kwargs={
+ "model_kwargs": {"do_sample": True, "temperature": 0.5, "top_p": 0.9}
+ },
+)
+
+generator = DatasetGenerator(prompt_node)
+generated_dataset = generator.generate(
+ prompt_template=prompt,
+ max_prompt_calls=500,
+ num_samples_to_generate=500,
+)
+
+generated_dataset.push_to_hub("500-cola-sentences-baseline")

src/small_model_training/project_experiments/cola/cola_gold_downstream.py

@@ -1,3 +1,81 @@
+import numpy as np
+import datetime
+import os
+from transformers import AutoTokenizer, TrainingArguments
+from transformers import DataCollatorWithPadding
+from transformers import AutoModelForSequenceClassification, Trainer
 from datasets import load_dataset
+import evaluate
 
-gold_dataset = load_dataset("glue", "cola", split="train")
+train_split = load_dataset("glue", "cola", split="train")
+val_split = load_dataset("glue", "cola", split="validation")
+test_split = load_dataset("glue", "cola", split="test")
+
+model_name = "bert-base-uncased"
+
+label2id = {"correct": 1, "wrong": 0, "acceptable": 1, "unacceptable": 0}
+id2label = {1: "acceptable", 0: "unacceptable"}
+
+# setup preprocessing
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+
+def preprocess_text(batch):
+ preprocessed_tokens = tokenizer(batch["text"], truncation=True, padding=True)
+ return preprocessed_tokens
+
+
+# setup compute_metrics
+accuracy = evaluate.load("accuracy")
+
+
+def compute_metrics(eval_pred):
+ predictions, labels = eval_pred
+ predictions = np.argmax(predictions, axis=1)
+ return accuracy.compute(predictions=predictions, references=labels)
+
+
+tokenized_train = train_split.map(preprocess_text, batched=True)
+tokenized_val = val_split.map(preprocess_text, batched=True)
+tokenized_test = test_split.map(preprocess_text, batched=True)
+
+data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+
+model = AutoModelForSequenceClassification.from_pretrained(
+ model_name, num_labels=2, id2label=id2label, label2id=label2id
+)
+
+# initialize training
+trainer = Trainer(
+ model=model,
+ args=TrainingArguments(
+ output_dir="gold_downstream_model",
+ learning_rate=2e-5,
+ per_device_train_batch_size=16,
+ per_device_eval_batch_size=16,
+ num_train_epochs=2,
+ weight_decay=0.01,
+ evaluation_strategy="epoch",
+ save_strategy="epoch",
+ load_best_model_at_end=True,
+ ),
+ train_dataset=tokenized_train,
+ eval_dataset=tokenized_val,
+ tokenizer=tokenizer,
+ data_collator=data_collator,
+ compute_metrics=compute_metrics,
+)
+
+trainer.train()
+outputs = trainer.predict(tokenized_test)
+test_accuracy = outputs[2]["test_accuracy"]
+
+results_file = "results.txt"
+experiments_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+results_path = f"{experiments_directory}/{results_file}"
+
+results_timestamp = datetime.datetime.now()
+
+with open(results_path, "a") as file:
+ file.write(f"{results_timestamp} - cola_gold\n")
+ file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/cola/cola_smt_downstream.py

@@ -0,0 +1,90 @@
+import numpy as np
+import datetime
+import os
+from transformers import AutoTokenizer, TrainingArguments
+from transformers import DataCollatorWithPadding
+from transformers import AutoModelForSequenceClassification, Trainer
+from datasets import load_dataset
+import evaluate
+
+generated_dataset = load_dataset("stolzenp/500-cola-sentences-smt", split="train")
+test_split = load_dataset("glue", "cola", split="test")
+
+model_name = "bert-base-uncased"
+
+label2id = {"correct": 1, "wrong": 0, "acceptable": 1, "unacceptable": 0}
+id2label = {1: "acceptable", 0: "unacceptable"}
+
+# setup preprocessing
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+
+def preprocess_text(batch):
+ preprocessed_tokens = tokenizer(batch["text"], truncation=True, padding=True)
+ return preprocessed_tokens
+
+
+def preprocess_text_and_labels(batch):
+ preprocessed_tokens = tokenizer(batch["text"], truncation=True, padding=True)
+ preprocessed_tokens["label"] = [label2id[label] for label in batch["label"]]
+ return preprocessed_tokens
+
+
+# setup compute_metrics
+accuracy = evaluate.load("accuracy")
+
+
+def compute_metrics(eval_pred):
+ predictions, labels = eval_pred
+ predictions = np.argmax(predictions, axis=1)
+ return accuracy.compute(predictions=predictions, references=labels)
+
+
+generated_dataset = generated_dataset.train_test_split(test_size=0.1)
+train_split = generated_dataset["train"]
+val_split = generated_dataset["test"]
+
+tokenized_train = train_split.map(preprocess_text_and_labels, batched=True)
+tokenized_val = val_split.map(preprocess_text_and_labels, batched=True)
+tokenized_test = test_split.map(preprocess_text, batched=True)
+
+data_collator = DataCollatorWithPadding(tokenizer=tokenizer)
+
+model = AutoModelForSequenceClassification.from_pretrained(
+ model_name, num_labels=2, id2label=id2label, label2id=label2id
+)
+
+# initialize training
+trainer = Trainer(
+ model=model,
+ args=TrainingArguments(
+ output_dir="smt_downstream_model",
+ learning_rate=2e-5,
+ per_device_train_batch_size=16,
+ per_device_eval_batch_size=16,
+ num_train_epochs=2,
+ weight_decay=0.01,
+ evaluation_strategy="epoch",
+ save_strategy="epoch",
+ load_best_model_at_end=True,
+ ),
+ train_dataset=tokenized_train,
+ eval_dataset=tokenized_val,
+ tokenizer=tokenizer,
+ data_collator=data_collator,
+ compute_metrics=compute_metrics,
+)
+
+trainer.train()
+outputs = trainer.predict(tokenized_test)
+test_accuracy = outputs[2]["test_accuracy"]
+
+results_file = "results.txt"
+experiments_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+results_path = f"{experiments_directory}/{results_file}"
+
+results_timestamp = datetime.datetime.now()
+
+with open(results_path, "a") as file:
+ file.write(f"{results_timestamp} - cola_smt\n")
+ file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/cola/cola_smt_generation.py

@@ -0,0 +1,30 @@
+from haystack.nodes import PromptNode
+from fabricator import DatasetGenerator
+from fabricator.prompts import BasePrompt
+
+label_options = ["correct", "wrong"]
+
+prompt = BasePrompt(
+ task_description="Generate a grammatically {} sentence similar to this:",
+ label_options=label_options,
+ target_formatting_template="",
+)
+
+prompt_node = PromptNode(
+ model_name_or_path="mistralai/Mistral-7B-Instruct-v0.1",
+ max_length=50,
+ model_kwargs={
+ "model_kwargs": {"do_sample": True, "temperature": 0.5, "top_p": 0.9}
+ },
+)
+
+generator = DatasetGenerator(prompt_node)
+generated_dataset = generator.generate(
+ prompt_template=prompt,
+ max_prompt_calls=500,
+ num_samples_to_generate=500,
+ small_model_training="text_classification",
+ train_small_model_every_X_generations=50,
+)
+
+generated_dataset.push_to_hub("500-cola-sentences-smt")

src/small_model_training/project_experiments/imdb/imdb_baseline_downstream.py

@@ -88,5 +88,5 @@ def compute_metrics(eval_pred):
 results_timestamp = datetime.datetime.now()
 
 with open(results_path, "a") as file:
- file.write(f"{results_timestamp} - imdb_baseline_downstream\n")
+ file.write(f"{results_timestamp} - imdb_baseline\n")
  file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/imdb/imdb_gold_downstream.py

@@ -79,5 +79,5 @@ def compute_metrics(eval_pred):
 results_timestamp = datetime.datetime.now()
 
 with open(results_path, "a") as file:
- file.write(f"{results_timestamp} - imdb_gold_downstream\n")
+ file.write(f"{results_timestamp} - imdb_gold\n")
  file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/imdb/imdb_smt_downstream.py

@@ -86,5 +86,5 @@ def compute_metrics(eval_pred):
 results_timestamp = datetime.datetime.now()
 
 with open(results_path, "a") as file:
- file.write(f"{results_timestamp} - imdb_smt_downstream\n")
+ file.write(f"{results_timestamp} - imdb_smt\n")
  file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/iterative_dataset_synthesis_movies.py

@@ -2,19 +2,19 @@
 from fabricator import DatasetGenerator
 from fabricator.prompts import BasePrompt
 
-label_options = ["positive", "negative"]
+label_options = ["correct", "wrong"]
 
 prompt = BasePrompt(
- task_description="Generate an excerpt from a {} movie review similar to these:",
+ task_description="Generate a grammatically {} sentence similar to this:",
  label_options=label_options,
  target_formatting_template="",
 )
 
 prompt_node = PromptNode(
  model_name_or_path="mistralai/Mistral-7B-Instruct-v0.1",
- max_length=100,
+ max_length=50,
  model_kwargs={
- "model_kwargs": {"do_sample": True, "temperature": 0.4, "top_p": 0.9}
+ "model_kwargs": {"do_sample": True, "temperature": 0.5, "top_p": 0.9}
  },
 )
 

src/small_model_training/project_experiments/sst2/sst2_baseline_downstream.py

@@ -88,5 +88,5 @@ def compute_metrics(eval_pred):
 results_timestamp = datetime.datetime.now()
 
 with open(results_path, "a") as file:
- file.write(f"{results_timestamp} - sst2_baseline_downstream\n")
+ file.write(f"{results_timestamp} - sst2_baseline\n")
  file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/sst2/sst2_gold_downstream.py

@@ -77,5 +77,5 @@ def compute_metrics(eval_pred):
 results_timestamp = datetime.datetime.now()
 
 with open(results_path, "a") as file:
- file.write(f"{results_timestamp} - sst2_gold_downstream\n")
+ file.write(f"{results_timestamp} - sst2_gold\n")
  file.write(f"accuracy: {test_accuracy}\n")

src/small_model_training/project_experiments/sst2/sst2_smt_downstream.py

@@ -86,5 +86,5 @@ def compute_metrics(eval_pred):
 results_timestamp = datetime.datetime.now()
 
 with open(results_path, "a") as file:
- file.write(f"{results_timestamp} - sst2_smt_downstream\n")
+ file.write(f"{results_timestamp} - sst2_smt\n")
  file.write(f"accuracy: {test_accuracy}\n")