diff --git a/examples/models/llama/evaluate/eager_eval.py b/examples/models/llama/evaluate/eager_eval.py
index e931c60c18..d7f4dae78f 100644
--- a/examples/models/llama/evaluate/eager_eval.py
+++ b/examples/models/llama/evaluate/eager_eval.py
@@ -31,7 +31,7 @@ def __init__(
         use_kv_cache: bool = False,
     ):
         device = "cuda" if torch.cuda.is_available() else "cpu"
-        super().__init__(device=device)
+        super().__init__(device=device, pretrained="gpt2")
         self._model = model
         self._tokenizer = tokenizer
         self._device = torch.device(device)
diff --git a/examples/models/llama/install_requirements.sh b/examples/models/llama/install_requirements.sh
index 470e804c2d..3103daeb7d 100755
--- a/examples/models/llama/install_requirements.sh
+++ b/examples/models/llama/install_requirements.sh
@@ -15,7 +15,7 @@ pip install --no-use-pep517 "git+https://github.com/pytorch/ao.git@${TORCHAO_VER
 
 # Install lm-eval for Model Evaluation with lm-evalution-harness
 # Install tiktoken for tokenizer
-pip install lm_eval==0.4.2
+pip install lm_eval==0.4.5
 pip install tiktoken blobfile
 
 # Call the install helper for further setup