From 6cad5397f7c54939330bc6c8f5c35cb9595275d6 Mon Sep 17 00:00:00 2001
From: Johannes Wesch <83402561+JohannesWesch@users.noreply.github.com>
Date: Tue, 15 Oct 2024 10:40:49 +0200
Subject: [PATCH] fix: documentation for run_is_already_computed (#1084)

---
 .../parameter_optimization.ipynb              | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/documentation/parameter_optimization.ipynb b/src/documentation/parameter_optimization.ipynb
index dc550d81a..a542dec7c 100644
--- a/src/documentation/parameter_optimization.ipynb
+++ b/src/documentation/parameter_optimization.ipynb
@@ -318,10 +318,8 @@
     "# Add a new model to the run configuration\n",
     "model_list.append(\"model d\")\n",
     "\n",
-    "# Loop over all combinations of parameters and run the `Task` for each combination.\n",
-    "# We set `recompute_if_metadata_changed` to `True` to only run the new model `model d` with the existing prompts.\n",
-    "# The previous runs are not recomputed and keep their old results, unless the metadata of the run has changed.\n",
-    "# Note, that currently the evaluations are recomputed for all runs.\n",
+    "# We use `run_is_already_computed` to only run the new model `model d` with the existing prompts.\n",
+    "# The previous runs/evaluations/aggregations are not recomputed and keep their old results, unless the metadata of the run has changed.\n",
     "for i, (model, prompt) in enumerate(itertools.product(model_list, prompt_list)):\n",
     "    dummy_task = DummyTask(model=model, prompt=prompt)\n",
     "\n",
@@ -348,12 +346,15 @@
     "            eval_overview.id, metadata=metadata, description=description, labels=labels\n",
     "        )\n",
     "\n",
-    "# Let's print the number of run overviews. Because we have not recomputed the 12 (3 models times 4 examples) runs we now expect 16 runs in total.\n",
-    "# Without the `recompute_if_metadata_changed` flag we would get 28 runs!\n",
-    "print(len(list(run_repository.run_overviews())))\n",
-    "print(len(list(evaluation_repository.evaluation_overviews())))\n",
+    "# Let's print the number of run/evaluation/aggregation overviews.\n",
+    "# Because we have not recomputed the 12 (3 models times 4 examples) runs/evaluations/aggregations we now expect 16 for each in total.\n",
+    "# Without the `run_is_already_computed` check we would get 28 runs/evaluations/aggregations!\n",
+    "print(f\"Number of Runs: {len(list(run_repository.run_overviews()))}\")\n",
+    "print(\n",
+    "    f\"Number of Evaluations: {len(list(evaluation_repository.evaluation_overviews()))}\"\n",
+    ")\n",
     "print(\n",
-    "    len(list(aggregation_repository.aggregation_overviews(DummyAggregatedEvaluation)))\n",
+    "    f\"Number of Aggregations: {len(list(aggregation_repository.aggregation_overviews(DummyAggregatedEvaluation)))}\"\n",
     ")"
    ]
   }