From fb83968a556829a2bace8c751b9773aa2e56f9f6 Mon Sep 17 00:00:00 2001 From: Merlin Kallenborn Date: Thu, 16 May 2024 16:22:18 +0200 Subject: [PATCH] WIP: docs: Add elo how to TASK: IL-502 --- .../hot_to_implement_elo_evaluations.ipynb | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/documentation/how_tos/hot_to_implement_elo_evaluations.ipynb diff --git a/src/documentation/how_tos/hot_to_implement_elo_evaluations.ipynb b/src/documentation/how_tos/hot_to_implement_elo_evaluations.ipynb new file mode 100644 index 000000000..56c0b7ccc --- /dev/null +++ b/src/documentation/how_tos/hot_to_implement_elo_evaluations.ipynb @@ -0,0 +1,23 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# How to implement elo evaluation runs\n", + "0. Run your tasks on the datasets you want to evaluate (see [here](./how_to_run_a_task_on_a_dataset.ipynb))\n", + " - When evaluating multiple runs, all of them need the same data types \n", + "2. Initialize all necessary repositories for the `IncrementalEvaluator`, and an `IncrementalEvaluationLogic`.\n", + "3. Run the evaluator to evaluate all examples and create a single `EvaluationOverview`\n", + "4. (Optional) Save the evaluation id for later use" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}