IL-238 jupyter pretty printing for lineages

src/examples/test.ipynb

@@ -0,0 +1,159 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from intelligence_layer.evaluation.dataset.domain import Example\n",
+    "from intelligence_layer.evaluation.run.domain import ExampleOutput\n",
+    "from intelligence_layer.evaluation.infrastructure.repository_navigator import (\n",
+    "    RunLineage,\n",
+    "    EvaluationLineage,\n",
+    "    RepositoryNavigator,\n",
+    ")\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lorem = \"\"\"Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. \n",
+    "\n",
+    "Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. \n",
+    "\n",
+    "Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. \n",
+    "\n",
+    "Nam liber tempor cum soluta nobis eleifend option congue nihil imperdiet doming id quod mazim placerat facer possim assum. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. \n",
+    "\n",
+    "Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis. \n",
+    "\n",
+    "At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, At accusam aliquyam diam diam dolore dolores duo eirmod eos erat, et nonumy sed tempor et et invidunt justo labore Stet clita ea et gubergren, kasd magna no rebum. sanctus sea sed takimata ut vero voluptua. est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat. \n",
+    "\n",
+    "Consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus. \n",
+    "\n",
+    "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. \n",
+    "\n",
+    "Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. \n",
+    "\n",
+    "Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. \"\"\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pydantic import BaseModel\n",
+    "from intelligence_layer.evaluation.evaluation.domain import ExampleEvaluation\n",
+    "\n",
+    "\n",
+    "class DummyEval(BaseModel):\n",
+    "    eval: str\n",
+    "\n",
+    "\n",
+    "outputs = [\n",
+    "    ExampleOutput(run_id=\"run-id\", example_id=\"example-id\", output=\"output\"),\n",
+    "    ExampleOutput(run_id=\"run-id\", example_id=\"example-id\", output=\"output\"),\n",
+    "]\n",
+    "lineage = EvaluationLineage(\n",
+    "    example=Example(\n",
+    "        input=lorem, expected_output=\"expected_output0\", data=\"data0\", id=\"example-id\"\n",
+    "    ),\n",
+    "    outputs=[\n",
+    "        ExampleOutput(run_id=\"run-id\", example_id=\"example-id\", output=\"output\"),\n",
+    "        ExampleOutput(run_id=\"run-id\", example_id=\"example-id\", output=\"output\"),\n",
+    "    ],\n",
+    "    evaluation=ExampleEvaluation(\n",
+    "        evaluation_id=\"eval-id\",\n",
+    "        example_id=\"example-id\",\n",
+    "        result=DummyEval(eval=\"Result\"),\n",
+    "    ),\n",
+    ")\n",
+    "lineage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from intelligence_layer.evaluation.infrastructure.repository_navigator import (\n",
+    "    evaluation_lineages_to_pandas,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "df = evaluation_lineages_to_pandas([lineage, lineage, lineage])\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.loc[(\"example-id\", \"eval-id\")].iloc[0].lineage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "lineage = RunLineage(\n",
+    "    example=Example(\n",
+    "        input=\"input0\",\n",
+    "        expected_output=\"expected_output0\",\n",
+    "        data=\"data0\",\n",
+    "        id=\"example-id\",\n",
+    "    ),\n",
+    "    output=ExampleOutput(run_id=\"run-id\", example_id=\"example-id\", output=\"output\"),\n",
+    ")\n",
+    "lineage"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from intelligence_layer.evaluation.infrastructure.repository_navigator import (\n",
+    "    run_lineages_to_pandas,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "run_lineages_to_pandas([lineage, lineage, lineage])"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "intelligence-layer-d3iSWYpm-py3.10",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/intelligence_layer/evaluation/dataset/domain.py

@@ -2,6 +2,7 @@
 from uuid import uuid4
 
 from pydantic import BaseModel, Field
+from rich.tree import Tree
 
 from intelligence_layer.core.task import Input
 from intelligence_layer.core.tracer.tracer import PydanticSerializable
@@ -40,6 +41,12 @@ def __str__(self) -> str:
             f'Expected output = "{self.expected_output}"\n'
         )
 
+    def _rich_render(self):
+        example_tree = Tree(f"Example: {self.id}")
+        example_tree.add(f"Input").add(str(self.input))
+        example_tree.add(f"Expected Output").add(str(self.expected_output))
+        return example_tree
+
 
 class Dataset(BaseModel):
     """Represents a dataset linked to multiple examples

src/intelligence_layer/evaluation/evaluation/domain.py

@@ -2,6 +2,7 @@
 from typing import Generic, Optional, TypeVar
 
 from pydantic import BaseModel, SerializeAsAny
+from rich.tree import Tree
 
 from intelligence_layer.evaluation.run.domain import RunOverview
 
@@ -55,6 +56,13 @@ def __str__(self) -> str:
             f"Result = {self.result}\n"
         )
 
+    def _rich_render(self, skip_example_id: bool = False):
+        tree = Tree(f"Evaluation: {self.evaluation_id}")
+        if not skip_example_id:
+            tree.add(f"Example ID: {self.example_id}")
+        tree.add(str(self.result))
+        return tree
+
 
 class EvaluationOverview(BaseModel, frozen=True):
     """Overview of the un-aggregated results of evaluating a :class:`Task` on a dataset.

src/intelligence_layer/evaluation/infrastructure/repository_navigator.py

@@ -1,7 +1,10 @@
 import itertools
 from typing import Generic, Iterable, Sequence
 
+import pandas as pd
+import rich
 from pydantic import BaseModel
+from rich.tree import Tree
 
 from intelligence_layer.core.task import Input, Output
 from intelligence_layer.evaluation.dataset.dataset_repository import DatasetRepository
@@ -21,12 +24,62 @@ class RunLineage(BaseModel, Generic[Input, ExpectedOutput, Output]):
     example: Example[Input, ExpectedOutput]
     output: ExampleOutput[Output]
 
+    def _ipython_display_(self):
+        tree = Tree("Run Lineage")
+        tree.add(self.example._rich_render())
+        tree.add(self.output._rich_render(skip_example_id=True))
+        rich.print(tree)
+
+
+def run_lineages_to_pandas(
+    evaluation_lineages: Sequence[RunLineage[Input, ExpectedOutput, Output]]
+):
+    df = pd.DataFrame(
+        [
+            vars(lineage.example) | vars(lineage.output) | {"lineage": lineage}
+            for lineage in evaluation_lineages
+        ]
+    )
+    df = df.drop(columns="id")
+    df = df.set_index(["example_id", "run_id"])
+    return df
+
 
 class EvaluationLineage(BaseModel, Generic[Input, ExpectedOutput, Output, Evaluation]):
     example: Example[Input, ExpectedOutput]
     outputs: Sequence[ExampleOutput[Output]]
     evaluation: ExampleEvaluation[Evaluation]
 
+    def _ipython_display_(self):
+        tree = Tree("Run Lineage")
+        tree.add(self.example._rich_render())
+        output_tree = Tree("Outputs")
+        for output in self.outputs:
+            output_tree.add(output._rich_render(skip_example_id=True))
+        tree.add(output_tree)
+        tree.add(self.evaluation._rich_render(skip_example_id=True))
+        rich.print(tree)
+
+
+def evaluation_lineages_to_pandas(
+    evaluation_lineages: Sequence[
+        EvaluationLineage[Input, ExpectedOutput, Output, Evaluation]
+    ]
+):
+    df = pd.DataFrame(
+        [
+            vars(lineage.example)
+            | vars(output)
+            | vars(lineage.evaluation)
+            | {"lineage": lineage}
+            for lineage in evaluation_lineages
+            for output in lineage.outputs
+        ]
+    )
+    df = df.drop(columns="id")
+    df = df.set_index(["example_id", "evaluation_id", "run_id"])
+    return df
+
 
 class RepositoryNavigator:
     """The `RepositoryNavigator` is used to retrieve coupled data from multiple repositories."""

src/intelligence_layer/evaluation/run/domain.py

@@ -2,6 +2,7 @@
 from typing import Generic
 
 from pydantic import BaseModel
+from rich.tree import Tree
 
 from intelligence_layer.core.task import Output
 
@@ -49,6 +50,13 @@ def __str__(self) -> str:
             f'Output="{self.output}"\n'
         )
 
+    def _rich_render(self, skip_example_id: bool = False):
+        tree = Tree(f"Output: {self.run_id}")
+        if not skip_example_id:
+            tree.add(f"Example ID: {self.example_id}")
+        tree.add(str(self.output))
+        return tree
+
 
 class SuccessfulExampleOutput(BaseModel, Generic[Output]):
     """Successful output of a single evaluated :class:`Example`

tests/evaluation/test_repository_navigator.py

@@ -261,3 +261,23 @@ def test_get_eval_lineage_for_non_existent_eval_id_returns_none(
         repository_navigator.evaluation_lineage(
             "non-existent-id", "irrelevant", str, str, str, DummyEval
         )
+
+
+def test_run_lineage_tree_view() -> None:
+    # TODO
+    ...
+
+
+def test_evaluation_lineage_tree_view() -> None:
+    # TODO
+    ...
+
+
+def test_run_lineages_to_pandas() -> None:
+    # TODO
+    ...
+
+
+def test_evaluation_lineages_to_pandas() -> None:
+    # TODO
+    ...