Merge pull request #73 from CambioML/comparison-dashboard

Comparison dashboard

pykoi/chat/db/comparator_database.py

@@ -4,6 +4,9 @@
 
 from typing import List, Tuple
 
+import pandas as pd
+
+
 from pykoi.chat.db.abs_database import AbsDatabase
 
 
@@ -64,9 +67,7 @@ def update(self, **kwargs) -> None:
         """
         Updates the database.
         """
-        raise NotImplementedError(
-            "ComparatorQuestionDatabase does not support update."
-        )
+        raise NotImplementedError("ComparatorQuestionDatabase does not support update.")
 
     def retrieve_all(self) -> List[Tuple]:
         """
@@ -175,9 +176,7 @@ def update(self, **kwargs) -> None:
         """
         with self._lock:
             cursor = self.get_cursor()
-            cursor.execute(
-                query, (kwargs["rank"], kwargs["qid"], kwargs["model"])
-            )
+            cursor.execute(query, (kwargs["rank"], kwargs["qid"], kwargs["model"]))
             self.get_connection().commit()
         if self._debug:
             rows = self.retrieve_all()
@@ -217,3 +216,35 @@ def print_table(self, rows: List[Tuple]) -> None:
                 f"Answer: {row[4]}, "
                 f"Timestamp: {row[5]}"
             )
+
+    def retrieve_all_question_answers_as_pandas(self) -> pd.DataFrame:
+        """
+        Retrieves all data by joining the comparator and comparator_question tables as a pandas dataframe.
+
+        Returns:
+            DataFrame: A pandas dataframe.
+        """
+        join_query = """
+        SELECT 
+            comparator.id, 
+            comparator.model, 
+            comparator.qid, 
+            comparator_question.question, 
+            comparator.rank, 
+            comparator.answer, 
+            comparator.timestamp
+        FROM comparator
+        INNER JOIN comparator_question 
+        ON comparator.qid = comparator_question.id;
+        """
+
+        with self._lock:
+            cursor = self.get_cursor()
+            cursor.execute(join_query)
+            rows = cursor.fetchall()
+
+        df = pd.DataFrame(
+            rows,
+            columns=["ID", "Model", "QID", "Question", "Rank", "Answer", "Timestamp"],
+        )
+        return df

pykoi/component/base.py

@@ -4,6 +4,7 @@
 
 from pykoi.component.chatbot_database_factory import ChatbotDatabaseFactory
 from pykoi.component.constants import FeedbackType
+from pykoi.chat.db.comparator_database import ComparatorDatabase
 from pykoi.chat.db.qa_database import QuestionAnswerDatabase
 from pykoi.chat.db.rag_database import RAGDatabase
 from pykoi.chat.db.ranking_database import RankingDatabase
@@ -42,9 +43,7 @@ class Component:
         props (Dict[str, Any]): Additional properties for the component.
     """
 
-    def __init__(
-        self, fetch_func: Optional[Callable], svelte_component: str, **kwargs
-    ):
+    def __init__(self, fetch_func: Optional[Callable], svelte_component: str, **kwargs):
         """
         Initialize a new instance of Component.
 
@@ -54,9 +53,7 @@ def __init__(
             kwargs: Additional properties for the component.
         """
         self.id = str(uuid.uuid4())  # Generate a unique ID
-        self.data_source = (
-            DataSource(self.id, fetch_func) if fetch_func else None
-        )
+        self.data_source = DataSource(self.id, fetch_func) if fetch_func else None
         self.svelte_component = svelte_component
         self.props = kwargs
 
@@ -81,6 +78,7 @@ def __init__(self, fetch_func: Callable, value_column: List[str], **kwargs):
         super().__init__(fetch_func, "Dropdown", **kwargs)
         self.value_column = value_column
 
+
 class RAG(Component):
     """
     RAG class represents a RAG component.
@@ -138,5 +136,8 @@ def __init__(self, database: any, **kwargs):
             database (QuestionAnswerDatabase): The database to use for the dashboard.
             kwargs: Additional properties for the dashboard.
         """
-        super().__init__(None, "Feedback", **kwargs)
+        if isinstance(database, ComparatorDatabase):
+            super().__init__(None, "CompareDashboard", **kwargs)
+        else:
+            super().__init__(None, "Feedback", **kwargs)
         self.database = database

pykoi/frontend/dist/index.html

@@ -5,8 +5,8 @@
     <link rel="icon" type="image/svg+xml" href="/vite.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>Vite + Svelte</title>
-    <script type="module" crossorigin src="/assets/index-e352a05f.js"></script>
-    <link rel="stylesheet" href="/assets/index-6e8a727f.css">
+    <script type="module" crossorigin src="/assets/index-42540648.js"></script>
+    <link rel="stylesheet" href="/assets/index-c2d4a9bc.css">
   </head>
   <body>
     <div id="app"></div>

pykoi/frontend/src/App.svelte

@@ -8,7 +8,7 @@
   import QuestionRating from "./lib/Annotations/QuestionRating.svelte";
   import RankedChatbot from "./lib/Chatbots/RankedChatbot.svelte";
   import RAG from "./lib/RAG/RAG.svelte";
-    import FeedbackWrapper from "./lib/Dashboards/FeedbackWrapper.svelte";
+  import FeedbackWrapper from "./lib/Dashboards/FeedbackWrapper.svelte";
 
   const components = writable([]);
   const selectedPage = writable(null);
@@ -18,6 +18,7 @@
     Dropdown: Dropdown,
     Feedback: FeedbackWrapper,
     Compare: ComparisonChat,
+    CompareDashboard: ComparisonChart,
     RetrievalQA: RAG,
   };
 

pykoi/frontend/src/lib/Comparison/BumpChart.svelte

@@ -3,14 +3,7 @@
   import { line } from "d3-shape";
   import { comparisonData } from "./store";
 
-  $: firstData = $comparisonData
-    .filter((d) => d.qid === 3)
-    .map((d) => ({ model: d.model, rank: d.rank }));
-
-  $: console.log("firstData", $comparisonData);
-
   $: models = Array.from(new Set($comparisonData.map((d) => d.model)));
-  $: console.log("models", models);
 
   let outerHeight;
   let outerWidth;
@@ -46,18 +39,11 @@
   $: pathLine = line()
     .x((d) => xScale(d.qid))
     .y((d) => yScale(d.rank));
-  // .curve(curveBasis)
 
   $: modelData = models.map((model) =>
     $comparisonData.filter((d) => d.model === model)
   );
 
-  $: console.log("md", modelData);
-  $: console.log(
-    "ranks",
-    $comparisonData.map((d) => d.rank)
-  );
-
   $: xTickArray =
     xScale.domain().length > 10
       ? xScale.domain().filter((_, index) => index % 2 === 0)
@@ -122,21 +108,6 @@
         </g>
       {/if}
     {/each}
-    <!-- y-ticks -->
-    {#each yScale.domain() as tick}
-      <g transform={`translate(${margin.left} ${yScale(tick) + 0})`}>
-        <!-- <text
-          class="axis-text"
-          x="-5"
-          y="0"
-          text-anchor="end"
-          dominant-baseline="middle"
-          >{firstData
-            .filter((d) => d.rank == tick)
-            .map((d) => d.model)[0]}</text
-        > -->
-      </g>
-    {/each}
 
     <!-- axis labels -->
     <text

pykoi/frontend/src/lib/Comparison/ComparisonChart.svelte

@@ -3,16 +3,11 @@
   import { scaleOrdinal } from "d3-scale";
   import { onMount } from "svelte";
   import HorizontalBar from "./HorizontalBar.svelte";
-  import { writable } from "svelte/store";
   import Table from "./Table.svelte";
-  import { data } from "./data";
   import { comparisonData } from "./store";
 
   import Heatmap from "./Heatmap.svelte";
 
-  let options = {
-    /* Your options here */
-  };
   $: models = Array.from(new Set($comparisonData.map((d) => d.model)));
   $: colorScale = scaleOrdinal()
     .domain($comparisonData.map((d) => d.model))
@@ -33,7 +28,7 @@
   }
 
   onMount(() => {
-    // retrieveDBData();
+    retrieveDBData();
   });
 
   function highLight(i) {
@@ -61,21 +56,12 @@
 </script>
 
 <div class="main-container">
-  <div class="instructions">
-    <h5 class="underline bold">Comparisons</h5>
-    <p>This panes represents the training performance of your model.</p>
-    <button>Download Data</button>
-  </div>
-
   {#if $comparisonData.length > 0}
     <div class="eval-container">
       <div class="left-charts">
         <div class="chart-captions">
           <h4>Model Comparisons</h4>
-          <!-- <p>
-            View the performance of your model over time. GPU stats are
-            available to the right.
-          </p> -->
+
           {#each models as model, i}
             <button
               data-model={model}
@@ -114,8 +100,6 @@
   {/if}
 </div>
 
-<!-- <Linechart /> -->
-
 <style>
   .holder {
     height: 100vh;
@@ -124,30 +108,12 @@
     justify-content: center;
     align-items: center;
   }
-  .buttons {
-    justify-content: center;
-  }
 
-  .rating-button {
-    text-transform: uppercase;
-    margin: 0;
-    /* transition: all 0.1s; */
-  }
   button {
     margin: 0;
   }
 
-  .rating-button:hover {
-    color: var(--white);
-    background: var(--black);
-  }
-
-  /* Remove the space between buttons */
-  .rating-button + .rating-button {
-    margin: 0;
-  }
   .chart-captions {
-    /* border: 1px solid black; */
     margin: auto;
     width: 100%;
     text-align: left;
@@ -157,45 +123,9 @@
     padding: 0;
     margin: 0;
   }
-  .chart-captions p {
-    font-size: var(--smallText);
-  }
-  .instructions {
-    text-align: left;
-    /* padding: 5%; */
-    padding-left: 0;
-  }
-
-  .instructions h4 {
-    text-align: left;
-  }
-
-  .instructions p {
-    font-size: var(--smallText);
-    text-align: left;
-  }
-
-  .instructions button {
-    font-size: var(--smallText);
-  }
-
-  .underline {
-    border-bottom: var(--line);
-  }
-
-  .bold {
-    font-weight: bold;
-    font-size: var(--smallText);
-    margin: 0;
-    padding: 0;
-  }
 
-  .instructions {
-    border-right: 1px solid #eee;
-  }
   .main-container {
-    display: grid;
-    grid-template-columns: 20% 80%;
+    margin: auto;
   }
   .eval-container {
     display: grid;
@@ -216,24 +146,9 @@
   }
 
   .right-charts {
-    /* display: grid;
-    grid-template-columns: 100%;
-    grid-template-rows: 33.3% 33.3% 33.3%;
-    height: 100vh; */
-    /* border: 2px solid black; */
     display: grid;
     grid-template-columns: 100%;
     grid-template-rows: 10% 33% 33%;
     gap: 1%;
   }
-
-  .right-chart-1 {
-    /* border: 1px solid black; */
-  }
-  .right-chart-2 {
-    /* border: 1px solid black; */
-  }
-  .right-chart-3 {
-    /* border: 1px solid black; */
-  }
 </style>

pykoi/frontend/src/lib/Comparison/store.js

@@ -20,34 +20,4 @@ export const comparisonData = writable([
   },
   { model: "gpt4", qid: 2, rank: 4, answer: "GPT-4's second unique answer." },
   { model: "claude", qid: 2, rank: 3, answer: "'s second unique answer." },
-
-  { model: "llama", qid: 3, rank: 1, answer: "Llama's third unique answer." },
-  {
-    model: "gpt3.5",
-    qid: 3,
-    rank: 4,
-    answer: "GPT-3.5's third unique answer.",
-  },
-  { model: "gpt4", qid: 3, rank: 4, answer: "GPT-4's third unique answer." },
-  { model: "claude", qid: 3, rank: 2, answer: "'s third unique answer." },
-
-  { model: "llama", qid: 4, rank: 1, answer: "Llama's fourth unique answer." },
-  { model: "gpt3.5", qid: 4, rank: 3, answer: "GPT-3.5's  unique answer." },
-  { model: "gpt4", qid: 4, rank: 2, answer: "GPT-4's  unique answer." },
-  { model: "claude", qid: 4, rank: 4, answer: "'s  unique answer." },
-
-  { model: "llama", qid: 5, rank: 1, answer: "Llama's  unique answer." },
-  { model: "gpt3.5", qid: 5, rank: 2, answer: "GPT-3.5's nique answer." },
-  { model: "gpt4", qid: 5, rank: 3, answer: "GPT-4's fifth unique answer." },
-  { model: "claude", qid: 5, rank: 4, answer: "'s fifthnswer." },
-
-  { model: "llama", qid: 6, rank: 1, answer: "Llama's sixth unique answer." },
-  {
-    model: "gpt3.5",
-    qid: 6,
-    rank: 2,
-    answer: "GPT-3.5's sixth unique answer.",
-  },
-  { model: "gpt4", qid: 6, rank: 3, answer: "GPT-4's sixth unique answer." },
-  { model: "claude", qid: 6, rank: 4, answer: "'s sixth uniq." },
 ]);