Adds test case

castorini · Nov 21, 2024 · 5b0a073 · 5b0a073
1 parent 07264a6
commit 5b0a073
Showing 1 changed file with 22 additions and 0 deletions.
diff --git a/src/test/java/io/anserini/eval/RelevanceJudgmentsTest.java b/src/test/java/io/anserini/eval/RelevanceJudgmentsTest.java
@@ -386,6 +386,28 @@ public void testTREC24_RAG_RAGGY_DEV() throws IOException{
     assertEquals(1, qrels.getRelevanceGrade("253263", "msmarco_v2.1_doc_46_843492186"));
   }
 
+  @Test
+  public void testTREC24_RAG_UMBRELA() throws IOException{
+    // % cut -f 1 -d ' ' tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt | uniq | wc
+    //      301     301    3448
+    // % wc tools/topics-and-qrels/qrels.rag24.raggy-dev.txt
+    //   108479  433916 6475451 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt
+
+    RelevanceJudgments qrels = new RelevanceJudgments("tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt");
+    assertNotNull(qrels);
+    assertEquals(301, qrels.getQids().size());
+    assertEquals(108479, getQrelsCount(qrels));
+    assertEquals(1, qrels.getRelevanceGrade("2024-145979", "msmarco_v2.1_doc_25_771726319#13_1477564195"));
+    assertEquals(1, qrels.getRelevanceGrade("2024-216592", "msmarco_v2.1_doc_52_1092442741#3_2165187686"));
+
+    qrels = RelevanceJudgments.fromQrels(Qrels.TREC2024_RAG_UMBRELA);
+    assertNotNull(qrels);
+    assertEquals(301, qrels.getQids().size());
+    assertEquals(108479, getQrelsCount(qrels));
+    assertEquals(1, qrels.getRelevanceGrade("2024-145979", "msmarco_v2.1_doc_25_771726319#13_1477564195"));
+    assertEquals(1, qrels.getRelevanceGrade("2024-216592", "msmarco_v2.1_doc_52_1092442741#3_2165187686"));
+  }
+
   @Test
   public void testMsmarcoDocDev() throws IOException{
     RelevanceJudgments qrels = new RelevanceJudgments("tools/topics-and-qrels/qrels.msmarco-doc.dev.txt");