diff --git a/src/main/java/io/anserini/eval/Qrels.java b/src/main/java/io/anserini/eval/Qrels.java
index 31adf04a4..fdf0393fd 100644
--- a/src/main/java/io/anserini/eval/Qrels.java
+++ b/src/main/java/io/anserini/eval/Qrels.java
@@ -56,6 +56,7 @@ public enum Qrels {
   TREC2022_DL_DOC_MSMARCO_V21("qrels.dl22-doc-msmarco-v2.1.txt"),
   TREC2023_DL_DOC_MSMARCO_V21("qrels.dl23-doc-msmarco-v2.1.txt"),
   TREC2024_RAG_RAGGY_DEV("qrels.rag24.raggy-dev.txt"),
+  TREC2024_RAG_UMBRELA("qrels.rag24.test-umbrela-all.txt"),
   MSMARCO_DOC_DEV("qrels.msmarco-doc.dev.txt"),
   MSMARCO_PASSAGE_DEV_SUBSET("qrels.msmarco-passage.dev-subset.txt"),
   MSMARCO_V2_DOC_DEV("qrels.msmarco-v2-doc.dev.txt"),
diff --git a/src/test/java/io/anserini/eval/RelevanceJudgmentsTest.java b/src/test/java/io/anserini/eval/RelevanceJudgmentsTest.java
index 25dddcffd..d807ff5a9 100644
--- a/src/test/java/io/anserini/eval/RelevanceJudgmentsTest.java
+++ b/src/test/java/io/anserini/eval/RelevanceJudgmentsTest.java
@@ -38,7 +38,7 @@ public int getQrelsCount(RelevanceJudgments qrels) throws IOException{
 
   @Test
   public void testTotalCount() {
-    assertEquals(170, Qrels.values().length);
+    assertEquals(171, Qrels.values().length);
   }
 
   @Test(expected = IOException.class)
@@ -386,6 +386,28 @@ public void testTREC24_RAG_RAGGY_DEV() throws IOException{
     assertEquals(1, qrels.getRelevanceGrade("253263", "msmarco_v2.1_doc_46_843492186"));
   }
 
+  @Test
+  public void testTREC24_RAG_UMBRELA() throws IOException{
+    // % cut -f 1 -d ' ' tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt | uniq | wc
+    //      301     301    3448
+    // % wc tools/topics-and-qrels/qrels.rag24.raggy-dev.txt
+    //   108479  433916 6475451 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt
+
+    RelevanceJudgments qrels = new RelevanceJudgments("tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt");
+    assertNotNull(qrels);
+    assertEquals(301, qrels.getQids().size());
+    assertEquals(108479, getQrelsCount(qrels));
+    assertEquals(1, qrels.getRelevanceGrade("2024-145979", "msmarco_v2.1_doc_25_771726319#13_1477564195"));
+    assertEquals(1, qrels.getRelevanceGrade("2024-216592", "msmarco_v2.1_doc_52_1092442741#3_2165187686"));
+
+    qrels = RelevanceJudgments.fromQrels(Qrels.TREC2024_RAG_UMBRELA);
+    assertNotNull(qrels);
+    assertEquals(301, qrels.getQids().size());
+    assertEquals(108479, getQrelsCount(qrels));
+    assertEquals(1, qrels.getRelevanceGrade("2024-145979", "msmarco_v2.1_doc_25_771726319#13_1477564195"));
+    assertEquals(1, qrels.getRelevanceGrade("2024-216592", "msmarco_v2.1_doc_52_1092442741#3_2165187686"));
+  }
+
   @Test
   public void testMsmarcoDocDev() throws IOException{
     RelevanceJudgments qrels = new RelevanceJudgments("tools/topics-and-qrels/qrels.msmarco-doc.dev.txt");
diff --git a/tools b/tools
index a8563573f..1c463184d 160000
--- a/tools
+++ b/tools
@@ -1 +1 @@
-Subproject commit a8563573fc8a288718a9f87662b4a5a16944c9ce
+Subproject commit 1c463184d53d3735c3f0bcee2c3e9509be83973d