From 8c2eeea2ebc6367a63586e05da303e78223a6d96 Mon Sep 17 00:00:00 2001 From: Shivani Upadhyay Date: Mon, 25 Nov 2024 18:55:39 -0500 Subject: [PATCH 1/2] Fixes type reader --- docs/regressions/regressions-rag24-doc-segmented-test.md | 6 +++--- src/main/resources/regression/rag24-doc-segmented-test.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/regressions/regressions-rag24-doc-segmented-test.md b/docs/regressions/regressions-rag24-doc-segmented-test.md index 378f87232f..b3d929afc8 100644 --- a/docs/regressions/regressions-rag24-doc-segmented-test.md +++ b/docs/regressions/regressions-rag24-doc-segmented-test.md @@ -46,21 +46,21 @@ After indexing has completed, you should be able to perform retrieval as follows bin/run.sh io.anserini.search.SearchCollection \ -index indexes/lucene-inverted.msmarco-v2.1-doc-segmented/ \ -topics tools/topics-and-qrels/topics.rag24.test.txt \ - -topicReader TsvInt \ + -topicReader TsvString \ -output runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt \ -bm25 & bin/run.sh io.anserini.search.SearchCollection \ -index indexes/lucene-inverted.msmarco-v2.1-doc-segmented/ \ -topics tools/topics-and-qrels/topics.rag24.test.txt \ - -topicReader TsvInt \ + -topicReader TsvString \ -output runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt \ -bm25 -rm3 -collection MsMarcoV2DocCollection & bin/run.sh io.anserini.search.SearchCollection \ -index indexes/lucene-inverted.msmarco-v2.1-doc-segmented/ \ -topics tools/topics-and-qrels/topics.rag24.test.txt \ - -topicReader TsvInt \ + -topicReader TsvString \ -output runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt \ -bm25 -rocchio -collection MsMarcoV2DocCollection & ``` diff --git a/src/main/resources/regression/rag24-doc-segmented-test.yaml b/src/main/resources/regression/rag24-doc-segmented-test.yaml index 2b90cfa291..20a470e89d 100644 --- a/src/main/resources/regression/rag24-doc-segmented-test.yaml +++ b/src/main/resources/regression/rag24-doc-segmented-test.yaml @@ -49,7 +49,7 @@ metrics: metric_precision: 4 can_combine: false -topic_reader: TsvInt +topic_reader: TsvString topics: - name: "RAG 24: Test queries" id: rag24.test From b989fa623604aa543773b0918fc35a8a554fefcc Mon Sep 17 00:00:00 2001 From: Shivani Upadhyay Date: Mon, 25 Nov 2024 23:06:22 -0500 Subject: [PATCH 2/2] Updates metrics --- .../regressions-rag24-doc-segmented-test.md | 36 +++++++++---------- .../regression/rag24-doc-segmented-test.yaml | 34 +++++++++--------- 2 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/regressions/regressions-rag24-doc-segmented-test.md b/docs/regressions/regressions-rag24-doc-segmented-test.md index b3d929afc8..b113c1060a 100644 --- a/docs/regressions/regressions-rag24-doc-segmented-test.md +++ b/docs/regressions/regressions-rag24-doc-segmented-test.md @@ -68,20 +68,20 @@ bin/run.sh io.anserini.search.SearchCollection \ Evaluation can be performed using `trec_eval`: ``` -bin/trec_eval -c -M 100 -m map tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt -bin/trec_eval -c -m recall.100 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt -bin/trec_eval -c -m recall.1000 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt -bin/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt - -bin/trec_eval -c -M 100 -m map tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt -bin/trec_eval -c -m recall.100 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt -bin/trec_eval -c -m recall.1000 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt -bin/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt - -bin/trec_eval -c -M 100 -m map tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt -bin/trec_eval -c -m recall.100 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt -bin/trec_eval -c -m recall.1000 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt -bin/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt +bin/trec_eval -c -l 2 -M 100 -m map tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt +bin/trec_eval -c -l 2 -m recall.100 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt +bin/trec_eval -c -l 2 -m recall.1000 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt +bin/trec_eval -c -l 2 -M 100 -m recip_rank -c -l 2 -m ndcg_cut.10 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default.topics.rag24.test.txt + +bin/trec_eval -c -l 2 -M 100 -m map tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt +bin/trec_eval -c -l 2 -m recall.100 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt +bin/trec_eval -c -l 2 -m recall.1000 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt +bin/trec_eval -c -l 2 -M 100 -m recip_rank -c -l 2 -m ndcg_cut.10 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rm3.topics.rag24.test.txt + +bin/trec_eval -c -l 2 -M 100 -m map tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt +bin/trec_eval -c -l 2 -m recall.100 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt +bin/trec_eval -c -l 2 -m recall.1000 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt +bin/trec_eval -c -l 2 -M 100 -m recip_rank -c -l 2 -m ndcg_cut.10 tools/topics-and-qrels/qrels.rag24.test-umbrela-all.txt runs/run.msmarco-v2.1-doc-segmented.bm25-default+rocchio.topics.rag24.test.txt ``` ## Effectiveness @@ -90,12 +90,12 @@ With the above commands, you should be able to reproduce the following results: | **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| |:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| -| RAG 24: Test queries | 0.0861 | 0.0873 | 0.0929 | +| RAG 24: Test queries | 0.0582 | 0.0604 | 0.0642 | | **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| RAG 24: Test queries | 0.7010 | 0.6687 | 0.6791 | +| RAG 24: Test queries | 0.3833 | 0.3601 | 0.3715 | | **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| | RAG 24: Test queries | 0.3290 | 0.3256 | 0.3307 | | **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| RAG 24: Test queries | 0.1395 | 0.1318 | 0.1384 | +| RAG 24: Test queries | 0.1396 | 0.1347 | 0.1402 | | **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| -| RAG 24: Test queries | 0.3467 | 0.3521 | 0.3667 | +| RAG 24: Test queries | 0.3364 | 0.3318 | 0.3469 | diff --git a/src/main/resources/regression/rag24-doc-segmented-test.yaml b/src/main/resources/regression/rag24-doc-segmented-test.yaml index 20a470e89d..b2b60d5e1f 100644 --- a/src/main/resources/regression/rag24-doc-segmented-test.yaml +++ b/src/main/resources/regression/rag24-doc-segmented-test.yaml @@ -15,35 +15,35 @@ index_stats: metrics: - metric: MAP@100 command: bin/trec_eval - params: -c -M 100 -m map + params: -c -l 2 -M 100 -m map separator: "\t" parse_index: 2 metric_precision: 4 can_combine: no - metric: MRR@100 command: bin/trec_eval - params: -c -M 100 -m recip_rank + params: -c -l 2 -M 100 -m recip_rank separator: "\t" parse_index: 2 metric_precision: 4 can_combine: true - metric: nDCG@10 command: bin/trec_eval - params: -c -m ndcg_cut.10 + params: -c -l 2 -m ndcg_cut.10 separator: "\t" parse_index: 2 metric_precision: 4 can_combine: true - metric: R@100 command: bin/trec_eval - params: -c -m recall.100 + params: -c -l 2 -m recall.100 separator: "\t" parse_index: 2 metric_precision: 4 can_combine: false - metric: R@1000 command: bin/trec_eval - params: -c -m recall.1000 + params: -c -l 2 -m recall.1000 separator: "\t" parse_index: 2 metric_precision: 4 @@ -62,40 +62,40 @@ models: params: -bm25 results: MAP@100: - - 0.0861 + - 0.0582 MRR@100: - - 0.7010 + - 0.3833 nDCG@10: - 0.3290 R@100: - - 0.1395 + - 0.1396 R@1000: - - 0.3467 + - 0.3364 - name: bm25-default+rm3 display: +RM3 params: -bm25 -rm3 -collection MsMarcoV2DocCollection results: MAP@100: - - 0.0873 + - 0.0604 MRR@100: - - 0.6687 + - 0.3601 nDCG@10: - 0.3256 R@100: - - 0.1318 + - 0.1347 R@1000: - - 0.3521 + - 0.3318 - name: bm25-default+rocchio display: +Rocchio params: -bm25 -rocchio -collection MsMarcoV2DocCollection results: MAP@100: - - 0.0929 + - 0.0642 MRR@100: - - 0.6791 + - 0.3715 nDCG@10: - 0.3307 R@100: - - 0.1384 + - 0.1402 R@1000: - - 0.3667 + - 0.3469