From 5996a8e9599897d417b4025bd1fb7296ad53598f Mon Sep 17 00:00:00 2001 From: lintool Date: Fri, 13 Sep 2024 17:16:04 -0400 Subject: [PATCH] more tolerance tweaks, removed dead code. --- src/main/python/run_regression.py | 231 +----------------- ...ssage.bge-base-en-v1.5.hnsw-int8.onnx.yaml | 2 +- ...e-embed-english-v3.0.hnsw-int8.cached.yaml | 8 +- ...cohere-embed-english-v3.0.hnsw.cached.yaml | 8 +- ...passage.cos-dpr-distil.hnsw-int8.onnx.yaml | 8 +- ...19-passage.cos-dpr-distil.hnsw.cached.yaml | 6 +- ...dl19-passage.cos-dpr-distil.hnsw.onnx.yaml | 6 +- ...-passage.openai-ada2.hnsw-int8.cached.yaml | 6 +- .../dl19-passage.openai-ada2.hnsw.cached.yaml | 2 +- ...age.bge-base-en-v1.5.hnsw-int8.cached.yaml | 4 +- ...ssage.bge-base-en-v1.5.hnsw-int8.onnx.yaml | 2 +- ...cohere-embed-english-v3.0.hnsw.cached.yaml | 6 +- ...20-passage.cos-dpr-distil.hnsw.cached.yaml | 6 +- ...dl20-passage.cos-dpr-distil.hnsw.onnx.yaml | 6 +- ...-passage.openai-ada2.hnsw-int8.cached.yaml | 2 +- ...-passage.bge-base-en-v1.5.hnsw.cached.yaml | 4 +- ...v1-passage.bge-base-en-v1.5.hnsw.onnx.yaml | 4 +- ...e-embed-english-v3.0.hnsw-int8.cached.yaml | 4 +- ...v1-passage.cos-dpr-distil.hnsw.cached.yaml | 2 +- ...-passage.openai-ada2.hnsw-int8.cached.yaml | 2 +- 20 files changed, 46 insertions(+), 273 deletions(-) diff --git a/src/main/python/run_regression.py b/src/main/python/run_regression.py index 4ec7ddae2..abb7b7acd 100644 --- a/src/main/python/run_regression.py +++ b/src/main/python/run_regression.py @@ -26,7 +26,6 @@ import tarfile import time import yaml -from collections import defaultdict from multiprocessing import Pool from subprocess import call, Popen, PIPE from tqdm import tqdm @@ -144,10 +143,10 @@ def construct_indexing_command(yaml_data, args): def construct_runfile_path(index, id, model_name): - # If the index is 'indexes/lucene-inverted.msmarco-passage-ca/', we pull out 'msmarco-passage-ca'. + # If the index is 'indexes/lucene-inverted.msmarco-passage-ca/', we pull out 'inverted.msmarco-passage-ca'. # 'indexes/lucene-hnsw-int8.msmarco-v1-passage.cos-dpr-distil/' -> 'hnsw-int8.msmarco-v1-passage.cos-dpr-distil' # 'indexes/lucene-hnsw.msmarco-v1-passage.cos-dpr-distil/' -> 'hnsw.msmarco-v1-passage.cos-dpr-distil/' - # Be careful, for 'indexes/lucene-inverted.mrtydi-v1.1-arabic/', we want to pull out 'inverted-mrtydi-v1.1-arabic'. + # Be careful, for 'indexes/lucene-inverted.mrtydi-v1.1-arabic/', we want to pull out 'inverted.mrtydi-v1.1-arabic'. index_part = index.split('/')[1].split('-', 1)[1] return os.path.join('runs/', 'run.{0}.{1}.{2}'.format(index_part, id, model_name)) @@ -183,184 +182,6 @@ def construct_convert_commands(yaml_data): return converting_commands -# beir_flat_int8_onnx = defaultdict(lambda: 0.004) -# beir_flat_int8_onnx['ArguAna'] = 0.03 -# beir_flat_int8_onnx['NFCorpus'] = 0.007 -# beir_flat_int8_onnx['Signal-1M'] = 0.006 -# beir_flat_int8_onnx['TREC-NEWS'] = 0.01 -# beir_flat_int8_onnx['Webis-Touche2020'] = 0.007 -# -# beir_flat_int8_cached = defaultdict(lambda: 0.004) -# beir_flat_int8_cached['BioASQ'] = 0.005 -# beir_flat_int8_cached['NFCorpus'] = 0.006 -# beir_flat_int8_cached['Signal-1M'] = 0.007 -# beir_flat_int8_cached['TREC-NEWS'] = 0.01 -# beir_flat_int8_cached['Webis-Touche2020'] = 0.007 -# -# beir_flat_onnx = defaultdict(lambda: 0.001) -# beir_flat_onnx['ArguAna'] = 0.02 -# beir_flat_onnx['CQADupStack-wordpress'] = 0.002 -# beir_flat_onnx['Quora'] = 0.002 -# beir_flat_onnx['Robust04'] = 0.004 -# -# beir_flat_cached = defaultdict(lambda: 1e-9) -# -# beir_flat_tolerance = { -# 'flat-int8-onnx': beir_flat_int8_onnx, -# 'flat-int8-cached': beir_flat_int8_cached, -# 'flat-onnx': beir_flat_onnx, -# 'flat-cached': beir_flat_cached, -# } - -# beir_hnsw_int8_onnx = defaultdict(lambda: 0.005) -# beir_hnsw_int8_onnx['ArguAna'] = 0.03 -# beir_hnsw_int8_onnx['BioASQ'] = 0.02 -# beir_hnsw_int8_onnx['DBPedia'] = 0.007 -# beir_hnsw_int8_onnx['FiQA-2018'] = 0.007 -# beir_hnsw_int8_onnx['HotpotQA'] = 0.008 -# beir_hnsw_int8_onnx['NFCorpus'] = 0.006 -# beir_hnsw_int8_onnx['Robust04'] = 0.006 -# beir_hnsw_int8_onnx['Signal-1M'] = 0.04 -# beir_hnsw_int8_onnx['TREC-NEWS'] = 0.02 -# beir_hnsw_int8_onnx['Webis-Touche2020'] = 0.01 -# -# beir_hnsw_int8_cached = defaultdict(lambda: 0.005) -# beir_hnsw_int8_cached['BioASQ'] = 0.02 -# beir_hnsw_int8_cached['FiQA-2018'] = 0.007 -# beir_hnsw_int8_cached['HotpotQA'] = 0.007 -# beir_hnsw_int8_cached['Signal-1M'] = 0.04 -# beir_hnsw_int8_cached['TREC-NEWS'] = 0.02 -# beir_hnsw_int8_cached['Webis-Touche2020'] = 0.006 -# -# beir_hnsw_onnx = defaultdict(lambda: 0.003) -# beir_hnsw_onnx['ArguAna'] = 0.02 -# beir_hnsw_onnx['BioASQ'] = 0.01 -# beir_hnsw_onnx['CQADupStack-wordpress'] = 0.004 -# beir_hnsw_onnx['DBPedia'] = 0.006 -# beir_hnsw_onnx['FEVER'] = 0.007 -# beir_hnsw_onnx['FiQA-2018'] = 0.007 -# beir_hnsw_onnx['HotpotQA'] = 0.007 -# beir_hnsw_onnx['Robust04'] = 0.004 -# beir_hnsw_onnx['Signal-1M'] = 0.05 -# beir_hnsw_onnx['TREC-NEWS'] = 0.02 -# -# beir_hnsw_cached = defaultdict(lambda: 0.003) -# beir_hnsw_cached['BioASQ'] = 0.01 -# beir_hnsw_cached['DBPedia'] = 0.006 -# beir_hnsw_cached['FEVER'] = 0.008 -# beir_hnsw_cached['FiQA-2018'] = 0.008 -# beir_hnsw_cached['HotpotQA'] = 0.007 -# beir_hnsw_cached['Signal-1M'] = 0.05 -# beir_hnsw_cached['TREC-NEWS'] = 0.025 -# -# beir_hnsw_tolerance = { -# 'hnsw-int8-onnx': beir_hnsw_int8_onnx, -# 'hnsw-int8-cached': beir_hnsw_int8_cached, -# 'hnsw-onnx': beir_hnsw_onnx, -# 'hnsw-cached': beir_hnsw_cached, -# } - -#flat_model_type_pattern = re.compile(r'(flat-int8-onnx|flat-int8-cached|flat-onnx|flat-cached)$') -# hnsw_model_type_pattern = re.compile(r'(hnsw-int8-onnx|hnsw-int8-cached|hnsw-onnx|hnsw-cached)$') -# -# beir_dataset_pattern = re.compile(r'BEIR \(v1.0.0\): (.*)$') - -# msmarco_v1_flat_int8_onnx = defaultdict(lambda: 0.002) -# msmarco_v1_flat_int8_cached = defaultdict(lambda: 0.002) -# msmarco_v1_flat_int8_cached['openai-ada2-flat-int8-cached'] = 0.008 -# msmarco_v1_flat_onnx = defaultdict(lambda: 0.0001) -# msmarco_v1_flat_cached = defaultdict(lambda: 1e-9) -# -# msmarco_v1_flat_tolerance = { -# 'flat-int8-onnx': msmarco_v1_flat_int8_onnx, -# 'flat-int8-cached': msmarco_v1_flat_int8_cached, -# 'flat-onnx': msmarco_v1_flat_onnx, -# 'flat-cached': msmarco_v1_flat_cached, -# } -# -# dl19_flat_int8_onnx = defaultdict(lambda: 0.002) -# dl19_flat_int8_onnx['bge-flat-int8-onnx'] = 0.008 -# dl19_flat_int8_cached = defaultdict(lambda: 0.002) -# dl19_flat_int8_cached['bge-flat-int8-cached'] = 0.005 -# dl19_flat_int8_cached['openai-ada2-flat-int8-cached'] = 0.008 -# dl19_flat_onnx = defaultdict(lambda: 0.0001) -# dl19_flat_onnx['bge-flat-onnx'] = 0.008 -# dl19_flat_cached = defaultdict(lambda: 1e-9) -# -# dl19_flat_tolerance = { -# 'flat-int8-onnx': dl19_flat_int8_onnx, -# 'flat-int8-cached': dl19_flat_int8_cached, -# 'flat-onnx': dl19_flat_onnx, -# 'flat-cached': dl19_flat_cached, -# } -# -# dl20_flat_int8_onnx = defaultdict(lambda: 0.002) -# dl20_flat_int8_onnx['bge-flat-int8-onnx'] = 0.004 -# dl20_flat_int8_onnx['cos-dpr-distil-flat-int8-onnx'] = 0.004 -# dl20_flat_int8_cached = defaultdict(lambda: 0.002) -# dl20_flat_int8_cached['bge-flat-int8-cached'] = 0.005 -# dl20_flat_int8_cached['cos-dpr-distil-flat-int8-cached'] = 0.004 -# dl20_flat_int8_cached['cohere-embed-english-v3.0-flat-int8-cached'] = 0.004 -# dl20_flat_int8_cached['openai-ada2-flat-int8-cached'] = 0.003 -# dl20_flat_onnx = defaultdict(lambda: 0.0001) -# dl20_flat_onnx['bge-flat-onnx'] = 0.005 -# dl20_flat_cached = defaultdict(lambda: 1e-9) -# -# dl20_flat_tolerance = { -# 'flat-int8-onnx': dl20_flat_int8_onnx, -# 'flat-int8-cached': dl20_flat_int8_cached, -# 'flat-onnx': dl20_flat_onnx, -# 'flat-cached': dl20_flat_cached, -# } - -# msmarco_v1_hnsw_int8_onnx = defaultdict(lambda: 0.01) -# msmarco_v1_hnsw_int8_cached = defaultdict(lambda: 0.01) -# msmarco_v1_hnsw_onnx = defaultdict(lambda: 0.01) -# msmarco_v1_hnsw_onnx['cos-dpr-distil-hnsw-onnx'] = 0.015 -# msmarco_v1_hnsw_cached = defaultdict(lambda: 0.01) -# msmarco_v1_hnsw_cached['cos-dpr-distil-hnsw-cached'] = 0.015 -# -# msmarco_v1_hnsw_tolerance = { -# 'hnsw-int8-onnx': msmarco_v1_hnsw_int8_onnx, -# 'hnsw-int8-cached': msmarco_v1_hnsw_int8_cached, -# 'hnsw-onnx': msmarco_v1_hnsw_onnx, -# 'hnsw-cached': msmarco_v1_hnsw_cached, -# } -# -# dl19_hnsw_int8_onnx = defaultdict(lambda: 0.01) -# dl19_hnsw_int8_onnx['bge-hnsw-int8-onnx'] = 0.025 -# dl19_hnsw_int8_onnx['cos-dpr-distil-hnsw-int8-onnx'] = 0.025 -# dl19_hnsw_int8_cached = defaultdict(lambda: 0.01) -# dl19_hnsw_int8_cached['bge-hnsw-int8-cached'] = 0.02 -# dl19_hnsw_int8_cached['cohere-embed-english-v3.0-hnsw-int8-cached'] = 0.02 -# dl19_hnsw_int8_cached['cos-dpr-distil-hnsw-int8-cached'] = 0.025 -# dl19_hnsw_int8_cached['openai-ada2-hnsw-int8-cached'] = 0.015 -# dl19_hnsw_onnx = defaultdict(lambda: 0.015) -# dl19_hnsw_onnx['bge-hnsw-onnx'] = 0.02 -# dl19_hnsw_cached = defaultdict(lambda: 0.015) -# dl19_hnsw_cached['cohere-embed-english-v3.0-hnsw-cached'] = 0.02 -# -# dl19_hnsw_tolerance = { -# 'hnsw-int8-onnx': dl19_hnsw_int8_onnx, -# 'hnsw-int8-cached': dl19_hnsw_int8_cached, -# 'hnsw-onnx': dl19_hnsw_onnx, -# 'hnsw-cached': dl19_hnsw_cached, -# } -# -# dl20_hnsw_int8_onnx = defaultdict(lambda: 0.02) -# dl20_hnsw_int8_cached = defaultdict(lambda: 0.02) -# dl20_hnsw_onnx = defaultdict(lambda: 0.015) -# dl20_hnsw_cached = defaultdict(lambda: 0.015) -# dl20_hnsw_cached['cohere-embed-english-v3.0-hnsw-cached'] = 0.025 -# -# dl20_hnsw_tolerance = { -# 'hnsw-int8-onnx': dl20_hnsw_int8_onnx, -# 'hnsw-int8-cached': dl20_hnsw_int8_cached, -# 'hnsw-onnx': dl20_hnsw_onnx, -# 'hnsw-cached': dl20_hnsw_cached, -# } - - def evaluate_and_verify(yaml_data, dry_run): fail_str = '\033[91m[FAIL]\033[0m ' ok_str = ' [OK] ' @@ -397,54 +218,6 @@ def evaluate_and_verify(yaml_data, dry_run): else: tolerance_ok = 0 - # if using_flat: - # if 'tolerance' in model: - # #print(model['tolerance']) - # #print(metric) - # tolerance_ok = model['tolerance'][metric['metric']][i] - # else: - # tolerance_ok = 0 - # else: - # # Extract model - # match = flat_model_type_pattern.search(model['name']) - # model_type = match.group(1) - # - # if 'BEIR' in topic_set['name']: - # # Extract BEIR dataset - # match = beir_dataset_pattern.search(topic_set['name']) - # beir_dataset = match.group(1) - # - # tolerance_ok = beir_flat_tolerance[model_type][beir_dataset] - # elif 'MS MARCO Passage' in topic_set['name']: - # tolerance_ok = msmarco_v1_flat_tolerance[model_type][model['name']] - # elif 'DL19' in topic_set['name']: - # tolerance_ok = dl19_flat_tolerance[model_type][model['name']] - # elif using_flat and 'DL20' in topic_set['name']: - # tolerance_ok = dl20_flat_tolerance[model_type][model['name']] - - # if using_hnsw: - # if 'tolerance' in model: - # tolerance_ok = model['tolerance'][metric['metric']][i] - # else: - # tolerance_ok = 0 - # else: - # # Extract model - # match = hnsw_model_type_pattern.search(model['name']) - # model_type = match.group(1) - # - # if 'BEIR' in topic_set['name']: - # # Extract BEIR dataset - # match = beir_dataset_pattern.search(topic_set['name']) - # beir_dataset = match.group(1) - # - # tolerance_ok = beir_hnsw_tolerance[model_type][beir_dataset] - # elif 'MS MARCO Passage' in topic_set['name']: - # tolerance_ok = msmarco_v1_hnsw_tolerance[model_type][model['name']] - # elif 'DL19' in topic_set['name']: - # tolerance_ok = dl19_hnsw_tolerance[model_type][model['name']] - # elif 'DL20' in topic_set['name']: - # tolerance_ok = dl20_hnsw_tolerance[model_type][model['name']] - if using_flat or using_hnsw: result_str = (f'expected: {expected:.4f} actual: {actual:.4f} ' f'(delta={expected-actual:.4f}, tolerance={tolerance_ok:.4f}) - ' diff --git a/src/main/resources/regression/dl19-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml b/src/main/resources/regression/dl19-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml index 8cc7a8857..1fa139a0c 100644 --- a/src/main/resources/regression/dl19-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml +++ b/src/main/resources/regression/dl19-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml @@ -65,7 +65,7 @@ models: - 0.8472 tolerance: AP@1000: - - 0.006 + - 0.008 nDCG@10: - 0.015 R@100: diff --git a/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml b/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml index 24d51c3fb..a06e7215e 100644 --- a/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml +++ b/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml @@ -65,10 +65,10 @@ models: - 0.8630 tolerance: AP@1000: - - 0.001 + - 0.015 nDCG@10: - - 0.003 + - 0.015 R@100: - - 0.003 - R@1000: - 0.015 + R@1000: + - 0.025 diff --git a/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw.cached.yaml b/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw.cached.yaml index 2c9e28297..8169959f5 100644 --- a/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw.cached.yaml +++ b/src/main/resources/regression/dl19-passage.cohere-embed-english-v3.0.hnsw.cached.yaml @@ -65,10 +65,10 @@ models: - 0.8630 tolerance: AP@1000: - - 0.001 + - 0.007 nDCG@10: - - 0.004 + - 0.015 R@100: - - 0.006 + - 0.007 R@1000: - - 0.02 + - 0.025 diff --git a/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw-int8.onnx.yaml b/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw-int8.onnx.yaml index 4bf3b95fb..603f704ac 100644 --- a/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw-int8.onnx.yaml +++ b/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw-int8.onnx.yaml @@ -65,10 +65,10 @@ models: - 0.8201 tolerance: AP@1000: - - 0.005 + - 0.003 nDCG@10: - - 0.005 + - 0.015 R@100: - - 0.005 + - 0.009 R@1000: - - 0.005 + - 0.015 diff --git a/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.cached.yaml b/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.cached.yaml index 7f5800a8e..934a586f5 100644 --- a/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.cached.yaml +++ b/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.cached.yaml @@ -67,8 +67,8 @@ models: AP@1000: - 0.007 nDCG@10: - - 0.015 + - 0.025 R@100: - - 0.009 - R@1000: - 0.015 + R@1000: + - 0.02 diff --git a/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.onnx.yaml b/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.onnx.yaml index 887378951..92ce70b7a 100644 --- a/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.onnx.yaml +++ b/src/main/resources/regression/dl19-passage.cos-dpr-distil.hnsw.onnx.yaml @@ -67,8 +67,8 @@ models: AP@1000: - 0.007 nDCG@10: - - 0.015 + - 0.025 R@100: - - 0.008 - R@1000: - 0.015 + R@1000: + - 0.02 diff --git a/src/main/resources/regression/dl19-passage.openai-ada2.hnsw-int8.cached.yaml b/src/main/resources/regression/dl19-passage.openai-ada2.hnsw-int8.cached.yaml index 5b05ac76f..d4fa68aaf 100644 --- a/src/main/resources/regression/dl19-passage.openai-ada2.hnsw-int8.cached.yaml +++ b/src/main/resources/regression/dl19-passage.openai-ada2.hnsw-int8.cached.yaml @@ -65,10 +65,10 @@ models: - 0.8629 tolerance: AP@1000: - - 0.003 + - 0.005 nDCG@10: - - 0.003 + - 0.015 R@100: - 0.01 R@1000: - - 0.01 + - 0.015 diff --git a/src/main/resources/regression/dl19-passage.openai-ada2.hnsw.cached.yaml b/src/main/resources/regression/dl19-passage.openai-ada2.hnsw.cached.yaml index 164c239df..a4fb3b391 100644 --- a/src/main/resources/regression/dl19-passage.openai-ada2.hnsw.cached.yaml +++ b/src/main/resources/regression/dl19-passage.openai-ada2.hnsw.cached.yaml @@ -71,4 +71,4 @@ models: R@100: - 0.002 R@1000: - - 0.005 + - 0.006 diff --git a/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.cached.yaml b/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.cached.yaml index 4d9ff19d0..e9d157192 100644 --- a/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.cached.yaml +++ b/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.cached.yaml @@ -67,8 +67,8 @@ models: AP@1000: - 0.007 nDCG@10: - - 0.005 + - 0.008 R@100: - 0.015 R@1000: - - 0.02 + - 0.025 diff --git a/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml b/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml index 11642ef19..969e745ea 100644 --- a/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml +++ b/src/main/resources/regression/dl20-passage.bge-base-en-v1.5.hnsw-int8.onnx.yaml @@ -71,4 +71,4 @@ models: R@100: - 0.015 R@1000: - - 0.015 + - 0.02 diff --git a/src/main/resources/regression/dl20-passage.cohere-embed-english-v3.0.hnsw.cached.yaml b/src/main/resources/regression/dl20-passage.cohere-embed-english-v3.0.hnsw.cached.yaml index 6d1ab2416..c54b16f0e 100644 --- a/src/main/resources/regression/dl20-passage.cohere-embed-english-v3.0.hnsw.cached.yaml +++ b/src/main/resources/regression/dl20-passage.cohere-embed-english-v3.0.hnsw.cached.yaml @@ -65,10 +65,10 @@ models: - 0.8682 tolerance: AP@1000: - - 0.002 + - 0.004 nDCG@10: - 0.001 R@100: - - 0.01 - R@1000: - 0.015 + R@1000: + - 0.02 diff --git a/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.cached.yaml b/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.cached.yaml index 92b6722b0..018cb535c 100644 --- a/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.cached.yaml +++ b/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.cached.yaml @@ -65,10 +65,10 @@ models: - 0.8533 tolerance: AP@1000: - - 0.002 + - 0.005 nDCG@10: - 0.001 R@100: - - 0.004 + - 0.008 R@1000: - - 0.007 + - 0.01 diff --git a/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.onnx.yaml b/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.onnx.yaml index bd9398ad9..5c2fbac0b 100644 --- a/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.onnx.yaml +++ b/src/main/resources/regression/dl20-passage.cos-dpr-distil.hnsw.onnx.yaml @@ -65,10 +65,10 @@ models: - 0.8533 tolerance: AP@1000: - - 0.003 + - 0.005 nDCG@10: - 0.001 R@100: - - 0.004 + - 0.008 R@1000: - - 0.006 + - 0.01 diff --git a/src/main/resources/regression/dl20-passage.openai-ada2.hnsw-int8.cached.yaml b/src/main/resources/regression/dl20-passage.openai-ada2.hnsw-int8.cached.yaml index 7a2aef345..8489693ae 100644 --- a/src/main/resources/regression/dl20-passage.openai-ada2.hnsw-int8.cached.yaml +++ b/src/main/resources/regression/dl20-passage.openai-ada2.hnsw-int8.cached.yaml @@ -71,4 +71,4 @@ models: R@100: - 0.002 R@1000: - - 0.001 + - 0.006 diff --git a/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.cached.yaml b/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.cached.yaml index 79a52621e..94ca552d2 100644 --- a/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.cached.yaml +++ b/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.cached.yaml @@ -69,6 +69,6 @@ models: RR@10: - 0.002 R@100: - - 0.006 - R@1000: - 0.007 + R@1000: + - 0.008 diff --git a/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.onnx.yaml b/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.onnx.yaml index fbd22bc54..7facc99cf 100644 --- a/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.onnx.yaml +++ b/src/main/resources/regression/msmarco-v1-passage.bge-base-en-v1.5.hnsw.onnx.yaml @@ -69,6 +69,6 @@ models: RR@10: - 0.002 R@100: - - 0.006 - R@1000: - 0.007 + R@1000: + - 0.008 diff --git a/src/main/resources/regression/msmarco-v1-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml b/src/main/resources/regression/msmarco-v1-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml index cc78b36b4..50bd920ee 100644 --- a/src/main/resources/regression/msmarco-v1-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml +++ b/src/main/resources/regression/msmarco-v1-passage.cohere-embed-english-v3.0.hnsw-int8.cached.yaml @@ -67,8 +67,8 @@ models: nDCG@10: - 0.003 AP@1000: - - 0.002 + - 0.003 RR@10: - 0.002 R@1000: - - 0.006 + - 0.008 diff --git a/src/main/resources/regression/msmarco-v1-passage.cos-dpr-distil.hnsw.cached.yaml b/src/main/resources/regression/msmarco-v1-passage.cos-dpr-distil.hnsw.cached.yaml index c7b7e81ec..d64aca624 100644 --- a/src/main/resources/regression/msmarco-v1-passage.cos-dpr-distil.hnsw.cached.yaml +++ b/src/main/resources/regression/msmarco-v1-passage.cos-dpr-distil.hnsw.cached.yaml @@ -71,4 +71,4 @@ models: R@100: - 0.009 R@1000: - - 0.01 + - 0.015 diff --git a/src/main/resources/regression/msmarco-v1-passage.openai-ada2.hnsw-int8.cached.yaml b/src/main/resources/regression/msmarco-v1-passage.openai-ada2.hnsw-int8.cached.yaml index fff80a5b7..a16fd9136 100644 --- a/src/main/resources/regression/msmarco-v1-passage.openai-ada2.hnsw-int8.cached.yaml +++ b/src/main/resources/regression/msmarco-v1-passage.openai-ada2.hnsw-int8.cached.yaml @@ -71,4 +71,4 @@ models: R@100: - 0.009 R@1000: - - 0.005 + - 0.006