diff --git a/algos-2021.yaml b/algos-2021.yaml index 05a19298..7698f4d4 100644 --- a/algos-2021.yaml +++ b/algos-2021.yaml @@ -474,7 +474,7 @@ deep-1B: base: args: | [{"R":100, "L":100, "B":50, "M":110, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/deep-1B/R100_L100_B50_M110" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/deep-1B/R100_L100_B50_M110" }] query-args: | [{"Ls":30, "BW":4, "T":16}, @@ -522,7 +522,7 @@ msspacev-1B: base: args: | [{"R":100, "L":100, "B":47, "M":100, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100" }] query-args: | [{"Ls":60, "BW":4, "T":16, "HE":3}, @@ -582,7 +582,7 @@ msspacev-1B: base: args: | [{"R":100, "L":100, "B":47, "M":100, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msspacev-1B/R100_L100_B47_M100" }] query-args: | [{"Ls":40, "BW":4, "T":16}, @@ -654,7 +654,7 @@ msturing-1B: base: args: | [{"R":100, "L":100, "B":50, "M":80, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80" }] query-args: | [{"Ls":30, "BW":4, "T":16}, @@ -758,7 +758,7 @@ msturing-1B: base: args: | [{"R":100, "L":100, "B":50, "M":80, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/msturing-1B/R100_L100_B50_M80" }] query-args: | [{"Ls":30, "BW":4, "T":16}, @@ -806,7 +806,7 @@ bigann-1B: base: args: | [{"R":100, "L":100, "B":50, "M":80, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80" }] query-args: | [{"Ls":30, "BW":4, "T":16}, @@ -938,7 +938,7 @@ bigann-1B: base: args: | [{"R":100, "L":100, "B":50, "M":80, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/bigann-1B/R100_L100_B50_M80" }] query-args: | [{"Ls":30, "BW":4, "T":16}, @@ -961,7 +961,7 @@ ssnpp-1B: base: args: | [{"R":100, "L":100, "B":60, "M":100, "C":500000, "CM":2, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100" }] query-args: | [{"Lmin":9, "Lmax":10000, "BW":4, "T":16}, @@ -1023,7 +1023,7 @@ ssnpp-1B: base: args: | [{"R":100, "L":100, "B":60, "M":100, "C":500000, "CM":2, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/fbssnpp-1B/R100_L100_B60_M100" }] query-args: | [{"Lmin":8, "Lmax":10000, "BW":4, "T":16}, @@ -1129,7 +1129,7 @@ text2image-1B: base: args: | [{"R":100, "L":100, "B":60, "M":115, "PQ":200, "C":500000, "CM":2, - "url": "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/diskann-T2-baseline-indices/text2image-1B/R100_L100_B60_M115_PQ200" + "url": "https://comp21storage.z5.web.core.windows.net/comp21/diskann-T2-baseline-indices/text2image-1B/R100_L100_B60_M115_PQ200" }] query-args: | [{"Ls":10, "BW":10, "T":16}, diff --git a/benchmark/datasets.py b/benchmark/datasets.py index 77eb63e3..e841a974 100644 --- a/benchmark/datasets.py +++ b/benchmark/datasets.py @@ -314,7 +314,7 @@ def __init__(self, nb_M=1000): subset_url + "GT_10M/bigann-10M" if self.nb_M == 10 else None ) - # self.gt_fn = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/bigann/public_query_gt100.bin" if self.nb == 10**9 else None + # self.gt_fn = "https://comp21storage.z5.web.core.windows.net/comp21/bigann/public_query_gt100.bin" if self.nb == 10**9 else None self.base_url = "https://dl.fbaipublicfiles.com/billion-scale-ann-benchmarks/bigann/" self.basedir = os.path.join(BASEDIR, "bigann") @@ -345,8 +345,8 @@ def __init__(self, nb_M=1000): self.basedir = os.path.join(BASEDIR, "deep1b") self.private_nq = 30000 - self.private_qs_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/deep1b/query.heldout.30K.fbin" - self.private_gt_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/deep1b/gt100-heldout.30K.fbin" + self.private_qs_url = "https://comp21storage.z5.web.core.windows.net/comp21/deep1b/query.heldout.30K.fbin" + self.private_gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/deep1b/gt100-heldout.30K.fbin" self.private_nq_large = 1000000 self.private_qs_large_url = "https://storage.yandexcloud.net/yr-secret-share/ann-datasets-5ac0659e27/DEEP/query.private.1M.fbin" @@ -406,16 +406,16 @@ def __init__(self, nb_M=1000): "msturing-gt-1M" if self.nb_M == 1 else None ) - self.base_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/" + self.base_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/" self.basedir = os.path.join(BASEDIR, "MSTuringANNS") self.private_nq = 10000 - self.private_qs_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/testQuery10K.fbin" - self.private_gt_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/gt100-private10K-queries.bin" + self.private_qs_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/testQuery10K.fbin" + self.private_gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/gt100-private10K-queries.bin" self.private_nq_large = 99605 - self.private_qs_large_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/testQuery99605.fbin" - self.private_gt_large_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/MSFT-TURING-ANNS/gt100-private99605-queries.bin" + self.private_qs_large_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/testQuery99605.fbin" + self.private_gt_large_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/gt100-private99605-queries.bin" def distance(self): return "euclidean" @@ -430,7 +430,7 @@ def __init__(self): self.qs_fn = "testQuery10K.fbin" self.gt_fn = "clu_msturing10M_gt100" - self.base_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-10M-clustered/" + self.base_url = "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-10M-clustered/" self.basedir = os.path.join(BASEDIR, "MSTuring-10M-clustered") self.private_gt_url = None @@ -480,12 +480,12 @@ def __init__(self, nb_M=1000): "msspacev-gt-1M" if self.nb_M == 1 else None ) - self.base_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/spacev1b/" + self.base_url = "https://comp21storage.z5.web.core.windows.net/comp21/spacev1b/" self.basedir = os.path.join(BASEDIR, "MSSPACEV1B") self.private_nq = 30000 - self.private_qs_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/spacev1b/private_query_30k.bin" - self.private_gt_url = "https://comp21storage.blob.core.windows.net/publiccontainer/comp21/spacev1b/gt100_private_query_30k.bin" + self.private_qs_url = "https://comp21storage.z5.web.core.windows.net/comp21/spacev1b/private_query_30k.bin" + self.private_gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/spacev1b/gt100_private_query_30k.bin" def distance(self): return "euclidean" @@ -500,7 +500,7 @@ def __init__(self, basedir="random-clustered"): self.qs_fn = f"queries_1000_20.fbin" self.gt_fn = f"clu_random_gt100" - self.base_url="https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/random-xs-clustered/" + self.base_url="https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/random-xs-clustered/" self.basedir = os.path.join(BASEDIR, f"{basedir}{self.nb}") if not os.path.exists(self.basedir): diff --git a/neurips23/streaming/clustered_runbook.yaml b/neurips23/streaming/clustered_runbook.yaml index 368fc19e..c0323e5e 100644 --- a/neurips23/streaming/clustered_runbook.yaml +++ b/neurips23/streaming/clustered_runbook.yaml @@ -192,7 +192,7 @@ random-xs-clustered: end: 10000 64: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/random-xs-clustered/clustered_runboook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/random-xs-clustered/clustered_runboook.yaml" msturing-10M-clustered: max_pts: 10000000 # On Azure D8lds v5, runtime: 14 minutes @@ -389,4 +389,4 @@ msturing-10M-clustered: end: 10000000 64: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-10M-clustered/clustered_runboook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-10M-clustered/clustered_runboook.yaml" diff --git a/neurips23/streaming/delete_runbook.yaml b/neurips23/streaming/delete_runbook.yaml index ac625d1b..04dae331 100644 --- a/neurips23/streaming/delete_runbook.yaml +++ b/neurips23/streaming/delete_runbook.yaml @@ -240,7 +240,7 @@ random-xs-clustered: end: 8000 76: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/random-xs-clustered/delete_runbook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/random-xs-clustered/delete_runbook.yaml" msturing-10M-clustered: max_pts: 6000000 # On Azure D8lds v5, runtime: ~23 minutes @@ -479,4 +479,4 @@ msturing-10M-clustered: end: 9200000 75: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/clustered_data/msturing-10M-clustered/delete_runbook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/clustered_data/msturing-10M-clustered/delete_runbook.yaml" diff --git a/neurips23/streaming/simple_runbook.yaml b/neurips23/streaming/simple_runbook.yaml index 21986c12..796a950c 100644 --- a/neurips23/streaming/simple_runbook.yaml +++ b/neurips23/streaming/simple_runbook.yaml @@ -18,7 +18,7 @@ random-xs: end: 5000 6: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/random10000/10000/simple_runbook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/random10000/10000/simple_runbook.yaml" msturing-10M: max_pts: 10000000 # On Azure D8lds v5 with "R":50, "L":50, "insert_threads":16, "consolidate_threads":16 @@ -41,7 +41,7 @@ msturing-10M: end: 5000000 6: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSTuringANNS/10000000/simple_runbook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSTuringANNS/10000000/simple_runbook.yaml" msturing-1M: max_pts: 1000000 # On Azure D8lds v5 with "R":50, "L":50, "insert_threads":16, "consolidate_threads":16 @@ -65,7 +65,7 @@ msturing-1M: end: 500000 6: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSTuringANNS/1000000/simple_runbook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSTuringANNS/1000000/simple_runbook.yaml" msspacev-10M: max_pts: 10000000 1: @@ -86,7 +86,7 @@ msspacev-10M: end: 5000000 6: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSSPACEV1B/10000000/simple_runbook.yaml" + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSSPACEV1B/10000000/simple_runbook.yaml" msspacev-1M: max_pts: 1000000 1: @@ -107,4 +107,4 @@ msspacev-1M: end: 500000 6: operation: "search" - gt_url: "https://comp21storage.blob.core.windows.net/publiccontainer/comp23/str_gt/MSSPACEV1B/1000000/simple_runbook.yaml" \ No newline at end of file + gt_url: "https://comp21storage.z5.web.core.windows.net/comp23/str_gt/MSSPACEV1B/1000000/simple_runbook.yaml" \ No newline at end of file