diff --git a/benchmark/datasets.py b/benchmark/datasets.py index fe59e08b..b6345621 100644 --- a/benchmark/datasets.py +++ b/benchmark/datasets.py @@ -430,6 +430,30 @@ def __init__(self, nb_M=1000): def distance(self): return "euclidean" +# 1M slice of MSTuring dataset, with ground truth corresponding to the 10K query set +# this is needed for backwards compatibility with the streaming code +class MSTuringANNSPQ(BillionScaleDatasetCompetitionFormat): + def __init__(self, nb_M=1): + self.nb_M = nb_M + self.nb = 10**6 * nb_M + self.d = 100 + self.nq = 10000 + self.dtype = "float32" + self.ds_fn = "base1b.fbin" + self.qs_fn = "testQuery10K.fbin" + self.gt_fn = ( + "msturing-1M-private-gt100" if self.nb_M == 1 else + None + ) + self.base_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/" + self.basedir = os.path.join(BASEDIR, "MSTuringANNSPQ") + + self.private_qs_url = None + self.private_gt_url = None + + def distance(self): + return "euclidean" + class MSTuringClustered10M(DatasetCompetitionFormat): def __init__(self): self.nb = 10**6 * 10 @@ -1285,6 +1309,8 @@ def short_name(self): 'msturing-10M': lambda : MSTuringANNS(10), 'msturing-1M': lambda : MSTuringANNS(1), + 'msturingpq-1M': lambda : MSTuringANNSPQ(1), + 'msturing-10M-clustered': lambda: MSTuringClustered10M(), 'msturing-30M-clustered': lambda: MSTuringClustered30M(), diff --git a/data_export.py b/data_export.py index 6063f979..083538cd 100644 --- a/data_export.py +++ b/data_export.py @@ -105,6 +105,9 @@ def cleaned_run_metric(run_metrics): 'neurips23/runbooks/delete_runbook.yaml', 'neurips23/runbooks/final_runbook.yaml', 'neurips23/runbooks/msturing-10M_slidingwindow_runbook.yaml', + 'neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml', + 'neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml', + 'neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml', 'neurips23/runbooks/wikipedia-35M_expirationtime_runbook.yaml', 'neurips23/runbooks/wikipedia-1M_expiration_time_runbook.yaml', 'neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml', diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py index 23eb5001..2ffd55ed 100644 --- a/neurips23/runbooks/gen_expiration_time_runbook.py +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -11,8 +11,10 @@ timesteps: how long to wait before deleting for each ratio seed: seed given to random generator do_replace: whether to include replace in runbook or not +do_delete: whether to include delete in the runbook or not +max_pts: whether to set a max_pts higher than the max that occurs in the runbook (default None) ''' -def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ratios, timesteps, seed = 0, do_replace = False, gt_url = None, do_delete = True): +def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ratios, timesteps, seed = 0, do_replace = False, gt_url = None, do_delete = True, max_points = None): random.seed(seed) data = {dataset_name: {}} @@ -99,7 +101,13 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra } t+=1 - data[dataset_name]["max_pts"]=max_num_points + if max_points != None: + if max_points >= max_num_points: + data[dataset_name]["max_pts"]=max_points + else: + data[dataset_name]["max_pts"]=max_num_points + else: + data[dataset_name]["max_pts"]=max_num_points if gt_url is not None: data[dataset_name]["gt_url"] = gt_url @@ -176,3 +184,36 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra max_t = 1000 gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, None) +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 5554 +dataset_file = 'msturingpq-1M_expiration_time_runbook.yaml' +dataset_name = 'msturingpq-1M' +dataset_size = 1000000 +max_t = 100 +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_runbook.yaml" +max_points = 450000 +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url, max_points) + +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 762 +dataset_file = 'msturingpq-1M_expiration_time_replace_only_runbook.yaml' +dataset_name = 'msturing-1M' +dataset_size = 1000000 +max_t = 100 +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_only_runbook.yaml" +max_points = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False, max_points) + +ratios = (1, 8, 18) +timesteps = (0, 100, 20) +seed = 83 +dataset_file = 'msturingpq-1M_expiration_time_replace_delete_runbook.yaml' +dataset_name = 'msturingpq-1M' +dataset_size = 1000000 +max_t = 100 +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_delete_runbook.yaml" +max_points = 270000 +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, max_points) + diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml new file mode 100644 index 00000000..178b2c85 --- /dev/null +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml @@ -0,0 +1,1213 @@ +msturingpq-1M: + 1: + end: 6984 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 15354 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 25137 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 35756 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 45330 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 56642 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 66205 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 76901 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 88285 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 95177 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 105079 + operation: insert + start: 100000 + 22: + ids_end: 10000 + ids_start: 6984 + operation: replace + tags_end: 3016 + tags_start: 0 + 23: + operation: search + 24: + end: 116396 + operation: insert + start: 110000 + 25: + ids_end: 20000 + ids_start: 15354 + operation: replace + tags_end: 14646 + tags_start: 10000 + 26: + operation: search + 27: + end: 127894 + operation: insert + start: 120000 + 28: + operation: search + 29: + end: 138727 + operation: insert + start: 130000 + 30: + ids_end: 40000 + ids_start: 35756 + operation: replace + tags_end: 34244 + tags_start: 30000 + 31: + operation: search + 32: + end: 146908 + operation: insert + start: 140000 + 33: + operation: search + 34: + end: 156762 + operation: insert + start: 150000 + 35: + ids_end: 60000 + ids_start: 56642 + operation: replace + tags_end: 53358 + tags_start: 50000 + 36: + operation: search + 37: + end: 168642 + operation: insert + start: 160000 + 38: + operation: search + 39: + end: 177514 + operation: insert + start: 170000 + 40: + operation: search + 41: + end: 185743 + operation: insert + start: 180000 + 42: + operation: search + 43: + end: 195510 + operation: insert + start: 190000 + 44: + operation: search + 45: + end: 206769 + operation: insert + start: 200000 + 46: + end: 6984 + operation: delete + start: 0 + 47: + ids_end: 110000 + ids_start: 105079 + operation: replace + tags_end: 104921 + tags_start: 100000 + 48: + operation: search + 49: + end: 217627 + operation: insert + start: 210000 + 50: + end: 15354 + operation: delete + start: 10000 + 51: + ids_end: 120000 + ids_start: 116396 + operation: replace + tags_end: 113604 + tags_start: 110000 + 52: + operation: search + 53: + end: 227628 + operation: insert + start: 220000 + 54: + ids_end: 130000 + ids_start: 127894 + operation: replace + tags_end: 122106 + tags_start: 120000 + 55: + operation: search + 56: + end: 237364 + operation: insert + start: 230000 + 57: + end: 35756 + operation: delete + start: 30000 + 58: + ids_end: 140000 + ids_start: 138727 + operation: replace + tags_end: 131273 + tags_start: 130000 + 59: + operation: search + 60: + end: 245586 + operation: insert + start: 240000 + 61: + ids_end: 150000 + ids_start: 146908 + operation: replace + tags_end: 143092 + tags_start: 140000 + 62: + operation: search + 63: + end: 256089 + operation: insert + start: 250000 + 64: + end: 56642 + operation: delete + start: 50000 + 65: + operation: search + 66: + end: 265390 + operation: insert + start: 260000 + 67: + ids_end: 170000 + ids_start: 168642 + operation: replace + tags_end: 161358 + tags_start: 160000 + 68: + operation: search + 69: + end: 275633 + operation: insert + start: 270000 + 70: + ids_end: 180000 + ids_start: 177514 + operation: replace + tags_end: 172486 + tags_start: 170000 + 71: + operation: search + 72: + end: 287312 + operation: insert + start: 280000 + 73: + ids_end: 190000 + ids_start: 185743 + operation: replace + tags_end: 184257 + tags_start: 180000 + 74: + operation: search + 75: + end: 295727 + operation: insert + start: 290000 + 76: + ids_end: 200000 + ids_start: 195510 + operation: replace + tags_end: 194490 + tags_start: 190000 + 77: + operation: search + 78: + end: 306078 + operation: insert + start: 300000 + 79: + end: 105079 + operation: delete + start: 100000 + 80: + operation: search + 81: + end: 316989 + operation: insert + start: 310000 + 82: + end: 116396 + operation: delete + start: 110000 + 83: + operation: search + 84: + end: 328885 + operation: insert + start: 320000 + 85: + end: 127894 + operation: delete + start: 120000 + 86: + ids_end: 230000 + ids_start: 227628 + operation: replace + tags_end: 222372 + tags_start: 220000 + 87: + operation: search + 88: + end: 335996 + operation: insert + start: 330000 + 89: + end: 138727 + operation: delete + start: 130000 + 90: + ids_end: 240000 + ids_start: 237364 + operation: replace + tags_end: 232636 + tags_start: 230000 + 91: + operation: search + 92: + end: 347569 + operation: insert + start: 340000 + 93: + end: 146908 + operation: delete + start: 140000 + 94: + operation: search + 95: + end: 357017 + operation: insert + start: 350000 + 96: + ids_end: 260000 + ids_start: 256089 + operation: replace + tags_end: 253911 + tags_start: 250000 + 97: + operation: search + 98: + end: 367528 + operation: insert + start: 360000 + 99: + end: 168642 + operation: delete + start: 160000 + 100: + operation: search + 101: + end: 378628 + operation: insert + start: 370000 + 102: + end: 177514 + operation: delete + start: 170000 + 103: + ids_end: 280000 + ids_start: 275633 + operation: replace + tags_end: 274367 + tags_start: 270000 + 104: + ids_end: 350000 + ids_start: 347569 + operation: replace + tags_end: 342431 + tags_start: 340000 + 105: + operation: search + 106: + end: 385445 + operation: insert + start: 380000 + 107: + end: 185743 + operation: delete + start: 180000 + 108: + operation: search + 109: + end: 395650 + operation: insert + start: 390000 + 110: + end: 195510 + operation: delete + start: 190000 + 111: + ids_end: 300000 + ids_start: 295727 + operation: replace + tags_end: 294273 + tags_start: 290000 + 112: + operation: search + 113: + end: 406069 + operation: insert + start: 400000 + 114: + ids_end: 310000 + ids_start: 306078 + operation: replace + tags_end: 303922 + tags_start: 300000 + 115: + operation: search + 116: + end: 417088 + operation: insert + start: 410000 + 117: + ids_end: 320000 + ids_start: 316989 + operation: replace + tags_end: 313011 + tags_start: 310000 + 118: + operation: search + 119: + end: 426737 + operation: insert + start: 420000 + 120: + end: 227628 + operation: delete + start: 220000 + 121: + ids_end: 330000 + ids_start: 328885 + operation: replace + tags_end: 321115 + tags_start: 320000 + 122: + operation: search + 123: + end: 437352 + operation: insert + start: 430000 + 124: + end: 237364 + operation: delete + start: 230000 + 125: + ids_end: 340000 + ids_start: 335996 + operation: replace + tags_end: 334004 + tags_start: 330000 + 126: + operation: search + 127: + end: 446515 + operation: insert + start: 440000 + 128: + operation: search + 129: + end: 456511 + operation: insert + start: 450000 + 130: + end: 256089 + operation: delete + start: 250000 + 131: + ids_end: 360000 + ids_start: 357017 + operation: replace + tags_end: 352983 + tags_start: 350000 + 132: + operation: search + 133: + end: 466520 + operation: insert + start: 460000 + 134: + ids_end: 220000 + ids_start: 217627 + operation: replace + tags_end: 212373 + tags_start: 210000 + 135: + operation: search + 136: + end: 478342 + operation: insert + start: 470000 + 137: + end: 275633 + operation: delete + start: 270000 + 138: + ids_end: 380000 + ids_start: 378628 + operation: replace + tags_end: 371372 + tags_start: 370000 + 139: + operation: search + 140: + end: 488707 + operation: insert + start: 480000 + 141: + ids_end: 390000 + ids_start: 385445 + operation: replace + tags_end: 384555 + tags_start: 380000 + 142: + operation: search + 143: + end: 498810 + operation: insert + start: 490000 + 144: + end: 295727 + operation: delete + start: 290000 + 145: + ids_end: 50000 + ids_start: 45330 + operation: replace + tags_end: 44670 + tags_start: 40000 + 146: + ids_end: 100000 + ids_start: 95177 + operation: replace + tags_end: 94823 + tags_start: 90000 + 147: + ids_end: 400000 + ids_start: 395650 + operation: replace + tags_end: 394350 + tags_start: 390000 + 148: + operation: search + 149: + end: 508747 + operation: insert + start: 500000 + 150: + end: 306078 + operation: delete + start: 300000 + 151: + operation: search + 152: + end: 518567 + operation: insert + start: 510000 + 153: + end: 316989 + operation: delete + start: 310000 + 154: + operation: search + 155: + end: 526963 + operation: insert + start: 520000 + 156: + end: 328885 + operation: delete + start: 320000 + 157: + ids_end: 30000 + ids_start: 25137 + operation: replace + tags_end: 24863 + tags_start: 20000 + 158: + operation: search + 159: + end: 537943 + operation: insert + start: 530000 + 160: + end: 335996 + operation: delete + start: 330000 + 161: + ids_end: 440000 + ids_start: 437352 + operation: replace + tags_end: 432648 + tags_start: 430000 + 162: + operation: search + 163: + end: 547352 + operation: insert + start: 540000 + 164: + ids_end: 450000 + ids_start: 446515 + operation: replace + tags_end: 443485 + tags_start: 440000 + 165: + operation: search + 166: + end: 558691 + operation: insert + start: 550000 + 167: + end: 357017 + operation: delete + start: 350000 + 168: + ids_end: 460000 + ids_start: 456511 + operation: replace + tags_end: 453489 + tags_start: 450000 + 169: + operation: search + 170: + end: 568362 + operation: insert + start: 560000 + 171: + ids_end: 70000 + ids_start: 66205 + operation: replace + tags_end: 63795 + tags_start: 60000 + 172: + operation: search + 173: + end: 578083 + operation: insert + start: 570000 + 174: + end: 378628 + operation: delete + start: 370000 + 175: + ids_end: 80000 + ids_start: 76901 + operation: replace + tags_end: 73099 + tags_start: 70000 + 176: + operation: search + 177: + end: 586128 + operation: insert + start: 580000 + 178: + end: 385445 + operation: delete + start: 380000 + 179: + ids_end: 90000 + ids_start: 88285 + operation: replace + tags_end: 81715 + tags_start: 80000 + 180: + operation: search + 181: + end: 598526 + operation: insert + start: 590000 + 182: + end: 395650 + operation: delete + start: 390000 + 183: + ids_end: 500000 + ids_start: 498810 + operation: replace + tags_end: 491190 + tags_start: 490000 + 184: + operation: search + 185: + end: 606875 + operation: insert + start: 600000 + 186: + operation: search + 187: + end: 615613 + operation: insert + start: 610000 + 188: + operation: search + 189: + end: 628233 + operation: insert + start: 620000 + 190: + operation: search + 191: + end: 635882 + operation: insert + start: 630000 + 192: + end: 437352 + operation: delete + start: 430000 + 193: + operation: search + 194: + end: 646348 + operation: insert + start: 640000 + 195: + end: 446515 + operation: delete + start: 440000 + 196: + operation: search + 197: + end: 658918 + operation: insert + start: 650000 + 198: + end: 456511 + operation: delete + start: 450000 + 199: + ids_end: 160000 + ids_start: 156762 + operation: replace + tags_end: 153238 + tags_start: 150000 + 200: + ids_end: 560000 + ids_start: 558691 + operation: replace + tags_end: 551309 + tags_start: 550000 + 201: + operation: search + 202: + end: 666780 + operation: insert + start: 660000 + 203: + operation: search + 204: + end: 678499 + operation: insert + start: 670000 + 205: + ids_end: 580000 + ids_start: 578083 + operation: replace + tags_end: 571917 + tags_start: 570000 + 206: + operation: search + 207: + end: 687858 + operation: insert + start: 680000 + 208: + ids_end: 590000 + ids_start: 586128 + operation: replace + tags_end: 583872 + tags_start: 580000 + 209: + operation: search + 210: + end: 697313 + operation: insert + start: 690000 + 211: + end: 498810 + operation: delete + start: 490000 + 212: + ids_end: 600000 + ids_start: 598526 + operation: replace + tags_end: 591474 + tags_start: 590000 + 213: + operation: search + 214: + end: 708960 + operation: insert + start: 700000 + 215: + ids_end: 210000 + ids_start: 206769 + operation: replace + tags_end: 203231 + tags_start: 200000 + 216: + ids_end: 610000 + ids_start: 606875 + operation: replace + tags_end: 603125 + tags_start: 600000 + 217: + operation: search + 218: + end: 716744 + operation: insert + start: 710000 + 219: + ids_end: 620000 + ids_start: 615613 + operation: replace + tags_end: 614387 + tags_start: 610000 + 220: + operation: search + 221: + end: 726852 + operation: insert + start: 720000 + 222: + ids_end: 630000 + ids_start: 628233 + operation: replace + tags_end: 621767 + tags_start: 620000 + 223: + operation: search + 224: + end: 736435 + operation: insert + start: 730000 + 225: + operation: search + 226: + end: 746156 + operation: insert + start: 740000 + 227: + ids_end: 250000 + ids_start: 245586 + operation: replace + tags_end: 244414 + tags_start: 240000 + 228: + operation: search + 229: + end: 758269 + operation: insert + start: 750000 + 230: + end: 558691 + operation: delete + start: 550000 + 231: + ids_end: 660000 + ids_start: 658918 + operation: replace + tags_end: 651082 + tags_start: 650000 + 232: + operation: search + 233: + end: 767633 + operation: insert + start: 760000 + 234: + ids_end: 270000 + ids_start: 265390 + operation: replace + tags_end: 264610 + tags_start: 260000 + 235: + ids_end: 670000 + ids_start: 666780 + operation: replace + tags_end: 663220 + tags_start: 660000 + 236: + operation: search + 237: + end: 778228 + operation: insert + start: 770000 + 238: + end: 578083 + operation: delete + start: 570000 + 239: + operation: search + 240: + end: 785943 + operation: insert + start: 780000 + 241: + end: 586128 + operation: delete + start: 580000 + 242: + ids_end: 290000 + ids_start: 287312 + operation: replace + tags_end: 282688 + tags_start: 280000 + 243: + ids_end: 690000 + ids_start: 687858 + operation: replace + tags_end: 682142 + tags_start: 680000 + 244: + operation: search + 245: + end: 795221 + operation: insert + start: 790000 + 246: + end: 598526 + operation: delete + start: 590000 + 247: + ids_end: 700000 + ids_start: 697313 + operation: replace + tags_end: 692687 + tags_start: 690000 + 248: + operation: search + 249: + end: 808959 + operation: insert + start: 800000 + 250: + end: 606875 + operation: delete + start: 600000 + 251: + ids_end: 710000 + ids_start: 708960 + operation: replace + tags_end: 701040 + tags_start: 700000 + 252: + operation: search + 253: + end: 816098 + operation: insert + start: 810000 + 254: + end: 615613 + operation: delete + start: 610000 + 255: + ids_end: 720000 + ids_start: 716744 + operation: replace + tags_end: 713256 + tags_start: 710000 + 256: + operation: search + 257: + end: 827128 + operation: insert + start: 820000 + 258: + end: 628233 + operation: delete + start: 620000 + 259: + operation: search + 260: + end: 835574 + operation: insert + start: 830000 + 261: + operation: search + 262: + end: 845675 + operation: insert + start: 840000 + 263: + operation: search + 264: + end: 856686 + operation: insert + start: 850000 + 265: + end: 658918 + operation: delete + start: 650000 + 266: + ids_end: 760000 + ids_start: 758269 + operation: replace + tags_end: 751731 + tags_start: 750000 + 267: + operation: search + 268: + end: 868969 + operation: insert + start: 860000 + 269: + end: 666780 + operation: delete + start: 660000 + 270: + ids_end: 370000 + ids_start: 367528 + operation: replace + tags_end: 362472 + tags_start: 360000 + 271: + operation: search + 272: + end: 878008 + operation: insert + start: 870000 + 273: + ids_end: 780000 + ids_start: 778228 + operation: replace + tags_end: 771772 + tags_start: 770000 + 274: + operation: search + 275: + end: 887151 + operation: insert + start: 880000 + 276: + end: 687858 + operation: delete + start: 680000 + 277: + ids_end: 790000 + ids_start: 785943 + operation: replace + tags_end: 784057 + tags_start: 780000 + 278: + operation: search + 279: + end: 898954 + operation: insert + start: 890000 + 280: + end: 697313 + operation: delete + start: 690000 + 281: + operation: search + 282: + end: 908889 + operation: insert + start: 900000 + 283: + end: 708960 + operation: delete + start: 700000 + 284: + ids_end: 410000 + ids_start: 406069 + operation: replace + tags_end: 403931 + tags_start: 400000 + 285: + ids_end: 810000 + ids_start: 808959 + operation: replace + tags_end: 801041 + tags_start: 800000 + 286: + operation: search + 287: + end: 916825 + operation: insert + start: 910000 + 288: + end: 716744 + operation: delete + start: 710000 + 289: + operation: search + 290: + end: 928354 + operation: insert + start: 920000 + 291: + ids_end: 430000 + ids_start: 426737 + operation: replace + tags_end: 423263 + tags_start: 420000 + 292: + ids_end: 830000 + ids_start: 827128 + operation: replace + tags_end: 822872 + tags_start: 820000 + 293: + operation: search + 294: + end: 935094 + operation: insert + start: 930000 + 295: + operation: search + 296: + end: 945059 + operation: insert + start: 940000 + 297: + operation: search + 298: + end: 955249 + operation: insert + start: 950000 + 299: + end: 758269 + operation: delete + start: 750000 + 300: + ids_end: 860000 + ids_start: 856686 + operation: replace + tags_end: 853314 + tags_start: 850000 + 301: + operation: search + 302: + end: 966129 + operation: insert + start: 960000 + 303: + ids_end: 470000 + ids_start: 466520 + operation: replace + tags_end: 463480 + tags_start: 460000 + 304: + ids_end: 870000 + ids_start: 868969 + operation: replace + tags_end: 861031 + tags_start: 860000 + 305: + operation: search + 306: + end: 978693 + operation: insert + start: 970000 + 307: + end: 778228 + operation: delete + start: 770000 + 308: + ids_end: 480000 + ids_start: 478342 + operation: replace + tags_end: 471658 + tags_start: 470000 + 309: + ids_end: 880000 + ids_start: 878008 + operation: replace + tags_end: 871992 + tags_start: 870000 + 310: + operation: search + 311: + end: 988547 + operation: insert + start: 980000 + 312: + end: 785943 + operation: delete + start: 780000 + 313: + ids_end: 490000 + ids_start: 488707 + operation: replace + tags_end: 481293 + tags_start: 480000 + 314: + ids_end: 910000 + ids_start: 908889 + operation: replace + tags_end: 901111 + tags_start: 900000 + 315: + operation: search + 316: + end: 996357 + operation: insert + start: 990000 + 317: + operation: search + max_pts: 248123 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_delete_runbook.yaml diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml new file mode 100644 index 00000000..09af04ac --- /dev/null +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml @@ -0,0 +1,1095 @@ +msturing-1M: + 1: + end: 8844 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 18924 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 25996 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 36330 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 45116 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 57938 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 65638 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 75848 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 86273 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 96408 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 108673 + operation: insert + start: 100000 + 22: + ids_end: 10000 + ids_start: 8844 + operation: replace + tags_end: 1156 + tags_start: 0 + 23: + operation: search + 24: + end: 115146 + operation: insert + start: 110000 + 25: + ids_end: 20000 + ids_start: 18924 + operation: replace + tags_end: 11076 + tags_start: 10000 + 26: + operation: search + 27: + end: 127355 + operation: insert + start: 120000 + 28: + operation: search + 29: + end: 136667 + operation: insert + start: 130000 + 30: + ids_end: 40000 + ids_start: 36330 + operation: replace + tags_end: 33670 + tags_start: 30000 + 31: + operation: search + 32: + end: 147698 + operation: insert + start: 140000 + 33: + ids_end: 50000 + ids_start: 45116 + operation: replace + tags_end: 44884 + tags_start: 40000 + 34: + operation: search + 35: + end: 158979 + operation: insert + start: 150000 + 36: + operation: search + 37: + end: 167999 + operation: insert + start: 160000 + 38: + operation: search + 39: + end: 176900 + operation: insert + start: 170000 + 40: + ids_end: 80000 + ids_start: 75848 + operation: replace + tags_end: 74152 + tags_start: 70000 + 41: + operation: search + 42: + end: 188688 + operation: insert + start: 180000 + 43: + ids_end: 90000 + ids_start: 86273 + operation: replace + tags_end: 83727 + tags_start: 80000 + 44: + operation: search + 45: + end: 198898 + operation: insert + start: 190000 + 46: + operation: search + 47: + end: 206136 + operation: insert + start: 200000 + 48: + ids_end: 110000 + ids_start: 108673 + operation: replace + tags_end: 101327 + tags_start: 100000 + 49: + operation: search + 50: + end: 216825 + operation: insert + start: 210000 + 51: + ids_end: 120000 + ids_start: 115146 + operation: replace + tags_end: 114854 + tags_start: 110000 + 52: + operation: search + 53: + end: 227243 + operation: insert + start: 220000 + 54: + operation: search + 55: + end: 238280 + operation: insert + start: 230000 + 56: + ids_end: 140000 + ids_start: 136667 + operation: replace + tags_end: 133333 + tags_start: 130000 + 57: + operation: search + 58: + end: 247749 + operation: insert + start: 240000 + 59: + operation: search + 60: + end: 257034 + operation: insert + start: 250000 + 61: + ids_end: 160000 + ids_start: 158979 + operation: replace + tags_end: 151021 + tags_start: 150000 + 62: + operation: search + 63: + end: 265957 + operation: insert + start: 260000 + 64: + ids_end: 170000 + ids_start: 167999 + operation: replace + tags_end: 162001 + tags_start: 160000 + 65: + operation: search + 66: + end: 277335 + operation: insert + start: 270000 + 67: + operation: search + 68: + end: 285359 + operation: insert + start: 280000 + 69: + ids_end: 190000 + ids_start: 188688 + operation: replace + tags_end: 181312 + tags_start: 180000 + 70: + operation: search + 71: + end: 298081 + operation: insert + start: 290000 + 72: + ids_end: 200000 + ids_start: 198898 + operation: replace + tags_end: 191102 + tags_start: 190000 + 73: + operation: search + 74: + end: 308638 + operation: insert + start: 300000 + 75: + ids_end: 210000 + ids_start: 206136 + operation: replace + tags_end: 203864 + tags_start: 200000 + 76: + operation: search + 77: + end: 318941 + operation: insert + start: 310000 + 78: + ids_end: 220000 + ids_start: 216825 + operation: replace + tags_end: 213175 + tags_start: 210000 + 79: + operation: search + 80: + end: 325288 + operation: insert + start: 320000 + 81: + ids_end: 230000 + ids_start: 227243 + operation: replace + tags_end: 222757 + tags_start: 220000 + 82: + operation: search + 83: + end: 338313 + operation: insert + start: 330000 + 84: + ids_end: 240000 + ids_start: 238280 + operation: replace + tags_end: 231720 + tags_start: 230000 + 85: + operation: search + 86: + end: 346984 + operation: insert + start: 340000 + 87: + ids_end: 250000 + ids_start: 247749 + operation: replace + tags_end: 242251 + tags_start: 240000 + 88: + operation: search + 89: + end: 358981 + operation: insert + start: 350000 + 90: + operation: search + 91: + end: 368169 + operation: insert + start: 360000 + 92: + operation: search + 93: + end: 375661 + operation: insert + start: 370000 + 94: + ids_end: 280000 + ids_start: 277335 + operation: replace + tags_end: 272665 + tags_start: 270000 + 95: + operation: search + 96: + end: 386309 + operation: insert + start: 380000 + 97: + ids_end: 290000 + ids_start: 285359 + operation: replace + tags_end: 284641 + tags_start: 280000 + 98: + operation: search + 99: + end: 395017 + operation: insert + start: 390000 + 100: + ids_end: 390000 + ids_start: 386309 + operation: replace + tags_end: 383691 + tags_start: 380000 + 101: + operation: search + 102: + end: 405729 + operation: insert + start: 400000 + 103: + ids_end: 310000 + ids_start: 308638 + operation: replace + tags_end: 301362 + tags_start: 300000 + 104: + operation: search + 105: + end: 416663 + operation: insert + start: 410000 + 106: + ids_end: 320000 + ids_start: 318941 + operation: replace + tags_end: 311059 + tags_start: 310000 + 107: + operation: search + 108: + end: 426477 + operation: insert + start: 420000 + 109: + ids_end: 330000 + ids_start: 325288 + operation: replace + tags_end: 324712 + tags_start: 320000 + 110: + operation: search + 111: + end: 436771 + operation: insert + start: 430000 + 112: + ids_end: 340000 + ids_start: 338313 + operation: replace + tags_end: 331687 + tags_start: 330000 + 113: + operation: search + 114: + end: 447512 + operation: insert + start: 440000 + 115: + ids_end: 350000 + ids_start: 346984 + operation: replace + tags_end: 343016 + tags_start: 340000 + 116: + operation: search + 117: + end: 455625 + operation: insert + start: 450000 + 118: + operation: search + 119: + end: 465802 + operation: insert + start: 460000 + 120: + ids_end: 370000 + ids_start: 368169 + operation: replace + tags_end: 361831 + tags_start: 360000 + 121: + operation: search + 122: + end: 478856 + operation: insert + start: 470000 + 123: + ids_end: 180000 + ids_start: 176900 + operation: replace + tags_end: 173100 + tags_start: 170000 + 124: + ids_end: 380000 + ids_start: 375661 + operation: replace + tags_end: 374339 + tags_start: 370000 + 125: + operation: search + 126: + end: 488196 + operation: insert + start: 480000 + 127: + operation: search + 128: + end: 498985 + operation: insert + start: 490000 + 129: + ids_end: 400000 + ids_start: 395017 + operation: replace + tags_end: 394983 + tags_start: 390000 + 130: + operation: search + 131: + end: 506281 + operation: insert + start: 500000 + 132: + ids_end: 410000 + ids_start: 405729 + operation: replace + tags_end: 404271 + tags_start: 400000 + 133: + operation: search + 134: + end: 517184 + operation: insert + start: 510000 + 135: + operation: search + 136: + end: 527275 + operation: insert + start: 520000 + 137: + ids_end: 30000 + ids_start: 25996 + operation: replace + tags_end: 24004 + tags_start: 20000 + 138: + ids_end: 430000 + ids_start: 426477 + operation: replace + tags_end: 423523 + tags_start: 420000 + 139: + operation: search + 140: + end: 538153 + operation: insert + start: 530000 + 141: + ids_end: 440000 + ids_start: 436771 + operation: replace + tags_end: 433229 + tags_start: 430000 + 142: + operation: search + 143: + end: 545662 + operation: insert + start: 540000 + 144: + ids_end: 450000 + ids_start: 447512 + operation: replace + tags_end: 442488 + tags_start: 440000 + 145: + operation: search + 146: + end: 557936 + operation: insert + start: 550000 + 147: + ids_end: 60000 + ids_start: 57938 + operation: replace + tags_end: 52062 + tags_start: 50000 + 148: + ids_end: 460000 + ids_start: 455625 + operation: replace + tags_end: 454375 + tags_start: 450000 + 149: + operation: search + 150: + end: 568765 + operation: insert + start: 560000 + 151: + ids_end: 70000 + ids_start: 65638 + operation: replace + tags_end: 64362 + tags_start: 60000 + 152: + ids_end: 470000 + ids_start: 465802 + operation: replace + tags_end: 464198 + tags_start: 460000 + 153: + operation: search + 154: + end: 575634 + operation: insert + start: 570000 + 155: + ids_end: 480000 + ids_start: 478856 + operation: replace + tags_end: 471144 + tags_start: 470000 + 156: + operation: search + 157: + end: 585561 + operation: insert + start: 580000 + 158: + ids_end: 490000 + ids_start: 488196 + operation: replace + tags_end: 481804 + tags_start: 480000 + 159: + operation: search + 160: + end: 595733 + operation: insert + start: 590000 + 161: + ids_end: 100000 + ids_start: 96408 + operation: replace + tags_end: 93592 + tags_start: 90000 + 162: + ids_end: 500000 + ids_start: 498985 + operation: replace + tags_end: 491015 + tags_start: 490000 + 163: + operation: search + 164: + end: 607223 + operation: insert + start: 600000 + 165: + operation: search + 166: + end: 616949 + operation: insert + start: 610000 + 167: + ids_end: 520000 + ids_start: 517184 + operation: replace + tags_end: 512816 + tags_start: 510000 + 168: + operation: search + 169: + end: 627143 + operation: insert + start: 620000 + 170: + ids_end: 130000 + ids_start: 127355 + operation: replace + tags_end: 122645 + tags_start: 120000 + 171: + ids_end: 530000 + ids_start: 527275 + operation: replace + tags_end: 522725 + tags_start: 520000 + 172: + operation: search + 173: + end: 636104 + operation: insert + start: 630000 + 174: + ids_end: 540000 + ids_start: 538153 + operation: replace + tags_end: 531847 + tags_start: 530000 + 175: + operation: search + 176: + end: 647869 + operation: insert + start: 640000 + 177: + ids_end: 150000 + ids_start: 147698 + operation: replace + tags_end: 142302 + tags_start: 140000 + 178: + ids_end: 550000 + ids_start: 545662 + operation: replace + tags_end: 544338 + tags_start: 540000 + 179: + operation: search + 180: + end: 656712 + operation: insert + start: 650000 + 181: + operation: search + 182: + end: 667029 + operation: insert + start: 660000 + 183: + operation: search + 184: + end: 675811 + operation: insert + start: 670000 + 185: + ids_end: 580000 + ids_start: 575634 + operation: replace + tags_end: 574366 + tags_start: 570000 + 186: + operation: search + 187: + end: 685846 + operation: insert + start: 680000 + 188: + operation: search + 189: + end: 697956 + operation: insert + start: 690000 + 190: + ids_end: 600000 + ids_start: 595733 + operation: replace + tags_end: 594267 + tags_start: 590000 + 191: + operation: search + 192: + end: 708233 + operation: insert + start: 700000 + 193: + ids_end: 610000 + ids_start: 607223 + operation: replace + tags_end: 602777 + tags_start: 600000 + 194: + operation: search + 195: + end: 718522 + operation: insert + start: 710000 + 196: + ids_end: 620000 + ids_start: 616949 + operation: replace + tags_end: 613051 + tags_start: 610000 + 197: + operation: search + 198: + end: 727277 + operation: insert + start: 720000 + 199: + ids_end: 630000 + ids_start: 627143 + operation: replace + tags_end: 622857 + tags_start: 620000 + 200: + operation: search + 201: + end: 737232 + operation: insert + start: 730000 + 202: + ids_end: 640000 + ids_start: 636104 + operation: replace + tags_end: 633896 + tags_start: 630000 + 203: + operation: search + 204: + end: 745364 + operation: insert + start: 740000 + 205: + ids_end: 650000 + ids_start: 647869 + operation: replace + tags_end: 642131 + tags_start: 640000 + 206: + operation: search + 207: + end: 756547 + operation: insert + start: 750000 + 208: + ids_end: 260000 + ids_start: 257034 + operation: replace + tags_end: 252966 + tags_start: 250000 + 209: + operation: search + 210: + end: 766299 + operation: insert + start: 760000 + 211: + ids_end: 270000 + ids_start: 265957 + operation: replace + tags_end: 264043 + tags_start: 260000 + 212: + ids_end: 670000 + ids_start: 667029 + operation: replace + tags_end: 662971 + tags_start: 660000 + 213: + operation: search + 214: + end: 776459 + operation: insert + start: 770000 + 215: + operation: search + 216: + end: 788998 + operation: insert + start: 780000 + 217: + ids_end: 690000 + ids_start: 685846 + operation: replace + tags_end: 684154 + tags_start: 680000 + 218: + operation: search + 219: + end: 797483 + operation: insert + start: 790000 + 220: + ids_end: 300000 + ids_start: 298081 + operation: replace + tags_end: 291919 + tags_start: 290000 + 221: + ids_end: 700000 + ids_start: 697956 + operation: replace + tags_end: 692044 + tags_start: 690000 + 222: + operation: search + 223: + end: 806394 + operation: insert + start: 800000 + 224: + ids_end: 710000 + ids_start: 708233 + operation: replace + tags_end: 701767 + tags_start: 700000 + 225: + operation: search + 226: + end: 815353 + operation: insert + start: 810000 + 227: + operation: search + 228: + end: 826206 + operation: insert + start: 820000 + 229: + ids_end: 730000 + ids_start: 727277 + operation: replace + tags_end: 722723 + tags_start: 720000 + 230: + operation: search + 231: + end: 835334 + operation: insert + start: 830000 + 232: + ids_end: 740000 + ids_start: 737232 + operation: replace + tags_end: 732768 + tags_start: 730000 + 233: + operation: search + 234: + end: 846259 + operation: insert + start: 840000 + 235: + ids_end: 750000 + ids_start: 745364 + operation: replace + tags_end: 744636 + tags_start: 740000 + 236: + operation: search + 237: + end: 857065 + operation: insert + start: 850000 + 238: + ids_end: 360000 + ids_start: 358981 + operation: replace + tags_end: 351019 + tags_start: 350000 + 239: + ids_end: 760000 + ids_start: 756547 + operation: replace + tags_end: 753453 + tags_start: 750000 + 240: + operation: search + 241: + end: 865379 + operation: insert + start: 860000 + 242: + ids_end: 770000 + ids_start: 766299 + operation: replace + tags_end: 763701 + tags_start: 760000 + 243: + operation: search + 244: + end: 875198 + operation: insert + start: 870000 + 245: + ids_end: 780000 + ids_start: 776459 + operation: replace + tags_end: 773541 + tags_start: 770000 + 246: + operation: search + 247: + end: 885741 + operation: insert + start: 880000 + 248: + ids_end: 790000 + ids_start: 788998 + operation: replace + tags_end: 781002 + tags_start: 780000 + 249: + operation: search + 250: + end: 895870 + operation: insert + start: 890000 + 251: + ids_end: 800000 + ids_start: 797483 + operation: replace + tags_end: 792517 + tags_start: 790000 + 252: + operation: search + 253: + end: 906094 + operation: insert + start: 900000 + 254: + ids_end: 810000 + ids_start: 806394 + operation: replace + tags_end: 803606 + tags_start: 800000 + 255: + operation: search + 256: + end: 916822 + operation: insert + start: 910000 + 257: + ids_end: 420000 + ids_start: 416663 + operation: replace + tags_end: 413337 + tags_start: 410000 + 258: + ids_end: 820000 + ids_start: 815353 + operation: replace + tags_end: 814647 + tags_start: 810000 + 259: + operation: search + 260: + end: 925862 + operation: insert + start: 920000 + 261: + ids_end: 830000 + ids_start: 826206 + operation: replace + tags_end: 823794 + tags_start: 820000 + 262: + operation: search + 263: + end: 938967 + operation: insert + start: 930000 + 264: + ids_end: 840000 + ids_start: 835334 + operation: replace + tags_end: 834666 + tags_start: 830000 + 265: + operation: search + 266: + end: 946541 + operation: insert + start: 940000 + 267: + ids_end: 850000 + ids_start: 846259 + operation: replace + tags_end: 843741 + tags_start: 840000 + 268: + operation: search + 269: + end: 956682 + operation: insert + start: 950000 + 270: + ids_end: 860000 + ids_start: 857065 + operation: replace + tags_end: 852935 + tags_start: 850000 + 271: + operation: search + 272: + end: 966997 + operation: insert + start: 960000 + 273: + ids_end: 870000 + ids_start: 865379 + operation: replace + tags_end: 864621 + tags_start: 860000 + 274: + operation: search + 275: + end: 976819 + operation: insert + start: 970000 + 276: + operation: search + 277: + end: 988752 + operation: insert + start: 980000 + 278: + ids_end: 890000 + ids_start: 885741 + operation: replace + tags_end: 884259 + tags_start: 880000 + 279: + operation: search + 280: + end: 997302 + operation: insert + start: 990000 + 281: + ids_end: 900000 + ids_start: 895870 + operation: replace + tags_end: 894130 + tags_start: 890000 + 282: + operation: search + max_pts: 696111 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_only_runbook.yaml diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml new file mode 100644 index 00000000..61db003a --- /dev/null +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml @@ -0,0 +1,847 @@ +msturingpq-1M: + 1: + end: 10000 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 20000 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 30000 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 40000 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 50000 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 60000 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 70000 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 80000 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 90000 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 100000 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 110000 + operation: insert + start: 100000 + 22: + operation: search + 23: + end: 120000 + operation: insert + start: 110000 + 24: + operation: search + 25: + end: 130000 + operation: insert + start: 120000 + 26: + operation: search + 27: + end: 140000 + operation: insert + start: 130000 + 28: + operation: search + 29: + end: 150000 + operation: insert + start: 140000 + 30: + operation: search + 31: + end: 160000 + operation: insert + start: 150000 + 32: + operation: search + 33: + end: 170000 + operation: insert + start: 160000 + 34: + operation: search + 35: + end: 180000 + operation: insert + start: 170000 + 36: + operation: search + 37: + end: 190000 + operation: insert + start: 180000 + 38: + operation: search + 39: + end: 200000 + operation: insert + start: 190000 + 40: + operation: search + 41: + end: 210000 + operation: insert + start: 200000 + 42: + end: 10000 + operation: delete + start: 0 + 43: + operation: search + 44: + end: 220000 + operation: insert + start: 210000 + 45: + operation: search + 46: + end: 230000 + operation: insert + start: 220000 + 47: + end: 30000 + operation: delete + start: 20000 + 48: + operation: search + 49: + end: 240000 + operation: insert + start: 230000 + 50: + operation: search + 51: + end: 250000 + operation: insert + start: 240000 + 52: + end: 50000 + operation: delete + start: 40000 + 53: + operation: search + 54: + end: 260000 + operation: insert + start: 250000 + 55: + end: 60000 + operation: delete + start: 50000 + 56: + operation: search + 57: + end: 270000 + operation: insert + start: 260000 + 58: + end: 70000 + operation: delete + start: 60000 + 59: + operation: search + 60: + end: 280000 + operation: insert + start: 270000 + 61: + end: 80000 + operation: delete + start: 70000 + 62: + operation: search + 63: + end: 290000 + operation: insert + start: 280000 + 64: + end: 90000 + operation: delete + start: 80000 + 65: + operation: search + 66: + end: 300000 + operation: insert + start: 290000 + 67: + end: 100000 + operation: delete + start: 90000 + 68: + operation: search + 69: + end: 310000 + operation: insert + start: 300000 + 70: + end: 110000 + operation: delete + start: 100000 + 71: + operation: search + 72: + end: 320000 + operation: insert + start: 310000 + 73: + end: 120000 + operation: delete + start: 110000 + 74: + operation: search + 75: + end: 330000 + operation: insert + start: 320000 + 76: + end: 130000 + operation: delete + start: 120000 + 77: + operation: search + 78: + end: 340000 + operation: insert + start: 330000 + 79: + end: 140000 + operation: delete + start: 130000 + 80: + operation: search + 81: + end: 350000 + operation: insert + start: 340000 + 82: + end: 150000 + operation: delete + start: 140000 + 83: + operation: search + 84: + end: 360000 + operation: insert + start: 350000 + 85: + end: 160000 + operation: delete + start: 150000 + 86: + operation: search + 87: + end: 370000 + operation: insert + start: 360000 + 88: + end: 170000 + operation: delete + start: 160000 + 89: + operation: search + 90: + end: 380000 + operation: insert + start: 370000 + 91: + end: 180000 + operation: delete + start: 170000 + 92: + operation: search + 93: + end: 390000 + operation: insert + start: 380000 + 94: + end: 190000 + operation: delete + start: 180000 + 95: + operation: search + 96: + end: 400000 + operation: insert + start: 390000 + 97: + end: 200000 + operation: delete + start: 190000 + 98: + operation: search + 99: + end: 410000 + operation: insert + start: 400000 + 100: + end: 210000 + operation: delete + start: 200000 + 101: + operation: search + 102: + end: 420000 + operation: insert + start: 410000 + 103: + end: 220000 + operation: delete + start: 210000 + 104: + operation: search + 105: + end: 430000 + operation: insert + start: 420000 + 106: + operation: search + 107: + end: 440000 + operation: insert + start: 430000 + 108: + end: 240000 + operation: delete + start: 230000 + 109: + operation: search + 110: + end: 450000 + operation: insert + start: 440000 + 111: + end: 250000 + operation: delete + start: 240000 + 112: + operation: search + 113: + end: 460000 + operation: insert + start: 450000 + 114: + end: 260000 + operation: delete + start: 250000 + 115: + operation: search + 116: + end: 470000 + operation: insert + start: 460000 + 117: + end: 270000 + operation: delete + start: 260000 + 118: + operation: search + 119: + end: 480000 + operation: insert + start: 470000 + 120: + end: 280000 + operation: delete + start: 270000 + 121: + operation: search + 122: + end: 490000 + operation: insert + start: 480000 + 123: + end: 290000 + operation: delete + start: 280000 + 124: + operation: search + 125: + end: 500000 + operation: insert + start: 490000 + 126: + end: 300000 + operation: delete + start: 290000 + 127: + operation: search + 128: + end: 510000 + operation: insert + start: 500000 + 129: + end: 310000 + operation: delete + start: 300000 + 130: + operation: search + 131: + end: 520000 + operation: insert + start: 510000 + 132: + operation: search + 133: + end: 530000 + operation: insert + start: 520000 + 134: + end: 330000 + operation: delete + start: 320000 + 135: + operation: search + 136: + end: 540000 + operation: insert + start: 530000 + 137: + end: 340000 + operation: delete + start: 330000 + 138: + operation: search + 139: + end: 550000 + operation: insert + start: 540000 + 140: + operation: search + 141: + end: 560000 + operation: insert + start: 550000 + 142: + operation: search + 143: + end: 570000 + operation: insert + start: 560000 + 144: + end: 370000 + operation: delete + start: 360000 + 145: + operation: search + 146: + end: 580000 + operation: insert + start: 570000 + 147: + operation: search + 148: + end: 590000 + operation: insert + start: 580000 + 149: + operation: search + 150: + end: 600000 + operation: insert + start: 590000 + 151: + end: 400000 + operation: delete + start: 390000 + 152: + operation: search + 153: + end: 610000 + operation: insert + start: 600000 + 154: + end: 410000 + operation: delete + start: 400000 + 155: + operation: search + 156: + end: 620000 + operation: insert + start: 610000 + 157: + end: 420000 + operation: delete + start: 410000 + 158: + operation: search + 159: + end: 630000 + operation: insert + start: 620000 + 160: + end: 430000 + operation: delete + start: 420000 + 161: + operation: search + 162: + end: 640000 + operation: insert + start: 630000 + 163: + end: 440000 + operation: delete + start: 430000 + 164: + operation: search + 165: + end: 650000 + operation: insert + start: 640000 + 166: + end: 450000 + operation: delete + start: 440000 + 167: + operation: search + 168: + end: 660000 + operation: insert + start: 650000 + 169: + end: 460000 + operation: delete + start: 450000 + 170: + operation: search + 171: + end: 670000 + operation: insert + start: 660000 + 172: + operation: search + 173: + end: 680000 + operation: insert + start: 670000 + 174: + end: 480000 + operation: delete + start: 470000 + 175: + operation: search + 176: + end: 690000 + operation: insert + start: 680000 + 177: + end: 490000 + operation: delete + start: 480000 + 178: + operation: search + 179: + end: 700000 + operation: insert + start: 690000 + 180: + end: 500000 + operation: delete + start: 490000 + 181: + operation: search + 182: + end: 710000 + operation: insert + start: 700000 + 183: + end: 510000 + operation: delete + start: 500000 + 184: + operation: search + 185: + end: 720000 + operation: insert + start: 710000 + 186: + end: 520000 + operation: delete + start: 510000 + 187: + operation: search + 188: + end: 730000 + operation: insert + start: 720000 + 189: + end: 530000 + operation: delete + start: 520000 + 190: + operation: search + 191: + end: 740000 + operation: insert + start: 730000 + 192: + end: 540000 + operation: delete + start: 530000 + 193: + operation: search + 194: + end: 750000 + operation: insert + start: 740000 + 195: + end: 550000 + operation: delete + start: 540000 + 196: + operation: search + 197: + end: 760000 + operation: insert + start: 750000 + 198: + operation: search + 199: + end: 770000 + operation: insert + start: 760000 + 200: + end: 570000 + operation: delete + start: 560000 + 201: + operation: search + 202: + end: 780000 + operation: insert + start: 770000 + 203: + end: 580000 + operation: delete + start: 570000 + 204: + operation: search + 205: + end: 790000 + operation: insert + start: 780000 + 206: + end: 590000 + operation: delete + start: 580000 + 207: + operation: search + 208: + end: 800000 + operation: insert + start: 790000 + 209: + operation: search + 210: + end: 810000 + operation: insert + start: 800000 + 211: + operation: search + 212: + end: 820000 + operation: insert + start: 810000 + 213: + operation: search + 214: + end: 830000 + operation: insert + start: 820000 + 215: + end: 630000 + operation: delete + start: 620000 + 216: + operation: search + 217: + end: 840000 + operation: insert + start: 830000 + 218: + end: 640000 + operation: delete + start: 630000 + 219: + operation: search + 220: + end: 850000 + operation: insert + start: 840000 + 221: + operation: search + 222: + end: 860000 + operation: insert + start: 850000 + 223: + operation: search + 224: + end: 870000 + operation: insert + start: 860000 + 225: + operation: search + 226: + end: 880000 + operation: insert + start: 870000 + 227: + end: 680000 + operation: delete + start: 670000 + 228: + operation: search + 229: + end: 890000 + operation: insert + start: 880000 + 230: + end: 690000 + operation: delete + start: 680000 + 231: + operation: search + 232: + end: 900000 + operation: insert + start: 890000 + 233: + operation: search + 234: + end: 910000 + operation: insert + start: 900000 + 235: + operation: search + 236: + end: 920000 + operation: insert + start: 910000 + 237: + end: 720000 + operation: delete + start: 710000 + 238: + operation: search + 239: + end: 930000 + operation: insert + start: 920000 + 240: + end: 730000 + operation: delete + start: 720000 + 241: + operation: search + 242: + end: 940000 + operation: insert + start: 930000 + 243: + end: 740000 + operation: delete + start: 730000 + 244: + operation: search + 245: + end: 950000 + operation: insert + start: 940000 + 246: + end: 750000 + operation: delete + start: 740000 + 247: + operation: search + 248: + end: 960000 + operation: insert + start: 950000 + 249: + end: 760000 + operation: delete + start: 750000 + 250: + operation: search + 251: + end: 970000 + operation: insert + start: 960000 + 252: + end: 770000 + operation: delete + start: 760000 + 253: + operation: search + 254: + end: 980000 + operation: insert + start: 970000 + 255: + end: 780000 + operation: delete + start: 770000 + 256: + operation: search + 257: + end: 990000 + operation: insert + start: 980000 + 258: + operation: search + 259: + end: 1000000 + operation: insert + start: 990000 + 260: + end: 800000 + operation: delete + start: 790000 + 261: + operation: search + max_pts: 400000 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml index 322e4326..9532c667 100644 --- a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml +++ b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml @@ -1208,4 +1208,4 @@ wikipedia-1M: 316: operation: search max_pts: 293233 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_delete_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_delete_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml index a26fbe27..7e325a28 100644 --- a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml +++ b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml @@ -1068,4 +1068,4 @@ wikipedia-1M: 278: operation: search max_pts: 698369 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_only_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_only_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml index a9b3aa49..86c5b595 100644 --- a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml +++ b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml @@ -4436,4 +4436,4 @@ wikipedia-35M: 1150: operation: search max_pts: 6682767 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_delete_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_delete_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml index ad5a38be..fe2660d7 100644 --- a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml +++ b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml @@ -852,4 +852,4 @@ wikipedia-35M: 222: operation: search max_pts: 5548955 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_only_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_only_runbook.yaml diff --git a/neurips23/streaming/diskann/config.yaml b/neurips23/streaming/diskann/config.yaml index a0d2b9ab..114d4536 100644 --- a/neurips23/streaming/diskann/config.yaml +++ b/neurips23/streaming/diskann/config.yaml @@ -35,7 +35,7 @@ msspacev-1M: query-args: | [{"Ls":300, "T":16}, {"Ls":100, "T":16}] -msturing-1M: +msturingpq-1M: diskann: docker-tag: neurips23-streaming-diskann module: neurips23.streaming.diskann.diskann-str @@ -44,10 +44,9 @@ msturing-1M: run-groups: base: args: | - [{"R":50, "L":50, "insert_threads":16, "consolidate_threads":16}] + [{"R":64, "L":128, "insert_threads":32, "consolidate_threads":32}] query-args: | - [{"Ls":300, "T":16}, - {"Ls":100, "T":16}] + [{"Ls":128, "T":16}] wikipedia-1M: diskann: docker-tag: neurips23-streaming-diskann diff --git a/requirements_py3.10.txt b/requirements_py3.10.txt index 3c5b5490..6e451c69 100644 --- a/requirements_py3.10.txt +++ b/requirements_py3.10.txt @@ -1,7 +1,7 @@ ansicolors==1.1.8 docker==7.1.0 h5py==3.10.0 -matplotlib==3.3.4 + numpy==1.24.2 pyyaml==6.0 psutil==5.9.4