From d95de3284a5ec6186600e52440d14006e2b50236 Mon Sep 17 00:00:00 2001 From: Magdalen Date: Thu, 14 Nov 2024 14:44:36 -0800 Subject: [PATCH 1/6] added runbooks --- .../runbooks/gen_expiration_time_runbook.py | 20 +++++++++++++++++++ ...xpiration_time_replace_delete_runbook.yaml | 2 +- ..._expiration_time_replace_only_runbook.yaml | 2 +- ...xpiration_time_replace_delete_runbook.yaml | 2 +- ..._expiration_time_replace_only_runbook.yaml | 2 +- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py index 23eb5001..655727ac 100644 --- a/neurips23/runbooks/gen_expiration_time_runbook.py +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -176,3 +176,23 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra max_t = 1000 gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, None) +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 732 +dataset_file = 'msturing_35M_expiration_time_runbook.yaml' +dataset_name = 'msturing-100M' +dataset_size = 35000000 +max_t = 350 +gt_url = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url) + +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 5554 +dataset_file = 'msturing-1M_expiration_time_runbook.yaml' +dataset_name = 'msturing-1M' +dataset_size = 1000000 +max_t = 100 +gt_url = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url) + diff --git a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml index 322e4326..9532c667 100644 --- a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml +++ b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml @@ -1208,4 +1208,4 @@ wikipedia-1M: 316: operation: search max_pts: 293233 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_delete_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_delete_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml index a26fbe27..7e325a28 100644 --- a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml +++ b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml @@ -1068,4 +1068,4 @@ wikipedia-1M: 278: operation: search max_pts: 698369 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_only_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_replace_only_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml index a9b3aa49..86c5b595 100644 --- a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml +++ b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml @@ -4436,4 +4436,4 @@ wikipedia-35M: 1150: operation: search max_pts: 6682767 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_delete_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_delete_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml index ad5a38be..fe2660d7 100644 --- a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml +++ b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml @@ -852,4 +852,4 @@ wikipedia-35M: 222: operation: search max_pts: 5548955 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_only_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expiration_time_replace_only_runbook.yaml From 1ac1618d03a50a937bae5b4d1098b1cb6c35574c Mon Sep 17 00:00:00 2001 From: Magdalen Date: Thu, 14 Nov 2024 14:45:56 -0800 Subject: [PATCH 2/6] add msturing runbooks --- .../runbooks/gen_expiration_time_runbook.py | 2 +- .../msturing-1M_expiration_time_runbook.yaml | 846 +++++ .../msturing-35M_expiration_time_runbook.yaml | 3290 +++++++++++++++++ 3 files changed, 4137 insertions(+), 1 deletion(-) create mode 100644 neurips23/runbooks/msturing-1M_expiration_time_runbook.yaml create mode 100644 neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py index 655727ac..57cf83ba 100644 --- a/neurips23/runbooks/gen_expiration_time_runbook.py +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -179,7 +179,7 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra ratios = (0, 4, 18) timesteps = (0, 100, 20) seed = 732 -dataset_file = 'msturing_35M_expiration_time_runbook.yaml' +dataset_file = 'msturing-35M_expiration_time_runbook.yaml' dataset_name = 'msturing-100M' dataset_size = 35000000 max_t = 350 diff --git a/neurips23/runbooks/msturing-1M_expiration_time_runbook.yaml b/neurips23/runbooks/msturing-1M_expiration_time_runbook.yaml new file mode 100644 index 00000000..770128dd --- /dev/null +++ b/neurips23/runbooks/msturing-1M_expiration_time_runbook.yaml @@ -0,0 +1,846 @@ +msturing-1M: + 1: + end: 10000 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 20000 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 30000 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 40000 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 50000 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 60000 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 70000 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 80000 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 90000 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 100000 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 110000 + operation: insert + start: 100000 + 22: + operation: search + 23: + end: 120000 + operation: insert + start: 110000 + 24: + operation: search + 25: + end: 130000 + operation: insert + start: 120000 + 26: + operation: search + 27: + end: 140000 + operation: insert + start: 130000 + 28: + operation: search + 29: + end: 150000 + operation: insert + start: 140000 + 30: + operation: search + 31: + end: 160000 + operation: insert + start: 150000 + 32: + operation: search + 33: + end: 170000 + operation: insert + start: 160000 + 34: + operation: search + 35: + end: 180000 + operation: insert + start: 170000 + 36: + operation: search + 37: + end: 190000 + operation: insert + start: 180000 + 38: + operation: search + 39: + end: 200000 + operation: insert + start: 190000 + 40: + operation: search + 41: + end: 210000 + operation: insert + start: 200000 + 42: + end: 10000 + operation: delete + start: 0 + 43: + operation: search + 44: + end: 220000 + operation: insert + start: 210000 + 45: + operation: search + 46: + end: 230000 + operation: insert + start: 220000 + 47: + end: 30000 + operation: delete + start: 20000 + 48: + operation: search + 49: + end: 240000 + operation: insert + start: 230000 + 50: + operation: search + 51: + end: 250000 + operation: insert + start: 240000 + 52: + end: 50000 + operation: delete + start: 40000 + 53: + operation: search + 54: + end: 260000 + operation: insert + start: 250000 + 55: + end: 60000 + operation: delete + start: 50000 + 56: + operation: search + 57: + end: 270000 + operation: insert + start: 260000 + 58: + end: 70000 + operation: delete + start: 60000 + 59: + operation: search + 60: + end: 280000 + operation: insert + start: 270000 + 61: + end: 80000 + operation: delete + start: 70000 + 62: + operation: search + 63: + end: 290000 + operation: insert + start: 280000 + 64: + end: 90000 + operation: delete + start: 80000 + 65: + operation: search + 66: + end: 300000 + operation: insert + start: 290000 + 67: + end: 100000 + operation: delete + start: 90000 + 68: + operation: search + 69: + end: 310000 + operation: insert + start: 300000 + 70: + end: 110000 + operation: delete + start: 100000 + 71: + operation: search + 72: + end: 320000 + operation: insert + start: 310000 + 73: + end: 120000 + operation: delete + start: 110000 + 74: + operation: search + 75: + end: 330000 + operation: insert + start: 320000 + 76: + end: 130000 + operation: delete + start: 120000 + 77: + operation: search + 78: + end: 340000 + operation: insert + start: 330000 + 79: + end: 140000 + operation: delete + start: 130000 + 80: + operation: search + 81: + end: 350000 + operation: insert + start: 340000 + 82: + end: 150000 + operation: delete + start: 140000 + 83: + operation: search + 84: + end: 360000 + operation: insert + start: 350000 + 85: + end: 160000 + operation: delete + start: 150000 + 86: + operation: search + 87: + end: 370000 + operation: insert + start: 360000 + 88: + end: 170000 + operation: delete + start: 160000 + 89: + operation: search + 90: + end: 380000 + operation: insert + start: 370000 + 91: + end: 180000 + operation: delete + start: 170000 + 92: + operation: search + 93: + end: 390000 + operation: insert + start: 380000 + 94: + end: 190000 + operation: delete + start: 180000 + 95: + operation: search + 96: + end: 400000 + operation: insert + start: 390000 + 97: + end: 200000 + operation: delete + start: 190000 + 98: + operation: search + 99: + end: 410000 + operation: insert + start: 400000 + 100: + end: 210000 + operation: delete + start: 200000 + 101: + operation: search + 102: + end: 420000 + operation: insert + start: 410000 + 103: + end: 220000 + operation: delete + start: 210000 + 104: + operation: search + 105: + end: 430000 + operation: insert + start: 420000 + 106: + operation: search + 107: + end: 440000 + operation: insert + start: 430000 + 108: + end: 240000 + operation: delete + start: 230000 + 109: + operation: search + 110: + end: 450000 + operation: insert + start: 440000 + 111: + end: 250000 + operation: delete + start: 240000 + 112: + operation: search + 113: + end: 460000 + operation: insert + start: 450000 + 114: + end: 260000 + operation: delete + start: 250000 + 115: + operation: search + 116: + end: 470000 + operation: insert + start: 460000 + 117: + end: 270000 + operation: delete + start: 260000 + 118: + operation: search + 119: + end: 480000 + operation: insert + start: 470000 + 120: + end: 280000 + operation: delete + start: 270000 + 121: + operation: search + 122: + end: 490000 + operation: insert + start: 480000 + 123: + end: 290000 + operation: delete + start: 280000 + 124: + operation: search + 125: + end: 500000 + operation: insert + start: 490000 + 126: + end: 300000 + operation: delete + start: 290000 + 127: + operation: search + 128: + end: 510000 + operation: insert + start: 500000 + 129: + end: 310000 + operation: delete + start: 300000 + 130: + operation: search + 131: + end: 520000 + operation: insert + start: 510000 + 132: + operation: search + 133: + end: 530000 + operation: insert + start: 520000 + 134: + end: 330000 + operation: delete + start: 320000 + 135: + operation: search + 136: + end: 540000 + operation: insert + start: 530000 + 137: + end: 340000 + operation: delete + start: 330000 + 138: + operation: search + 139: + end: 550000 + operation: insert + start: 540000 + 140: + operation: search + 141: + end: 560000 + operation: insert + start: 550000 + 142: + operation: search + 143: + end: 570000 + operation: insert + start: 560000 + 144: + end: 370000 + operation: delete + start: 360000 + 145: + operation: search + 146: + end: 580000 + operation: insert + start: 570000 + 147: + operation: search + 148: + end: 590000 + operation: insert + start: 580000 + 149: + operation: search + 150: + end: 600000 + operation: insert + start: 590000 + 151: + end: 400000 + operation: delete + start: 390000 + 152: + operation: search + 153: + end: 610000 + operation: insert + start: 600000 + 154: + end: 410000 + operation: delete + start: 400000 + 155: + operation: search + 156: + end: 620000 + operation: insert + start: 610000 + 157: + end: 420000 + operation: delete + start: 410000 + 158: + operation: search + 159: + end: 630000 + operation: insert + start: 620000 + 160: + end: 430000 + operation: delete + start: 420000 + 161: + operation: search + 162: + end: 640000 + operation: insert + start: 630000 + 163: + end: 440000 + operation: delete + start: 430000 + 164: + operation: search + 165: + end: 650000 + operation: insert + start: 640000 + 166: + end: 450000 + operation: delete + start: 440000 + 167: + operation: search + 168: + end: 660000 + operation: insert + start: 650000 + 169: + end: 460000 + operation: delete + start: 450000 + 170: + operation: search + 171: + end: 670000 + operation: insert + start: 660000 + 172: + operation: search + 173: + end: 680000 + operation: insert + start: 670000 + 174: + end: 480000 + operation: delete + start: 470000 + 175: + operation: search + 176: + end: 690000 + operation: insert + start: 680000 + 177: + end: 490000 + operation: delete + start: 480000 + 178: + operation: search + 179: + end: 700000 + operation: insert + start: 690000 + 180: + end: 500000 + operation: delete + start: 490000 + 181: + operation: search + 182: + end: 710000 + operation: insert + start: 700000 + 183: + end: 510000 + operation: delete + start: 500000 + 184: + operation: search + 185: + end: 720000 + operation: insert + start: 710000 + 186: + end: 520000 + operation: delete + start: 510000 + 187: + operation: search + 188: + end: 730000 + operation: insert + start: 720000 + 189: + end: 530000 + operation: delete + start: 520000 + 190: + operation: search + 191: + end: 740000 + operation: insert + start: 730000 + 192: + end: 540000 + operation: delete + start: 530000 + 193: + operation: search + 194: + end: 750000 + operation: insert + start: 740000 + 195: + end: 550000 + operation: delete + start: 540000 + 196: + operation: search + 197: + end: 760000 + operation: insert + start: 750000 + 198: + operation: search + 199: + end: 770000 + operation: insert + start: 760000 + 200: + end: 570000 + operation: delete + start: 560000 + 201: + operation: search + 202: + end: 780000 + operation: insert + start: 770000 + 203: + end: 580000 + operation: delete + start: 570000 + 204: + operation: search + 205: + end: 790000 + operation: insert + start: 780000 + 206: + end: 590000 + operation: delete + start: 580000 + 207: + operation: search + 208: + end: 800000 + operation: insert + start: 790000 + 209: + operation: search + 210: + end: 810000 + operation: insert + start: 800000 + 211: + operation: search + 212: + end: 820000 + operation: insert + start: 810000 + 213: + operation: search + 214: + end: 830000 + operation: insert + start: 820000 + 215: + end: 630000 + operation: delete + start: 620000 + 216: + operation: search + 217: + end: 840000 + operation: insert + start: 830000 + 218: + end: 640000 + operation: delete + start: 630000 + 219: + operation: search + 220: + end: 850000 + operation: insert + start: 840000 + 221: + operation: search + 222: + end: 860000 + operation: insert + start: 850000 + 223: + operation: search + 224: + end: 870000 + operation: insert + start: 860000 + 225: + operation: search + 226: + end: 880000 + operation: insert + start: 870000 + 227: + end: 680000 + operation: delete + start: 670000 + 228: + operation: search + 229: + end: 890000 + operation: insert + start: 880000 + 230: + end: 690000 + operation: delete + start: 680000 + 231: + operation: search + 232: + end: 900000 + operation: insert + start: 890000 + 233: + operation: search + 234: + end: 910000 + operation: insert + start: 900000 + 235: + operation: search + 236: + end: 920000 + operation: insert + start: 910000 + 237: + end: 720000 + operation: delete + start: 710000 + 238: + operation: search + 239: + end: 930000 + operation: insert + start: 920000 + 240: + end: 730000 + operation: delete + start: 720000 + 241: + operation: search + 242: + end: 940000 + operation: insert + start: 930000 + 243: + end: 740000 + operation: delete + start: 730000 + 244: + operation: search + 245: + end: 950000 + operation: insert + start: 940000 + 246: + end: 750000 + operation: delete + start: 740000 + 247: + operation: search + 248: + end: 960000 + operation: insert + start: 950000 + 249: + end: 760000 + operation: delete + start: 750000 + 250: + operation: search + 251: + end: 970000 + operation: insert + start: 960000 + 252: + end: 770000 + operation: delete + start: 760000 + 253: + operation: search + 254: + end: 980000 + operation: insert + start: 970000 + 255: + end: 780000 + operation: delete + start: 770000 + 256: + operation: search + 257: + end: 990000 + operation: insert + start: 980000 + 258: + operation: search + 259: + end: 1000000 + operation: insert + start: 990000 + 260: + end: 800000 + operation: delete + start: 790000 + 261: + operation: search + max_pts: 500000 diff --git a/neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml b/neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml new file mode 100644 index 00000000..b1fdb772 --- /dev/null +++ b/neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml @@ -0,0 +1,3290 @@ +msturing-100M: + 1: + end: 100000 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 200000 + operation: insert + start: 100000 + 4: + operation: search + 5: + end: 300000 + operation: insert + start: 200000 + 6: + operation: search + 7: + end: 400000 + operation: insert + start: 300000 + 8: + operation: search + 9: + end: 500000 + operation: insert + start: 400000 + 10: + operation: search + 11: + end: 600000 + operation: insert + start: 500000 + 12: + operation: search + 13: + end: 700000 + operation: insert + start: 600000 + 14: + operation: search + 15: + end: 800000 + operation: insert + start: 700000 + 16: + operation: search + 17: + end: 900000 + operation: insert + start: 800000 + 18: + operation: search + 19: + end: 1000000 + operation: insert + start: 900000 + 20: + operation: search + 21: + end: 1100000 + operation: insert + start: 1000000 + 22: + operation: search + 23: + end: 1200000 + operation: insert + start: 1100000 + 24: + operation: search + 25: + end: 1300000 + operation: insert + start: 1200000 + 26: + operation: search + 27: + end: 1400000 + operation: insert + start: 1300000 + 28: + operation: search + 29: + end: 1500000 + operation: insert + start: 1400000 + 30: + operation: search + 31: + end: 1600000 + operation: insert + start: 1500000 + 32: + operation: search + 33: + end: 1700000 + operation: insert + start: 1600000 + 34: + operation: search + 35: + end: 1800000 + operation: insert + start: 1700000 + 36: + operation: search + 37: + end: 1900000 + operation: insert + start: 1800000 + 38: + operation: search + 39: + end: 2000000 + operation: insert + start: 1900000 + 40: + operation: search + 41: + end: 2100000 + operation: insert + start: 2000000 + 42: + operation: search + 43: + end: 2200000 + operation: insert + start: 2100000 + 44: + end: 200000 + operation: delete + start: 100000 + 45: + operation: search + 46: + end: 2300000 + operation: insert + start: 2200000 + 47: + end: 300000 + operation: delete + start: 200000 + 48: + operation: search + 49: + end: 2400000 + operation: insert + start: 2300000 + 50: + operation: search + 51: + end: 2500000 + operation: insert + start: 2400000 + 52: + operation: search + 53: + end: 2600000 + operation: insert + start: 2500000 + 54: + end: 600000 + operation: delete + start: 500000 + 55: + operation: search + 56: + end: 2700000 + operation: insert + start: 2600000 + 57: + end: 700000 + operation: delete + start: 600000 + 58: + operation: search + 59: + end: 2800000 + operation: insert + start: 2700000 + 60: + end: 800000 + operation: delete + start: 700000 + 61: + operation: search + 62: + end: 2900000 + operation: insert + start: 2800000 + 63: + operation: search + 64: + end: 3000000 + operation: insert + start: 2900000 + 65: + end: 1000000 + operation: delete + start: 900000 + 66: + operation: search + 67: + end: 3100000 + operation: insert + start: 3000000 + 68: + end: 1100000 + operation: delete + start: 1000000 + 69: + operation: search + 70: + end: 3200000 + operation: insert + start: 3100000 + 71: + end: 1200000 + operation: delete + start: 1100000 + 72: + operation: search + 73: + end: 3300000 + operation: insert + start: 3200000 + 74: + end: 1300000 + operation: delete + start: 1200000 + 75: + operation: search + 76: + end: 3400000 + operation: insert + start: 3300000 + 77: + end: 1400000 + operation: delete + start: 1300000 + 78: + operation: search + 79: + end: 3500000 + operation: insert + start: 3400000 + 80: + end: 1500000 + operation: delete + start: 1400000 + 81: + operation: search + 82: + end: 3600000 + operation: insert + start: 3500000 + 83: + operation: search + 84: + end: 3700000 + operation: insert + start: 3600000 + 85: + end: 1700000 + operation: delete + start: 1600000 + 86: + operation: search + 87: + end: 3800000 + operation: insert + start: 3700000 + 88: + operation: search + 89: + end: 3900000 + operation: insert + start: 3800000 + 90: + end: 1900000 + operation: delete + start: 1800000 + 91: + operation: search + 92: + end: 4000000 + operation: insert + start: 3900000 + 93: + end: 2000000 + operation: delete + start: 1900000 + 94: + operation: search + 95: + end: 4100000 + operation: insert + start: 4000000 + 96: + end: 2100000 + operation: delete + start: 2000000 + 97: + operation: search + 98: + end: 4200000 + operation: insert + start: 4100000 + 99: + end: 2200000 + operation: delete + start: 2100000 + 100: + operation: search + 101: + end: 4300000 + operation: insert + start: 4200000 + 102: + end: 2300000 + operation: delete + start: 2200000 + 103: + operation: search + 104: + end: 4400000 + operation: insert + start: 4300000 + 105: + end: 2400000 + operation: delete + start: 2300000 + 106: + operation: search + 107: + end: 4500000 + operation: insert + start: 4400000 + 108: + end: 2500000 + operation: delete + start: 2400000 + 109: + operation: search + 110: + end: 4600000 + operation: insert + start: 4500000 + 111: + end: 2600000 + operation: delete + start: 2500000 + 112: + operation: search + 113: + end: 4700000 + operation: insert + start: 4600000 + 114: + end: 2700000 + operation: delete + start: 2600000 + 115: + operation: search + 116: + end: 4800000 + operation: insert + start: 4700000 + 117: + end: 2800000 + operation: delete + start: 2700000 + 118: + operation: search + 119: + end: 4900000 + operation: insert + start: 4800000 + 120: + end: 2900000 + operation: delete + start: 2800000 + 121: + operation: search + 122: + end: 5000000 + operation: insert + start: 4900000 + 123: + end: 3000000 + operation: delete + start: 2900000 + 124: + operation: search + 125: + end: 5100000 + operation: insert + start: 5000000 + 126: + operation: search + 127: + end: 5200000 + operation: insert + start: 5100000 + 128: + operation: search + 129: + end: 5300000 + operation: insert + start: 5200000 + 130: + end: 3300000 + operation: delete + start: 3200000 + 131: + operation: search + 132: + end: 5400000 + operation: insert + start: 5300000 + 133: + operation: search + 134: + end: 5500000 + operation: insert + start: 5400000 + 135: + end: 3500000 + operation: delete + start: 3400000 + 136: + operation: search + 137: + end: 5600000 + operation: insert + start: 5500000 + 138: + end: 3600000 + operation: delete + start: 3500000 + 139: + operation: search + 140: + end: 5700000 + operation: insert + start: 5600000 + 141: + end: 3700000 + operation: delete + start: 3600000 + 142: + operation: search + 143: + end: 5800000 + operation: insert + start: 5700000 + 144: + end: 3800000 + operation: delete + start: 3700000 + 145: + operation: search + 146: + end: 5900000 + operation: insert + start: 5800000 + 147: + end: 3900000 + operation: delete + start: 3800000 + 148: + operation: search + 149: + end: 6000000 + operation: insert + start: 5900000 + 150: + end: 4000000 + operation: delete + start: 3900000 + 151: + operation: search + 152: + end: 6100000 + operation: insert + start: 6000000 + 153: + operation: search + 154: + end: 6200000 + operation: insert + start: 6100000 + 155: + end: 4200000 + operation: delete + start: 4100000 + 156: + operation: search + 157: + end: 6300000 + operation: insert + start: 6200000 + 158: + end: 4300000 + operation: delete + start: 4200000 + 159: + operation: search + 160: + end: 6400000 + operation: insert + start: 6300000 + 161: + end: 4400000 + operation: delete + start: 4300000 + 162: + operation: search + 163: + end: 6500000 + operation: insert + start: 6400000 + 164: + end: 4500000 + operation: delete + start: 4400000 + 165: + operation: search + 166: + end: 6600000 + operation: insert + start: 6500000 + 167: + end: 4600000 + operation: delete + start: 4500000 + 168: + operation: search + 169: + end: 6700000 + operation: insert + start: 6600000 + 170: + end: 4700000 + operation: delete + start: 4600000 + 171: + operation: search + 172: + end: 6800000 + operation: insert + start: 6700000 + 173: + end: 4800000 + operation: delete + start: 4700000 + 174: + operation: search + 175: + end: 6900000 + operation: insert + start: 6800000 + 176: + end: 4900000 + operation: delete + start: 4800000 + 177: + operation: search + 178: + end: 7000000 + operation: insert + start: 6900000 + 179: + end: 5000000 + operation: delete + start: 4900000 + 180: + operation: search + 181: + end: 7100000 + operation: insert + start: 7000000 + 182: + end: 5100000 + operation: delete + start: 5000000 + 183: + operation: search + 184: + end: 7200000 + operation: insert + start: 7100000 + 185: + end: 5200000 + operation: delete + start: 5100000 + 186: + operation: search + 187: + end: 7300000 + operation: insert + start: 7200000 + 188: + end: 5300000 + operation: delete + start: 5200000 + 189: + operation: search + 190: + end: 7400000 + operation: insert + start: 7300000 + 191: + end: 5400000 + operation: delete + start: 5300000 + 192: + operation: search + 193: + end: 7500000 + operation: insert + start: 7400000 + 194: + end: 5500000 + operation: delete + start: 5400000 + 195: + operation: search + 196: + end: 7600000 + operation: insert + start: 7500000 + 197: + end: 5600000 + operation: delete + start: 5500000 + 198: + operation: search + 199: + end: 7700000 + operation: insert + start: 7600000 + 200: + end: 5700000 + operation: delete + start: 5600000 + 201: + operation: search + 202: + end: 7800000 + operation: insert + start: 7700000 + 203: + operation: search + 204: + end: 7900000 + operation: insert + start: 7800000 + 205: + operation: search + 206: + end: 8000000 + operation: insert + start: 7900000 + 207: + operation: search + 208: + end: 8100000 + operation: insert + start: 8000000 + 209: + end: 6100000 + operation: delete + start: 6000000 + 210: + operation: search + 211: + end: 8200000 + operation: insert + start: 8100000 + 212: + end: 6200000 + operation: delete + start: 6100000 + 213: + operation: search + 214: + end: 8300000 + operation: insert + start: 8200000 + 215: + end: 6300000 + operation: delete + start: 6200000 + 216: + operation: search + 217: + end: 8400000 + operation: insert + start: 8300000 + 218: + end: 6400000 + operation: delete + start: 6300000 + 219: + operation: search + 220: + end: 8500000 + operation: insert + start: 8400000 + 221: + end: 6500000 + operation: delete + start: 6400000 + 222: + operation: search + 223: + end: 8600000 + operation: insert + start: 8500000 + 224: + end: 6600000 + operation: delete + start: 6500000 + 225: + operation: search + 226: + end: 8700000 + operation: insert + start: 8600000 + 227: + end: 6700000 + operation: delete + start: 6600000 + 228: + operation: search + 229: + end: 8800000 + operation: insert + start: 8700000 + 230: + end: 6800000 + operation: delete + start: 6700000 + 231: + operation: search + 232: + end: 8900000 + operation: insert + start: 8800000 + 233: + end: 6900000 + operation: delete + start: 6800000 + 234: + operation: search + 235: + end: 9000000 + operation: insert + start: 8900000 + 236: + end: 7000000 + operation: delete + start: 6900000 + 237: + operation: search + 238: + end: 9100000 + operation: insert + start: 9000000 + 239: + end: 7100000 + operation: delete + start: 7000000 + 240: + operation: search + 241: + end: 9200000 + operation: insert + start: 9100000 + 242: + end: 7200000 + operation: delete + start: 7100000 + 243: + operation: search + 244: + end: 9300000 + operation: insert + start: 9200000 + 245: + operation: search + 246: + end: 9400000 + operation: insert + start: 9300000 + 247: + end: 7400000 + operation: delete + start: 7300000 + 248: + operation: search + 249: + end: 9500000 + operation: insert + start: 9400000 + 250: + end: 7500000 + operation: delete + start: 7400000 + 251: + operation: search + 252: + end: 9600000 + operation: insert + start: 9500000 + 253: + operation: search + 254: + end: 9700000 + operation: insert + start: 9600000 + 255: + end: 7700000 + operation: delete + start: 7600000 + 256: + operation: search + 257: + end: 9800000 + operation: insert + start: 9700000 + 258: + end: 7800000 + operation: delete + start: 7700000 + 259: + operation: search + 260: + end: 9900000 + operation: insert + start: 9800000 + 261: + end: 7900000 + operation: delete + start: 7800000 + 262: + operation: search + 263: + end: 10000000 + operation: insert + start: 9900000 + 264: + end: 8000000 + operation: delete + start: 7900000 + 265: + operation: search + 266: + end: 10100000 + operation: insert + start: 10000000 + 267: + end: 100000 + operation: delete + start: 0 + 268: + end: 8100000 + operation: delete + start: 8000000 + 269: + operation: search + 270: + end: 10200000 + operation: insert + start: 10100000 + 271: + end: 8200000 + operation: delete + start: 8100000 + 272: + operation: search + 273: + end: 10300000 + operation: insert + start: 10200000 + 274: + end: 8300000 + operation: delete + start: 8200000 + 275: + operation: search + 276: + end: 10400000 + operation: insert + start: 10300000 + 277: + end: 400000 + operation: delete + start: 300000 + 278: + end: 8400000 + operation: delete + start: 8300000 + 279: + operation: search + 280: + end: 10500000 + operation: insert + start: 10400000 + 281: + end: 500000 + operation: delete + start: 400000 + 282: + end: 8500000 + operation: delete + start: 8400000 + 283: + operation: search + 284: + end: 10600000 + operation: insert + start: 10500000 + 285: + end: 8600000 + operation: delete + start: 8500000 + 286: + operation: search + 287: + end: 10700000 + operation: insert + start: 10600000 + 288: + end: 8700000 + operation: delete + start: 8600000 + 289: + operation: search + 290: + end: 10800000 + operation: insert + start: 10700000 + 291: + operation: search + 292: + end: 10900000 + operation: insert + start: 10800000 + 293: + end: 900000 + operation: delete + start: 800000 + 294: + end: 8900000 + operation: delete + start: 8800000 + 295: + operation: search + 296: + end: 11000000 + operation: insert + start: 10900000 + 297: + operation: search + 298: + end: 11100000 + operation: insert + start: 11000000 + 299: + end: 9100000 + operation: delete + start: 9000000 + 300: + operation: search + 301: + end: 11200000 + operation: insert + start: 11100000 + 302: + end: 9200000 + operation: delete + start: 9100000 + 303: + operation: search + 304: + end: 11300000 + operation: insert + start: 11200000 + 305: + operation: search + 306: + end: 11400000 + operation: insert + start: 11300000 + 307: + end: 9400000 + operation: delete + start: 9300000 + 308: + operation: search + 309: + end: 11500000 + operation: insert + start: 11400000 + 310: + operation: search + 311: + end: 11600000 + operation: insert + start: 11500000 + 312: + end: 9600000 + operation: delete + start: 9500000 + 313: + operation: search + 314: + end: 11700000 + operation: insert + start: 11600000 + 315: + end: 9700000 + operation: delete + start: 9600000 + 316: + operation: search + 317: + end: 11800000 + operation: insert + start: 11700000 + 318: + end: 9800000 + operation: delete + start: 9700000 + 319: + operation: search + 320: + end: 11900000 + operation: insert + start: 11800000 + 321: + end: 9900000 + operation: delete + start: 9800000 + 322: + operation: search + 323: + end: 12000000 + operation: insert + start: 11900000 + 324: + end: 10000000 + operation: delete + start: 9900000 + 325: + operation: search + 326: + end: 12100000 + operation: insert + start: 12000000 + 327: + end: 10100000 + operation: delete + start: 10000000 + 328: + operation: search + 329: + end: 12200000 + operation: insert + start: 12100000 + 330: + end: 10200000 + operation: delete + start: 10100000 + 331: + operation: search + 332: + end: 12300000 + operation: insert + start: 12200000 + 333: + end: 10300000 + operation: delete + start: 10200000 + 334: + operation: search + 335: + end: 12400000 + operation: insert + start: 12300000 + 336: + end: 10400000 + operation: delete + start: 10300000 + 337: + operation: search + 338: + end: 12500000 + operation: insert + start: 12400000 + 339: + operation: search + 340: + end: 12600000 + operation: insert + start: 12500000 + 341: + end: 10600000 + operation: delete + start: 10500000 + 342: + operation: search + 343: + end: 12700000 + operation: insert + start: 12600000 + 344: + end: 10700000 + operation: delete + start: 10600000 + 345: + operation: search + 346: + end: 12800000 + operation: insert + start: 12700000 + 347: + end: 10800000 + operation: delete + start: 10700000 + 348: + operation: search + 349: + end: 12900000 + operation: insert + start: 12800000 + 350: + operation: search + 351: + end: 13000000 + operation: insert + start: 12900000 + 352: + end: 11000000 + operation: delete + start: 10900000 + 353: + operation: search + 354: + end: 13100000 + operation: insert + start: 13000000 + 355: + end: 3100000 + operation: delete + start: 3000000 + 356: + end: 11100000 + operation: delete + start: 11000000 + 357: + operation: search + 358: + end: 13200000 + operation: insert + start: 13100000 + 359: + end: 11200000 + operation: delete + start: 11100000 + 360: + operation: search + 361: + end: 13300000 + operation: insert + start: 13200000 + 362: + end: 11300000 + operation: delete + start: 11200000 + 363: + operation: search + 364: + end: 13400000 + operation: insert + start: 13300000 + 365: + end: 3400000 + operation: delete + start: 3300000 + 366: + end: 11400000 + operation: delete + start: 11300000 + 367: + operation: search + 368: + end: 13500000 + operation: insert + start: 13400000 + 369: + end: 11500000 + operation: delete + start: 11400000 + 370: + operation: search + 371: + end: 13600000 + operation: insert + start: 13500000 + 372: + end: 11600000 + operation: delete + start: 11500000 + 373: + operation: search + 374: + end: 13700000 + operation: insert + start: 13600000 + 375: + end: 11700000 + operation: delete + start: 11600000 + 376: + operation: search + 377: + end: 13800000 + operation: insert + start: 13700000 + 378: + end: 11800000 + operation: delete + start: 11700000 + 379: + operation: search + 380: + end: 13900000 + operation: insert + start: 13800000 + 381: + end: 11900000 + operation: delete + start: 11800000 + 382: + operation: search + 383: + end: 14000000 + operation: insert + start: 13900000 + 384: + end: 12000000 + operation: delete + start: 11900000 + 385: + operation: search + 386: + end: 14100000 + operation: insert + start: 14000000 + 387: + end: 4100000 + operation: delete + start: 4000000 + 388: + end: 12100000 + operation: delete + start: 12000000 + 389: + operation: search + 390: + end: 14200000 + operation: insert + start: 14100000 + 391: + end: 12200000 + operation: delete + start: 12100000 + 392: + operation: search + 393: + end: 14300000 + operation: insert + start: 14200000 + 394: + end: 12300000 + operation: delete + start: 12200000 + 395: + operation: search + 396: + end: 14400000 + operation: insert + start: 14300000 + 397: + end: 12400000 + operation: delete + start: 12300000 + 398: + operation: search + 399: + end: 14500000 + operation: insert + start: 14400000 + 400: + end: 12500000 + operation: delete + start: 12400000 + 401: + operation: search + 402: + end: 14600000 + operation: insert + start: 14500000 + 403: + operation: search + 404: + end: 14700000 + operation: insert + start: 14600000 + 405: + operation: search + 406: + end: 14800000 + operation: insert + start: 14700000 + 407: + operation: search + 408: + end: 14900000 + operation: insert + start: 14800000 + 409: + operation: search + 410: + end: 15000000 + operation: insert + start: 14900000 + 411: + end: 13000000 + operation: delete + start: 12900000 + 412: + operation: search + 413: + end: 15100000 + operation: insert + start: 15000000 + 414: + operation: search + 415: + end: 15200000 + operation: insert + start: 15100000 + 416: + operation: search + 417: + end: 15300000 + operation: insert + start: 15200000 + 418: + end: 13300000 + operation: delete + start: 13200000 + 419: + operation: search + 420: + end: 15400000 + operation: insert + start: 15300000 + 421: + end: 13400000 + operation: delete + start: 13300000 + 422: + operation: search + 423: + end: 15500000 + operation: insert + start: 15400000 + 424: + end: 13500000 + operation: delete + start: 13400000 + 425: + operation: search + 426: + end: 15600000 + operation: insert + start: 15500000 + 427: + end: 13600000 + operation: delete + start: 13500000 + 428: + operation: search + 429: + end: 15700000 + operation: insert + start: 15600000 + 430: + operation: search + 431: + end: 15800000 + operation: insert + start: 15700000 + 432: + end: 5800000 + operation: delete + start: 5700000 + 433: + end: 13800000 + operation: delete + start: 13700000 + 434: + operation: search + 435: + end: 15900000 + operation: insert + start: 15800000 + 436: + end: 5900000 + operation: delete + start: 5800000 + 437: + operation: search + 438: + end: 16000000 + operation: insert + start: 15900000 + 439: + end: 6000000 + operation: delete + start: 5900000 + 440: + end: 14000000 + operation: delete + start: 13900000 + 441: + operation: search + 442: + end: 16100000 + operation: insert + start: 16000000 + 443: + operation: search + 444: + end: 16200000 + operation: insert + start: 16100000 + 445: + end: 14200000 + operation: delete + start: 14100000 + 446: + operation: search + 447: + end: 16300000 + operation: insert + start: 16200000 + 448: + end: 14300000 + operation: delete + start: 14200000 + 449: + operation: search + 450: + end: 16400000 + operation: insert + start: 16300000 + 451: + end: 14400000 + operation: delete + start: 14300000 + 452: + operation: search + 453: + end: 16500000 + operation: insert + start: 16400000 + 454: + operation: search + 455: + end: 16600000 + operation: insert + start: 16500000 + 456: + end: 14600000 + operation: delete + start: 14500000 + 457: + operation: search + 458: + end: 16700000 + operation: insert + start: 16600000 + 459: + operation: search + 460: + end: 16800000 + operation: insert + start: 16700000 + 461: + end: 14800000 + operation: delete + start: 14700000 + 462: + operation: search + 463: + end: 16900000 + operation: insert + start: 16800000 + 464: + operation: search + 465: + end: 17000000 + operation: insert + start: 16900000 + 466: + end: 15000000 + operation: delete + start: 14900000 + 467: + operation: search + 468: + end: 17100000 + operation: insert + start: 17000000 + 469: + end: 15100000 + operation: delete + start: 15000000 + 470: + operation: search + 471: + end: 17200000 + operation: insert + start: 17100000 + 472: + end: 15200000 + operation: delete + start: 15100000 + 473: + operation: search + 474: + end: 17300000 + operation: insert + start: 17200000 + 475: + end: 15300000 + operation: delete + start: 15200000 + 476: + operation: search + 477: + end: 17400000 + operation: insert + start: 17300000 + 478: + end: 15400000 + operation: delete + start: 15300000 + 479: + operation: search + 480: + end: 17500000 + operation: insert + start: 17400000 + 481: + operation: search + 482: + end: 17600000 + operation: insert + start: 17500000 + 483: + end: 7600000 + operation: delete + start: 7500000 + 484: + end: 15600000 + operation: delete + start: 15500000 + 485: + operation: search + 486: + end: 17700000 + operation: insert + start: 17600000 + 487: + end: 15700000 + operation: delete + start: 15600000 + 488: + operation: search + 489: + end: 17800000 + operation: insert + start: 17700000 + 490: + end: 15800000 + operation: delete + start: 15700000 + 491: + operation: search + 492: + end: 17900000 + operation: insert + start: 17800000 + 493: + end: 15900000 + operation: delete + start: 15800000 + 494: + operation: search + 495: + end: 18000000 + operation: insert + start: 17900000 + 496: + end: 16000000 + operation: delete + start: 15900000 + 497: + operation: search + 498: + end: 18100000 + operation: insert + start: 18000000 + 499: + operation: search + 500: + end: 18200000 + operation: insert + start: 18100000 + 501: + end: 16200000 + operation: delete + start: 16100000 + 502: + operation: search + 503: + end: 18300000 + operation: insert + start: 18200000 + 504: + end: 16300000 + operation: delete + start: 16200000 + 505: + operation: search + 506: + end: 18400000 + operation: insert + start: 18300000 + 507: + end: 16400000 + operation: delete + start: 16300000 + 508: + operation: search + 509: + end: 18500000 + operation: insert + start: 18400000 + 510: + operation: search + 511: + end: 18600000 + operation: insert + start: 18500000 + 512: + end: 16600000 + operation: delete + start: 16500000 + 513: + operation: search + 514: + end: 18700000 + operation: insert + start: 18600000 + 515: + end: 16700000 + operation: delete + start: 16600000 + 516: + operation: search + 517: + end: 18800000 + operation: insert + start: 18700000 + 518: + end: 8800000 + operation: delete + start: 8700000 + 519: + end: 16800000 + operation: delete + start: 16700000 + 520: + operation: search + 521: + end: 18900000 + operation: insert + start: 18800000 + 522: + end: 16900000 + operation: delete + start: 16800000 + 523: + operation: search + 524: + end: 19000000 + operation: insert + start: 18900000 + 525: + end: 9000000 + operation: delete + start: 8900000 + 526: + end: 17000000 + operation: delete + start: 16900000 + 527: + operation: search + 528: + end: 19100000 + operation: insert + start: 19000000 + 529: + end: 17100000 + operation: delete + start: 17000000 + 530: + operation: search + 531: + end: 19200000 + operation: insert + start: 19100000 + 532: + end: 17200000 + operation: delete + start: 17100000 + 533: + operation: search + 534: + end: 19300000 + operation: insert + start: 19200000 + 535: + end: 17300000 + operation: delete + start: 17200000 + 536: + operation: search + 537: + end: 19400000 + operation: insert + start: 19300000 + 538: + end: 17400000 + operation: delete + start: 17300000 + 539: + operation: search + 540: + end: 19500000 + operation: insert + start: 19400000 + 541: + end: 9500000 + operation: delete + start: 9400000 + 542: + end: 17500000 + operation: delete + start: 17400000 + 543: + operation: search + 544: + end: 19600000 + operation: insert + start: 19500000 + 545: + end: 17600000 + operation: delete + start: 17500000 + 546: + operation: search + 547: + end: 19700000 + operation: insert + start: 19600000 + 548: + end: 17700000 + operation: delete + start: 17600000 + 549: + operation: search + 550: + end: 19800000 + operation: insert + start: 19700000 + 551: + end: 17800000 + operation: delete + start: 17700000 + 552: + operation: search + 553: + end: 19900000 + operation: insert + start: 19800000 + 554: + end: 17900000 + operation: delete + start: 17800000 + 555: + operation: search + 556: + end: 20000000 + operation: insert + start: 19900000 + 557: + end: 18000000 + operation: delete + start: 17900000 + 558: + operation: search + 559: + end: 20100000 + operation: insert + start: 20000000 + 560: + end: 18100000 + operation: delete + start: 18000000 + 561: + operation: search + 562: + end: 20200000 + operation: insert + start: 20100000 + 563: + end: 18200000 + operation: delete + start: 18100000 + 564: + operation: search + 565: + end: 20300000 + operation: insert + start: 20200000 + 566: + end: 18300000 + operation: delete + start: 18200000 + 567: + operation: search + 568: + end: 20400000 + operation: insert + start: 20300000 + 569: + end: 18400000 + operation: delete + start: 18300000 + 570: + operation: search + 571: + end: 20500000 + operation: insert + start: 20400000 + 572: + end: 10500000 + operation: delete + start: 10400000 + 573: + end: 18500000 + operation: delete + start: 18400000 + 574: + operation: search + 575: + end: 20600000 + operation: insert + start: 20500000 + 576: + operation: search + 577: + end: 20700000 + operation: insert + start: 20600000 + 578: + end: 18700000 + operation: delete + start: 18600000 + 579: + operation: search + 580: + end: 20800000 + operation: insert + start: 20700000 + 581: + end: 18800000 + operation: delete + start: 18700000 + 582: + operation: search + 583: + end: 20900000 + operation: insert + start: 20800000 + 584: + end: 10900000 + operation: delete + start: 10800000 + 585: + end: 18900000 + operation: delete + start: 18800000 + 586: + operation: search + 587: + end: 21000000 + operation: insert + start: 20900000 + 588: + end: 19000000 + operation: delete + start: 18900000 + 589: + operation: search + 590: + end: 21100000 + operation: insert + start: 21000000 + 591: + operation: search + 592: + end: 21200000 + operation: insert + start: 21100000 + 593: + end: 19200000 + operation: delete + start: 19100000 + 594: + operation: search + 595: + end: 21300000 + operation: insert + start: 21200000 + 596: + end: 19300000 + operation: delete + start: 19200000 + 597: + operation: search + 598: + end: 21400000 + operation: insert + start: 21300000 + 599: + end: 19400000 + operation: delete + start: 19300000 + 600: + operation: search + 601: + end: 21500000 + operation: insert + start: 21400000 + 602: + end: 19500000 + operation: delete + start: 19400000 + 603: + operation: search + 604: + end: 21600000 + operation: insert + start: 21500000 + 605: + end: 19600000 + operation: delete + start: 19500000 + 606: + operation: search + 607: + end: 21700000 + operation: insert + start: 21600000 + 608: + end: 19700000 + operation: delete + start: 19600000 + 609: + operation: search + 610: + end: 21800000 + operation: insert + start: 21700000 + 611: + end: 19800000 + operation: delete + start: 19700000 + 612: + operation: search + 613: + end: 21900000 + operation: insert + start: 21800000 + 614: + operation: search + 615: + end: 22000000 + operation: insert + start: 21900000 + 616: + end: 20000000 + operation: delete + start: 19900000 + 617: + operation: search + 618: + end: 22100000 + operation: insert + start: 22000000 + 619: + end: 20100000 + operation: delete + start: 20000000 + 620: + operation: search + 621: + end: 22200000 + operation: insert + start: 22100000 + 622: + end: 20200000 + operation: delete + start: 20100000 + 623: + operation: search + 624: + end: 22300000 + operation: insert + start: 22200000 + 625: + end: 20300000 + operation: delete + start: 20200000 + 626: + operation: search + 627: + end: 22400000 + operation: insert + start: 22300000 + 628: + end: 20400000 + operation: delete + start: 20300000 + 629: + operation: search + 630: + end: 22500000 + operation: insert + start: 22400000 + 631: + end: 20500000 + operation: delete + start: 20400000 + 632: + operation: search + 633: + end: 22600000 + operation: insert + start: 22500000 + 634: + end: 12600000 + operation: delete + start: 12500000 + 635: + end: 20600000 + operation: delete + start: 20500000 + 636: + operation: search + 637: + end: 22700000 + operation: insert + start: 22600000 + 638: + operation: search + 639: + end: 22800000 + operation: insert + start: 22700000 + 640: + end: 12800000 + operation: delete + start: 12700000 + 641: + end: 20800000 + operation: delete + start: 20700000 + 642: + operation: search + 643: + end: 22900000 + operation: insert + start: 22800000 + 644: + end: 12900000 + operation: delete + start: 12800000 + 645: + operation: search + 646: + end: 23000000 + operation: insert + start: 22900000 + 647: + end: 21000000 + operation: delete + start: 20900000 + 648: + operation: search + 649: + end: 23100000 + operation: insert + start: 23000000 + 650: + end: 13100000 + operation: delete + start: 13000000 + 651: + end: 21100000 + operation: delete + start: 21000000 + 652: + operation: search + 653: + end: 23200000 + operation: insert + start: 23100000 + 654: + end: 21200000 + operation: delete + start: 21100000 + 655: + operation: search + 656: + end: 23300000 + operation: insert + start: 23200000 + 657: + operation: search + 658: + end: 23400000 + operation: insert + start: 23300000 + 659: + end: 21400000 + operation: delete + start: 21300000 + 660: + operation: search + 661: + end: 23500000 + operation: insert + start: 23400000 + 662: + end: 21500000 + operation: delete + start: 21400000 + 663: + operation: search + 664: + end: 23600000 + operation: insert + start: 23500000 + 665: + operation: search + 666: + end: 23700000 + operation: insert + start: 23600000 + 667: + end: 13700000 + operation: delete + start: 13600000 + 668: + end: 21700000 + operation: delete + start: 21600000 + 669: + operation: search + 670: + end: 23800000 + operation: insert + start: 23700000 + 671: + end: 21800000 + operation: delete + start: 21700000 + 672: + operation: search + 673: + end: 23900000 + operation: insert + start: 23800000 + 674: + end: 21900000 + operation: delete + start: 21800000 + 675: + operation: search + 676: + end: 24000000 + operation: insert + start: 23900000 + 677: + end: 22000000 + operation: delete + start: 21900000 + 678: + operation: search + 679: + end: 24100000 + operation: insert + start: 24000000 + 680: + end: 22100000 + operation: delete + start: 22000000 + 681: + operation: search + 682: + end: 24200000 + operation: insert + start: 24100000 + 683: + end: 22200000 + operation: delete + start: 22100000 + 684: + operation: search + 685: + end: 24300000 + operation: insert + start: 24200000 + 686: + operation: search + 687: + end: 24400000 + operation: insert + start: 24300000 + 688: + end: 22400000 + operation: delete + start: 22300000 + 689: + operation: search + 690: + end: 24500000 + operation: insert + start: 24400000 + 691: + end: 14500000 + operation: delete + start: 14400000 + 692: + operation: search + 693: + end: 24600000 + operation: insert + start: 24500000 + 694: + end: 22600000 + operation: delete + start: 22500000 + 695: + operation: search + 696: + end: 24700000 + operation: insert + start: 24600000 + 697: + end: 22700000 + operation: delete + start: 22600000 + 698: + operation: search + 699: + end: 24800000 + operation: insert + start: 24700000 + 700: + end: 22800000 + operation: delete + start: 22700000 + 701: + operation: search + 702: + end: 24900000 + operation: insert + start: 24800000 + 703: + end: 14900000 + operation: delete + start: 14800000 + 704: + end: 22900000 + operation: delete + start: 22800000 + 705: + operation: search + 706: + end: 25000000 + operation: insert + start: 24900000 + 707: + end: 23000000 + operation: delete + start: 22900000 + 708: + operation: search + 709: + end: 25100000 + operation: insert + start: 25000000 + 710: + end: 23100000 + operation: delete + start: 23000000 + 711: + operation: search + 712: + end: 25200000 + operation: insert + start: 25100000 + 713: + end: 23200000 + operation: delete + start: 23100000 + 714: + operation: search + 715: + end: 25300000 + operation: insert + start: 25200000 + 716: + end: 23300000 + operation: delete + start: 23200000 + 717: + operation: search + 718: + end: 25400000 + operation: insert + start: 25300000 + 719: + end: 23400000 + operation: delete + start: 23300000 + 720: + operation: search + 721: + end: 25500000 + operation: insert + start: 25400000 + 722: + operation: search + 723: + end: 25600000 + operation: insert + start: 25500000 + 724: + operation: search + 725: + end: 25700000 + operation: insert + start: 25600000 + 726: + end: 23700000 + operation: delete + start: 23600000 + 727: + operation: search + 728: + end: 25800000 + operation: insert + start: 25700000 + 729: + end: 23800000 + operation: delete + start: 23700000 + 730: + operation: search + 731: + end: 25900000 + operation: insert + start: 25800000 + 732: + end: 23900000 + operation: delete + start: 23800000 + 733: + operation: search + 734: + end: 26000000 + operation: insert + start: 25900000 + 735: + operation: search + 736: + end: 26100000 + operation: insert + start: 26000000 + 737: + end: 24100000 + operation: delete + start: 24000000 + 738: + operation: search + 739: + end: 26200000 + operation: insert + start: 26100000 + 740: + end: 24200000 + operation: delete + start: 24100000 + 741: + operation: search + 742: + end: 26300000 + operation: insert + start: 26200000 + 743: + end: 24300000 + operation: delete + start: 24200000 + 744: + operation: search + 745: + end: 26400000 + operation: insert + start: 26300000 + 746: + end: 24400000 + operation: delete + start: 24300000 + 747: + operation: search + 748: + end: 26500000 + operation: insert + start: 26400000 + 749: + end: 24500000 + operation: delete + start: 24400000 + 750: + operation: search + 751: + end: 26600000 + operation: insert + start: 26500000 + 752: + end: 24600000 + operation: delete + start: 24500000 + 753: + operation: search + 754: + end: 26700000 + operation: insert + start: 26600000 + 755: + end: 24700000 + operation: delete + start: 24600000 + 756: + operation: search + 757: + end: 26800000 + operation: insert + start: 26700000 + 758: + end: 24800000 + operation: delete + start: 24700000 + 759: + operation: search + 760: + end: 26900000 + operation: insert + start: 26800000 + 761: + end: 24900000 + operation: delete + start: 24800000 + 762: + operation: search + 763: + end: 27000000 + operation: insert + start: 26900000 + 764: + end: 25000000 + operation: delete + start: 24900000 + 765: + operation: search + 766: + end: 27100000 + operation: insert + start: 27000000 + 767: + operation: search + 768: + end: 27200000 + operation: insert + start: 27100000 + 769: + end: 25200000 + operation: delete + start: 25100000 + 770: + operation: search + 771: + end: 27300000 + operation: insert + start: 27200000 + 772: + end: 25300000 + operation: delete + start: 25200000 + 773: + operation: search + 774: + end: 27400000 + operation: insert + start: 27300000 + 775: + end: 25400000 + operation: delete + start: 25300000 + 776: + operation: search + 777: + end: 27500000 + operation: insert + start: 27400000 + 778: + end: 25500000 + operation: delete + start: 25400000 + 779: + operation: search + 780: + end: 27600000 + operation: insert + start: 27500000 + 781: + end: 25600000 + operation: delete + start: 25500000 + 782: + operation: search + 783: + end: 27700000 + operation: insert + start: 27600000 + 784: + operation: search + 785: + end: 27800000 + operation: insert + start: 27700000 + 786: + end: 25800000 + operation: delete + start: 25700000 + 787: + operation: search + 788: + end: 27900000 + operation: insert + start: 27800000 + 789: + end: 25900000 + operation: delete + start: 25800000 + 790: + operation: search + 791: + end: 28000000 + operation: insert + start: 27900000 + 792: + operation: search + 793: + end: 28100000 + operation: insert + start: 28000000 + 794: + end: 26100000 + operation: delete + start: 26000000 + 795: + operation: search + 796: + end: 28200000 + operation: insert + start: 28100000 + 797: + end: 26200000 + operation: delete + start: 26100000 + 798: + operation: search + 799: + end: 28300000 + operation: insert + start: 28200000 + 800: + end: 26300000 + operation: delete + start: 26200000 + 801: + operation: search + 802: + end: 28400000 + operation: insert + start: 28300000 + 803: + end: 26400000 + operation: delete + start: 26300000 + 804: + operation: search + 805: + end: 28500000 + operation: insert + start: 28400000 + 806: + end: 26500000 + operation: delete + start: 26400000 + 807: + operation: search + 808: + end: 28600000 + operation: insert + start: 28500000 + 809: + end: 18600000 + operation: delete + start: 18500000 + 810: + end: 26600000 + operation: delete + start: 26500000 + 811: + operation: search + 812: + end: 28700000 + operation: insert + start: 28600000 + 813: + end: 26700000 + operation: delete + start: 26600000 + 814: + operation: search + 815: + end: 28800000 + operation: insert + start: 28700000 + 816: + end: 26800000 + operation: delete + start: 26700000 + 817: + operation: search + 818: + end: 28900000 + operation: insert + start: 28800000 + 819: + operation: search + 820: + end: 29000000 + operation: insert + start: 28900000 + 821: + end: 27000000 + operation: delete + start: 26900000 + 822: + operation: search + 823: + end: 29100000 + operation: insert + start: 29000000 + 824: + end: 19100000 + operation: delete + start: 19000000 + 825: + end: 27100000 + operation: delete + start: 27000000 + 826: + operation: search + 827: + end: 29200000 + operation: insert + start: 29100000 + 828: + end: 27200000 + operation: delete + start: 27100000 + 829: + operation: search + 830: + end: 29300000 + operation: insert + start: 29200000 + 831: + end: 27300000 + operation: delete + start: 27200000 + 832: + operation: search + 833: + end: 29400000 + operation: insert + start: 29300000 + 834: + end: 27400000 + operation: delete + start: 27300000 + 835: + operation: search + 836: + end: 29500000 + operation: insert + start: 29400000 + 837: + end: 27500000 + operation: delete + start: 27400000 + 838: + operation: search + 839: + end: 29600000 + operation: insert + start: 29500000 + 840: + end: 27600000 + operation: delete + start: 27500000 + 841: + operation: search + 842: + end: 29700000 + operation: insert + start: 29600000 + 843: + end: 27700000 + operation: delete + start: 27600000 + 844: + operation: search + 845: + end: 29800000 + operation: insert + start: 29700000 + 846: + end: 27800000 + operation: delete + start: 27700000 + 847: + operation: search + 848: + end: 29900000 + operation: insert + start: 29800000 + 849: + end: 19900000 + operation: delete + start: 19800000 + 850: + operation: search + 851: + end: 30000000 + operation: insert + start: 29900000 + 852: + end: 28000000 + operation: delete + start: 27900000 + 853: + operation: search + 854: + end: 30100000 + operation: insert + start: 30000000 + 855: + operation: search + 856: + end: 30200000 + operation: insert + start: 30100000 + 857: + operation: search + 858: + end: 30300000 + operation: insert + start: 30200000 + 859: + end: 28300000 + operation: delete + start: 28200000 + 860: + operation: search + 861: + end: 30400000 + operation: insert + start: 30300000 + 862: + end: 28400000 + operation: delete + start: 28300000 + 863: + operation: search + 864: + end: 30500000 + operation: insert + start: 30400000 + 865: + end: 28500000 + operation: delete + start: 28400000 + 866: + operation: search + 867: + end: 30600000 + operation: insert + start: 30500000 + 868: + end: 28600000 + operation: delete + start: 28500000 + 869: + operation: search + 870: + end: 30700000 + operation: insert + start: 30600000 + 871: + end: 20700000 + operation: delete + start: 20600000 + 872: + end: 28700000 + operation: delete + start: 28600000 + 873: + operation: search + 874: + end: 30800000 + operation: insert + start: 30700000 + 875: + end: 28800000 + operation: delete + start: 28700000 + 876: + operation: search + 877: + end: 30900000 + operation: insert + start: 30800000 + 878: + end: 20900000 + operation: delete + start: 20800000 + 879: + end: 28900000 + operation: delete + start: 28800000 + 880: + operation: search + 881: + end: 31000000 + operation: insert + start: 30900000 + 882: + operation: search + 883: + end: 31100000 + operation: insert + start: 31000000 + 884: + operation: search + 885: + end: 31200000 + operation: insert + start: 31100000 + 886: + end: 29200000 + operation: delete + start: 29100000 + 887: + operation: search + 888: + end: 31300000 + operation: insert + start: 31200000 + 889: + end: 21300000 + operation: delete + start: 21200000 + 890: + end: 29300000 + operation: delete + start: 29200000 + 891: + operation: search + 892: + end: 31400000 + operation: insert + start: 31300000 + 893: + end: 29400000 + operation: delete + start: 29300000 + 894: + operation: search + 895: + end: 31500000 + operation: insert + start: 31400000 + 896: + end: 29500000 + operation: delete + start: 29400000 + 897: + operation: search + 898: + end: 31600000 + operation: insert + start: 31500000 + 899: + end: 21600000 + operation: delete + start: 21500000 + 900: + operation: search + 901: + end: 31700000 + operation: insert + start: 31600000 + 902: + end: 29700000 + operation: delete + start: 29600000 + 903: + operation: search + 904: + end: 31800000 + operation: insert + start: 31700000 + 905: + operation: search + 906: + end: 31900000 + operation: insert + start: 31800000 + 907: + end: 29900000 + operation: delete + start: 29800000 + 908: + operation: search + 909: + end: 32000000 + operation: insert + start: 31900000 + 910: + operation: search + 911: + end: 32100000 + operation: insert + start: 32000000 + 912: + end: 30100000 + operation: delete + start: 30000000 + 913: + operation: search + 914: + end: 32200000 + operation: insert + start: 32100000 + 915: + end: 30200000 + operation: delete + start: 30100000 + 916: + operation: search + 917: + end: 32300000 + operation: insert + start: 32200000 + 918: + end: 22300000 + operation: delete + start: 22200000 + 919: + operation: search + 920: + end: 32400000 + operation: insert + start: 32300000 + 921: + end: 30400000 + operation: delete + start: 30300000 + 922: + operation: search + 923: + end: 32500000 + operation: insert + start: 32400000 + 924: + end: 22500000 + operation: delete + start: 22400000 + 925: + end: 30500000 + operation: delete + start: 30400000 + 926: + operation: search + 927: + end: 32600000 + operation: insert + start: 32500000 + 928: + end: 30600000 + operation: delete + start: 30500000 + 929: + operation: search + 930: + end: 32700000 + operation: insert + start: 32600000 + 931: + end: 30700000 + operation: delete + start: 30600000 + 932: + operation: search + 933: + end: 32800000 + operation: insert + start: 32700000 + 934: + end: 30800000 + operation: delete + start: 30700000 + 935: + operation: search + 936: + end: 32900000 + operation: insert + start: 32800000 + 937: + end: 30900000 + operation: delete + start: 30800000 + 938: + operation: search + 939: + end: 33000000 + operation: insert + start: 32900000 + 940: + operation: search + 941: + end: 33100000 + operation: insert + start: 33000000 + 942: + operation: search + 943: + end: 33200000 + operation: insert + start: 33100000 + 944: + end: 31200000 + operation: delete + start: 31100000 + 945: + operation: search + 946: + end: 33300000 + operation: insert + start: 33200000 + 947: + end: 31300000 + operation: delete + start: 31200000 + 948: + operation: search + 949: + end: 33400000 + operation: insert + start: 33300000 + 950: + operation: search + 951: + end: 33500000 + operation: insert + start: 33400000 + 952: + end: 23500000 + operation: delete + start: 23400000 + 953: + end: 31500000 + operation: delete + start: 31400000 + 954: + operation: search + 955: + end: 33600000 + operation: insert + start: 33500000 + 956: + end: 31600000 + operation: delete + start: 31500000 + 957: + operation: search + 958: + end: 33700000 + operation: insert + start: 33600000 + 959: + end: 31700000 + operation: delete + start: 31600000 + 960: + operation: search + 961: + end: 33800000 + operation: insert + start: 33700000 + 962: + end: 31800000 + operation: delete + start: 31700000 + 963: + operation: search + 964: + end: 33900000 + operation: insert + start: 33800000 + 965: + operation: search + 966: + end: 34000000 + operation: insert + start: 33900000 + 967: + end: 24000000 + operation: delete + start: 23900000 + 968: + end: 32000000 + operation: delete + start: 31900000 + 969: + operation: search + 970: + end: 34100000 + operation: insert + start: 34000000 + 971: + end: 32100000 + operation: delete + start: 32000000 + 972: + operation: search + 973: + end: 34200000 + operation: insert + start: 34100000 + 974: + end: 32200000 + operation: delete + start: 32100000 + 975: + operation: search + 976: + end: 34300000 + operation: insert + start: 34200000 + 977: + end: 32300000 + operation: delete + start: 32200000 + 978: + operation: search + 979: + end: 34400000 + operation: insert + start: 34300000 + 980: + operation: search + 981: + end: 34500000 + operation: insert + start: 34400000 + 982: + end: 32500000 + operation: delete + start: 32400000 + 983: + operation: search + 984: + end: 34600000 + operation: insert + start: 34500000 + 985: + operation: search + 986: + end: 34700000 + operation: insert + start: 34600000 + 987: + end: 32700000 + operation: delete + start: 32600000 + 988: + operation: search + 989: + end: 34800000 + operation: insert + start: 34700000 + 990: + end: 32800000 + operation: delete + start: 32700000 + 991: + operation: search + 992: + end: 34900000 + operation: insert + start: 34800000 + 993: + end: 32900000 + operation: delete + start: 32800000 + 994: + operation: search + 995: + end: 35000000 + operation: insert + start: 34900000 + 996: + end: 33000000 + operation: delete + start: 32900000 + 997: + operation: search + max_pts: 6500000 From 38e4c08ab1a7300cf667fcc4fe3a615cec543df1 Mon Sep 17 00:00:00 2001 From: Magdalen Date: Tue, 19 Nov 2024 14:05:55 -0800 Subject: [PATCH 3/6] renamed runbooks, increased max pts on runbooks --- benchmark/datasets.py | 26 + data_export.py | 3 + .../runbooks/gen_expiration_time_runbook.py | 26 +- .../msturing-35M_expiration_time_runbook.yaml | 3290 ----------------- ...xpiration_time_replace_delete_runbook.yaml | 1212 ++++++ ..._expiration_time_replace_only_runbook.yaml | 1094 ++++++ ...sturingpq-1M_expiration_time_runbook.yaml} | 4 +- neurips23/streaming/diskann/config.yaml | 7 +- 8 files changed, 2358 insertions(+), 3304 deletions(-) delete mode 100644 neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml create mode 100644 neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml create mode 100644 neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml rename neurips23/runbooks/{msturing-1M_expiration_time_runbook.yaml => msturingpq-1M_expiration_time_runbook.yaml} (99%) diff --git a/benchmark/datasets.py b/benchmark/datasets.py index fe59e08b..a7307e13 100644 --- a/benchmark/datasets.py +++ b/benchmark/datasets.py @@ -430,6 +430,30 @@ def __init__(self, nb_M=1000): def distance(self): return "euclidean" +# 1M slice of MSTuring dataset, with ground truth corresponding to the 10K query set +# this is needed for backwards compatibility with the streaming code +class MSTuringANNSPQ(BillionScaleDatasetCompetitionFormat): + def __init__(self, nb_M=1): + self.nb_M = nb_M + self.nb = 10**6 * nb_M + self.d = 100 + self.nq = 10000 + self.dtype = "float32" + self.ds_fn = "base1b.fbin" + self.qs_fn = "testQuery10K.fbin" + self.gt_fn = ( + "msturing-1M-private-gt100" if self.nb_M == 1 else + None + ) + self.base_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/" + self.basedir = os.path.join(BASEDIR, "MSTuringANNSPQ") + + self.private_qs_url = None + self.private_gt_url = None + + def distance(self): + return "euclidean" + class MSTuringClustered10M(DatasetCompetitionFormat): def __init__(self): self.nb = 10**6 * 10 @@ -1285,6 +1309,8 @@ def short_name(self): 'msturing-10M': lambda : MSTuringANNS(10), 'msturing-1M': lambda : MSTuringANNS(1), + 'msturingpq-1M': lambda : MSTuringANNSPQ(1), + 'msturing-10M-clustered': lambda: MSTuringClustered10M(), 'msturing-30M-clustered': lambda: MSTuringClustered30M(), diff --git a/data_export.py b/data_export.py index 6063f979..083538cd 100644 --- a/data_export.py +++ b/data_export.py @@ -105,6 +105,9 @@ def cleaned_run_metric(run_metrics): 'neurips23/runbooks/delete_runbook.yaml', 'neurips23/runbooks/final_runbook.yaml', 'neurips23/runbooks/msturing-10M_slidingwindow_runbook.yaml', + 'neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml', + 'neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml', + 'neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml', 'neurips23/runbooks/wikipedia-35M_expirationtime_runbook.yaml', 'neurips23/runbooks/wikipedia-1M_expiration_time_runbook.yaml', 'neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml', diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py index 57cf83ba..23ee7ad2 100644 --- a/neurips23/runbooks/gen_expiration_time_runbook.py +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -178,21 +178,31 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra ratios = (0, 4, 18) timesteps = (0, 100, 20) -seed = 732 -dataset_file = 'msturing-35M_expiration_time_runbook.yaml' -dataset_name = 'msturing-100M' -dataset_size = 35000000 -max_t = 350 +seed = 5554 +dataset_file = 'msturingpq-1M_expiration_time_runbook.yaml' +dataset_name = 'msturingpq-1M' +dataset_size = 1000000 +max_t = 100 gt_url = None gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url) ratios = (0, 4, 18) timesteps = (0, 100, 20) -seed = 5554 -dataset_file = 'msturing-1M_expiration_time_runbook.yaml' +seed = 762 +dataset_file = 'msturingpq-1M_expiration_time_replace_only_runbook.yaml' dataset_name = 'msturing-1M' dataset_size = 1000000 max_t = 100 gt_url = None -gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url) +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False) + +ratios = (1, 8, 18) +timesteps = (0, 100, 20) +seed = 83 +dataset_file = 'msturingpq-1M_expiration_time_replace_delete_runbook.yaml' +dataset_name = 'msturingpq-1M' +dataset_size = 1000000 +max_t = 100 +gt_url = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url) diff --git a/neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml b/neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml deleted file mode 100644 index b1fdb772..00000000 --- a/neurips23/runbooks/msturing-35M_expiration_time_runbook.yaml +++ /dev/null @@ -1,3290 +0,0 @@ -msturing-100M: - 1: - end: 100000 - operation: insert - start: 0 - 2: - operation: search - 3: - end: 200000 - operation: insert - start: 100000 - 4: - operation: search - 5: - end: 300000 - operation: insert - start: 200000 - 6: - operation: search - 7: - end: 400000 - operation: insert - start: 300000 - 8: - operation: search - 9: - end: 500000 - operation: insert - start: 400000 - 10: - operation: search - 11: - end: 600000 - operation: insert - start: 500000 - 12: - operation: search - 13: - end: 700000 - operation: insert - start: 600000 - 14: - operation: search - 15: - end: 800000 - operation: insert - start: 700000 - 16: - operation: search - 17: - end: 900000 - operation: insert - start: 800000 - 18: - operation: search - 19: - end: 1000000 - operation: insert - start: 900000 - 20: - operation: search - 21: - end: 1100000 - operation: insert - start: 1000000 - 22: - operation: search - 23: - end: 1200000 - operation: insert - start: 1100000 - 24: - operation: search - 25: - end: 1300000 - operation: insert - start: 1200000 - 26: - operation: search - 27: - end: 1400000 - operation: insert - start: 1300000 - 28: - operation: search - 29: - end: 1500000 - operation: insert - start: 1400000 - 30: - operation: search - 31: - end: 1600000 - operation: insert - start: 1500000 - 32: - operation: search - 33: - end: 1700000 - operation: insert - start: 1600000 - 34: - operation: search - 35: - end: 1800000 - operation: insert - start: 1700000 - 36: - operation: search - 37: - end: 1900000 - operation: insert - start: 1800000 - 38: - operation: search - 39: - end: 2000000 - operation: insert - start: 1900000 - 40: - operation: search - 41: - end: 2100000 - operation: insert - start: 2000000 - 42: - operation: search - 43: - end: 2200000 - operation: insert - start: 2100000 - 44: - end: 200000 - operation: delete - start: 100000 - 45: - operation: search - 46: - end: 2300000 - operation: insert - start: 2200000 - 47: - end: 300000 - operation: delete - start: 200000 - 48: - operation: search - 49: - end: 2400000 - operation: insert - start: 2300000 - 50: - operation: search - 51: - end: 2500000 - operation: insert - start: 2400000 - 52: - operation: search - 53: - end: 2600000 - operation: insert - start: 2500000 - 54: - end: 600000 - operation: delete - start: 500000 - 55: - operation: search - 56: - end: 2700000 - operation: insert - start: 2600000 - 57: - end: 700000 - operation: delete - start: 600000 - 58: - operation: search - 59: - end: 2800000 - operation: insert - start: 2700000 - 60: - end: 800000 - operation: delete - start: 700000 - 61: - operation: search - 62: - end: 2900000 - operation: insert - start: 2800000 - 63: - operation: search - 64: - end: 3000000 - operation: insert - start: 2900000 - 65: - end: 1000000 - operation: delete - start: 900000 - 66: - operation: search - 67: - end: 3100000 - operation: insert - start: 3000000 - 68: - end: 1100000 - operation: delete - start: 1000000 - 69: - operation: search - 70: - end: 3200000 - operation: insert - start: 3100000 - 71: - end: 1200000 - operation: delete - start: 1100000 - 72: - operation: search - 73: - end: 3300000 - operation: insert - start: 3200000 - 74: - end: 1300000 - operation: delete - start: 1200000 - 75: - operation: search - 76: - end: 3400000 - operation: insert - start: 3300000 - 77: - end: 1400000 - operation: delete - start: 1300000 - 78: - operation: search - 79: - end: 3500000 - operation: insert - start: 3400000 - 80: - end: 1500000 - operation: delete - start: 1400000 - 81: - operation: search - 82: - end: 3600000 - operation: insert - start: 3500000 - 83: - operation: search - 84: - end: 3700000 - operation: insert - start: 3600000 - 85: - end: 1700000 - operation: delete - start: 1600000 - 86: - operation: search - 87: - end: 3800000 - operation: insert - start: 3700000 - 88: - operation: search - 89: - end: 3900000 - operation: insert - start: 3800000 - 90: - end: 1900000 - operation: delete - start: 1800000 - 91: - operation: search - 92: - end: 4000000 - operation: insert - start: 3900000 - 93: - end: 2000000 - operation: delete - start: 1900000 - 94: - operation: search - 95: - end: 4100000 - operation: insert - start: 4000000 - 96: - end: 2100000 - operation: delete - start: 2000000 - 97: - operation: search - 98: - end: 4200000 - operation: insert - start: 4100000 - 99: - end: 2200000 - operation: delete - start: 2100000 - 100: - operation: search - 101: - end: 4300000 - operation: insert - start: 4200000 - 102: - end: 2300000 - operation: delete - start: 2200000 - 103: - operation: search - 104: - end: 4400000 - operation: insert - start: 4300000 - 105: - end: 2400000 - operation: delete - start: 2300000 - 106: - operation: search - 107: - end: 4500000 - operation: insert - start: 4400000 - 108: - end: 2500000 - operation: delete - start: 2400000 - 109: - operation: search - 110: - end: 4600000 - operation: insert - start: 4500000 - 111: - end: 2600000 - operation: delete - start: 2500000 - 112: - operation: search - 113: - end: 4700000 - operation: insert - start: 4600000 - 114: - end: 2700000 - operation: delete - start: 2600000 - 115: - operation: search - 116: - end: 4800000 - operation: insert - start: 4700000 - 117: - end: 2800000 - operation: delete - start: 2700000 - 118: - operation: search - 119: - end: 4900000 - operation: insert - start: 4800000 - 120: - end: 2900000 - operation: delete - start: 2800000 - 121: - operation: search - 122: - end: 5000000 - operation: insert - start: 4900000 - 123: - end: 3000000 - operation: delete - start: 2900000 - 124: - operation: search - 125: - end: 5100000 - operation: insert - start: 5000000 - 126: - operation: search - 127: - end: 5200000 - operation: insert - start: 5100000 - 128: - operation: search - 129: - end: 5300000 - operation: insert - start: 5200000 - 130: - end: 3300000 - operation: delete - start: 3200000 - 131: - operation: search - 132: - end: 5400000 - operation: insert - start: 5300000 - 133: - operation: search - 134: - end: 5500000 - operation: insert - start: 5400000 - 135: - end: 3500000 - operation: delete - start: 3400000 - 136: - operation: search - 137: - end: 5600000 - operation: insert - start: 5500000 - 138: - end: 3600000 - operation: delete - start: 3500000 - 139: - operation: search - 140: - end: 5700000 - operation: insert - start: 5600000 - 141: - end: 3700000 - operation: delete - start: 3600000 - 142: - operation: search - 143: - end: 5800000 - operation: insert - start: 5700000 - 144: - end: 3800000 - operation: delete - start: 3700000 - 145: - operation: search - 146: - end: 5900000 - operation: insert - start: 5800000 - 147: - end: 3900000 - operation: delete - start: 3800000 - 148: - operation: search - 149: - end: 6000000 - operation: insert - start: 5900000 - 150: - end: 4000000 - operation: delete - start: 3900000 - 151: - operation: search - 152: - end: 6100000 - operation: insert - start: 6000000 - 153: - operation: search - 154: - end: 6200000 - operation: insert - start: 6100000 - 155: - end: 4200000 - operation: delete - start: 4100000 - 156: - operation: search - 157: - end: 6300000 - operation: insert - start: 6200000 - 158: - end: 4300000 - operation: delete - start: 4200000 - 159: - operation: search - 160: - end: 6400000 - operation: insert - start: 6300000 - 161: - end: 4400000 - operation: delete - start: 4300000 - 162: - operation: search - 163: - end: 6500000 - operation: insert - start: 6400000 - 164: - end: 4500000 - operation: delete - start: 4400000 - 165: - operation: search - 166: - end: 6600000 - operation: insert - start: 6500000 - 167: - end: 4600000 - operation: delete - start: 4500000 - 168: - operation: search - 169: - end: 6700000 - operation: insert - start: 6600000 - 170: - end: 4700000 - operation: delete - start: 4600000 - 171: - operation: search - 172: - end: 6800000 - operation: insert - start: 6700000 - 173: - end: 4800000 - operation: delete - start: 4700000 - 174: - operation: search - 175: - end: 6900000 - operation: insert - start: 6800000 - 176: - end: 4900000 - operation: delete - start: 4800000 - 177: - operation: search - 178: - end: 7000000 - operation: insert - start: 6900000 - 179: - end: 5000000 - operation: delete - start: 4900000 - 180: - operation: search - 181: - end: 7100000 - operation: insert - start: 7000000 - 182: - end: 5100000 - operation: delete - start: 5000000 - 183: - operation: search - 184: - end: 7200000 - operation: insert - start: 7100000 - 185: - end: 5200000 - operation: delete - start: 5100000 - 186: - operation: search - 187: - end: 7300000 - operation: insert - start: 7200000 - 188: - end: 5300000 - operation: delete - start: 5200000 - 189: - operation: search - 190: - end: 7400000 - operation: insert - start: 7300000 - 191: - end: 5400000 - operation: delete - start: 5300000 - 192: - operation: search - 193: - end: 7500000 - operation: insert - start: 7400000 - 194: - end: 5500000 - operation: delete - start: 5400000 - 195: - operation: search - 196: - end: 7600000 - operation: insert - start: 7500000 - 197: - end: 5600000 - operation: delete - start: 5500000 - 198: - operation: search - 199: - end: 7700000 - operation: insert - start: 7600000 - 200: - end: 5700000 - operation: delete - start: 5600000 - 201: - operation: search - 202: - end: 7800000 - operation: insert - start: 7700000 - 203: - operation: search - 204: - end: 7900000 - operation: insert - start: 7800000 - 205: - operation: search - 206: - end: 8000000 - operation: insert - start: 7900000 - 207: - operation: search - 208: - end: 8100000 - operation: insert - start: 8000000 - 209: - end: 6100000 - operation: delete - start: 6000000 - 210: - operation: search - 211: - end: 8200000 - operation: insert - start: 8100000 - 212: - end: 6200000 - operation: delete - start: 6100000 - 213: - operation: search - 214: - end: 8300000 - operation: insert - start: 8200000 - 215: - end: 6300000 - operation: delete - start: 6200000 - 216: - operation: search - 217: - end: 8400000 - operation: insert - start: 8300000 - 218: - end: 6400000 - operation: delete - start: 6300000 - 219: - operation: search - 220: - end: 8500000 - operation: insert - start: 8400000 - 221: - end: 6500000 - operation: delete - start: 6400000 - 222: - operation: search - 223: - end: 8600000 - operation: insert - start: 8500000 - 224: - end: 6600000 - operation: delete - start: 6500000 - 225: - operation: search - 226: - end: 8700000 - operation: insert - start: 8600000 - 227: - end: 6700000 - operation: delete - start: 6600000 - 228: - operation: search - 229: - end: 8800000 - operation: insert - start: 8700000 - 230: - end: 6800000 - operation: delete - start: 6700000 - 231: - operation: search - 232: - end: 8900000 - operation: insert - start: 8800000 - 233: - end: 6900000 - operation: delete - start: 6800000 - 234: - operation: search - 235: - end: 9000000 - operation: insert - start: 8900000 - 236: - end: 7000000 - operation: delete - start: 6900000 - 237: - operation: search - 238: - end: 9100000 - operation: insert - start: 9000000 - 239: - end: 7100000 - operation: delete - start: 7000000 - 240: - operation: search - 241: - end: 9200000 - operation: insert - start: 9100000 - 242: - end: 7200000 - operation: delete - start: 7100000 - 243: - operation: search - 244: - end: 9300000 - operation: insert - start: 9200000 - 245: - operation: search - 246: - end: 9400000 - operation: insert - start: 9300000 - 247: - end: 7400000 - operation: delete - start: 7300000 - 248: - operation: search - 249: - end: 9500000 - operation: insert - start: 9400000 - 250: - end: 7500000 - operation: delete - start: 7400000 - 251: - operation: search - 252: - end: 9600000 - operation: insert - start: 9500000 - 253: - operation: search - 254: - end: 9700000 - operation: insert - start: 9600000 - 255: - end: 7700000 - operation: delete - start: 7600000 - 256: - operation: search - 257: - end: 9800000 - operation: insert - start: 9700000 - 258: - end: 7800000 - operation: delete - start: 7700000 - 259: - operation: search - 260: - end: 9900000 - operation: insert - start: 9800000 - 261: - end: 7900000 - operation: delete - start: 7800000 - 262: - operation: search - 263: - end: 10000000 - operation: insert - start: 9900000 - 264: - end: 8000000 - operation: delete - start: 7900000 - 265: - operation: search - 266: - end: 10100000 - operation: insert - start: 10000000 - 267: - end: 100000 - operation: delete - start: 0 - 268: - end: 8100000 - operation: delete - start: 8000000 - 269: - operation: search - 270: - end: 10200000 - operation: insert - start: 10100000 - 271: - end: 8200000 - operation: delete - start: 8100000 - 272: - operation: search - 273: - end: 10300000 - operation: insert - start: 10200000 - 274: - end: 8300000 - operation: delete - start: 8200000 - 275: - operation: search - 276: - end: 10400000 - operation: insert - start: 10300000 - 277: - end: 400000 - operation: delete - start: 300000 - 278: - end: 8400000 - operation: delete - start: 8300000 - 279: - operation: search - 280: - end: 10500000 - operation: insert - start: 10400000 - 281: - end: 500000 - operation: delete - start: 400000 - 282: - end: 8500000 - operation: delete - start: 8400000 - 283: - operation: search - 284: - end: 10600000 - operation: insert - start: 10500000 - 285: - end: 8600000 - operation: delete - start: 8500000 - 286: - operation: search - 287: - end: 10700000 - operation: insert - start: 10600000 - 288: - end: 8700000 - operation: delete - start: 8600000 - 289: - operation: search - 290: - end: 10800000 - operation: insert - start: 10700000 - 291: - operation: search - 292: - end: 10900000 - operation: insert - start: 10800000 - 293: - end: 900000 - operation: delete - start: 800000 - 294: - end: 8900000 - operation: delete - start: 8800000 - 295: - operation: search - 296: - end: 11000000 - operation: insert - start: 10900000 - 297: - operation: search - 298: - end: 11100000 - operation: insert - start: 11000000 - 299: - end: 9100000 - operation: delete - start: 9000000 - 300: - operation: search - 301: - end: 11200000 - operation: insert - start: 11100000 - 302: - end: 9200000 - operation: delete - start: 9100000 - 303: - operation: search - 304: - end: 11300000 - operation: insert - start: 11200000 - 305: - operation: search - 306: - end: 11400000 - operation: insert - start: 11300000 - 307: - end: 9400000 - operation: delete - start: 9300000 - 308: - operation: search - 309: - end: 11500000 - operation: insert - start: 11400000 - 310: - operation: search - 311: - end: 11600000 - operation: insert - start: 11500000 - 312: - end: 9600000 - operation: delete - start: 9500000 - 313: - operation: search - 314: - end: 11700000 - operation: insert - start: 11600000 - 315: - end: 9700000 - operation: delete - start: 9600000 - 316: - operation: search - 317: - end: 11800000 - operation: insert - start: 11700000 - 318: - end: 9800000 - operation: delete - start: 9700000 - 319: - operation: search - 320: - end: 11900000 - operation: insert - start: 11800000 - 321: - end: 9900000 - operation: delete - start: 9800000 - 322: - operation: search - 323: - end: 12000000 - operation: insert - start: 11900000 - 324: - end: 10000000 - operation: delete - start: 9900000 - 325: - operation: search - 326: - end: 12100000 - operation: insert - start: 12000000 - 327: - end: 10100000 - operation: delete - start: 10000000 - 328: - operation: search - 329: - end: 12200000 - operation: insert - start: 12100000 - 330: - end: 10200000 - operation: delete - start: 10100000 - 331: - operation: search - 332: - end: 12300000 - operation: insert - start: 12200000 - 333: - end: 10300000 - operation: delete - start: 10200000 - 334: - operation: search - 335: - end: 12400000 - operation: insert - start: 12300000 - 336: - end: 10400000 - operation: delete - start: 10300000 - 337: - operation: search - 338: - end: 12500000 - operation: insert - start: 12400000 - 339: - operation: search - 340: - end: 12600000 - operation: insert - start: 12500000 - 341: - end: 10600000 - operation: delete - start: 10500000 - 342: - operation: search - 343: - end: 12700000 - operation: insert - start: 12600000 - 344: - end: 10700000 - operation: delete - start: 10600000 - 345: - operation: search - 346: - end: 12800000 - operation: insert - start: 12700000 - 347: - end: 10800000 - operation: delete - start: 10700000 - 348: - operation: search - 349: - end: 12900000 - operation: insert - start: 12800000 - 350: - operation: search - 351: - end: 13000000 - operation: insert - start: 12900000 - 352: - end: 11000000 - operation: delete - start: 10900000 - 353: - operation: search - 354: - end: 13100000 - operation: insert - start: 13000000 - 355: - end: 3100000 - operation: delete - start: 3000000 - 356: - end: 11100000 - operation: delete - start: 11000000 - 357: - operation: search - 358: - end: 13200000 - operation: insert - start: 13100000 - 359: - end: 11200000 - operation: delete - start: 11100000 - 360: - operation: search - 361: - end: 13300000 - operation: insert - start: 13200000 - 362: - end: 11300000 - operation: delete - start: 11200000 - 363: - operation: search - 364: - end: 13400000 - operation: insert - start: 13300000 - 365: - end: 3400000 - operation: delete - start: 3300000 - 366: - end: 11400000 - operation: delete - start: 11300000 - 367: - operation: search - 368: - end: 13500000 - operation: insert - start: 13400000 - 369: - end: 11500000 - operation: delete - start: 11400000 - 370: - operation: search - 371: - end: 13600000 - operation: insert - start: 13500000 - 372: - end: 11600000 - operation: delete - start: 11500000 - 373: - operation: search - 374: - end: 13700000 - operation: insert - start: 13600000 - 375: - end: 11700000 - operation: delete - start: 11600000 - 376: - operation: search - 377: - end: 13800000 - operation: insert - start: 13700000 - 378: - end: 11800000 - operation: delete - start: 11700000 - 379: - operation: search - 380: - end: 13900000 - operation: insert - start: 13800000 - 381: - end: 11900000 - operation: delete - start: 11800000 - 382: - operation: search - 383: - end: 14000000 - operation: insert - start: 13900000 - 384: - end: 12000000 - operation: delete - start: 11900000 - 385: - operation: search - 386: - end: 14100000 - operation: insert - start: 14000000 - 387: - end: 4100000 - operation: delete - start: 4000000 - 388: - end: 12100000 - operation: delete - start: 12000000 - 389: - operation: search - 390: - end: 14200000 - operation: insert - start: 14100000 - 391: - end: 12200000 - operation: delete - start: 12100000 - 392: - operation: search - 393: - end: 14300000 - operation: insert - start: 14200000 - 394: - end: 12300000 - operation: delete - start: 12200000 - 395: - operation: search - 396: - end: 14400000 - operation: insert - start: 14300000 - 397: - end: 12400000 - operation: delete - start: 12300000 - 398: - operation: search - 399: - end: 14500000 - operation: insert - start: 14400000 - 400: - end: 12500000 - operation: delete - start: 12400000 - 401: - operation: search - 402: - end: 14600000 - operation: insert - start: 14500000 - 403: - operation: search - 404: - end: 14700000 - operation: insert - start: 14600000 - 405: - operation: search - 406: - end: 14800000 - operation: insert - start: 14700000 - 407: - operation: search - 408: - end: 14900000 - operation: insert - start: 14800000 - 409: - operation: search - 410: - end: 15000000 - operation: insert - start: 14900000 - 411: - end: 13000000 - operation: delete - start: 12900000 - 412: - operation: search - 413: - end: 15100000 - operation: insert - start: 15000000 - 414: - operation: search - 415: - end: 15200000 - operation: insert - start: 15100000 - 416: - operation: search - 417: - end: 15300000 - operation: insert - start: 15200000 - 418: - end: 13300000 - operation: delete - start: 13200000 - 419: - operation: search - 420: - end: 15400000 - operation: insert - start: 15300000 - 421: - end: 13400000 - operation: delete - start: 13300000 - 422: - operation: search - 423: - end: 15500000 - operation: insert - start: 15400000 - 424: - end: 13500000 - operation: delete - start: 13400000 - 425: - operation: search - 426: - end: 15600000 - operation: insert - start: 15500000 - 427: - end: 13600000 - operation: delete - start: 13500000 - 428: - operation: search - 429: - end: 15700000 - operation: insert - start: 15600000 - 430: - operation: search - 431: - end: 15800000 - operation: insert - start: 15700000 - 432: - end: 5800000 - operation: delete - start: 5700000 - 433: - end: 13800000 - operation: delete - start: 13700000 - 434: - operation: search - 435: - end: 15900000 - operation: insert - start: 15800000 - 436: - end: 5900000 - operation: delete - start: 5800000 - 437: - operation: search - 438: - end: 16000000 - operation: insert - start: 15900000 - 439: - end: 6000000 - operation: delete - start: 5900000 - 440: - end: 14000000 - operation: delete - start: 13900000 - 441: - operation: search - 442: - end: 16100000 - operation: insert - start: 16000000 - 443: - operation: search - 444: - end: 16200000 - operation: insert - start: 16100000 - 445: - end: 14200000 - operation: delete - start: 14100000 - 446: - operation: search - 447: - end: 16300000 - operation: insert - start: 16200000 - 448: - end: 14300000 - operation: delete - start: 14200000 - 449: - operation: search - 450: - end: 16400000 - operation: insert - start: 16300000 - 451: - end: 14400000 - operation: delete - start: 14300000 - 452: - operation: search - 453: - end: 16500000 - operation: insert - start: 16400000 - 454: - operation: search - 455: - end: 16600000 - operation: insert - start: 16500000 - 456: - end: 14600000 - operation: delete - start: 14500000 - 457: - operation: search - 458: - end: 16700000 - operation: insert - start: 16600000 - 459: - operation: search - 460: - end: 16800000 - operation: insert - start: 16700000 - 461: - end: 14800000 - operation: delete - start: 14700000 - 462: - operation: search - 463: - end: 16900000 - operation: insert - start: 16800000 - 464: - operation: search - 465: - end: 17000000 - operation: insert - start: 16900000 - 466: - end: 15000000 - operation: delete - start: 14900000 - 467: - operation: search - 468: - end: 17100000 - operation: insert - start: 17000000 - 469: - end: 15100000 - operation: delete - start: 15000000 - 470: - operation: search - 471: - end: 17200000 - operation: insert - start: 17100000 - 472: - end: 15200000 - operation: delete - start: 15100000 - 473: - operation: search - 474: - end: 17300000 - operation: insert - start: 17200000 - 475: - end: 15300000 - operation: delete - start: 15200000 - 476: - operation: search - 477: - end: 17400000 - operation: insert - start: 17300000 - 478: - end: 15400000 - operation: delete - start: 15300000 - 479: - operation: search - 480: - end: 17500000 - operation: insert - start: 17400000 - 481: - operation: search - 482: - end: 17600000 - operation: insert - start: 17500000 - 483: - end: 7600000 - operation: delete - start: 7500000 - 484: - end: 15600000 - operation: delete - start: 15500000 - 485: - operation: search - 486: - end: 17700000 - operation: insert - start: 17600000 - 487: - end: 15700000 - operation: delete - start: 15600000 - 488: - operation: search - 489: - end: 17800000 - operation: insert - start: 17700000 - 490: - end: 15800000 - operation: delete - start: 15700000 - 491: - operation: search - 492: - end: 17900000 - operation: insert - start: 17800000 - 493: - end: 15900000 - operation: delete - start: 15800000 - 494: - operation: search - 495: - end: 18000000 - operation: insert - start: 17900000 - 496: - end: 16000000 - operation: delete - start: 15900000 - 497: - operation: search - 498: - end: 18100000 - operation: insert - start: 18000000 - 499: - operation: search - 500: - end: 18200000 - operation: insert - start: 18100000 - 501: - end: 16200000 - operation: delete - start: 16100000 - 502: - operation: search - 503: - end: 18300000 - operation: insert - start: 18200000 - 504: - end: 16300000 - operation: delete - start: 16200000 - 505: - operation: search - 506: - end: 18400000 - operation: insert - start: 18300000 - 507: - end: 16400000 - operation: delete - start: 16300000 - 508: - operation: search - 509: - end: 18500000 - operation: insert - start: 18400000 - 510: - operation: search - 511: - end: 18600000 - operation: insert - start: 18500000 - 512: - end: 16600000 - operation: delete - start: 16500000 - 513: - operation: search - 514: - end: 18700000 - operation: insert - start: 18600000 - 515: - end: 16700000 - operation: delete - start: 16600000 - 516: - operation: search - 517: - end: 18800000 - operation: insert - start: 18700000 - 518: - end: 8800000 - operation: delete - start: 8700000 - 519: - end: 16800000 - operation: delete - start: 16700000 - 520: - operation: search - 521: - end: 18900000 - operation: insert - start: 18800000 - 522: - end: 16900000 - operation: delete - start: 16800000 - 523: - operation: search - 524: - end: 19000000 - operation: insert - start: 18900000 - 525: - end: 9000000 - operation: delete - start: 8900000 - 526: - end: 17000000 - operation: delete - start: 16900000 - 527: - operation: search - 528: - end: 19100000 - operation: insert - start: 19000000 - 529: - end: 17100000 - operation: delete - start: 17000000 - 530: - operation: search - 531: - end: 19200000 - operation: insert - start: 19100000 - 532: - end: 17200000 - operation: delete - start: 17100000 - 533: - operation: search - 534: - end: 19300000 - operation: insert - start: 19200000 - 535: - end: 17300000 - operation: delete - start: 17200000 - 536: - operation: search - 537: - end: 19400000 - operation: insert - start: 19300000 - 538: - end: 17400000 - operation: delete - start: 17300000 - 539: - operation: search - 540: - end: 19500000 - operation: insert - start: 19400000 - 541: - end: 9500000 - operation: delete - start: 9400000 - 542: - end: 17500000 - operation: delete - start: 17400000 - 543: - operation: search - 544: - end: 19600000 - operation: insert - start: 19500000 - 545: - end: 17600000 - operation: delete - start: 17500000 - 546: - operation: search - 547: - end: 19700000 - operation: insert - start: 19600000 - 548: - end: 17700000 - operation: delete - start: 17600000 - 549: - operation: search - 550: - end: 19800000 - operation: insert - start: 19700000 - 551: - end: 17800000 - operation: delete - start: 17700000 - 552: - operation: search - 553: - end: 19900000 - operation: insert - start: 19800000 - 554: - end: 17900000 - operation: delete - start: 17800000 - 555: - operation: search - 556: - end: 20000000 - operation: insert - start: 19900000 - 557: - end: 18000000 - operation: delete - start: 17900000 - 558: - operation: search - 559: - end: 20100000 - operation: insert - start: 20000000 - 560: - end: 18100000 - operation: delete - start: 18000000 - 561: - operation: search - 562: - end: 20200000 - operation: insert - start: 20100000 - 563: - end: 18200000 - operation: delete - start: 18100000 - 564: - operation: search - 565: - end: 20300000 - operation: insert - start: 20200000 - 566: - end: 18300000 - operation: delete - start: 18200000 - 567: - operation: search - 568: - end: 20400000 - operation: insert - start: 20300000 - 569: - end: 18400000 - operation: delete - start: 18300000 - 570: - operation: search - 571: - end: 20500000 - operation: insert - start: 20400000 - 572: - end: 10500000 - operation: delete - start: 10400000 - 573: - end: 18500000 - operation: delete - start: 18400000 - 574: - operation: search - 575: - end: 20600000 - operation: insert - start: 20500000 - 576: - operation: search - 577: - end: 20700000 - operation: insert - start: 20600000 - 578: - end: 18700000 - operation: delete - start: 18600000 - 579: - operation: search - 580: - end: 20800000 - operation: insert - start: 20700000 - 581: - end: 18800000 - operation: delete - start: 18700000 - 582: - operation: search - 583: - end: 20900000 - operation: insert - start: 20800000 - 584: - end: 10900000 - operation: delete - start: 10800000 - 585: - end: 18900000 - operation: delete - start: 18800000 - 586: - operation: search - 587: - end: 21000000 - operation: insert - start: 20900000 - 588: - end: 19000000 - operation: delete - start: 18900000 - 589: - operation: search - 590: - end: 21100000 - operation: insert - start: 21000000 - 591: - operation: search - 592: - end: 21200000 - operation: insert - start: 21100000 - 593: - end: 19200000 - operation: delete - start: 19100000 - 594: - operation: search - 595: - end: 21300000 - operation: insert - start: 21200000 - 596: - end: 19300000 - operation: delete - start: 19200000 - 597: - operation: search - 598: - end: 21400000 - operation: insert - start: 21300000 - 599: - end: 19400000 - operation: delete - start: 19300000 - 600: - operation: search - 601: - end: 21500000 - operation: insert - start: 21400000 - 602: - end: 19500000 - operation: delete - start: 19400000 - 603: - operation: search - 604: - end: 21600000 - operation: insert - start: 21500000 - 605: - end: 19600000 - operation: delete - start: 19500000 - 606: - operation: search - 607: - end: 21700000 - operation: insert - start: 21600000 - 608: - end: 19700000 - operation: delete - start: 19600000 - 609: - operation: search - 610: - end: 21800000 - operation: insert - start: 21700000 - 611: - end: 19800000 - operation: delete - start: 19700000 - 612: - operation: search - 613: - end: 21900000 - operation: insert - start: 21800000 - 614: - operation: search - 615: - end: 22000000 - operation: insert - start: 21900000 - 616: - end: 20000000 - operation: delete - start: 19900000 - 617: - operation: search - 618: - end: 22100000 - operation: insert - start: 22000000 - 619: - end: 20100000 - operation: delete - start: 20000000 - 620: - operation: search - 621: - end: 22200000 - operation: insert - start: 22100000 - 622: - end: 20200000 - operation: delete - start: 20100000 - 623: - operation: search - 624: - end: 22300000 - operation: insert - start: 22200000 - 625: - end: 20300000 - operation: delete - start: 20200000 - 626: - operation: search - 627: - end: 22400000 - operation: insert - start: 22300000 - 628: - end: 20400000 - operation: delete - start: 20300000 - 629: - operation: search - 630: - end: 22500000 - operation: insert - start: 22400000 - 631: - end: 20500000 - operation: delete - start: 20400000 - 632: - operation: search - 633: - end: 22600000 - operation: insert - start: 22500000 - 634: - end: 12600000 - operation: delete - start: 12500000 - 635: - end: 20600000 - operation: delete - start: 20500000 - 636: - operation: search - 637: - end: 22700000 - operation: insert - start: 22600000 - 638: - operation: search - 639: - end: 22800000 - operation: insert - start: 22700000 - 640: - end: 12800000 - operation: delete - start: 12700000 - 641: - end: 20800000 - operation: delete - start: 20700000 - 642: - operation: search - 643: - end: 22900000 - operation: insert - start: 22800000 - 644: - end: 12900000 - operation: delete - start: 12800000 - 645: - operation: search - 646: - end: 23000000 - operation: insert - start: 22900000 - 647: - end: 21000000 - operation: delete - start: 20900000 - 648: - operation: search - 649: - end: 23100000 - operation: insert - start: 23000000 - 650: - end: 13100000 - operation: delete - start: 13000000 - 651: - end: 21100000 - operation: delete - start: 21000000 - 652: - operation: search - 653: - end: 23200000 - operation: insert - start: 23100000 - 654: - end: 21200000 - operation: delete - start: 21100000 - 655: - operation: search - 656: - end: 23300000 - operation: insert - start: 23200000 - 657: - operation: search - 658: - end: 23400000 - operation: insert - start: 23300000 - 659: - end: 21400000 - operation: delete - start: 21300000 - 660: - operation: search - 661: - end: 23500000 - operation: insert - start: 23400000 - 662: - end: 21500000 - operation: delete - start: 21400000 - 663: - operation: search - 664: - end: 23600000 - operation: insert - start: 23500000 - 665: - operation: search - 666: - end: 23700000 - operation: insert - start: 23600000 - 667: - end: 13700000 - operation: delete - start: 13600000 - 668: - end: 21700000 - operation: delete - start: 21600000 - 669: - operation: search - 670: - end: 23800000 - operation: insert - start: 23700000 - 671: - end: 21800000 - operation: delete - start: 21700000 - 672: - operation: search - 673: - end: 23900000 - operation: insert - start: 23800000 - 674: - end: 21900000 - operation: delete - start: 21800000 - 675: - operation: search - 676: - end: 24000000 - operation: insert - start: 23900000 - 677: - end: 22000000 - operation: delete - start: 21900000 - 678: - operation: search - 679: - end: 24100000 - operation: insert - start: 24000000 - 680: - end: 22100000 - operation: delete - start: 22000000 - 681: - operation: search - 682: - end: 24200000 - operation: insert - start: 24100000 - 683: - end: 22200000 - operation: delete - start: 22100000 - 684: - operation: search - 685: - end: 24300000 - operation: insert - start: 24200000 - 686: - operation: search - 687: - end: 24400000 - operation: insert - start: 24300000 - 688: - end: 22400000 - operation: delete - start: 22300000 - 689: - operation: search - 690: - end: 24500000 - operation: insert - start: 24400000 - 691: - end: 14500000 - operation: delete - start: 14400000 - 692: - operation: search - 693: - end: 24600000 - operation: insert - start: 24500000 - 694: - end: 22600000 - operation: delete - start: 22500000 - 695: - operation: search - 696: - end: 24700000 - operation: insert - start: 24600000 - 697: - end: 22700000 - operation: delete - start: 22600000 - 698: - operation: search - 699: - end: 24800000 - operation: insert - start: 24700000 - 700: - end: 22800000 - operation: delete - start: 22700000 - 701: - operation: search - 702: - end: 24900000 - operation: insert - start: 24800000 - 703: - end: 14900000 - operation: delete - start: 14800000 - 704: - end: 22900000 - operation: delete - start: 22800000 - 705: - operation: search - 706: - end: 25000000 - operation: insert - start: 24900000 - 707: - end: 23000000 - operation: delete - start: 22900000 - 708: - operation: search - 709: - end: 25100000 - operation: insert - start: 25000000 - 710: - end: 23100000 - operation: delete - start: 23000000 - 711: - operation: search - 712: - end: 25200000 - operation: insert - start: 25100000 - 713: - end: 23200000 - operation: delete - start: 23100000 - 714: - operation: search - 715: - end: 25300000 - operation: insert - start: 25200000 - 716: - end: 23300000 - operation: delete - start: 23200000 - 717: - operation: search - 718: - end: 25400000 - operation: insert - start: 25300000 - 719: - end: 23400000 - operation: delete - start: 23300000 - 720: - operation: search - 721: - end: 25500000 - operation: insert - start: 25400000 - 722: - operation: search - 723: - end: 25600000 - operation: insert - start: 25500000 - 724: - operation: search - 725: - end: 25700000 - operation: insert - start: 25600000 - 726: - end: 23700000 - operation: delete - start: 23600000 - 727: - operation: search - 728: - end: 25800000 - operation: insert - start: 25700000 - 729: - end: 23800000 - operation: delete - start: 23700000 - 730: - operation: search - 731: - end: 25900000 - operation: insert - start: 25800000 - 732: - end: 23900000 - operation: delete - start: 23800000 - 733: - operation: search - 734: - end: 26000000 - operation: insert - start: 25900000 - 735: - operation: search - 736: - end: 26100000 - operation: insert - start: 26000000 - 737: - end: 24100000 - operation: delete - start: 24000000 - 738: - operation: search - 739: - end: 26200000 - operation: insert - start: 26100000 - 740: - end: 24200000 - operation: delete - start: 24100000 - 741: - operation: search - 742: - end: 26300000 - operation: insert - start: 26200000 - 743: - end: 24300000 - operation: delete - start: 24200000 - 744: - operation: search - 745: - end: 26400000 - operation: insert - start: 26300000 - 746: - end: 24400000 - operation: delete - start: 24300000 - 747: - operation: search - 748: - end: 26500000 - operation: insert - start: 26400000 - 749: - end: 24500000 - operation: delete - start: 24400000 - 750: - operation: search - 751: - end: 26600000 - operation: insert - start: 26500000 - 752: - end: 24600000 - operation: delete - start: 24500000 - 753: - operation: search - 754: - end: 26700000 - operation: insert - start: 26600000 - 755: - end: 24700000 - operation: delete - start: 24600000 - 756: - operation: search - 757: - end: 26800000 - operation: insert - start: 26700000 - 758: - end: 24800000 - operation: delete - start: 24700000 - 759: - operation: search - 760: - end: 26900000 - operation: insert - start: 26800000 - 761: - end: 24900000 - operation: delete - start: 24800000 - 762: - operation: search - 763: - end: 27000000 - operation: insert - start: 26900000 - 764: - end: 25000000 - operation: delete - start: 24900000 - 765: - operation: search - 766: - end: 27100000 - operation: insert - start: 27000000 - 767: - operation: search - 768: - end: 27200000 - operation: insert - start: 27100000 - 769: - end: 25200000 - operation: delete - start: 25100000 - 770: - operation: search - 771: - end: 27300000 - operation: insert - start: 27200000 - 772: - end: 25300000 - operation: delete - start: 25200000 - 773: - operation: search - 774: - end: 27400000 - operation: insert - start: 27300000 - 775: - end: 25400000 - operation: delete - start: 25300000 - 776: - operation: search - 777: - end: 27500000 - operation: insert - start: 27400000 - 778: - end: 25500000 - operation: delete - start: 25400000 - 779: - operation: search - 780: - end: 27600000 - operation: insert - start: 27500000 - 781: - end: 25600000 - operation: delete - start: 25500000 - 782: - operation: search - 783: - end: 27700000 - operation: insert - start: 27600000 - 784: - operation: search - 785: - end: 27800000 - operation: insert - start: 27700000 - 786: - end: 25800000 - operation: delete - start: 25700000 - 787: - operation: search - 788: - end: 27900000 - operation: insert - start: 27800000 - 789: - end: 25900000 - operation: delete - start: 25800000 - 790: - operation: search - 791: - end: 28000000 - operation: insert - start: 27900000 - 792: - operation: search - 793: - end: 28100000 - operation: insert - start: 28000000 - 794: - end: 26100000 - operation: delete - start: 26000000 - 795: - operation: search - 796: - end: 28200000 - operation: insert - start: 28100000 - 797: - end: 26200000 - operation: delete - start: 26100000 - 798: - operation: search - 799: - end: 28300000 - operation: insert - start: 28200000 - 800: - end: 26300000 - operation: delete - start: 26200000 - 801: - operation: search - 802: - end: 28400000 - operation: insert - start: 28300000 - 803: - end: 26400000 - operation: delete - start: 26300000 - 804: - operation: search - 805: - end: 28500000 - operation: insert - start: 28400000 - 806: - end: 26500000 - operation: delete - start: 26400000 - 807: - operation: search - 808: - end: 28600000 - operation: insert - start: 28500000 - 809: - end: 18600000 - operation: delete - start: 18500000 - 810: - end: 26600000 - operation: delete - start: 26500000 - 811: - operation: search - 812: - end: 28700000 - operation: insert - start: 28600000 - 813: - end: 26700000 - operation: delete - start: 26600000 - 814: - operation: search - 815: - end: 28800000 - operation: insert - start: 28700000 - 816: - end: 26800000 - operation: delete - start: 26700000 - 817: - operation: search - 818: - end: 28900000 - operation: insert - start: 28800000 - 819: - operation: search - 820: - end: 29000000 - operation: insert - start: 28900000 - 821: - end: 27000000 - operation: delete - start: 26900000 - 822: - operation: search - 823: - end: 29100000 - operation: insert - start: 29000000 - 824: - end: 19100000 - operation: delete - start: 19000000 - 825: - end: 27100000 - operation: delete - start: 27000000 - 826: - operation: search - 827: - end: 29200000 - operation: insert - start: 29100000 - 828: - end: 27200000 - operation: delete - start: 27100000 - 829: - operation: search - 830: - end: 29300000 - operation: insert - start: 29200000 - 831: - end: 27300000 - operation: delete - start: 27200000 - 832: - operation: search - 833: - end: 29400000 - operation: insert - start: 29300000 - 834: - end: 27400000 - operation: delete - start: 27300000 - 835: - operation: search - 836: - end: 29500000 - operation: insert - start: 29400000 - 837: - end: 27500000 - operation: delete - start: 27400000 - 838: - operation: search - 839: - end: 29600000 - operation: insert - start: 29500000 - 840: - end: 27600000 - operation: delete - start: 27500000 - 841: - operation: search - 842: - end: 29700000 - operation: insert - start: 29600000 - 843: - end: 27700000 - operation: delete - start: 27600000 - 844: - operation: search - 845: - end: 29800000 - operation: insert - start: 29700000 - 846: - end: 27800000 - operation: delete - start: 27700000 - 847: - operation: search - 848: - end: 29900000 - operation: insert - start: 29800000 - 849: - end: 19900000 - operation: delete - start: 19800000 - 850: - operation: search - 851: - end: 30000000 - operation: insert - start: 29900000 - 852: - end: 28000000 - operation: delete - start: 27900000 - 853: - operation: search - 854: - end: 30100000 - operation: insert - start: 30000000 - 855: - operation: search - 856: - end: 30200000 - operation: insert - start: 30100000 - 857: - operation: search - 858: - end: 30300000 - operation: insert - start: 30200000 - 859: - end: 28300000 - operation: delete - start: 28200000 - 860: - operation: search - 861: - end: 30400000 - operation: insert - start: 30300000 - 862: - end: 28400000 - operation: delete - start: 28300000 - 863: - operation: search - 864: - end: 30500000 - operation: insert - start: 30400000 - 865: - end: 28500000 - operation: delete - start: 28400000 - 866: - operation: search - 867: - end: 30600000 - operation: insert - start: 30500000 - 868: - end: 28600000 - operation: delete - start: 28500000 - 869: - operation: search - 870: - end: 30700000 - operation: insert - start: 30600000 - 871: - end: 20700000 - operation: delete - start: 20600000 - 872: - end: 28700000 - operation: delete - start: 28600000 - 873: - operation: search - 874: - end: 30800000 - operation: insert - start: 30700000 - 875: - end: 28800000 - operation: delete - start: 28700000 - 876: - operation: search - 877: - end: 30900000 - operation: insert - start: 30800000 - 878: - end: 20900000 - operation: delete - start: 20800000 - 879: - end: 28900000 - operation: delete - start: 28800000 - 880: - operation: search - 881: - end: 31000000 - operation: insert - start: 30900000 - 882: - operation: search - 883: - end: 31100000 - operation: insert - start: 31000000 - 884: - operation: search - 885: - end: 31200000 - operation: insert - start: 31100000 - 886: - end: 29200000 - operation: delete - start: 29100000 - 887: - operation: search - 888: - end: 31300000 - operation: insert - start: 31200000 - 889: - end: 21300000 - operation: delete - start: 21200000 - 890: - end: 29300000 - operation: delete - start: 29200000 - 891: - operation: search - 892: - end: 31400000 - operation: insert - start: 31300000 - 893: - end: 29400000 - operation: delete - start: 29300000 - 894: - operation: search - 895: - end: 31500000 - operation: insert - start: 31400000 - 896: - end: 29500000 - operation: delete - start: 29400000 - 897: - operation: search - 898: - end: 31600000 - operation: insert - start: 31500000 - 899: - end: 21600000 - operation: delete - start: 21500000 - 900: - operation: search - 901: - end: 31700000 - operation: insert - start: 31600000 - 902: - end: 29700000 - operation: delete - start: 29600000 - 903: - operation: search - 904: - end: 31800000 - operation: insert - start: 31700000 - 905: - operation: search - 906: - end: 31900000 - operation: insert - start: 31800000 - 907: - end: 29900000 - operation: delete - start: 29800000 - 908: - operation: search - 909: - end: 32000000 - operation: insert - start: 31900000 - 910: - operation: search - 911: - end: 32100000 - operation: insert - start: 32000000 - 912: - end: 30100000 - operation: delete - start: 30000000 - 913: - operation: search - 914: - end: 32200000 - operation: insert - start: 32100000 - 915: - end: 30200000 - operation: delete - start: 30100000 - 916: - operation: search - 917: - end: 32300000 - operation: insert - start: 32200000 - 918: - end: 22300000 - operation: delete - start: 22200000 - 919: - operation: search - 920: - end: 32400000 - operation: insert - start: 32300000 - 921: - end: 30400000 - operation: delete - start: 30300000 - 922: - operation: search - 923: - end: 32500000 - operation: insert - start: 32400000 - 924: - end: 22500000 - operation: delete - start: 22400000 - 925: - end: 30500000 - operation: delete - start: 30400000 - 926: - operation: search - 927: - end: 32600000 - operation: insert - start: 32500000 - 928: - end: 30600000 - operation: delete - start: 30500000 - 929: - operation: search - 930: - end: 32700000 - operation: insert - start: 32600000 - 931: - end: 30700000 - operation: delete - start: 30600000 - 932: - operation: search - 933: - end: 32800000 - operation: insert - start: 32700000 - 934: - end: 30800000 - operation: delete - start: 30700000 - 935: - operation: search - 936: - end: 32900000 - operation: insert - start: 32800000 - 937: - end: 30900000 - operation: delete - start: 30800000 - 938: - operation: search - 939: - end: 33000000 - operation: insert - start: 32900000 - 940: - operation: search - 941: - end: 33100000 - operation: insert - start: 33000000 - 942: - operation: search - 943: - end: 33200000 - operation: insert - start: 33100000 - 944: - end: 31200000 - operation: delete - start: 31100000 - 945: - operation: search - 946: - end: 33300000 - operation: insert - start: 33200000 - 947: - end: 31300000 - operation: delete - start: 31200000 - 948: - operation: search - 949: - end: 33400000 - operation: insert - start: 33300000 - 950: - operation: search - 951: - end: 33500000 - operation: insert - start: 33400000 - 952: - end: 23500000 - operation: delete - start: 23400000 - 953: - end: 31500000 - operation: delete - start: 31400000 - 954: - operation: search - 955: - end: 33600000 - operation: insert - start: 33500000 - 956: - end: 31600000 - operation: delete - start: 31500000 - 957: - operation: search - 958: - end: 33700000 - operation: insert - start: 33600000 - 959: - end: 31700000 - operation: delete - start: 31600000 - 960: - operation: search - 961: - end: 33800000 - operation: insert - start: 33700000 - 962: - end: 31800000 - operation: delete - start: 31700000 - 963: - operation: search - 964: - end: 33900000 - operation: insert - start: 33800000 - 965: - operation: search - 966: - end: 34000000 - operation: insert - start: 33900000 - 967: - end: 24000000 - operation: delete - start: 23900000 - 968: - end: 32000000 - operation: delete - start: 31900000 - 969: - operation: search - 970: - end: 34100000 - operation: insert - start: 34000000 - 971: - end: 32100000 - operation: delete - start: 32000000 - 972: - operation: search - 973: - end: 34200000 - operation: insert - start: 34100000 - 974: - end: 32200000 - operation: delete - start: 32100000 - 975: - operation: search - 976: - end: 34300000 - operation: insert - start: 34200000 - 977: - end: 32300000 - operation: delete - start: 32200000 - 978: - operation: search - 979: - end: 34400000 - operation: insert - start: 34300000 - 980: - operation: search - 981: - end: 34500000 - operation: insert - start: 34400000 - 982: - end: 32500000 - operation: delete - start: 32400000 - 983: - operation: search - 984: - end: 34600000 - operation: insert - start: 34500000 - 985: - operation: search - 986: - end: 34700000 - operation: insert - start: 34600000 - 987: - end: 32700000 - operation: delete - start: 32600000 - 988: - operation: search - 989: - end: 34800000 - operation: insert - start: 34700000 - 990: - end: 32800000 - operation: delete - start: 32700000 - 991: - operation: search - 992: - end: 34900000 - operation: insert - start: 34800000 - 993: - end: 32900000 - operation: delete - start: 32800000 - 994: - operation: search - 995: - end: 35000000 - operation: insert - start: 34900000 - 996: - end: 33000000 - operation: delete - start: 32900000 - 997: - operation: search - max_pts: 6500000 diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml new file mode 100644 index 00000000..b55ba4a4 --- /dev/null +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml @@ -0,0 +1,1212 @@ +msturingpq-1M: + 1: + end: 6984 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 15354 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 25137 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 35756 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 45330 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 56642 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 66205 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 76901 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 88285 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 95177 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 105079 + operation: insert + start: 100000 + 22: + ids_end: 10000 + ids_start: 6984 + operation: replace + tags_end: 3016 + tags_start: 0 + 23: + operation: search + 24: + end: 116396 + operation: insert + start: 110000 + 25: + ids_end: 20000 + ids_start: 15354 + operation: replace + tags_end: 14646 + tags_start: 10000 + 26: + operation: search + 27: + end: 127894 + operation: insert + start: 120000 + 28: + operation: search + 29: + end: 138727 + operation: insert + start: 130000 + 30: + ids_end: 40000 + ids_start: 35756 + operation: replace + tags_end: 34244 + tags_start: 30000 + 31: + operation: search + 32: + end: 146908 + operation: insert + start: 140000 + 33: + operation: search + 34: + end: 156762 + operation: insert + start: 150000 + 35: + ids_end: 60000 + ids_start: 56642 + operation: replace + tags_end: 53358 + tags_start: 50000 + 36: + operation: search + 37: + end: 168642 + operation: insert + start: 160000 + 38: + operation: search + 39: + end: 177514 + operation: insert + start: 170000 + 40: + operation: search + 41: + end: 185743 + operation: insert + start: 180000 + 42: + operation: search + 43: + end: 195510 + operation: insert + start: 190000 + 44: + operation: search + 45: + end: 206769 + operation: insert + start: 200000 + 46: + end: 6984 + operation: delete + start: 0 + 47: + ids_end: 110000 + ids_start: 105079 + operation: replace + tags_end: 104921 + tags_start: 100000 + 48: + operation: search + 49: + end: 217627 + operation: insert + start: 210000 + 50: + end: 15354 + operation: delete + start: 10000 + 51: + ids_end: 120000 + ids_start: 116396 + operation: replace + tags_end: 113604 + tags_start: 110000 + 52: + operation: search + 53: + end: 227628 + operation: insert + start: 220000 + 54: + ids_end: 130000 + ids_start: 127894 + operation: replace + tags_end: 122106 + tags_start: 120000 + 55: + operation: search + 56: + end: 237364 + operation: insert + start: 230000 + 57: + end: 35756 + operation: delete + start: 30000 + 58: + ids_end: 140000 + ids_start: 138727 + operation: replace + tags_end: 131273 + tags_start: 130000 + 59: + operation: search + 60: + end: 245586 + operation: insert + start: 240000 + 61: + ids_end: 150000 + ids_start: 146908 + operation: replace + tags_end: 143092 + tags_start: 140000 + 62: + operation: search + 63: + end: 256089 + operation: insert + start: 250000 + 64: + end: 56642 + operation: delete + start: 50000 + 65: + operation: search + 66: + end: 265390 + operation: insert + start: 260000 + 67: + ids_end: 170000 + ids_start: 168642 + operation: replace + tags_end: 161358 + tags_start: 160000 + 68: + operation: search + 69: + end: 275633 + operation: insert + start: 270000 + 70: + ids_end: 180000 + ids_start: 177514 + operation: replace + tags_end: 172486 + tags_start: 170000 + 71: + operation: search + 72: + end: 287312 + operation: insert + start: 280000 + 73: + ids_end: 190000 + ids_start: 185743 + operation: replace + tags_end: 184257 + tags_start: 180000 + 74: + operation: search + 75: + end: 295727 + operation: insert + start: 290000 + 76: + ids_end: 200000 + ids_start: 195510 + operation: replace + tags_end: 194490 + tags_start: 190000 + 77: + operation: search + 78: + end: 306078 + operation: insert + start: 300000 + 79: + end: 105079 + operation: delete + start: 100000 + 80: + operation: search + 81: + end: 316989 + operation: insert + start: 310000 + 82: + end: 116396 + operation: delete + start: 110000 + 83: + operation: search + 84: + end: 328885 + operation: insert + start: 320000 + 85: + end: 127894 + operation: delete + start: 120000 + 86: + ids_end: 230000 + ids_start: 227628 + operation: replace + tags_end: 222372 + tags_start: 220000 + 87: + operation: search + 88: + end: 335996 + operation: insert + start: 330000 + 89: + end: 138727 + operation: delete + start: 130000 + 90: + ids_end: 240000 + ids_start: 237364 + operation: replace + tags_end: 232636 + tags_start: 230000 + 91: + operation: search + 92: + end: 347569 + operation: insert + start: 340000 + 93: + end: 146908 + operation: delete + start: 140000 + 94: + operation: search + 95: + end: 357017 + operation: insert + start: 350000 + 96: + ids_end: 260000 + ids_start: 256089 + operation: replace + tags_end: 253911 + tags_start: 250000 + 97: + operation: search + 98: + end: 367528 + operation: insert + start: 360000 + 99: + end: 168642 + operation: delete + start: 160000 + 100: + operation: search + 101: + end: 378628 + operation: insert + start: 370000 + 102: + end: 177514 + operation: delete + start: 170000 + 103: + ids_end: 280000 + ids_start: 275633 + operation: replace + tags_end: 274367 + tags_start: 270000 + 104: + ids_end: 350000 + ids_start: 347569 + operation: replace + tags_end: 342431 + tags_start: 340000 + 105: + operation: search + 106: + end: 385445 + operation: insert + start: 380000 + 107: + end: 185743 + operation: delete + start: 180000 + 108: + operation: search + 109: + end: 395650 + operation: insert + start: 390000 + 110: + end: 195510 + operation: delete + start: 190000 + 111: + ids_end: 300000 + ids_start: 295727 + operation: replace + tags_end: 294273 + tags_start: 290000 + 112: + operation: search + 113: + end: 406069 + operation: insert + start: 400000 + 114: + ids_end: 310000 + ids_start: 306078 + operation: replace + tags_end: 303922 + tags_start: 300000 + 115: + operation: search + 116: + end: 417088 + operation: insert + start: 410000 + 117: + ids_end: 320000 + ids_start: 316989 + operation: replace + tags_end: 313011 + tags_start: 310000 + 118: + operation: search + 119: + end: 426737 + operation: insert + start: 420000 + 120: + end: 227628 + operation: delete + start: 220000 + 121: + ids_end: 330000 + ids_start: 328885 + operation: replace + tags_end: 321115 + tags_start: 320000 + 122: + operation: search + 123: + end: 437352 + operation: insert + start: 430000 + 124: + end: 237364 + operation: delete + start: 230000 + 125: + ids_end: 340000 + ids_start: 335996 + operation: replace + tags_end: 334004 + tags_start: 330000 + 126: + operation: search + 127: + end: 446515 + operation: insert + start: 440000 + 128: + operation: search + 129: + end: 456511 + operation: insert + start: 450000 + 130: + end: 256089 + operation: delete + start: 250000 + 131: + ids_end: 360000 + ids_start: 357017 + operation: replace + tags_end: 352983 + tags_start: 350000 + 132: + operation: search + 133: + end: 466520 + operation: insert + start: 460000 + 134: + ids_end: 220000 + ids_start: 217627 + operation: replace + tags_end: 212373 + tags_start: 210000 + 135: + operation: search + 136: + end: 478342 + operation: insert + start: 470000 + 137: + end: 275633 + operation: delete + start: 270000 + 138: + ids_end: 380000 + ids_start: 378628 + operation: replace + tags_end: 371372 + tags_start: 370000 + 139: + operation: search + 140: + end: 488707 + operation: insert + start: 480000 + 141: + ids_end: 390000 + ids_start: 385445 + operation: replace + tags_end: 384555 + tags_start: 380000 + 142: + operation: search + 143: + end: 498810 + operation: insert + start: 490000 + 144: + end: 295727 + operation: delete + start: 290000 + 145: + ids_end: 50000 + ids_start: 45330 + operation: replace + tags_end: 44670 + tags_start: 40000 + 146: + ids_end: 100000 + ids_start: 95177 + operation: replace + tags_end: 94823 + tags_start: 90000 + 147: + ids_end: 400000 + ids_start: 395650 + operation: replace + tags_end: 394350 + tags_start: 390000 + 148: + operation: search + 149: + end: 508747 + operation: insert + start: 500000 + 150: + end: 306078 + operation: delete + start: 300000 + 151: + operation: search + 152: + end: 518567 + operation: insert + start: 510000 + 153: + end: 316989 + operation: delete + start: 310000 + 154: + operation: search + 155: + end: 526963 + operation: insert + start: 520000 + 156: + end: 328885 + operation: delete + start: 320000 + 157: + ids_end: 30000 + ids_start: 25137 + operation: replace + tags_end: 24863 + tags_start: 20000 + 158: + operation: search + 159: + end: 537943 + operation: insert + start: 530000 + 160: + end: 335996 + operation: delete + start: 330000 + 161: + ids_end: 440000 + ids_start: 437352 + operation: replace + tags_end: 432648 + tags_start: 430000 + 162: + operation: search + 163: + end: 547352 + operation: insert + start: 540000 + 164: + ids_end: 450000 + ids_start: 446515 + operation: replace + tags_end: 443485 + tags_start: 440000 + 165: + operation: search + 166: + end: 558691 + operation: insert + start: 550000 + 167: + end: 357017 + operation: delete + start: 350000 + 168: + ids_end: 460000 + ids_start: 456511 + operation: replace + tags_end: 453489 + tags_start: 450000 + 169: + operation: search + 170: + end: 568362 + operation: insert + start: 560000 + 171: + ids_end: 70000 + ids_start: 66205 + operation: replace + tags_end: 63795 + tags_start: 60000 + 172: + operation: search + 173: + end: 578083 + operation: insert + start: 570000 + 174: + end: 378628 + operation: delete + start: 370000 + 175: + ids_end: 80000 + ids_start: 76901 + operation: replace + tags_end: 73099 + tags_start: 70000 + 176: + operation: search + 177: + end: 586128 + operation: insert + start: 580000 + 178: + end: 385445 + operation: delete + start: 380000 + 179: + ids_end: 90000 + ids_start: 88285 + operation: replace + tags_end: 81715 + tags_start: 80000 + 180: + operation: search + 181: + end: 598526 + operation: insert + start: 590000 + 182: + end: 395650 + operation: delete + start: 390000 + 183: + ids_end: 500000 + ids_start: 498810 + operation: replace + tags_end: 491190 + tags_start: 490000 + 184: + operation: search + 185: + end: 606875 + operation: insert + start: 600000 + 186: + operation: search + 187: + end: 615613 + operation: insert + start: 610000 + 188: + operation: search + 189: + end: 628233 + operation: insert + start: 620000 + 190: + operation: search + 191: + end: 635882 + operation: insert + start: 630000 + 192: + end: 437352 + operation: delete + start: 430000 + 193: + operation: search + 194: + end: 646348 + operation: insert + start: 640000 + 195: + end: 446515 + operation: delete + start: 440000 + 196: + operation: search + 197: + end: 658918 + operation: insert + start: 650000 + 198: + end: 456511 + operation: delete + start: 450000 + 199: + ids_end: 160000 + ids_start: 156762 + operation: replace + tags_end: 153238 + tags_start: 150000 + 200: + ids_end: 560000 + ids_start: 558691 + operation: replace + tags_end: 551309 + tags_start: 550000 + 201: + operation: search + 202: + end: 666780 + operation: insert + start: 660000 + 203: + operation: search + 204: + end: 678499 + operation: insert + start: 670000 + 205: + ids_end: 580000 + ids_start: 578083 + operation: replace + tags_end: 571917 + tags_start: 570000 + 206: + operation: search + 207: + end: 687858 + operation: insert + start: 680000 + 208: + ids_end: 590000 + ids_start: 586128 + operation: replace + tags_end: 583872 + tags_start: 580000 + 209: + operation: search + 210: + end: 697313 + operation: insert + start: 690000 + 211: + end: 498810 + operation: delete + start: 490000 + 212: + ids_end: 600000 + ids_start: 598526 + operation: replace + tags_end: 591474 + tags_start: 590000 + 213: + operation: search + 214: + end: 708960 + operation: insert + start: 700000 + 215: + ids_end: 210000 + ids_start: 206769 + operation: replace + tags_end: 203231 + tags_start: 200000 + 216: + ids_end: 610000 + ids_start: 606875 + operation: replace + tags_end: 603125 + tags_start: 600000 + 217: + operation: search + 218: + end: 716744 + operation: insert + start: 710000 + 219: + ids_end: 620000 + ids_start: 615613 + operation: replace + tags_end: 614387 + tags_start: 610000 + 220: + operation: search + 221: + end: 726852 + operation: insert + start: 720000 + 222: + ids_end: 630000 + ids_start: 628233 + operation: replace + tags_end: 621767 + tags_start: 620000 + 223: + operation: search + 224: + end: 736435 + operation: insert + start: 730000 + 225: + operation: search + 226: + end: 746156 + operation: insert + start: 740000 + 227: + ids_end: 250000 + ids_start: 245586 + operation: replace + tags_end: 244414 + tags_start: 240000 + 228: + operation: search + 229: + end: 758269 + operation: insert + start: 750000 + 230: + end: 558691 + operation: delete + start: 550000 + 231: + ids_end: 660000 + ids_start: 658918 + operation: replace + tags_end: 651082 + tags_start: 650000 + 232: + operation: search + 233: + end: 767633 + operation: insert + start: 760000 + 234: + ids_end: 270000 + ids_start: 265390 + operation: replace + tags_end: 264610 + tags_start: 260000 + 235: + ids_end: 670000 + ids_start: 666780 + operation: replace + tags_end: 663220 + tags_start: 660000 + 236: + operation: search + 237: + end: 778228 + operation: insert + start: 770000 + 238: + end: 578083 + operation: delete + start: 570000 + 239: + operation: search + 240: + end: 785943 + operation: insert + start: 780000 + 241: + end: 586128 + operation: delete + start: 580000 + 242: + ids_end: 290000 + ids_start: 287312 + operation: replace + tags_end: 282688 + tags_start: 280000 + 243: + ids_end: 690000 + ids_start: 687858 + operation: replace + tags_end: 682142 + tags_start: 680000 + 244: + operation: search + 245: + end: 795221 + operation: insert + start: 790000 + 246: + end: 598526 + operation: delete + start: 590000 + 247: + ids_end: 700000 + ids_start: 697313 + operation: replace + tags_end: 692687 + tags_start: 690000 + 248: + operation: search + 249: + end: 808959 + operation: insert + start: 800000 + 250: + end: 606875 + operation: delete + start: 600000 + 251: + ids_end: 710000 + ids_start: 708960 + operation: replace + tags_end: 701040 + tags_start: 700000 + 252: + operation: search + 253: + end: 816098 + operation: insert + start: 810000 + 254: + end: 615613 + operation: delete + start: 610000 + 255: + ids_end: 720000 + ids_start: 716744 + operation: replace + tags_end: 713256 + tags_start: 710000 + 256: + operation: search + 257: + end: 827128 + operation: insert + start: 820000 + 258: + end: 628233 + operation: delete + start: 620000 + 259: + operation: search + 260: + end: 835574 + operation: insert + start: 830000 + 261: + operation: search + 262: + end: 845675 + operation: insert + start: 840000 + 263: + operation: search + 264: + end: 856686 + operation: insert + start: 850000 + 265: + end: 658918 + operation: delete + start: 650000 + 266: + ids_end: 760000 + ids_start: 758269 + operation: replace + tags_end: 751731 + tags_start: 750000 + 267: + operation: search + 268: + end: 868969 + operation: insert + start: 860000 + 269: + end: 666780 + operation: delete + start: 660000 + 270: + ids_end: 370000 + ids_start: 367528 + operation: replace + tags_end: 362472 + tags_start: 360000 + 271: + operation: search + 272: + end: 878008 + operation: insert + start: 870000 + 273: + ids_end: 780000 + ids_start: 778228 + operation: replace + tags_end: 771772 + tags_start: 770000 + 274: + operation: search + 275: + end: 887151 + operation: insert + start: 880000 + 276: + end: 687858 + operation: delete + start: 680000 + 277: + ids_end: 790000 + ids_start: 785943 + operation: replace + tags_end: 784057 + tags_start: 780000 + 278: + operation: search + 279: + end: 898954 + operation: insert + start: 890000 + 280: + end: 697313 + operation: delete + start: 690000 + 281: + operation: search + 282: + end: 908889 + operation: insert + start: 900000 + 283: + end: 708960 + operation: delete + start: 700000 + 284: + ids_end: 410000 + ids_start: 406069 + operation: replace + tags_end: 403931 + tags_start: 400000 + 285: + ids_end: 810000 + ids_start: 808959 + operation: replace + tags_end: 801041 + tags_start: 800000 + 286: + operation: search + 287: + end: 916825 + operation: insert + start: 910000 + 288: + end: 716744 + operation: delete + start: 710000 + 289: + operation: search + 290: + end: 928354 + operation: insert + start: 920000 + 291: + ids_end: 430000 + ids_start: 426737 + operation: replace + tags_end: 423263 + tags_start: 420000 + 292: + ids_end: 830000 + ids_start: 827128 + operation: replace + tags_end: 822872 + tags_start: 820000 + 293: + operation: search + 294: + end: 935094 + operation: insert + start: 930000 + 295: + operation: search + 296: + end: 945059 + operation: insert + start: 940000 + 297: + operation: search + 298: + end: 955249 + operation: insert + start: 950000 + 299: + end: 758269 + operation: delete + start: 750000 + 300: + ids_end: 860000 + ids_start: 856686 + operation: replace + tags_end: 853314 + tags_start: 850000 + 301: + operation: search + 302: + end: 966129 + operation: insert + start: 960000 + 303: + ids_end: 470000 + ids_start: 466520 + operation: replace + tags_end: 463480 + tags_start: 460000 + 304: + ids_end: 870000 + ids_start: 868969 + operation: replace + tags_end: 861031 + tags_start: 860000 + 305: + operation: search + 306: + end: 978693 + operation: insert + start: 970000 + 307: + end: 778228 + operation: delete + start: 770000 + 308: + ids_end: 480000 + ids_start: 478342 + operation: replace + tags_end: 471658 + tags_start: 470000 + 309: + ids_end: 880000 + ids_start: 878008 + operation: replace + tags_end: 871992 + tags_start: 870000 + 310: + operation: search + 311: + end: 988547 + operation: insert + start: 980000 + 312: + end: 785943 + operation: delete + start: 780000 + 313: + ids_end: 490000 + ids_start: 488707 + operation: replace + tags_end: 481293 + tags_start: 480000 + 314: + ids_end: 910000 + ids_start: 908889 + operation: replace + tags_end: 901111 + tags_start: 900000 + 315: + operation: search + 316: + end: 996357 + operation: insert + start: 990000 + 317: + operation: search + max_pts: 270000 diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml new file mode 100644 index 00000000..01349c42 --- /dev/null +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml @@ -0,0 +1,1094 @@ +msturing-1M: + 1: + end: 8844 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 18924 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 25996 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 36330 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 45116 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 57938 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 65638 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 75848 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 86273 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 96408 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 108673 + operation: insert + start: 100000 + 22: + ids_end: 10000 + ids_start: 8844 + operation: replace + tags_end: 1156 + tags_start: 0 + 23: + operation: search + 24: + end: 115146 + operation: insert + start: 110000 + 25: + ids_end: 20000 + ids_start: 18924 + operation: replace + tags_end: 11076 + tags_start: 10000 + 26: + operation: search + 27: + end: 127355 + operation: insert + start: 120000 + 28: + operation: search + 29: + end: 136667 + operation: insert + start: 130000 + 30: + ids_end: 40000 + ids_start: 36330 + operation: replace + tags_end: 33670 + tags_start: 30000 + 31: + operation: search + 32: + end: 147698 + operation: insert + start: 140000 + 33: + ids_end: 50000 + ids_start: 45116 + operation: replace + tags_end: 44884 + tags_start: 40000 + 34: + operation: search + 35: + end: 158979 + operation: insert + start: 150000 + 36: + operation: search + 37: + end: 167999 + operation: insert + start: 160000 + 38: + operation: search + 39: + end: 176900 + operation: insert + start: 170000 + 40: + ids_end: 80000 + ids_start: 75848 + operation: replace + tags_end: 74152 + tags_start: 70000 + 41: + operation: search + 42: + end: 188688 + operation: insert + start: 180000 + 43: + ids_end: 90000 + ids_start: 86273 + operation: replace + tags_end: 83727 + tags_start: 80000 + 44: + operation: search + 45: + end: 198898 + operation: insert + start: 190000 + 46: + operation: search + 47: + end: 206136 + operation: insert + start: 200000 + 48: + ids_end: 110000 + ids_start: 108673 + operation: replace + tags_end: 101327 + tags_start: 100000 + 49: + operation: search + 50: + end: 216825 + operation: insert + start: 210000 + 51: + ids_end: 120000 + ids_start: 115146 + operation: replace + tags_end: 114854 + tags_start: 110000 + 52: + operation: search + 53: + end: 227243 + operation: insert + start: 220000 + 54: + operation: search + 55: + end: 238280 + operation: insert + start: 230000 + 56: + ids_end: 140000 + ids_start: 136667 + operation: replace + tags_end: 133333 + tags_start: 130000 + 57: + operation: search + 58: + end: 247749 + operation: insert + start: 240000 + 59: + operation: search + 60: + end: 257034 + operation: insert + start: 250000 + 61: + ids_end: 160000 + ids_start: 158979 + operation: replace + tags_end: 151021 + tags_start: 150000 + 62: + operation: search + 63: + end: 265957 + operation: insert + start: 260000 + 64: + ids_end: 170000 + ids_start: 167999 + operation: replace + tags_end: 162001 + tags_start: 160000 + 65: + operation: search + 66: + end: 277335 + operation: insert + start: 270000 + 67: + operation: search + 68: + end: 285359 + operation: insert + start: 280000 + 69: + ids_end: 190000 + ids_start: 188688 + operation: replace + tags_end: 181312 + tags_start: 180000 + 70: + operation: search + 71: + end: 298081 + operation: insert + start: 290000 + 72: + ids_end: 200000 + ids_start: 198898 + operation: replace + tags_end: 191102 + tags_start: 190000 + 73: + operation: search + 74: + end: 308638 + operation: insert + start: 300000 + 75: + ids_end: 210000 + ids_start: 206136 + operation: replace + tags_end: 203864 + tags_start: 200000 + 76: + operation: search + 77: + end: 318941 + operation: insert + start: 310000 + 78: + ids_end: 220000 + ids_start: 216825 + operation: replace + tags_end: 213175 + tags_start: 210000 + 79: + operation: search + 80: + end: 325288 + operation: insert + start: 320000 + 81: + ids_end: 230000 + ids_start: 227243 + operation: replace + tags_end: 222757 + tags_start: 220000 + 82: + operation: search + 83: + end: 338313 + operation: insert + start: 330000 + 84: + ids_end: 240000 + ids_start: 238280 + operation: replace + tags_end: 231720 + tags_start: 230000 + 85: + operation: search + 86: + end: 346984 + operation: insert + start: 340000 + 87: + ids_end: 250000 + ids_start: 247749 + operation: replace + tags_end: 242251 + tags_start: 240000 + 88: + operation: search + 89: + end: 358981 + operation: insert + start: 350000 + 90: + operation: search + 91: + end: 368169 + operation: insert + start: 360000 + 92: + operation: search + 93: + end: 375661 + operation: insert + start: 370000 + 94: + ids_end: 280000 + ids_start: 277335 + operation: replace + tags_end: 272665 + tags_start: 270000 + 95: + operation: search + 96: + end: 386309 + operation: insert + start: 380000 + 97: + ids_end: 290000 + ids_start: 285359 + operation: replace + tags_end: 284641 + tags_start: 280000 + 98: + operation: search + 99: + end: 395017 + operation: insert + start: 390000 + 100: + ids_end: 390000 + ids_start: 386309 + operation: replace + tags_end: 383691 + tags_start: 380000 + 101: + operation: search + 102: + end: 405729 + operation: insert + start: 400000 + 103: + ids_end: 310000 + ids_start: 308638 + operation: replace + tags_end: 301362 + tags_start: 300000 + 104: + operation: search + 105: + end: 416663 + operation: insert + start: 410000 + 106: + ids_end: 320000 + ids_start: 318941 + operation: replace + tags_end: 311059 + tags_start: 310000 + 107: + operation: search + 108: + end: 426477 + operation: insert + start: 420000 + 109: + ids_end: 330000 + ids_start: 325288 + operation: replace + tags_end: 324712 + tags_start: 320000 + 110: + operation: search + 111: + end: 436771 + operation: insert + start: 430000 + 112: + ids_end: 340000 + ids_start: 338313 + operation: replace + tags_end: 331687 + tags_start: 330000 + 113: + operation: search + 114: + end: 447512 + operation: insert + start: 440000 + 115: + ids_end: 350000 + ids_start: 346984 + operation: replace + tags_end: 343016 + tags_start: 340000 + 116: + operation: search + 117: + end: 455625 + operation: insert + start: 450000 + 118: + operation: search + 119: + end: 465802 + operation: insert + start: 460000 + 120: + ids_end: 370000 + ids_start: 368169 + operation: replace + tags_end: 361831 + tags_start: 360000 + 121: + operation: search + 122: + end: 478856 + operation: insert + start: 470000 + 123: + ids_end: 180000 + ids_start: 176900 + operation: replace + tags_end: 173100 + tags_start: 170000 + 124: + ids_end: 380000 + ids_start: 375661 + operation: replace + tags_end: 374339 + tags_start: 370000 + 125: + operation: search + 126: + end: 488196 + operation: insert + start: 480000 + 127: + operation: search + 128: + end: 498985 + operation: insert + start: 490000 + 129: + ids_end: 400000 + ids_start: 395017 + operation: replace + tags_end: 394983 + tags_start: 390000 + 130: + operation: search + 131: + end: 506281 + operation: insert + start: 500000 + 132: + ids_end: 410000 + ids_start: 405729 + operation: replace + tags_end: 404271 + tags_start: 400000 + 133: + operation: search + 134: + end: 517184 + operation: insert + start: 510000 + 135: + operation: search + 136: + end: 527275 + operation: insert + start: 520000 + 137: + ids_end: 30000 + ids_start: 25996 + operation: replace + tags_end: 24004 + tags_start: 20000 + 138: + ids_end: 430000 + ids_start: 426477 + operation: replace + tags_end: 423523 + tags_start: 420000 + 139: + operation: search + 140: + end: 538153 + operation: insert + start: 530000 + 141: + ids_end: 440000 + ids_start: 436771 + operation: replace + tags_end: 433229 + tags_start: 430000 + 142: + operation: search + 143: + end: 545662 + operation: insert + start: 540000 + 144: + ids_end: 450000 + ids_start: 447512 + operation: replace + tags_end: 442488 + tags_start: 440000 + 145: + operation: search + 146: + end: 557936 + operation: insert + start: 550000 + 147: + ids_end: 60000 + ids_start: 57938 + operation: replace + tags_end: 52062 + tags_start: 50000 + 148: + ids_end: 460000 + ids_start: 455625 + operation: replace + tags_end: 454375 + tags_start: 450000 + 149: + operation: search + 150: + end: 568765 + operation: insert + start: 560000 + 151: + ids_end: 70000 + ids_start: 65638 + operation: replace + tags_end: 64362 + tags_start: 60000 + 152: + ids_end: 470000 + ids_start: 465802 + operation: replace + tags_end: 464198 + tags_start: 460000 + 153: + operation: search + 154: + end: 575634 + operation: insert + start: 570000 + 155: + ids_end: 480000 + ids_start: 478856 + operation: replace + tags_end: 471144 + tags_start: 470000 + 156: + operation: search + 157: + end: 585561 + operation: insert + start: 580000 + 158: + ids_end: 490000 + ids_start: 488196 + operation: replace + tags_end: 481804 + tags_start: 480000 + 159: + operation: search + 160: + end: 595733 + operation: insert + start: 590000 + 161: + ids_end: 100000 + ids_start: 96408 + operation: replace + tags_end: 93592 + tags_start: 90000 + 162: + ids_end: 500000 + ids_start: 498985 + operation: replace + tags_end: 491015 + tags_start: 490000 + 163: + operation: search + 164: + end: 607223 + operation: insert + start: 600000 + 165: + operation: search + 166: + end: 616949 + operation: insert + start: 610000 + 167: + ids_end: 520000 + ids_start: 517184 + operation: replace + tags_end: 512816 + tags_start: 510000 + 168: + operation: search + 169: + end: 627143 + operation: insert + start: 620000 + 170: + ids_end: 130000 + ids_start: 127355 + operation: replace + tags_end: 122645 + tags_start: 120000 + 171: + ids_end: 530000 + ids_start: 527275 + operation: replace + tags_end: 522725 + tags_start: 520000 + 172: + operation: search + 173: + end: 636104 + operation: insert + start: 630000 + 174: + ids_end: 540000 + ids_start: 538153 + operation: replace + tags_end: 531847 + tags_start: 530000 + 175: + operation: search + 176: + end: 647869 + operation: insert + start: 640000 + 177: + ids_end: 150000 + ids_start: 147698 + operation: replace + tags_end: 142302 + tags_start: 140000 + 178: + ids_end: 550000 + ids_start: 545662 + operation: replace + tags_end: 544338 + tags_start: 540000 + 179: + operation: search + 180: + end: 656712 + operation: insert + start: 650000 + 181: + operation: search + 182: + end: 667029 + operation: insert + start: 660000 + 183: + operation: search + 184: + end: 675811 + operation: insert + start: 670000 + 185: + ids_end: 580000 + ids_start: 575634 + operation: replace + tags_end: 574366 + tags_start: 570000 + 186: + operation: search + 187: + end: 685846 + operation: insert + start: 680000 + 188: + operation: search + 189: + end: 697956 + operation: insert + start: 690000 + 190: + ids_end: 600000 + ids_start: 595733 + operation: replace + tags_end: 594267 + tags_start: 590000 + 191: + operation: search + 192: + end: 708233 + operation: insert + start: 700000 + 193: + ids_end: 610000 + ids_start: 607223 + operation: replace + tags_end: 602777 + tags_start: 600000 + 194: + operation: search + 195: + end: 718522 + operation: insert + start: 710000 + 196: + ids_end: 620000 + ids_start: 616949 + operation: replace + tags_end: 613051 + tags_start: 610000 + 197: + operation: search + 198: + end: 727277 + operation: insert + start: 720000 + 199: + ids_end: 630000 + ids_start: 627143 + operation: replace + tags_end: 622857 + tags_start: 620000 + 200: + operation: search + 201: + end: 737232 + operation: insert + start: 730000 + 202: + ids_end: 640000 + ids_start: 636104 + operation: replace + tags_end: 633896 + tags_start: 630000 + 203: + operation: search + 204: + end: 745364 + operation: insert + start: 740000 + 205: + ids_end: 650000 + ids_start: 647869 + operation: replace + tags_end: 642131 + tags_start: 640000 + 206: + operation: search + 207: + end: 756547 + operation: insert + start: 750000 + 208: + ids_end: 260000 + ids_start: 257034 + operation: replace + tags_end: 252966 + tags_start: 250000 + 209: + operation: search + 210: + end: 766299 + operation: insert + start: 760000 + 211: + ids_end: 270000 + ids_start: 265957 + operation: replace + tags_end: 264043 + tags_start: 260000 + 212: + ids_end: 670000 + ids_start: 667029 + operation: replace + tags_end: 662971 + tags_start: 660000 + 213: + operation: search + 214: + end: 776459 + operation: insert + start: 770000 + 215: + operation: search + 216: + end: 788998 + operation: insert + start: 780000 + 217: + ids_end: 690000 + ids_start: 685846 + operation: replace + tags_end: 684154 + tags_start: 680000 + 218: + operation: search + 219: + end: 797483 + operation: insert + start: 790000 + 220: + ids_end: 300000 + ids_start: 298081 + operation: replace + tags_end: 291919 + tags_start: 290000 + 221: + ids_end: 700000 + ids_start: 697956 + operation: replace + tags_end: 692044 + tags_start: 690000 + 222: + operation: search + 223: + end: 806394 + operation: insert + start: 800000 + 224: + ids_end: 710000 + ids_start: 708233 + operation: replace + tags_end: 701767 + tags_start: 700000 + 225: + operation: search + 226: + end: 815353 + operation: insert + start: 810000 + 227: + operation: search + 228: + end: 826206 + operation: insert + start: 820000 + 229: + ids_end: 730000 + ids_start: 727277 + operation: replace + tags_end: 722723 + tags_start: 720000 + 230: + operation: search + 231: + end: 835334 + operation: insert + start: 830000 + 232: + ids_end: 740000 + ids_start: 737232 + operation: replace + tags_end: 732768 + tags_start: 730000 + 233: + operation: search + 234: + end: 846259 + operation: insert + start: 840000 + 235: + ids_end: 750000 + ids_start: 745364 + operation: replace + tags_end: 744636 + tags_start: 740000 + 236: + operation: search + 237: + end: 857065 + operation: insert + start: 850000 + 238: + ids_end: 360000 + ids_start: 358981 + operation: replace + tags_end: 351019 + tags_start: 350000 + 239: + ids_end: 760000 + ids_start: 756547 + operation: replace + tags_end: 753453 + tags_start: 750000 + 240: + operation: search + 241: + end: 865379 + operation: insert + start: 860000 + 242: + ids_end: 770000 + ids_start: 766299 + operation: replace + tags_end: 763701 + tags_start: 760000 + 243: + operation: search + 244: + end: 875198 + operation: insert + start: 870000 + 245: + ids_end: 780000 + ids_start: 776459 + operation: replace + tags_end: 773541 + tags_start: 770000 + 246: + operation: search + 247: + end: 885741 + operation: insert + start: 880000 + 248: + ids_end: 790000 + ids_start: 788998 + operation: replace + tags_end: 781002 + tags_start: 780000 + 249: + operation: search + 250: + end: 895870 + operation: insert + start: 890000 + 251: + ids_end: 800000 + ids_start: 797483 + operation: replace + tags_end: 792517 + tags_start: 790000 + 252: + operation: search + 253: + end: 906094 + operation: insert + start: 900000 + 254: + ids_end: 810000 + ids_start: 806394 + operation: replace + tags_end: 803606 + tags_start: 800000 + 255: + operation: search + 256: + end: 916822 + operation: insert + start: 910000 + 257: + ids_end: 420000 + ids_start: 416663 + operation: replace + tags_end: 413337 + tags_start: 410000 + 258: + ids_end: 820000 + ids_start: 815353 + operation: replace + tags_end: 814647 + tags_start: 810000 + 259: + operation: search + 260: + end: 925862 + operation: insert + start: 920000 + 261: + ids_end: 830000 + ids_start: 826206 + operation: replace + tags_end: 823794 + tags_start: 820000 + 262: + operation: search + 263: + end: 938967 + operation: insert + start: 930000 + 264: + ids_end: 840000 + ids_start: 835334 + operation: replace + tags_end: 834666 + tags_start: 830000 + 265: + operation: search + 266: + end: 946541 + operation: insert + start: 940000 + 267: + ids_end: 850000 + ids_start: 846259 + operation: replace + tags_end: 843741 + tags_start: 840000 + 268: + operation: search + 269: + end: 956682 + operation: insert + start: 950000 + 270: + ids_end: 860000 + ids_start: 857065 + operation: replace + tags_end: 852935 + tags_start: 850000 + 271: + operation: search + 272: + end: 966997 + operation: insert + start: 960000 + 273: + ids_end: 870000 + ids_start: 865379 + operation: replace + tags_end: 864621 + tags_start: 860000 + 274: + operation: search + 275: + end: 976819 + operation: insert + start: 970000 + 276: + operation: search + 277: + end: 988752 + operation: insert + start: 980000 + 278: + ids_end: 890000 + ids_start: 885741 + operation: replace + tags_end: 884259 + tags_start: 880000 + 279: + operation: search + 280: + end: 997302 + operation: insert + start: 990000 + 281: + ids_end: 900000 + ids_start: 895870 + operation: replace + tags_end: 894130 + tags_start: 890000 + 282: + operation: search + max_pts: 696111 diff --git a/neurips23/runbooks/msturing-1M_expiration_time_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml similarity index 99% rename from neurips23/runbooks/msturing-1M_expiration_time_runbook.yaml rename to neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml index 770128dd..6c299678 100644 --- a/neurips23/runbooks/msturing-1M_expiration_time_runbook.yaml +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml @@ -1,4 +1,4 @@ -msturing-1M: +msturingpq-1M: 1: end: 10000 operation: insert @@ -843,4 +843,4 @@ msturing-1M: start: 790000 261: operation: search - max_pts: 500000 + max_pts: 440000 diff --git a/neurips23/streaming/diskann/config.yaml b/neurips23/streaming/diskann/config.yaml index a0d2b9ab..114d4536 100644 --- a/neurips23/streaming/diskann/config.yaml +++ b/neurips23/streaming/diskann/config.yaml @@ -35,7 +35,7 @@ msspacev-1M: query-args: | [{"Ls":300, "T":16}, {"Ls":100, "T":16}] -msturing-1M: +msturingpq-1M: diskann: docker-tag: neurips23-streaming-diskann module: neurips23.streaming.diskann.diskann-str @@ -44,10 +44,9 @@ msturing-1M: run-groups: base: args: | - [{"R":50, "L":50, "insert_threads":16, "consolidate_threads":16}] + [{"R":64, "L":128, "insert_threads":32, "consolidate_threads":32}] query-args: | - [{"Ls":300, "T":16}, - {"Ls":100, "T":16}] + [{"Ls":128, "T":16}] wikipedia-1M: diskann: docker-tag: neurips23-streaming-diskann From 99067a927958f76550a14b46312982e620e8f798 Mon Sep 17 00:00:00 2001 From: Magdalen Date: Tue, 19 Nov 2024 14:33:54 -0800 Subject: [PATCH 4/6] changes --- benchmark/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/datasets.py b/benchmark/datasets.py index a7307e13..b6345621 100644 --- a/benchmark/datasets.py +++ b/benchmark/datasets.py @@ -435,7 +435,7 @@ def distance(self): class MSTuringANNSPQ(BillionScaleDatasetCompetitionFormat): def __init__(self, nb_M=1): self.nb_M = nb_M - self.nb = 10**6 * nb_M + self.nb = 10**6 * nb_M self.d = 100 self.nq = 10000 self.dtype = "float32" From fe7103e8862b8d2a07292fac94e37869cf27d6a2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 20 Nov 2024 16:22:49 +0000 Subject: [PATCH 5/6] added gt urls, added configurable max points --- .../runbooks/gen_expiration_time_runbook.py | 24 ++++++++++++------- requirements_py3.10.txt | 2 +- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py index 23ee7ad2..f9b3be65 100644 --- a/neurips23/runbooks/gen_expiration_time_runbook.py +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -11,8 +11,10 @@ timesteps: how long to wait before deleting for each ratio seed: seed given to random generator do_replace: whether to include replace in runbook or not +do_delete: whether to include delete in the runbook or not +max_pts: whether to set a max_pts higher than the max that occurs in the runbook (default None) ''' -def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ratios, timesteps, seed = 0, do_replace = False, gt_url = None, do_delete = True): +def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ratios, timesteps, seed = 0, do_replace = False, gt_url = None, do_delete = True, max_points = None): random.seed(seed) data = {dataset_name: {}} @@ -99,7 +101,10 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra } t+=1 - data[dataset_name]["max_pts"]=max_num_points + if max_points != None & max_points >= max_num_points: + data[dataset_name]["max_pts"]=max_points + else: + data[dataset_name]["max_pts"]=max_num_points if gt_url is not None: data[dataset_name]["gt_url"] = gt_url @@ -183,8 +188,9 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra dataset_name = 'msturingpq-1M' dataset_size = 1000000 max_t = 100 -gt_url = None -gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url) +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_runbook.yaml" +max_points = 450000 +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url, max_points) ratios = (0, 4, 18) timesteps = (0, 100, 20) @@ -193,8 +199,9 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra dataset_name = 'msturing-1M' dataset_size = 1000000 max_t = 100 -gt_url = None -gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False) +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_only_runbook.yaml" +max_points = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False, max_points) ratios = (1, 8, 18) timesteps = (0, 100, 20) @@ -203,6 +210,7 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra dataset_name = 'msturingpq-1M' dataset_size = 1000000 max_t = 100 -gt_url = None -gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url) +gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_delete_runbook.yaml" +max_points = 270000 +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, max_points) diff --git a/requirements_py3.10.txt b/requirements_py3.10.txt index 3c5b5490..6e451c69 100644 --- a/requirements_py3.10.txt +++ b/requirements_py3.10.txt @@ -1,7 +1,7 @@ ansicolors==1.1.8 docker==7.1.0 h5py==3.10.0 -matplotlib==3.3.4 + numpy==1.24.2 pyyaml==6.0 psutil==5.9.4 From f3a7ad97f59e8cc97649617fd3b8edbd6f8be25e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 27 Nov 2024 20:53:18 +0000 Subject: [PATCH 6/6] add gt urls' --- neurips23/runbooks/gen_expiration_time_runbook.py | 7 +++++-- ...turingpq-1M_expiration_time_replace_delete_runbook.yaml | 3 ++- ...msturingpq-1M_expiration_time_replace_only_runbook.yaml | 1 + .../runbooks/msturingpq-1M_expiration_time_runbook.yaml | 3 ++- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py index f9b3be65..2ffd55ed 100644 --- a/neurips23/runbooks/gen_expiration_time_runbook.py +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -101,8 +101,11 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra } t+=1 - if max_points != None & max_points >= max_num_points: - data[dataset_name]["max_pts"]=max_points + if max_points != None: + if max_points >= max_num_points: + data[dataset_name]["max_pts"]=max_points + else: + data[dataset_name]["max_pts"]=max_num_points else: data[dataset_name]["max_pts"]=max_num_points diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml index b55ba4a4..178b2c85 100644 --- a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml @@ -1209,4 +1209,5 @@ msturingpq-1M: start: 990000 317: operation: search - max_pts: 270000 + max_pts: 248123 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_delete_runbook.yaml diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml index 01349c42..09af04ac 100644 --- a/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml @@ -1092,3 +1092,4 @@ msturing-1M: 282: operation: search max_pts: 696111 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_only_runbook.yaml diff --git a/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml b/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml index 6c299678..61db003a 100644 --- a/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml +++ b/neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml @@ -843,4 +843,5 @@ msturingpq-1M: start: 790000 261: operation: search - max_pts: 440000 + max_pts: 400000 + gt_url: https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_runbook.yaml