diff --git a/benchmark/streaming/load_runbook.py b/benchmark/streaming/load_runbook.py index ec70e1da..e1c6eede 100644 --- a/benchmark/streaming/load_runbook.py +++ b/benchmark/streaming/load_runbook.py @@ -30,6 +30,7 @@ def load_runbook(dataset_name, max_pts, runbook_file): if entry['tags_start'] < 0 or entry ['tags_start'] >= max_pts: raise Exception('Start of indices to be replaced out of range in runbook') if entry['tags_end'] < 0 or entry ['tags_end'] > max_pts: + print(entry['tags_end']) raise Exception('End of indices to be replaced out of range in runbook') if entry['ids_start'] < 0 or entry ['ids_start'] >= max_pts: raise Exception('Start of indices to replace out of range in runbook') diff --git a/data_export.py b/data_export.py index 6c24216b..79abe2a5 100644 --- a/data_export.py +++ b/data_export.py @@ -96,17 +96,18 @@ def cleaned_run_metric(run_metrics): dataset = DATASETS[dataset_name]() runbook_paths = [None] if track == 'streaming': - runbook_paths = ['neurips23/streaming/simple_runbook.yaml', - 'neurips23/streaming/simple_replace_runbook.yaml', - 'neurips23/streaming/random_replace_runbook.yaml', - 'neurips23/streaming/clustered_replace_runbook.yaml', - 'neurips23/streaming/clustered_runbook.yaml', - 'neurips23/streaming/clustered_runbook.yaml', - 'neurips23/streaming/delete_runbook.yaml', - 'neurips23/streaming/final_runbook.yaml', - 'neurips23/streaming/msturing-10M_slidingwindow_runbook.yaml', - 'neurips23/streaming/wikipedia-35M_expirationtime_runbook.yaml', - 'neurips23/streaming/msmarco-100M_expirationtime_runbook.yaml'] + runbook_paths = ['neurips23/runbooks/streaming/simple_runbook.yaml', + 'neurips23/runbooks/streaming/simple_replace_runbook.yaml', + 'neurips23/runbooks/streaming/random_replace_runbook.yaml', + 'neurips23/runbooks/streaming/clustered_replace_runbook.yaml', + 'neurips23/runbooks/streaming/clustered_runbook.yaml', + 'neurips23/runbooks/streaming/clustered_runbook.yaml', + 'neurips23/runbooks/streaming/delete_runbook.yaml', + 'neurips23/runbooks/streaming/final_runbook.yaml', + 'neurips23/runbooks/streaming/msturing-10M_slidingwindow_runbook.yaml', + 'neurips23/runbooks/streaming/wikipedia-35M_expirationtime_runbook.yaml', + 'neurips23/runbooks/streaming/wikipedia-35M_expiration_time_replace_runbook.yaml', + 'neurips23/runbooks/streaming/msmarco-100M_expirationtime_runbook.yaml'] for runbook_path in runbook_paths: print("Looking for runbook ", runbook_path) results = load_all_results(dataset_name, neurips23track=track, runbook_path=runbook_path) diff --git a/neurips23/README.md b/neurips23/README.md index b6b448c2..11808816 100644 --- a/neurips23/README.md +++ b/neurips23/README.md @@ -46,8 +46,8 @@ The baselines were run on an Azure Standard D8lds v5 (8 vcpus, 16 GiB memory) ma |---------|-------------|-----------------------------|---------| |Sparse | Linear Scan | 101 | `python3 run.py --dataset sparse-full --algorithm linscan --neurips23track sparse` | |Filter | faiss | 3200 | `python3 run.py --dataset yfcc-10M --algorithm faiss --neurips23track filter` | -|Streaming| DiskANN | 0.924 (recall@10), 23 mins | `python3 run.py --dataset msturing-10M-clustered --algorithm diskann --neurips23track streaming --runbook_path neurips23/streaming/delete_runbook.yaml` | -|Streaming| DiskANN | 0.883 (recall@10), 45 mins | `python3 run.py --dataset msturing-30M-clustered --algorithm diskann --neurips23track streaming --runbook_path neurips23/streaming/final_runbook.yaml` | +|Streaming| DiskANN | 0.924 (recall@10), 23 mins | `python3 run.py --dataset msturing-10M-clustered --algorithm diskann --neurips23track streaming --runbook_path neurips23/runbooks/delete_runbook.yaml` | +|Streaming| DiskANN | 0.883 (recall@10), 45 mins | `python3 run.py --dataset msturing-30M-clustered --algorithm diskann --neurips23track streaming --runbook_path neurips23/runbooks/final_runbook.yaml` | |OOD | DiskANN | 4882 | `python3 run.py --dataset text2image-10M --algorithm diskann --neurips23track ood` | ## For_Participants @@ -99,7 +99,7 @@ Test the benchmark and baseline using the algorithm's definition file on small t python run.py --neurips23track filter --algorithm faiss --dataset random-filter-s python run.py --neurips23track sparse --algorithm linscan --dataset sparse-small python run.py --neurips23track ood --algorithm diskann --dataset random-xs -python run.py --neurips23track streaming --algorithm diskann --dataset random-xs --runbook_path neurips23/streaming/simple_runbook.yaml +python run.py --neurips23track streaming --algorithm diskann --dataset random-xs --runbook_path neurips23/runbooks/simple_runbook.yaml ``` For the competition dataset, run commands mentioned in the table above, for example: @@ -108,22 +108,22 @@ python run.py --neurips23track filter --algorithm faiss --dataset yfcc-10M python run.py --neurips23track sparse --algorithm linscan --dataset sparse-full python run.py --neurips23track ood --algorithm diskann --dataset text2image-10M # preliminary runbook for testing -python run.py --neurips23track streaming --algorithm diskann --dataset msturing-10M-clustered --runbook_path neurips23/streaming/delete_runbook.yaml +python run.py --neurips23track streaming --algorithm diskann --dataset msturing-10M-clustered --runbook_path neurips23/runbooks/delete_runbook.yaml #Final runbook for evaluation -python run.py --neurips23track streaming --algorithm diskann --dataset msturing-30M-clustered --runbook_path neurips23/streaming/final_runbook.yaml +python run.py --neurips23track streaming --algorithm diskann --dataset msturing-30M-clustered --runbook_path neurips23/runbooks/final_runbook.yaml ``` For streaming track, runbook specifies the order of operations to be executed by the algorithms. To download the ground truth for every search operation: (needs azcopy tool in your binary path): ``` -python -m benchmark.streaming.download_gt --runbook_file neurips23/streaming/simple_runbook.yaml --dataset msspacev-10M -python -m benchmark.streaming.download_gt --runbook_file neurips23/streaming/delete_runbook.yaml --dataset msturing-10M-clustered -python -m benchmark.streaming.download_gt --runbook_file neurips23/streaming/final_runbook.yaml --dataset msturing-30M-clustered +python -m benchmark.streaming.download_gt --runbook_file neurips23/runbooks/simple_runbook.yaml --dataset msspacev-10M +python -m benchmark.streaming.download_gt --runbook_file neurips23/runbooks/delete_runbook.yaml --dataset msturing-10M-clustered +python -m benchmark.streaming.download_gt --runbook_file neurips23/runbooks/final_runbook.yaml --dataset msturing-30M-clustered ``` Alternately, to compute ground truth for an arbitrary runbook, [clone and build DiskANN repo](https://github.com/Microsoft/DiskANN) and use the command line tool to compute ground truth at various search checkpoints. The `--gt_cmdline_tool` points to the directory with DiskANN commandline tools. ``` -python benchmark/streaming/compute_gt.py --dataset msspacev-10M --runbook neurips23/streaming/simple_runbook.yaml --gt_cmdline_tool ~/DiskANN/build/apps/utils/compute_groundtruth +python benchmark/streaming/compute_gt.py --dataset msspacev-10M --runbook neurips23/runbooks/simple_runbook.yaml --gt_cmdline_tool ~/DiskANN/build/apps/utils/compute_groundtruth ``` -Consider also the examples in runbooks [here]]neurips23/streaming/clustered_runbook.yaml) and [here](neurips23/streaming/delete_runbook.yaml). The datasets here are [generated](neurips23/streaming/clustered_data_gen.py) by clustering the original dataset with k-means and packing points in the same cluster into contiguous indices. Then insertions are then performed one cluster at a time. This runbook tests if an indexing algorithm can adapt to data draft. The `max_pts` entry for the dataset in the runbook indicates an upper bound on the number of active points that the index must support during the runbook execution. +Consider also the examples in runbooks [here]]neurips23/runbooks/clustered_runbook.yaml) and [here](neurips23/runbooks/delete_runbook.yaml). The datasets here are [generated](neurips23/runbooks/clustered_data_gen.py) by clustering the original dataset with k-means and packing points in the same cluster into contiguous indices. Then insertions are then performed one cluster at a time. This runbook tests if an indexing algorithm can adapt to data draft. The `max_pts` entry for the dataset in the runbook indicates an upper bound on the number of active points that the index must support during the runbook execution. To make the results available for post-processing, change permissions of the results folder diff --git a/neurips23/streaming/clustered_data_gen.py b/neurips23/runbooks/clustered_data_gen.py similarity index 100% rename from neurips23/streaming/clustered_data_gen.py rename to neurips23/runbooks/clustered_data_gen.py diff --git a/neurips23/streaming/clustered_replace_runbook.yaml b/neurips23/runbooks/clustered_replace_runbook.yaml similarity index 100% rename from neurips23/streaming/clustered_replace_runbook.yaml rename to neurips23/runbooks/clustered_replace_runbook.yaml diff --git a/neurips23/streaming/clustered_runbook.yaml b/neurips23/runbooks/clustered_runbook.yaml similarity index 100% rename from neurips23/streaming/clustered_runbook.yaml rename to neurips23/runbooks/clustered_runbook.yaml diff --git a/neurips23/streaming/delete_runbook.yaml b/neurips23/runbooks/delete_runbook.yaml similarity index 100% rename from neurips23/streaming/delete_runbook.yaml rename to neurips23/runbooks/delete_runbook.yaml diff --git a/neurips23/streaming/final_runbook.yaml b/neurips23/runbooks/final_runbook.yaml similarity index 100% rename from neurips23/streaming/final_runbook.yaml rename to neurips23/runbooks/final_runbook.yaml diff --git a/neurips23/streaming/final_runbook_gen.py b/neurips23/runbooks/final_runbook_gen.py similarity index 100% rename from neurips23/streaming/final_runbook_gen.py rename to neurips23/runbooks/final_runbook_gen.py diff --git a/neurips23/runbooks/gen_expiration_time_runbook.py b/neurips23/runbooks/gen_expiration_time_runbook.py new file mode 100644 index 00000000..bb5996bc --- /dev/null +++ b/neurips23/runbooks/gen_expiration_time_runbook.py @@ -0,0 +1,176 @@ +import yaml +import os +import random + +''' +dataset_name: dataset key as specified in benchmark/datasets.py +dataset_size: size of datasets +max_t: number of timesteps +runbook_filename: name to save the runbook to +ratios: tuple of three numbers indicating proportion of deletes/replaces assigned to each timestep +timesteps: how long to wait before deleting for each ratio +seed: seed given to random generator +do_replace: whether to include replace in runbook or not +''' +def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ratios, timesteps, seed = 0, do_replace = False, gt_url = None, do_delete = True): + random.seed(seed) + data = {dataset_name: {}} + + max_num_points=0 + num_points=0 + + batch_size = dataset_size//max_t + to_delete=[[] for _ in range(max_t)] + to_replace=[[] for _ in range(max_t)] + + t=1 + + for i in range(max_t): + if do_replace: + fraction = random.uniform(.5, .9) + else: + fraction = 1.0 + start = i*batch_size + end = start + int(fraction*batch_size) + ids_start = end + ids_end = (i+1)*batch_size + tags_start = i*batch_size + tags_end = tags_start + (ids_end - ids_start) + replace_info = (tags_start, tags_end, ids_start, ids_end) + delete_info = (tags_start, end) + data[dataset_name][t]={ + 'operation': 'insert', + 'start': i*(batch_size), + 'end': end + } + t+=1 + + num_points+=int(fraction*batch_size) + + max_num_points=max(max_num_points,num_points) + + + data_type = random.randint(0, ratios[2]) + if do_delete: + if data_type <= ratios[0]: + pass + elif data_type > ratios[0] and data_type < ratios[1]: + if (i+timesteps[1] < max_t): + to_delete[i+timesteps[1]].append(delete_info) + else: + if (i+timesteps[2] < max_t): + to_delete[i+timesteps[2]].append(delete_info) + + if do_replace: + if data_type <= ratios[0]: + remaining_steps = (max_t - t)//2 + to_replace[i+remaining_steps].append(replace_info) + # with probability 1/19, the points get replaced at t_max-t/2 steps + elif data_type > ratios[0] and data_type < ratios[1]: + if (i + timesteps[1]//2 < max_t): + to_replace[i+timesteps[1]//2].append(replace_info) + # with probability 3/19, the points get replaced after 50 steps + else: + if (i + timesteps[2]//2 < max_t): + to_replace[i+timesteps[2]//2].append(replace_info) + # with probability 15/19, the points get replaced after 10 steps + + for (start, end) in to_delete[i]: + data[dataset_name][t]={ + 'operation': 'delete', + 'start': start, + 'end': end + } + t+=1 + num_points-=batch_size + + for (tags_start, tags_end, ids_start, ids_end) in to_replace[i]: + data[dataset_name][t] ={ + 'operation' : 'replace', + 'tags_start': tags_start, + 'tags_end': tags_end, + 'ids_start': ids_start, + 'ids_end': ids_end + } + t += 1 + + data[dataset_name][t]={ + 'operation': 'search', + } + t+=1 + + data[dataset_name]["max_pts"]=max_num_points + + if gt_url is not None: + data[dataset_name]["gt_url"] = gt_url + + with open(runbook_filename, 'w') as outfile: + yaml.dump(data, outfile, default_flow_style=False) + +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 809 +dataset_file = 'wikipedia-35M_expirationtime_runbook.yaml' +dataset_name = 'wikipedia-35M' +dataset_size = 35000000 +max_t = 350 +gt_url = "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expirationtime_runbook.yaml" +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url) + +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 1232 +dataset_file = 'wikipedia-1M_expiration_time_runbook.yaml' +dataset_name = 'wikipedia-1M' +dataset_size = 1000000 +max_t = 100 +gt_url = "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_runbook.yaml/" +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url) + +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 10001 +dataset_file = 'wikipedia-35M_expiration_time_replace_only_runbook.yaml' +dataset_name = 'wikipedia-35M' +dataset_size = 8000000 #only use a prefix of the dataset +max_t = 80 +gt_url = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False) + +ratios = (0, 4, 18) +timesteps = (0, 100, 20) +seed = 754 +dataset_file = 'wikipedia-1M_expiration_time_replace_only_runbook.yaml' +dataset_name = 'wikipedia-1M' +dataset_size = 1000000 +max_t = 100 +gt_url = None +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False) + +ratios = (3, 8, 18) +timesteps = (0, 300, 50) +seed = 22 +dataset_file = 'wikipedia-35M_expiration_time_replace_delete_runbook.yaml' +dataset_name = 'wikipedia-35M' +dataset_size = 35000000 +max_t = 350 +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, None) + +ratios = (1, 8, 18) +timesteps = (0, 100, 20) +seed = 56 +dataset_file = 'wikipedia-1M_expiration_time_replace_delete_runbook.yaml' +dataset_name = 'wikipedia-1M' +dataset_size = 1000000 +max_t = 100 +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, None) + +ratios = (0, 6, 25) +timesteps = (0, 200, 50) +seed = 809 +dataset_file = 'msmarco-100M_expirationtime_runbook.yaml' +dataset_name = 'msmarco-100M' +dataset_size = 101070374 +max_t = 1000 +gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, None) + diff --git a/neurips23/streaming/gen_replace_runbooks.py b/neurips23/runbooks/gen_replace_runbooks.py similarity index 100% rename from neurips23/streaming/gen_replace_runbooks.py rename to neurips23/runbooks/gen_replace_runbooks.py diff --git a/neurips23/streaming/generate_msturing10m_runbooks.py b/neurips23/runbooks/generate_msturing10m_runbooks.py similarity index 100% rename from neurips23/streaming/generate_msturing10m_runbooks.py rename to neurips23/runbooks/generate_msturing10m_runbooks.py diff --git a/neurips23/streaming/msmarco-100M_expirationtime_runbook.yaml b/neurips23/runbooks/msmarco-100M_expirationtime_runbook.yaml similarity index 100% rename from neurips23/streaming/msmarco-100M_expirationtime_runbook.yaml rename to neurips23/runbooks/msmarco-100M_expirationtime_runbook.yaml diff --git a/neurips23/streaming/msturing-10M_slidingwindow_runbook.yaml b/neurips23/runbooks/msturing-10M_slidingwindow_runbook.yaml similarity index 100% rename from neurips23/streaming/msturing-10M_slidingwindow_runbook.yaml rename to neurips23/runbooks/msturing-10M_slidingwindow_runbook.yaml diff --git a/neurips23/streaming/random_replace_runbook.yaml b/neurips23/runbooks/random_replace_runbook.yaml similarity index 100% rename from neurips23/streaming/random_replace_runbook.yaml rename to neurips23/runbooks/random_replace_runbook.yaml diff --git a/neurips23/streaming/simple_replace_runbook.yaml b/neurips23/runbooks/simple_replace_runbook.yaml similarity index 100% rename from neurips23/streaming/simple_replace_runbook.yaml rename to neurips23/runbooks/simple_replace_runbook.yaml diff --git a/neurips23/streaming/simple_runbook.yaml b/neurips23/runbooks/simple_runbook.yaml similarity index 100% rename from neurips23/streaming/simple_runbook.yaml rename to neurips23/runbooks/simple_runbook.yaml diff --git a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml new file mode 100644 index 00000000..b12159b0 --- /dev/null +++ b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_delete_runbook.yaml @@ -0,0 +1,1210 @@ +wikipedia-1M: + 1: + end: 8863 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 15045 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 26204 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 38954 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 47829 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 56498 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 66349 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 78970 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 85089 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 95843 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 107227 + operation: insert + start: 100000 + 22: + ids_end: 10000 + ids_start: 8863 + operation: replace + tags_end: 1137 + tags_start: 0 + 23: + operation: search + 24: + end: 117719 + operation: insert + start: 110000 + 25: + ids_end: 20000 + ids_start: 15045 + operation: replace + tags_end: 14955 + tags_start: 10000 + 26: + operation: search + 27: + end: 126541 + operation: insert + start: 120000 + 28: + ids_end: 30000 + ids_start: 26204 + operation: replace + tags_end: 23796 + tags_start: 20000 + 29: + operation: search + 30: + end: 135746 + operation: insert + start: 130000 + 31: + ids_end: 40000 + ids_start: 38954 + operation: replace + tags_end: 31046 + tags_start: 30000 + 32: + operation: search + 33: + end: 147234 + operation: insert + start: 140000 + 34: + operation: search + 35: + end: 155738 + operation: insert + start: 150000 + 36: + operation: search + 37: + end: 165502 + operation: insert + start: 160000 + 38: + ids_end: 70000 + ids_start: 66349 + operation: replace + tags_end: 63651 + tags_start: 60000 + 39: + operation: search + 40: + end: 178120 + operation: insert + start: 170000 + 41: + operation: search + 42: + end: 187700 + operation: insert + start: 180000 + 43: + ids_end: 90000 + ids_start: 85089 + operation: replace + tags_end: 84911 + tags_start: 80000 + 44: + operation: search + 45: + end: 195702 + operation: insert + start: 190000 + 46: + ids_end: 100000 + ids_start: 95843 + operation: replace + tags_end: 94157 + tags_start: 90000 + 47: + operation: search + 48: + end: 206987 + operation: insert + start: 200000 + 49: + end: 8863 + operation: delete + start: 0 + 50: + ids_end: 110000 + ids_start: 107227 + operation: replace + tags_end: 102773 + tags_start: 100000 + 51: + operation: search + 52: + end: 215020 + operation: insert + start: 210000 + 53: + end: 15045 + operation: delete + start: 10000 + 54: + operation: search + 55: + end: 227333 + operation: insert + start: 220000 + 56: + end: 26204 + operation: delete + start: 20000 + 57: + operation: search + 58: + end: 238493 + operation: insert + start: 230000 + 59: + end: 38954 + operation: delete + start: 30000 + 60: + operation: search + 61: + end: 247104 + operation: insert + start: 240000 + 62: + ids_end: 150000 + ids_start: 147234 + operation: replace + tags_end: 142766 + tags_start: 140000 + 63: + operation: search + 64: + end: 257440 + operation: insert + start: 250000 + 65: + ids_end: 160000 + ids_start: 155738 + operation: replace + tags_end: 154262 + tags_start: 150000 + 66: + operation: search + 67: + end: 267710 + operation: insert + start: 260000 + 68: + end: 66349 + operation: delete + start: 60000 + 69: + operation: search + 70: + end: 278172 + operation: insert + start: 270000 + 71: + ids_end: 180000 + ids_start: 178120 + operation: replace + tags_end: 171880 + tags_start: 170000 + 72: + operation: search + 73: + end: 287054 + operation: insert + start: 280000 + 74: + end: 85089 + operation: delete + start: 80000 + 75: + operation: search + 76: + end: 296996 + operation: insert + start: 290000 + 77: + end: 95843 + operation: delete + start: 90000 + 78: + ids_end: 200000 + ids_start: 195702 + operation: replace + tags_end: 194298 + tags_start: 190000 + 79: + operation: search + 80: + end: 307729 + operation: insert + start: 300000 + 81: + end: 107227 + operation: delete + start: 100000 + 82: + ids_end: 210000 + ids_start: 206987 + operation: replace + tags_end: 203013 + tags_start: 200000 + 83: + operation: search + 84: + end: 315722 + operation: insert + start: 310000 + 85: + operation: search + 86: + end: 325570 + operation: insert + start: 320000 + 87: + operation: search + 88: + end: 336210 + operation: insert + start: 330000 + 89: + operation: search + 90: + end: 346006 + operation: insert + start: 340000 + 91: + end: 147234 + operation: delete + start: 140000 + 92: + operation: search + 93: + end: 358646 + operation: insert + start: 350000 + 94: + end: 155738 + operation: delete + start: 150000 + 95: + ids_end: 260000 + ids_start: 257440 + operation: replace + tags_end: 252560 + tags_start: 250000 + 96: + operation: search + 97: + end: 367418 + operation: insert + start: 360000 + 98: + ids_end: 270000 + ids_start: 267710 + operation: replace + tags_end: 262290 + tags_start: 260000 + 99: + operation: search + 100: + end: 378451 + operation: insert + start: 370000 + 101: + end: 178120 + operation: delete + start: 170000 + 102: + operation: search + 103: + end: 388086 + operation: insert + start: 380000 + 104: + ids_end: 290000 + ids_start: 287054 + operation: replace + tags_end: 282946 + tags_start: 280000 + 105: + ids_end: 350000 + ids_start: 346006 + operation: replace + tags_end: 343994 + tags_start: 340000 + 106: + ids_end: 360000 + ids_start: 358646 + operation: replace + tags_end: 351354 + tags_start: 350000 + 107: + operation: search + 108: + end: 395197 + operation: insert + start: 390000 + 109: + end: 195702 + operation: delete + start: 190000 + 110: + operation: search + 111: + end: 408414 + operation: insert + start: 400000 + 112: + end: 206987 + operation: delete + start: 200000 + 113: + ids_end: 300000 + ids_start: 296996 + operation: replace + tags_end: 293004 + tags_start: 290000 + 114: + operation: search + 115: + end: 417415 + operation: insert + start: 410000 + 116: + ids_end: 320000 + ids_start: 315722 + operation: replace + tags_end: 314278 + tags_start: 310000 + 117: + operation: search + 118: + end: 425973 + operation: insert + start: 420000 + 119: + operation: search + 120: + end: 437050 + operation: insert + start: 430000 + 121: + ids_end: 240000 + ids_start: 238493 + operation: replace + tags_end: 231507 + tags_start: 230000 + 122: + ids_end: 340000 + ids_start: 336210 + operation: replace + tags_end: 333790 + tags_start: 330000 + 123: + operation: search + 124: + end: 446204 + operation: insert + start: 440000 + 125: + ids_end: 220000 + ids_start: 215020 + operation: replace + tags_end: 214980 + tags_start: 210000 + 126: + ids_end: 230000 + ids_start: 227333 + operation: replace + tags_end: 222667 + tags_start: 220000 + 127: + operation: search + 128: + end: 455226 + operation: insert + start: 450000 + 129: + end: 257440 + operation: delete + start: 250000 + 130: + operation: search + 131: + end: 465531 + operation: insert + start: 460000 + 132: + end: 267710 + operation: delete + start: 260000 + 133: + ids_end: 370000 + ids_start: 367418 + operation: replace + tags_end: 362582 + tags_start: 360000 + 134: + operation: search + 135: + end: 477661 + operation: insert + start: 470000 + 136: + operation: search + 137: + end: 488161 + operation: insert + start: 480000 + 138: + end: 287054 + operation: delete + start: 280000 + 139: + operation: search + 140: + end: 497091 + operation: insert + start: 490000 + 141: + ids_end: 400000 + ids_start: 395197 + operation: replace + tags_end: 394803 + tags_start: 390000 + 142: + operation: search + 143: + end: 505164 + operation: insert + start: 500000 + 144: + ids_end: 410000 + ids_start: 408414 + operation: replace + tags_end: 401586 + tags_start: 400000 + 145: + operation: search + 146: + end: 516045 + operation: insert + start: 510000 + 147: + end: 315722 + operation: delete + start: 310000 + 148: + ids_end: 420000 + ids_start: 417415 + operation: replace + tags_end: 412585 + tags_start: 410000 + 149: + operation: search + 150: + end: 525538 + operation: insert + start: 520000 + 151: + operation: search + 152: + end: 538224 + operation: insert + start: 530000 + 153: + end: 336210 + operation: delete + start: 330000 + 154: + ids_end: 440000 + ids_start: 437050 + operation: replace + tags_end: 432950 + tags_start: 430000 + 155: + operation: search + 156: + end: 548076 + operation: insert + start: 540000 + 157: + ids_end: 50000 + ids_start: 47829 + operation: replace + tags_end: 42171 + tags_start: 40000 + 158: + operation: search + 159: + end: 558231 + operation: insert + start: 550000 + 160: + ids_end: 60000 + ids_start: 56498 + operation: replace + tags_end: 53502 + tags_start: 50000 + 161: + ids_end: 460000 + ids_start: 455226 + operation: replace + tags_end: 454774 + tags_start: 450000 + 162: + operation: search + 163: + end: 568942 + operation: insert + start: 560000 + 164: + end: 367418 + operation: delete + start: 360000 + 165: + operation: search + 166: + end: 578790 + operation: insert + start: 570000 + 167: + ids_end: 80000 + ids_start: 78970 + operation: replace + tags_end: 71030 + tags_start: 70000 + 168: + ids_end: 480000 + ids_start: 477661 + operation: replace + tags_end: 472339 + tags_start: 470000 + 169: + operation: search + 170: + end: 585829 + operation: insert + start: 580000 + 171: + operation: search + 172: + end: 598005 + operation: insert + start: 590000 + 173: + end: 395197 + operation: delete + start: 390000 + 174: + operation: search + 175: + end: 607151 + operation: insert + start: 600000 + 176: + end: 408414 + operation: delete + start: 400000 + 177: + ids_end: 510000 + ids_start: 505164 + operation: replace + tags_end: 504836 + tags_start: 500000 + 178: + operation: search + 179: + end: 615447 + operation: insert + start: 610000 + 180: + end: 417415 + operation: delete + start: 410000 + 181: + ids_end: 120000 + ids_start: 117719 + operation: replace + tags_end: 112281 + tags_start: 110000 + 182: + ids_end: 520000 + ids_start: 516045 + operation: replace + tags_end: 513955 + tags_start: 510000 + 183: + operation: search + 184: + end: 626681 + operation: insert + start: 620000 + 185: + ids_end: 130000 + ids_start: 126541 + operation: replace + tags_end: 123459 + tags_start: 120000 + 186: + ids_end: 530000 + ids_start: 525538 + operation: replace + tags_end: 524462 + tags_start: 520000 + 187: + operation: search + 188: + end: 637862 + operation: insert + start: 630000 + 189: + end: 437050 + operation: delete + start: 430000 + 190: + ids_end: 140000 + ids_start: 135746 + operation: replace + tags_end: 134254 + tags_start: 130000 + 191: + operation: search + 192: + end: 648116 + operation: insert + start: 640000 + 193: + ids_end: 550000 + ids_start: 548076 + operation: replace + tags_end: 541924 + tags_start: 540000 + 194: + operation: search + 195: + end: 655417 + operation: insert + start: 650000 + 196: + end: 455226 + operation: delete + start: 450000 + 197: + operation: search + 198: + end: 668757 + operation: insert + start: 660000 + 199: + ids_end: 170000 + ids_start: 165502 + operation: replace + tags_end: 164498 + tags_start: 160000 + 200: + ids_end: 570000 + ids_start: 568942 + operation: replace + tags_end: 561058 + tags_start: 560000 + 201: + operation: search + 202: + end: 675476 + operation: insert + start: 670000 + 203: + end: 477661 + operation: delete + start: 470000 + 204: + operation: search + 205: + end: 687145 + operation: insert + start: 680000 + 206: + ids_end: 190000 + ids_start: 187700 + operation: replace + tags_end: 182300 + tags_start: 180000 + 207: + operation: search + 208: + end: 698603 + operation: insert + start: 690000 + 209: + operation: search + 210: + end: 708379 + operation: insert + start: 700000 + 211: + end: 505164 + operation: delete + start: 500000 + 212: + ids_end: 610000 + ids_start: 607151 + operation: replace + tags_end: 602849 + tags_start: 600000 + 213: + operation: search + 214: + end: 717518 + operation: insert + start: 710000 + 215: + end: 516045 + operation: delete + start: 510000 + 216: + operation: search + 217: + end: 726507 + operation: insert + start: 720000 + 218: + end: 525538 + operation: delete + start: 520000 + 219: + ids_end: 630000 + ids_start: 626681 + operation: replace + tags_end: 623319 + tags_start: 620000 + 220: + operation: search + 221: + end: 738192 + operation: insert + start: 730000 + 222: + operation: search + 223: + end: 748167 + operation: insert + start: 740000 + 224: + end: 548076 + operation: delete + start: 540000 + 225: + ids_end: 250000 + ids_start: 247104 + operation: replace + tags_end: 242896 + tags_start: 240000 + 226: + ids_end: 650000 + ids_start: 648116 + operation: replace + tags_end: 641884 + tags_start: 640000 + 227: + operation: search + 228: + end: 756315 + operation: insert + start: 750000 + 229: + operation: search + 230: + end: 768528 + operation: insert + start: 760000 + 231: + end: 568942 + operation: delete + start: 560000 + 232: + operation: search + 233: + end: 778053 + operation: insert + start: 770000 + 234: + ids_end: 280000 + ids_start: 278172 + operation: replace + tags_end: 271828 + tags_start: 270000 + 235: + ids_end: 680000 + ids_start: 675476 + operation: replace + tags_end: 674524 + tags_start: 670000 + 236: + operation: search + 237: + end: 786230 + operation: insert + start: 780000 + 238: + ids_end: 690000 + ids_start: 687145 + operation: replace + tags_end: 682855 + tags_start: 680000 + 239: + operation: search + 240: + end: 795504 + operation: insert + start: 790000 + 241: + operation: search + 242: + end: 806837 + operation: insert + start: 800000 + 243: + end: 607151 + operation: delete + start: 600000 + 244: + ids_end: 310000 + ids_start: 307729 + operation: replace + tags_end: 302271 + tags_start: 300000 + 245: + ids_end: 710000 + ids_start: 708379 + operation: replace + tags_end: 701621 + tags_start: 700000 + 246: + operation: search + 247: + end: 817126 + operation: insert + start: 810000 + 248: + ids_end: 720000 + ids_start: 717518 + operation: replace + tags_end: 712482 + tags_start: 710000 + 249: + operation: search + 250: + end: 827853 + operation: insert + start: 820000 + 251: + end: 626681 + operation: delete + start: 620000 + 252: + ids_end: 330000 + ids_start: 325570 + operation: replace + tags_end: 324430 + tags_start: 320000 + 253: + ids_end: 730000 + ids_start: 726507 + operation: replace + tags_end: 723493 + tags_start: 720000 + 254: + operation: search + 255: + end: 835331 + operation: insert + start: 830000 + 256: + operation: search + 257: + end: 846151 + operation: insert + start: 840000 + 258: + end: 648116 + operation: delete + start: 640000 + 259: + ids_end: 750000 + ids_start: 748167 + operation: replace + tags_end: 741833 + tags_start: 740000 + 260: + operation: search + 261: + end: 858571 + operation: insert + start: 850000 + 262: + ids_end: 760000 + ids_start: 756315 + operation: replace + tags_end: 753685 + tags_start: 750000 + 263: + operation: search + 264: + end: 868465 + operation: insert + start: 860000 + 265: + ids_end: 770000 + ids_start: 768528 + operation: replace + tags_end: 761472 + tags_start: 760000 + 266: + operation: search + 267: + end: 876579 + operation: insert + start: 870000 + 268: + end: 675476 + operation: delete + start: 670000 + 269: + ids_end: 380000 + ids_start: 378451 + operation: replace + tags_end: 371549 + tags_start: 370000 + 270: + ids_end: 780000 + ids_start: 778053 + operation: replace + tags_end: 771947 + tags_start: 770000 + 271: + operation: search + 272: + end: 888993 + operation: insert + start: 880000 + 273: + end: 687145 + operation: delete + start: 680000 + 274: + ids_end: 390000 + ids_start: 388086 + operation: replace + tags_end: 381914 + tags_start: 380000 + 275: + ids_end: 790000 + ids_start: 786230 + operation: replace + tags_end: 783770 + tags_start: 780000 + 276: + operation: search + 277: + end: 898134 + operation: insert + start: 890000 + 278: + ids_end: 800000 + ids_start: 795504 + operation: replace + tags_end: 794496 + tags_start: 790000 + 279: + operation: search + 280: + end: 905208 + operation: insert + start: 900000 + 281: + end: 708379 + operation: delete + start: 700000 + 282: + operation: search + 283: + end: 918965 + operation: insert + start: 910000 + 284: + end: 717518 + operation: delete + start: 710000 + 285: + operation: search + 286: + end: 926429 + operation: insert + start: 920000 + 287: + end: 726507 + operation: delete + start: 720000 + 288: + ids_end: 830000 + ids_start: 827853 + operation: replace + tags_end: 822147 + tags_start: 820000 + 289: + operation: search + 290: + end: 936860 + operation: insert + start: 930000 + 291: + operation: search + 292: + end: 947896 + operation: insert + start: 940000 + 293: + end: 748167 + operation: delete + start: 740000 + 294: + ids_end: 450000 + ids_start: 446204 + operation: replace + tags_end: 443796 + tags_start: 440000 + 295: + operation: search + 296: + end: 958620 + operation: insert + start: 950000 + 297: + end: 756315 + operation: delete + start: 750000 + 298: + ids_end: 860000 + ids_start: 858571 + operation: replace + tags_end: 851429 + tags_start: 850000 + 299: + operation: search + 300: + end: 966413 + operation: insert + start: 960000 + 301: + end: 768528 + operation: delete + start: 760000 + 302: + ids_end: 470000 + ids_start: 465531 + operation: replace + tags_end: 464469 + tags_start: 460000 + 303: + operation: search + 304: + end: 977801 + operation: insert + start: 970000 + 305: + end: 778053 + operation: delete + start: 770000 + 306: + ids_end: 880000 + ids_start: 876579 + operation: replace + tags_end: 873421 + tags_start: 870000 + 307: + operation: search + 308: + end: 988745 + operation: insert + start: 980000 + 309: + end: 786230 + operation: delete + start: 780000 + 310: + ids_end: 490000 + ids_start: 488161 + operation: replace + tags_end: 481839 + tags_start: 480000 + 311: + ids_end: 930000 + ids_start: 926429 + operation: replace + tags_end: 923571 + tags_start: 920000 + 312: + operation: search + 313: + end: 998446 + operation: insert + start: 990000 + 314: + end: 795504 + operation: delete + start: 790000 + 315: + ids_end: 900000 + ids_start: 898134 + operation: replace + tags_end: 891866 + tags_start: 890000 + 316: + operation: search + max_pts: 293233 diff --git a/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml new file mode 100644 index 00000000..d036dc57 --- /dev/null +++ b/neurips23/runbooks/wikipedia-1M_expiration_time_replace_only_runbook.yaml @@ -0,0 +1,1070 @@ +wikipedia-1M: + 1: + end: 5717 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 18550 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 25471 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 35103 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 47914 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 55854 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 65428 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 76655 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 86469 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 96387 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 106086 + operation: insert + start: 100000 + 22: + ids_end: 10000 + ids_start: 5717 + operation: replace + tags_end: 4283 + tags_start: 0 + 23: + operation: search + 24: + end: 118994 + operation: insert + start: 110000 + 25: + ids_end: 20000 + ids_start: 18550 + operation: replace + tags_end: 11450 + tags_start: 10000 + 26: + operation: search + 27: + end: 126384 + operation: insert + start: 120000 + 28: + operation: search + 29: + end: 136115 + operation: insert + start: 130000 + 30: + ids_end: 40000 + ids_start: 35103 + operation: replace + tags_end: 34897 + tags_start: 30000 + 31: + operation: search + 32: + end: 148544 + operation: insert + start: 140000 + 33: + operation: search + 34: + end: 158561 + operation: insert + start: 150000 + 35: + ids_end: 60000 + ids_start: 55854 + operation: replace + tags_end: 54146 + tags_start: 50000 + 36: + operation: search + 37: + end: 166312 + operation: insert + start: 160000 + 38: + operation: search + 39: + end: 175474 + operation: insert + start: 170000 + 40: + ids_end: 80000 + ids_start: 76655 + operation: replace + tags_end: 73345 + tags_start: 70000 + 41: + operation: search + 42: + end: 186318 + operation: insert + start: 180000 + 43: + ids_end: 90000 + ids_start: 86469 + operation: replace + tags_end: 83531 + tags_start: 80000 + 44: + operation: search + 45: + end: 195139 + operation: insert + start: 190000 + 46: + ids_end: 100000 + ids_start: 96387 + operation: replace + tags_end: 93613 + tags_start: 90000 + 47: + operation: search + 48: + end: 207010 + operation: insert + start: 200000 + 49: + operation: search + 50: + end: 217340 + operation: insert + start: 210000 + 51: + ids_end: 120000 + ids_start: 118994 + operation: replace + tags_end: 111006 + tags_start: 110000 + 52: + operation: search + 53: + end: 225086 + operation: insert + start: 220000 + 54: + operation: search + 55: + end: 236306 + operation: insert + start: 230000 + 56: + operation: search + 57: + end: 248731 + operation: insert + start: 240000 + 58: + ids_end: 150000 + ids_start: 148544 + operation: replace + tags_end: 141456 + tags_start: 140000 + 59: + operation: search + 60: + end: 257095 + operation: insert + start: 250000 + 61: + ids_end: 160000 + ids_start: 158561 + operation: replace + tags_end: 151439 + tags_start: 150000 + 62: + operation: search + 63: + end: 266084 + operation: insert + start: 260000 + 64: + operation: search + 65: + end: 277913 + operation: insert + start: 270000 + 66: + ids_end: 180000 + ids_start: 175474 + operation: replace + tags_end: 174526 + tags_start: 170000 + 67: + operation: search + 68: + end: 286571 + operation: insert + start: 280000 + 69: + operation: search + 70: + end: 297573 + operation: insert + start: 290000 + 71: + ids_end: 200000 + ids_start: 195139 + operation: replace + tags_end: 194861 + tags_start: 190000 + 72: + operation: search + 73: + end: 307004 + operation: insert + start: 300000 + 74: + ids_end: 210000 + ids_start: 207010 + operation: replace + tags_end: 202990 + tags_start: 200000 + 75: + operation: search + 76: + end: 317166 + operation: insert + start: 310000 + 77: + ids_end: 220000 + ids_start: 217340 + operation: replace + tags_end: 212660 + tags_start: 210000 + 78: + operation: search + 79: + end: 327332 + operation: insert + start: 320000 + 80: + ids_end: 230000 + ids_start: 225086 + operation: replace + tags_end: 224914 + tags_start: 220000 + 81: + operation: search + 82: + end: 338452 + operation: insert + start: 330000 + 83: + ids_end: 240000 + ids_start: 236306 + operation: replace + tags_end: 233694 + tags_start: 230000 + 84: + operation: search + 85: + end: 347357 + operation: insert + start: 340000 + 86: + operation: search + 87: + end: 356294 + operation: insert + start: 350000 + 88: + operation: search + 89: + end: 365818 + operation: insert + start: 360000 + 90: + ids_end: 270000 + ids_start: 266084 + operation: replace + tags_end: 263916 + tags_start: 260000 + 91: + operation: search + 92: + end: 376290 + operation: insert + start: 370000 + 93: + ids_end: 280000 + ids_start: 277913 + operation: replace + tags_end: 272087 + tags_start: 270000 + 94: + operation: search + 95: + end: 386132 + operation: insert + start: 380000 + 96: + ids_end: 290000 + ids_start: 286571 + operation: replace + tags_end: 283429 + tags_start: 280000 + 97: + operation: search + 98: + end: 398893 + operation: insert + start: 390000 + 99: + ids_end: 300000 + ids_start: 297573 + operation: replace + tags_end: 292427 + tags_start: 290000 + 100: + operation: search + 101: + end: 406004 + operation: insert + start: 400000 + 102: + ids_end: 310000 + ids_start: 307004 + operation: replace + tags_end: 302996 + tags_start: 300000 + 103: + operation: search + 104: + end: 418900 + operation: insert + start: 410000 + 105: + ids_end: 340000 + ids_start: 338452 + operation: replace + tags_end: 331548 + tags_start: 330000 + 106: + ids_end: 360000 + ids_start: 356294 + operation: replace + tags_end: 353706 + tags_start: 350000 + 107: + operation: search + 108: + end: 427662 + operation: insert + start: 420000 + 109: + ids_end: 330000 + ids_start: 327332 + operation: replace + tags_end: 322668 + tags_start: 320000 + 110: + operation: search + 111: + end: 435236 + operation: insert + start: 430000 + 112: + operation: search + 113: + end: 446775 + operation: insert + start: 440000 + 114: + ids_end: 350000 + ids_start: 347357 + operation: replace + tags_end: 342643 + tags_start: 340000 + 115: + operation: search + 116: + end: 456947 + operation: insert + start: 450000 + 117: + ids_end: 250000 + ids_start: 248731 + operation: replace + tags_end: 241269 + tags_start: 240000 + 118: + operation: search + 119: + end: 468960 + operation: insert + start: 460000 + 120: + ids_end: 190000 + ids_start: 186318 + operation: replace + tags_end: 183682 + tags_start: 180000 + 121: + ids_end: 370000 + ids_start: 365818 + operation: replace + tags_end: 364182 + tags_start: 360000 + 122: + operation: search + 123: + end: 477977 + operation: insert + start: 470000 + 124: + ids_end: 170000 + ids_start: 166312 + operation: replace + tags_end: 163688 + tags_start: 160000 + 125: + ids_end: 380000 + ids_start: 376290 + operation: replace + tags_end: 373710 + tags_start: 370000 + 126: + operation: search + 127: + end: 487247 + operation: insert + start: 480000 + 128: + ids_end: 130000 + ids_start: 126384 + operation: replace + tags_end: 123616 + tags_start: 120000 + 129: + ids_end: 390000 + ids_start: 386132 + operation: replace + tags_end: 383868 + tags_start: 380000 + 130: + operation: search + 131: + end: 496461 + operation: insert + start: 490000 + 132: + ids_end: 70000 + ids_start: 65428 + operation: replace + tags_end: 64572 + tags_start: 60000 + 133: + ids_end: 400000 + ids_start: 398893 + operation: replace + tags_end: 391107 + tags_start: 390000 + 134: + operation: search + 135: + end: 507760 + operation: insert + start: 500000 + 136: + ids_end: 410000 + ids_start: 406004 + operation: replace + tags_end: 403996 + tags_start: 400000 + 137: + operation: search + 138: + end: 515942 + operation: insert + start: 510000 + 139: + ids_end: 420000 + ids_start: 418900 + operation: replace + tags_end: 411100 + tags_start: 410000 + 140: + operation: search + 141: + end: 527387 + operation: insert + start: 520000 + 142: + ids_end: 30000 + ids_start: 25471 + operation: replace + tags_end: 24529 + tags_start: 20000 + 143: + operation: search + 144: + end: 537002 + operation: insert + start: 530000 + 145: + ids_end: 440000 + ids_start: 435236 + operation: replace + tags_end: 434764 + tags_start: 430000 + 146: + operation: search + 147: + end: 546774 + operation: insert + start: 540000 + 148: + ids_end: 50000 + ids_start: 47914 + operation: replace + tags_end: 42086 + tags_start: 40000 + 149: + ids_end: 450000 + ids_start: 446775 + operation: replace + tags_end: 443225 + tags_start: 440000 + 150: + operation: search + 151: + end: 556503 + operation: insert + start: 550000 + 152: + ids_end: 460000 + ids_start: 456947 + operation: replace + tags_end: 453053 + tags_start: 450000 + 153: + operation: search + 154: + end: 566708 + operation: insert + start: 560000 + 155: + ids_end: 470000 + ids_start: 468960 + operation: replace + tags_end: 461040 + tags_start: 460000 + 156: + operation: search + 157: + end: 578101 + operation: insert + start: 570000 + 158: + ids_end: 480000 + ids_start: 477977 + operation: replace + tags_end: 472023 + tags_start: 470000 + 159: + operation: search + 160: + end: 588207 + operation: insert + start: 580000 + 161: + ids_end: 490000 + ids_start: 487247 + operation: replace + tags_end: 482753 + tags_start: 480000 + 162: + operation: search + 163: + end: 597377 + operation: insert + start: 590000 + 164: + operation: search + 165: + end: 606480 + operation: insert + start: 600000 + 166: + ids_end: 110000 + ids_start: 106086 + operation: replace + tags_end: 103914 + tags_start: 100000 + 167: + ids_end: 510000 + ids_start: 507760 + operation: replace + tags_end: 502240 + tags_start: 500000 + 168: + operation: search + 169: + end: 616347 + operation: insert + start: 610000 + 170: + operation: search + 171: + end: 628995 + operation: insert + start: 620000 + 172: + operation: search + 173: + end: 636353 + operation: insert + start: 630000 + 174: + ids_end: 140000 + ids_start: 136115 + operation: replace + tags_end: 133885 + tags_start: 130000 + 175: + ids_end: 540000 + ids_start: 537002 + operation: replace + tags_end: 532998 + tags_start: 530000 + 176: + operation: search + 177: + end: 646743 + operation: insert + start: 640000 + 178: + operation: search + 179: + end: 657082 + operation: insert + start: 650000 + 180: + ids_end: 560000 + ids_start: 556503 + operation: replace + tags_end: 553497 + tags_start: 550000 + 181: + operation: search + 182: + end: 665090 + operation: insert + start: 660000 + 183: + ids_end: 570000 + ids_start: 566708 + operation: replace + tags_end: 563292 + tags_start: 560000 + 184: + operation: search + 185: + end: 678873 + operation: insert + start: 670000 + 186: + ids_end: 580000 + ids_start: 578101 + operation: replace + tags_end: 571899 + tags_start: 570000 + 187: + operation: search + 188: + end: 688788 + operation: insert + start: 680000 + 189: + ids_end: 590000 + ids_start: 588207 + operation: replace + tags_end: 581793 + tags_start: 580000 + 190: + operation: search + 191: + end: 696832 + operation: insert + start: 690000 + 192: + operation: search + 193: + end: 705843 + operation: insert + start: 700000 + 194: + ids_end: 610000 + ids_start: 606480 + operation: replace + tags_end: 603520 + tags_start: 600000 + 195: + operation: search + 196: + end: 715357 + operation: insert + start: 710000 + 197: + operation: search + 198: + end: 727345 + operation: insert + start: 720000 + 199: + operation: search + 200: + end: 736932 + operation: insert + start: 730000 + 201: + ids_end: 640000 + ids_start: 636353 + operation: replace + tags_end: 633647 + tags_start: 630000 + 202: + operation: search + 203: + end: 748274 + operation: insert + start: 740000 + 204: + ids_end: 650000 + ids_start: 646743 + operation: replace + tags_end: 643257 + tags_start: 640000 + 205: + operation: search + 206: + end: 755014 + operation: insert + start: 750000 + 207: + ids_end: 260000 + ids_start: 257095 + operation: replace + tags_end: 252905 + tags_start: 250000 + 208: + ids_end: 660000 + ids_start: 657082 + operation: replace + tags_end: 652918 + tags_start: 650000 + 209: + operation: search + 210: + end: 767869 + operation: insert + start: 760000 + 211: + ids_end: 670000 + ids_start: 665090 + operation: replace + tags_end: 664910 + tags_start: 660000 + 212: + operation: search + 213: + end: 778779 + operation: insert + start: 770000 + 214: + ids_end: 680000 + ids_start: 678873 + operation: replace + tags_end: 671127 + tags_start: 670000 + 215: + operation: search + 216: + end: 788734 + operation: insert + start: 780000 + 217: + operation: search + 218: + end: 796576 + operation: insert + start: 790000 + 219: + ids_end: 700000 + ids_start: 696832 + operation: replace + tags_end: 693168 + tags_start: 690000 + 220: + operation: search + 221: + end: 808855 + operation: insert + start: 800000 + 222: + ids_end: 710000 + ids_start: 705843 + operation: replace + tags_end: 704157 + tags_start: 700000 + 223: + operation: search + 224: + end: 818590 + operation: insert + start: 810000 + 225: + ids_end: 320000 + ids_start: 317166 + operation: replace + tags_end: 312834 + tags_start: 310000 + 226: + ids_end: 720000 + ids_start: 715357 + operation: replace + tags_end: 714643 + tags_start: 710000 + 227: + operation: search + 228: + end: 827402 + operation: insert + start: 820000 + 229: + ids_end: 730000 + ids_start: 727345 + operation: replace + tags_end: 722655 + tags_start: 720000 + 230: + operation: search + 231: + end: 836430 + operation: insert + start: 830000 + 232: + ids_end: 740000 + ids_start: 736932 + operation: replace + tags_end: 733068 + tags_start: 730000 + 233: + operation: search + 234: + end: 845685 + operation: insert + start: 840000 + 235: + ids_end: 750000 + ids_start: 748274 + operation: replace + tags_end: 741726 + tags_start: 740000 + 236: + operation: search + 237: + end: 858690 + operation: insert + start: 850000 + 238: + ids_end: 760000 + ids_start: 755014 + operation: replace + tags_end: 754986 + tags_start: 750000 + 239: + operation: search + 240: + end: 865352 + operation: insert + start: 860000 + 241: + ids_end: 770000 + ids_start: 767869 + operation: replace + tags_end: 762131 + tags_start: 760000 + 242: + operation: search + 243: + end: 877344 + operation: insert + start: 870000 + 244: + operation: search + 245: + end: 885193 + operation: insert + start: 880000 + 246: + ids_end: 790000 + ids_start: 788734 + operation: replace + tags_end: 781266 + tags_start: 780000 + 247: + operation: search + 248: + end: 896545 + operation: insert + start: 890000 + 249: + operation: search + 250: + end: 908515 + operation: insert + start: 900000 + 251: + operation: search + 252: + end: 916753 + operation: insert + start: 910000 + 253: + ids_end: 820000 + ids_start: 818590 + operation: replace + tags_end: 811410 + tags_start: 810000 + 254: + operation: search + 255: + end: 925994 + operation: insert + start: 920000 + 256: + ids_end: 430000 + ids_start: 427662 + operation: replace + tags_end: 422338 + tags_start: 420000 + 257: + ids_end: 830000 + ids_start: 827402 + operation: replace + tags_end: 822598 + tags_start: 820000 + 258: + operation: search + 259: + end: 936044 + operation: insert + start: 930000 + 260: + operation: search + 261: + end: 948527 + operation: insert + start: 940000 + 262: + ids_end: 850000 + ids_start: 845685 + operation: replace + tags_end: 844315 + tags_start: 840000 + 263: + operation: search + 264: + end: 956903 + operation: insert + start: 950000 + 265: + operation: search + 266: + end: 966748 + operation: insert + start: 960000 + 267: + ids_end: 870000 + ids_start: 865352 + operation: replace + tags_end: 864648 + tags_start: 860000 + 268: + operation: search + 269: + end: 977296 + operation: insert + start: 970000 + 270: + ids_end: 880000 + ids_start: 877344 + operation: replace + tags_end: 872656 + tags_start: 870000 + 271: + operation: search + 272: + end: 987403 + operation: insert + start: 980000 + 273: + ids_end: 890000 + ids_start: 885193 + operation: replace + tags_end: 884807 + tags_start: 880000 + 274: + operation: search + 275: + end: 995942 + operation: insert + start: 990000 + 276: + ids_end: 500000 + ids_start: 496461 + operation: replace + tags_end: 493539 + tags_start: 490000 + 277: + ids_end: 900000 + ids_start: 896545 + operation: replace + tags_end: 893455 + tags_start: 890000 + 278: + operation: search + max_pts: 698369 diff --git a/neurips23/runbooks/wikipedia-1M_expiration_time_runbook.yaml b/neurips23/runbooks/wikipedia-1M_expiration_time_runbook.yaml new file mode 100644 index 00000000..0875cf40 --- /dev/null +++ b/neurips23/runbooks/wikipedia-1M_expiration_time_runbook.yaml @@ -0,0 +1,843 @@ +wikipedia-1M: + 1: + end: 10000 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 20000 + operation: insert + start: 10000 + 4: + operation: search + 5: + end: 30000 + operation: insert + start: 20000 + 6: + operation: search + 7: + end: 40000 + operation: insert + start: 30000 + 8: + operation: search + 9: + end: 50000 + operation: insert + start: 40000 + 10: + operation: search + 11: + end: 60000 + operation: insert + start: 50000 + 12: + operation: search + 13: + end: 70000 + operation: insert + start: 60000 + 14: + operation: search + 15: + end: 80000 + operation: insert + start: 70000 + 16: + operation: search + 17: + end: 90000 + operation: insert + start: 80000 + 18: + operation: search + 19: + end: 100000 + operation: insert + start: 90000 + 20: + operation: search + 21: + end: 110000 + operation: insert + start: 100000 + 22: + operation: search + 23: + end: 120000 + operation: insert + start: 110000 + 24: + operation: search + 25: + end: 130000 + operation: insert + start: 120000 + 26: + operation: search + 27: + end: 140000 + operation: insert + start: 130000 + 28: + operation: search + 29: + end: 150000 + operation: insert + start: 140000 + 30: + operation: search + 31: + end: 160000 + operation: insert + start: 150000 + 32: + operation: search + 33: + end: 170000 + operation: insert + start: 160000 + 34: + operation: search + 35: + end: 180000 + operation: insert + start: 170000 + 36: + operation: search + 37: + end: 190000 + operation: insert + start: 180000 + 38: + operation: search + 39: + end: 200000 + operation: insert + start: 190000 + 40: + operation: search + 41: + end: 210000 + operation: insert + start: 200000 + 42: + end: 10000 + operation: delete + start: 0 + 43: + operation: search + 44: + end: 220000 + operation: insert + start: 210000 + 45: + end: 20000 + operation: delete + start: 10000 + 46: + operation: search + 47: + end: 230000 + operation: insert + start: 220000 + 48: + end: 30000 + operation: delete + start: 20000 + 49: + operation: search + 50: + end: 240000 + operation: insert + start: 230000 + 51: + end: 40000 + operation: delete + start: 30000 + 52: + operation: search + 53: + end: 250000 + operation: insert + start: 240000 + 54: + end: 50000 + operation: delete + start: 40000 + 55: + operation: search + 56: + end: 260000 + operation: insert + start: 250000 + 57: + operation: search + 58: + end: 270000 + operation: insert + start: 260000 + 59: + end: 70000 + operation: delete + start: 60000 + 60: + operation: search + 61: + end: 280000 + operation: insert + start: 270000 + 62: + end: 80000 + operation: delete + start: 70000 + 63: + operation: search + 64: + end: 290000 + operation: insert + start: 280000 + 65: + end: 90000 + operation: delete + start: 80000 + 66: + operation: search + 67: + end: 300000 + operation: insert + start: 290000 + 68: + operation: search + 69: + end: 310000 + operation: insert + start: 300000 + 70: + operation: search + 71: + end: 320000 + operation: insert + start: 310000 + 72: + end: 120000 + operation: delete + start: 110000 + 73: + operation: search + 74: + end: 330000 + operation: insert + start: 320000 + 75: + operation: search + 76: + end: 340000 + operation: insert + start: 330000 + 77: + end: 140000 + operation: delete + start: 130000 + 78: + operation: search + 79: + end: 350000 + operation: insert + start: 340000 + 80: + end: 150000 + operation: delete + start: 140000 + 81: + operation: search + 82: + end: 360000 + operation: insert + start: 350000 + 83: + end: 160000 + operation: delete + start: 150000 + 84: + operation: search + 85: + end: 370000 + operation: insert + start: 360000 + 86: + operation: search + 87: + end: 380000 + operation: insert + start: 370000 + 88: + end: 180000 + operation: delete + start: 170000 + 89: + operation: search + 90: + end: 390000 + operation: insert + start: 380000 + 91: + end: 190000 + operation: delete + start: 180000 + 92: + operation: search + 93: + end: 400000 + operation: insert + start: 390000 + 94: + end: 200000 + operation: delete + start: 190000 + 95: + operation: search + 96: + end: 410000 + operation: insert + start: 400000 + 97: + end: 210000 + operation: delete + start: 200000 + 98: + operation: search + 99: + end: 420000 + operation: insert + start: 410000 + 100: + operation: search + 101: + end: 430000 + operation: insert + start: 420000 + 102: + end: 230000 + operation: delete + start: 220000 + 103: + operation: search + 104: + end: 440000 + operation: insert + start: 430000 + 105: + end: 240000 + operation: delete + start: 230000 + 106: + operation: search + 107: + end: 450000 + operation: insert + start: 440000 + 108: + end: 250000 + operation: delete + start: 240000 + 109: + operation: search + 110: + end: 460000 + operation: insert + start: 450000 + 111: + operation: search + 112: + end: 470000 + operation: insert + start: 460000 + 113: + end: 270000 + operation: delete + start: 260000 + 114: + operation: search + 115: + end: 480000 + operation: insert + start: 470000 + 116: + end: 280000 + operation: delete + start: 270000 + 117: + operation: search + 118: + end: 490000 + operation: insert + start: 480000 + 119: + end: 290000 + operation: delete + start: 280000 + 120: + operation: search + 121: + end: 500000 + operation: insert + start: 490000 + 122: + end: 300000 + operation: delete + start: 290000 + 123: + operation: search + 124: + end: 510000 + operation: insert + start: 500000 + 125: + end: 310000 + operation: delete + start: 300000 + 126: + operation: search + 127: + end: 520000 + operation: insert + start: 510000 + 128: + end: 320000 + operation: delete + start: 310000 + 129: + operation: search + 130: + end: 530000 + operation: insert + start: 520000 + 131: + end: 330000 + operation: delete + start: 320000 + 132: + operation: search + 133: + end: 540000 + operation: insert + start: 530000 + 134: + end: 340000 + operation: delete + start: 330000 + 135: + operation: search + 136: + end: 550000 + operation: insert + start: 540000 + 137: + end: 350000 + operation: delete + start: 340000 + 138: + operation: search + 139: + end: 560000 + operation: insert + start: 550000 + 140: + end: 360000 + operation: delete + start: 350000 + 141: + operation: search + 142: + end: 570000 + operation: insert + start: 560000 + 143: + operation: search + 144: + end: 580000 + operation: insert + start: 570000 + 145: + operation: search + 146: + end: 590000 + operation: insert + start: 580000 + 147: + end: 390000 + operation: delete + start: 380000 + 148: + operation: search + 149: + end: 600000 + operation: insert + start: 590000 + 150: + end: 400000 + operation: delete + start: 390000 + 151: + operation: search + 152: + end: 610000 + operation: insert + start: 600000 + 153: + end: 410000 + operation: delete + start: 400000 + 154: + operation: search + 155: + end: 620000 + operation: insert + start: 610000 + 156: + end: 420000 + operation: delete + start: 410000 + 157: + operation: search + 158: + end: 630000 + operation: insert + start: 620000 + 159: + end: 430000 + operation: delete + start: 420000 + 160: + operation: search + 161: + end: 640000 + operation: insert + start: 630000 + 162: + end: 440000 + operation: delete + start: 430000 + 163: + operation: search + 164: + end: 650000 + operation: insert + start: 640000 + 165: + end: 450000 + operation: delete + start: 440000 + 166: + operation: search + 167: + end: 660000 + operation: insert + start: 650000 + 168: + end: 460000 + operation: delete + start: 450000 + 169: + operation: search + 170: + end: 670000 + operation: insert + start: 660000 + 171: + end: 470000 + operation: delete + start: 460000 + 172: + operation: search + 173: + end: 680000 + operation: insert + start: 670000 + 174: + end: 480000 + operation: delete + start: 470000 + 175: + operation: search + 176: + end: 690000 + operation: insert + start: 680000 + 177: + end: 490000 + operation: delete + start: 480000 + 178: + operation: search + 179: + end: 700000 + operation: insert + start: 690000 + 180: + operation: search + 181: + end: 710000 + operation: insert + start: 700000 + 182: + operation: search + 183: + end: 720000 + operation: insert + start: 710000 + 184: + end: 520000 + operation: delete + start: 510000 + 185: + operation: search + 186: + end: 730000 + operation: insert + start: 720000 + 187: + operation: search + 188: + end: 740000 + operation: insert + start: 730000 + 189: + end: 540000 + operation: delete + start: 530000 + 190: + operation: search + 191: + end: 750000 + operation: insert + start: 740000 + 192: + operation: search + 193: + end: 760000 + operation: insert + start: 750000 + 194: + end: 560000 + operation: delete + start: 550000 + 195: + operation: search + 196: + end: 770000 + operation: insert + start: 760000 + 197: + end: 570000 + operation: delete + start: 560000 + 198: + operation: search + 199: + end: 780000 + operation: insert + start: 770000 + 200: + operation: search + 201: + end: 790000 + operation: insert + start: 780000 + 202: + end: 590000 + operation: delete + start: 580000 + 203: + operation: search + 204: + end: 800000 + operation: insert + start: 790000 + 205: + end: 600000 + operation: delete + start: 590000 + 206: + operation: search + 207: + end: 810000 + operation: insert + start: 800000 + 208: + end: 610000 + operation: delete + start: 600000 + 209: + operation: search + 210: + end: 820000 + operation: insert + start: 810000 + 211: + operation: search + 212: + end: 830000 + operation: insert + start: 820000 + 213: + end: 630000 + operation: delete + start: 620000 + 214: + operation: search + 215: + end: 840000 + operation: insert + start: 830000 + 216: + end: 640000 + operation: delete + start: 630000 + 217: + operation: search + 218: + end: 850000 + operation: insert + start: 840000 + 219: + end: 650000 + operation: delete + start: 640000 + 220: + operation: search + 221: + end: 860000 + operation: insert + start: 850000 + 222: + end: 660000 + operation: delete + start: 650000 + 223: + operation: search + 224: + end: 870000 + operation: insert + start: 860000 + 225: + operation: search + 226: + end: 880000 + operation: insert + start: 870000 + 227: + end: 680000 + operation: delete + start: 670000 + 228: + operation: search + 229: + end: 890000 + operation: insert + start: 880000 + 230: + end: 690000 + operation: delete + start: 680000 + 231: + operation: search + 232: + end: 900000 + operation: insert + start: 890000 + 233: + end: 700000 + operation: delete + start: 690000 + 234: + operation: search + 235: + end: 910000 + operation: insert + start: 900000 + 236: + end: 710000 + operation: delete + start: 700000 + 237: + operation: search + 238: + end: 920000 + operation: insert + start: 910000 + 239: + operation: search + 240: + end: 930000 + operation: insert + start: 920000 + 241: + end: 730000 + operation: delete + start: 720000 + 242: + operation: search + 243: + end: 940000 + operation: insert + start: 930000 + 244: + operation: search + 245: + end: 950000 + operation: insert + start: 940000 + 246: + end: 750000 + operation: delete + start: 740000 + 247: + operation: search + 248: + end: 960000 + operation: insert + start: 950000 + 249: + end: 760000 + operation: delete + start: 750000 + 250: + operation: search + 251: + end: 970000 + operation: insert + start: 960000 + 252: + end: 770000 + operation: delete + start: 760000 + 253: + operation: search + 254: + end: 980000 + operation: insert + start: 970000 + 255: + operation: search + 256: + end: 990000 + operation: insert + start: 980000 + 257: + operation: search + 258: + end: 1000000 + operation: insert + start: 990000 + 259: + end: 800000 + operation: delete + start: 790000 + 260: + operation: search + max_pts: 410000 + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-1M_expiration_time_runbook.yaml/ diff --git a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml new file mode 100644 index 00000000..499c6fab --- /dev/null +++ b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_delete_runbook.yaml @@ -0,0 +1,4438 @@ +wikipedia-35M: + 1: + end: 88328 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 159704 + operation: insert + start: 100000 + 4: + operation: search + 5: + end: 257370 + operation: insert + start: 200000 + 6: + operation: search + 7: + end: 379583 + operation: insert + start: 300000 + 8: + operation: search + 9: + end: 481703 + operation: insert + start: 400000 + 10: + operation: search + 11: + end: 559269 + operation: insert + start: 500000 + 12: + operation: search + 13: + end: 662778 + operation: insert + start: 600000 + 14: + operation: search + 15: + end: 787283 + operation: insert + start: 700000 + 16: + operation: search + 17: + end: 878324 + operation: insert + start: 800000 + 18: + operation: search + 19: + end: 973085 + operation: insert + start: 900000 + 20: + operation: search + 21: + end: 1073582 + operation: insert + start: 1000000 + 22: + operation: search + 23: + end: 1166860 + operation: insert + start: 1100000 + 24: + operation: search + 25: + end: 1283992 + operation: insert + start: 1200000 + 26: + operation: search + 27: + end: 1371233 + operation: insert + start: 1300000 + 28: + operation: search + 29: + end: 1452446 + operation: insert + start: 1400000 + 30: + operation: search + 31: + end: 1574492 + operation: insert + start: 1500000 + 32: + operation: search + 33: + end: 1657378 + operation: insert + start: 1600000 + 34: + operation: search + 35: + end: 1762498 + operation: insert + start: 1700000 + 36: + operation: search + 37: + end: 1855688 + operation: insert + start: 1800000 + 38: + operation: search + 39: + end: 1976700 + operation: insert + start: 1900000 + 40: + operation: search + 41: + end: 2067276 + operation: insert + start: 2000000 + 42: + operation: search + 43: + end: 2171272 + operation: insert + start: 2100000 + 44: + operation: search + 45: + end: 2266775 + operation: insert + start: 2200000 + 46: + operation: search + 47: + end: 2370251 + operation: insert + start: 2300000 + 48: + operation: search + 49: + end: 2485289 + operation: insert + start: 2400000 + 50: + operation: search + 51: + end: 2576439 + operation: insert + start: 2500000 + 52: + operation: search + 53: + end: 2662292 + operation: insert + start: 2600000 + 54: + ids_end: 200000 + ids_start: 159704 + operation: replace + tags_end: 140296 + tags_start: 100000 + 55: + operation: search + 56: + end: 2765888 + operation: insert + start: 2700000 + 57: + operation: search + 58: + end: 2861892 + operation: insert + start: 2800000 + 59: + ids_end: 400000 + ids_start: 379583 + operation: replace + tags_end: 320417 + tags_start: 300000 + 60: + operation: search + 61: + end: 2971724 + operation: insert + start: 2900000 + 62: + operation: search + 63: + end: 3080301 + operation: insert + start: 3000000 + 64: + operation: search + 65: + end: 3167093 + operation: insert + start: 3100000 + 66: + operation: search + 67: + end: 3289859 + operation: insert + start: 3200000 + 68: + ids_end: 800000 + ids_start: 787283 + operation: replace + tags_end: 712717 + tags_start: 700000 + 69: + operation: search + 70: + end: 3353942 + operation: insert + start: 3300000 + 71: + operation: search + 72: + end: 3452322 + operation: insert + start: 3400000 + 73: + operation: search + 74: + end: 3570479 + operation: insert + start: 3500000 + 75: + ids_end: 1100000 + ids_start: 1073582 + operation: replace + tags_end: 1026418 + tags_start: 1000000 + 76: + operation: search + 77: + end: 3650270 + operation: insert + start: 3600000 + 78: + operation: search + 79: + end: 3785820 + operation: insert + start: 3700000 + 80: + ids_end: 1300000 + ids_start: 1283992 + operation: replace + tags_end: 1216008 + tags_start: 1200000 + 81: + operation: search + 82: + end: 3851206 + operation: insert + start: 3800000 + 83: + ids_end: 1400000 + ids_start: 1371233 + operation: replace + tags_end: 1328767 + tags_start: 1300000 + 84: + operation: search + 85: + end: 3963965 + operation: insert + start: 3900000 + 86: + ids_end: 1500000 + ids_start: 1452446 + operation: replace + tags_end: 1447554 + tags_start: 1400000 + 87: + operation: search + 88: + end: 4086390 + operation: insert + start: 4000000 + 89: + ids_end: 1600000 + ids_start: 1574492 + operation: replace + tags_end: 1525508 + tags_start: 1500000 + 90: + operation: search + 91: + end: 4167743 + operation: insert + start: 4100000 + 92: + ids_end: 1700000 + ids_start: 1657378 + operation: replace + tags_end: 1642622 + tags_start: 1600000 + 93: + operation: search + 94: + end: 4271418 + operation: insert + start: 4200000 + 95: + ids_end: 1800000 + ids_start: 1762498 + operation: replace + tags_end: 1737502 + tags_start: 1700000 + 96: + operation: search + 97: + end: 4376223 + operation: insert + start: 4300000 + 98: + ids_end: 1900000 + ids_start: 1855688 + operation: replace + tags_end: 1844312 + tags_start: 1800000 + 99: + operation: search + 100: + end: 4452083 + operation: insert + start: 4400000 + 101: + operation: search + 102: + end: 4554879 + operation: insert + start: 4500000 + 103: + ids_end: 2100000 + ids_start: 2067276 + operation: replace + tags_end: 2032724 + tags_start: 2000000 + 104: + operation: search + 105: + end: 4686840 + operation: insert + start: 4600000 + 106: + operation: search + 107: + end: 4751728 + operation: insert + start: 4700000 + 108: + ids_end: 2300000 + ids_start: 2266775 + operation: replace + tags_end: 2233225 + tags_start: 2200000 + 109: + operation: search + 110: + end: 4869941 + operation: insert + start: 4800000 + 111: + ids_end: 2400000 + ids_start: 2370251 + operation: replace + tags_end: 2329749 + tags_start: 2300000 + 112: + operation: search + 113: + end: 4955422 + operation: insert + start: 4900000 + 114: + ids_end: 2500000 + ids_start: 2485289 + operation: replace + tags_end: 2414711 + tags_start: 2400000 + 115: + operation: search + 116: + end: 5089651 + operation: insert + start: 5000000 + 117: + operation: search + 118: + end: 5169046 + operation: insert + start: 5100000 + 119: + end: 159704 + operation: delete + start: 100000 + 120: + ids_end: 2700000 + ids_start: 2662292 + operation: replace + tags_end: 2637708 + tags_start: 2600000 + 121: + operation: search + 122: + end: 5275730 + operation: insert + start: 5200000 + 123: + operation: search + 124: + end: 5388189 + operation: insert + start: 5300000 + 125: + end: 379583 + operation: delete + start: 300000 + 126: + ids_end: 2900000 + ids_start: 2861892 + operation: replace + tags_end: 2838108 + tags_start: 2800000 + 127: + operation: search + 128: + end: 5482791 + operation: insert + start: 5400000 + 129: + ids_end: 3000000 + ids_start: 2971724 + operation: replace + tags_end: 2928276 + tags_start: 2900000 + 130: + operation: search + 131: + end: 5587709 + operation: insert + start: 5500000 + 132: + ids_end: 3100000 + ids_start: 3080301 + operation: replace + tags_end: 3019699 + tags_start: 3000000 + 133: + operation: search + 134: + end: 5657473 + operation: insert + start: 5600000 + 135: + operation: search + 136: + end: 5775383 + operation: insert + start: 5700000 + 137: + end: 787283 + operation: delete + start: 700000 + 138: + ids_end: 3300000 + ids_start: 3289859 + operation: replace + tags_end: 3210141 + tags_start: 3200000 + 139: + operation: search + 140: + end: 5853890 + operation: insert + start: 5800000 + 141: + ids_end: 3400000 + ids_start: 3353942 + operation: replace + tags_end: 3346058 + tags_start: 3300000 + 142: + operation: search + 143: + end: 5977997 + operation: insert + start: 5900000 + 144: + operation: search + 145: + end: 6051674 + operation: insert + start: 6000000 + 146: + end: 1073582 + operation: delete + start: 1000000 + 147: + ids_end: 3600000 + ids_start: 3570479 + operation: replace + tags_end: 3529521 + tags_start: 3500000 + 148: + operation: search + 149: + end: 6164091 + operation: insert + start: 6100000 + 150: + ids_end: 3700000 + ids_start: 3650270 + operation: replace + tags_end: 3649730 + tags_start: 3600000 + 151: + operation: search + 152: + end: 6289725 + operation: insert + start: 6200000 + 153: + end: 1283992 + operation: delete + start: 1200000 + 154: + operation: search + 155: + end: 6360197 + operation: insert + start: 6300000 + 156: + end: 1371233 + operation: delete + start: 1300000 + 157: + ids_end: 3900000 + ids_start: 3851206 + operation: replace + tags_end: 3848794 + tags_start: 3800000 + 158: + operation: search + 159: + end: 6457226 + operation: insert + start: 6400000 + 160: + end: 1452446 + operation: delete + start: 1400000 + 161: + ids_end: 4000000 + ids_start: 3963965 + operation: replace + tags_end: 3936035 + tags_start: 3900000 + 162: + operation: search + 163: + end: 6580101 + operation: insert + start: 6500000 + 164: + end: 1574492 + operation: delete + start: 1500000 + 165: + operation: search + 166: + end: 6668798 + operation: insert + start: 6600000 + 167: + end: 1657378 + operation: delete + start: 1600000 + 168: + operation: search + 169: + end: 6766110 + operation: insert + start: 6700000 + 170: + end: 1762498 + operation: delete + start: 1700000 + 171: + operation: search + 172: + end: 6879396 + operation: insert + start: 6800000 + 173: + end: 1855688 + operation: delete + start: 1800000 + 174: + operation: search + 175: + end: 6983366 + operation: insert + start: 6900000 + 176: + ids_end: 4500000 + ids_start: 4452083 + operation: replace + tags_end: 4447917 + tags_start: 4400000 + 177: + operation: search + 178: + end: 7059507 + operation: insert + start: 7000000 + 179: + end: 2067276 + operation: delete + start: 2000000 + 180: + operation: search + 181: + end: 7158207 + operation: insert + start: 7100000 + 182: + ids_end: 4700000 + ids_start: 4686840 + operation: replace + tags_end: 4613160 + tags_start: 4600000 + 183: + operation: search + 184: + end: 7275401 + operation: insert + start: 7200000 + 185: + end: 2266775 + operation: delete + start: 2200000 + 186: + operation: search + 187: + end: 7380011 + operation: insert + start: 7300000 + 188: + end: 2370251 + operation: delete + start: 2300000 + 189: + ids_end: 4900000 + ids_start: 4869941 + operation: replace + tags_end: 4830059 + tags_start: 4800000 + 190: + operation: search + 191: + end: 7468578 + operation: insert + start: 7400000 + 192: + end: 2485289 + operation: delete + start: 2400000 + 193: + ids_end: 5000000 + ids_start: 4955422 + operation: replace + tags_end: 4944578 + tags_start: 4900000 + 194: + operation: search + 195: + end: 7585571 + operation: insert + start: 7500000 + 196: + ids_end: 5100000 + ids_start: 5089651 + operation: replace + tags_end: 5010349 + tags_start: 5000000 + 197: + operation: search + 198: + end: 7660650 + operation: insert + start: 7600000 + 199: + end: 2662292 + operation: delete + start: 2600000 + 200: + operation: search + 201: + end: 7787231 + operation: insert + start: 7700000 + 202: + ids_end: 5300000 + ids_start: 5275730 + operation: replace + tags_end: 5224270 + tags_start: 5200000 + 203: + operation: search + 204: + end: 7856311 + operation: insert + start: 7800000 + 205: + end: 2861892 + operation: delete + start: 2800000 + 206: + operation: search + 207: + end: 7952271 + operation: insert + start: 7900000 + 208: + end: 2971724 + operation: delete + start: 2900000 + 209: + ids_end: 5500000 + ids_start: 5482791 + operation: replace + tags_end: 5417209 + tags_start: 5400000 + 210: + operation: search + 211: + end: 8056334 + operation: insert + start: 8000000 + 212: + end: 3080301 + operation: delete + start: 3000000 + 213: + operation: search + 214: + end: 8151690 + operation: insert + start: 8100000 + 215: + ids_end: 5700000 + ids_start: 5657473 + operation: replace + tags_end: 5642527 + tags_start: 5600000 + 216: + operation: search + 217: + end: 8278017 + operation: insert + start: 8200000 + 218: + end: 3289859 + operation: delete + start: 3200000 + 219: + ids_end: 5800000 + ids_start: 5775383 + operation: replace + tags_end: 5724617 + tags_start: 5700000 + 220: + operation: search + 221: + end: 8380651 + operation: insert + start: 8300000 + 222: + end: 3353942 + operation: delete + start: 3300000 + 223: + ids_end: 5900000 + ids_start: 5853890 + operation: replace + tags_end: 5846110 + tags_start: 5800000 + 224: + operation: search + 225: + end: 8458613 + operation: insert + start: 8400000 + 226: + operation: search + 227: + end: 8583855 + operation: insert + start: 8500000 + 228: + end: 3570479 + operation: delete + start: 3500000 + 229: + operation: search + 230: + end: 8667722 + operation: insert + start: 8600000 + 231: + end: 3650270 + operation: delete + start: 3600000 + 232: + operation: search + 233: + end: 8755593 + operation: insert + start: 8700000 + 234: + ids_end: 6300000 + ids_start: 6289725 + operation: replace + tags_end: 6210275 + tags_start: 6200000 + 235: + operation: search + 236: + end: 8860361 + operation: insert + start: 8800000 + 237: + end: 3851206 + operation: delete + start: 3800000 + 238: + operation: search + 239: + end: 8978115 + operation: insert + start: 8900000 + 240: + end: 3963965 + operation: delete + start: 3900000 + 241: + ids_end: 6500000 + ids_start: 6457226 + operation: replace + tags_end: 6442774 + tags_start: 6400000 + 242: + operation: search + 243: + end: 9057019 + operation: insert + start: 9000000 + 244: + ids_end: 6600000 + ids_start: 6580101 + operation: replace + tags_end: 6519899 + tags_start: 6500000 + 245: + operation: search + 246: + end: 9175891 + operation: insert + start: 9100000 + 247: + operation: search + 248: + end: 9283940 + operation: insert + start: 9200000 + 249: + operation: search + 250: + end: 9360523 + operation: insert + start: 9300000 + 251: + ids_end: 6900000 + ids_start: 6879396 + operation: replace + tags_end: 6820604 + tags_start: 6800000 + 252: + operation: search + 253: + end: 9461110 + operation: insert + start: 9400000 + 254: + end: 4452083 + operation: delete + start: 4400000 + 255: + operation: search + 256: + end: 9563916 + operation: insert + start: 9500000 + 257: + operation: search + 258: + end: 9669831 + operation: insert + start: 9600000 + 259: + end: 4686840 + operation: delete + start: 4600000 + 260: + ids_end: 7200000 + ids_start: 7158207 + operation: replace + tags_end: 7141793 + tags_start: 7100000 + 261: + operation: search + 262: + end: 9782694 + operation: insert + start: 9700000 + 263: + operation: search + 264: + end: 9857619 + operation: insert + start: 9800000 + 265: + end: 4869941 + operation: delete + start: 4800000 + 266: + ids_end: 7400000 + ids_start: 7380011 + operation: replace + tags_end: 7319989 + tags_start: 7300000 + 267: + operation: search + 268: + end: 9963731 + operation: insert + start: 9900000 + 269: + end: 4955422 + operation: delete + start: 4900000 + 270: + operation: search + 271: + end: 10065469 + operation: insert + start: 10000000 + 272: + end: 5089651 + operation: delete + start: 5000000 + 273: + operation: search + 274: + end: 10157799 + operation: insert + start: 10100000 + 275: + operation: search + 276: + end: 10253427 + operation: insert + start: 10200000 + 277: + end: 5275730 + operation: delete + start: 5200000 + 278: + ids_end: 7800000 + ids_start: 7787231 + operation: replace + tags_end: 7712769 + tags_start: 7700000 + 279: + operation: search + 280: + end: 10363530 + operation: insert + start: 10300000 + 281: + ids_end: 7900000 + ids_start: 7856311 + operation: replace + tags_end: 7843689 + tags_start: 7800000 + 282: + operation: search + 283: + end: 10470707 + operation: insert + start: 10400000 + 284: + end: 5482791 + operation: delete + start: 5400000 + 285: + ids_end: 8000000 + ids_start: 7952271 + operation: replace + tags_end: 7947729 + tags_start: 7900000 + 286: + operation: search + 287: + end: 10557874 + operation: insert + start: 10500000 + 288: + ids_end: 8100000 + ids_start: 8056334 + operation: replace + tags_end: 8043666 + tags_start: 8000000 + 289: + operation: search + 290: + end: 10684586 + operation: insert + start: 10600000 + 291: + end: 5657473 + operation: delete + start: 5600000 + 292: + ids_end: 8200000 + ids_start: 8151690 + operation: replace + tags_end: 8148310 + tags_start: 8100000 + 293: + operation: search + 294: + end: 10771087 + operation: insert + start: 10700000 + 295: + end: 5775383 + operation: delete + start: 5700000 + 296: + ids_end: 8300000 + ids_start: 8278017 + operation: replace + tags_end: 8221983 + tags_start: 8200000 + 297: + operation: search + 298: + end: 10854536 + operation: insert + start: 10800000 + 299: + end: 5853890 + operation: delete + start: 5800000 + 300: + operation: search + 301: + end: 10977458 + operation: insert + start: 10900000 + 302: + ids_end: 8500000 + ids_start: 8458613 + operation: replace + tags_end: 8441387 + tags_start: 8400000 + 303: + operation: search + 304: + end: 11053942 + operation: insert + start: 11000000 + 305: + operation: search + 306: + end: 11186854 + operation: insert + start: 11100000 + 307: + ids_end: 8700000 + ids_start: 8667722 + operation: replace + tags_end: 8632278 + tags_start: 8600000 + 308: + operation: search + 309: + end: 11270191 + operation: insert + start: 11200000 + 310: + end: 6289725 + operation: delete + start: 6200000 + 311: + ids_end: 8800000 + ids_start: 8755593 + operation: replace + tags_end: 8744407 + tags_start: 8700000 + 312: + operation: search + 313: + end: 11386372 + operation: insert + start: 11300000 + 314: + operation: search + 315: + end: 11473000 + operation: insert + start: 11400000 + 316: + end: 6457226 + operation: delete + start: 6400000 + 317: + operation: search + 318: + end: 11579593 + operation: insert + start: 11500000 + 319: + end: 6580101 + operation: delete + start: 6500000 + 320: + ids_end: 9100000 + ids_start: 9057019 + operation: replace + tags_end: 9042981 + tags_start: 9000000 + 321: + operation: search + 322: + end: 11662888 + operation: insert + start: 11600000 + 323: + ids_end: 9200000 + ids_start: 9175891 + operation: replace + tags_end: 9124109 + tags_start: 9100000 + 324: + operation: search + 325: + end: 11786057 + operation: insert + start: 11700000 + 326: + ids_end: 9300000 + ids_start: 9283940 + operation: replace + tags_end: 9216060 + tags_start: 9200000 + 327: + operation: search + 328: + end: 11876088 + operation: insert + start: 11800000 + 329: + end: 6879396 + operation: delete + start: 6800000 + 330: + ids_end: 9400000 + ids_start: 9360523 + operation: replace + tags_end: 9339477 + tags_start: 9300000 + 331: + operation: search + 332: + end: 11977164 + operation: insert + start: 11900000 + 333: + operation: search + 334: + end: 12059539 + operation: insert + start: 12000000 + 335: + ids_end: 9600000 + ids_start: 9563916 + operation: replace + tags_end: 9536084 + tags_start: 9500000 + 336: + operation: search + 337: + end: 12154716 + operation: insert + start: 12100000 + 338: + end: 7158207 + operation: delete + start: 7100000 + 339: + ids_end: 9700000 + ids_start: 9669831 + operation: replace + tags_end: 9630169 + tags_start: 9600000 + 340: + operation: search + 341: + end: 12261933 + operation: insert + start: 12200000 + 342: + ids_end: 9800000 + ids_start: 9782694 + operation: replace + tags_end: 9717306 + tags_start: 9700000 + 343: + operation: search + 344: + end: 12387471 + operation: insert + start: 12300000 + 345: + end: 7380011 + operation: delete + start: 7300000 + 346: + ids_end: 9900000 + ids_start: 9857619 + operation: replace + tags_end: 9842381 + tags_start: 9800000 + 347: + operation: search + 348: + end: 12454126 + operation: insert + start: 12400000 + 349: + operation: search + 350: + end: 12582984 + operation: insert + start: 12500000 + 351: + ids_end: 10100000 + ids_start: 10065469 + operation: replace + tags_end: 10034531 + tags_start: 10000000 + 352: + ids_end: 12400000 + ids_start: 12387471 + operation: replace + tags_end: 12312529 + tags_start: 12300000 + 353: + operation: search + 354: + end: 12656392 + operation: insert + start: 12600000 + 355: + ids_end: 12300000 + ids_start: 12261933 + operation: replace + tags_end: 12238067 + tags_start: 12200000 + 356: + operation: search + 357: + end: 12766593 + operation: insert + start: 12700000 + 358: + end: 7787231 + operation: delete + start: 7700000 + 359: + ids_end: 10300000 + ids_start: 10253427 + operation: replace + tags_end: 10246573 + tags_start: 10200000 + 360: + ids_end: 12000000 + ids_start: 11977164 + operation: replace + tags_end: 11922836 + tags_start: 11900000 + 361: + ids_end: 12100000 + ids_start: 12059539 + operation: replace + tags_end: 12040461 + tags_start: 12000000 + 362: + operation: search + 363: + end: 12855590 + operation: insert + start: 12800000 + 364: + end: 7856311 + operation: delete + start: 7800000 + 365: + operation: search + 366: + end: 12962271 + operation: insert + start: 12900000 + 367: + end: 7952271 + operation: delete + start: 7900000 + 368: + ids_end: 11700000 + ids_start: 11662888 + operation: replace + tags_end: 11637112 + tags_start: 11600000 + 369: + operation: search + 370: + end: 13059095 + operation: insert + start: 13000000 + 371: + end: 8056334 + operation: delete + start: 8000000 + 372: + ids_end: 10600000 + ids_start: 10557874 + operation: replace + tags_end: 10542126 + tags_start: 10500000 + 373: + operation: search + 374: + end: 13186142 + operation: insert + start: 13100000 + 375: + end: 8151690 + operation: delete + start: 8100000 + 376: + operation: search + 377: + end: 13288018 + operation: insert + start: 13200000 + 378: + end: 8278017 + operation: delete + start: 8200000 + 379: + ids_end: 11100000 + ids_start: 11053942 + operation: replace + tags_end: 11046058 + tags_start: 11000000 + 380: + operation: search + 381: + end: 13371714 + operation: insert + start: 13300000 + 382: + ids_end: 10900000 + ids_start: 10854536 + operation: replace + tags_end: 10845464 + tags_start: 10800000 + 383: + ids_end: 11000000 + ids_start: 10977458 + operation: replace + tags_end: 10922542 + tags_start: 10900000 + 384: + operation: search + 385: + end: 13455127 + operation: insert + start: 13400000 + 386: + end: 8458613 + operation: delete + start: 8400000 + 387: + ids_end: 10800000 + ids_start: 10771087 + operation: replace + tags_end: 10728913 + tags_start: 10700000 + 388: + operation: search + 389: + end: 13580060 + operation: insert + start: 13500000 + 390: + ids_end: 10700000 + ids_start: 10684586 + operation: replace + tags_end: 10615414 + tags_start: 10600000 + 391: + operation: search + 392: + end: 13655549 + operation: insert + start: 13600000 + 393: + end: 8667722 + operation: delete + start: 8600000 + 394: + ids_end: 11200000 + ids_start: 11186854 + operation: replace + tags_end: 11113146 + tags_start: 11100000 + 395: + operation: search + 396: + end: 13760287 + operation: insert + start: 13700000 + 397: + end: 8755593 + operation: delete + start: 8700000 + 398: + ids_end: 10400000 + ids_start: 10363530 + operation: replace + tags_end: 10336470 + tags_start: 10300000 + 399: + ids_end: 11300000 + ids_start: 11270191 + operation: replace + tags_end: 11229809 + tags_start: 11200000 + 400: + operation: search + 401: + end: 13882810 + operation: insert + start: 13800000 + 402: + ids_end: 11400000 + ids_start: 11386372 + operation: replace + tags_end: 11313628 + tags_start: 11300000 + 403: + operation: search + 404: + end: 13972370 + operation: insert + start: 13900000 + 405: + ids_end: 10000000 + ids_start: 9963731 + operation: replace + tags_end: 9936269 + tags_start: 9900000 + 406: + ids_end: 11500000 + ids_start: 11473000 + operation: replace + tags_end: 11427000 + tags_start: 11400000 + 407: + operation: search + 408: + end: 14082055 + operation: insert + start: 14000000 + 409: + end: 9057019 + operation: delete + start: 9000000 + 410: + ids_end: 11600000 + ids_start: 11579593 + operation: replace + tags_end: 11520407 + tags_start: 11500000 + 411: + operation: search + 412: + end: 14165070 + operation: insert + start: 14100000 + 413: + end: 9175891 + operation: delete + start: 9100000 + 414: + operation: search + 415: + end: 14258243 + operation: insert + start: 14200000 + 416: + end: 9283940 + operation: delete + start: 9200000 + 417: + ids_end: 11800000 + ids_start: 11786057 + operation: replace + tags_end: 11713943 + tags_start: 11700000 + 418: + operation: search + 419: + end: 14386580 + operation: insert + start: 14300000 + 420: + end: 9360523 + operation: delete + start: 9300000 + 421: + ids_end: 11900000 + ids_start: 11876088 + operation: replace + tags_end: 11823912 + tags_start: 11800000 + 422: + operation: search + 423: + end: 14485809 + operation: insert + start: 14400000 + 424: + ids_end: 8900000 + ids_start: 8860361 + operation: replace + tags_end: 8839639 + tags_start: 8800000 + 425: + ids_end: 9000000 + ids_start: 8978115 + operation: replace + tags_end: 8921885 + tags_start: 8900000 + 426: + operation: search + 427: + end: 14575226 + operation: insert + start: 14500000 + 428: + end: 9563916 + operation: delete + start: 9500000 + 429: + operation: search + 430: + end: 14665784 + operation: insert + start: 14600000 + 431: + end: 9669831 + operation: delete + start: 9600000 + 432: + ids_end: 12200000 + ids_start: 12154716 + operation: replace + tags_end: 12145284 + tags_start: 12100000 + 433: + operation: search + 434: + end: 14782319 + operation: insert + start: 14700000 + 435: + end: 9782694 + operation: delete + start: 9700000 + 436: + ids_end: 8400000 + ids_start: 8380651 + operation: replace + tags_end: 8319349 + tags_start: 8300000 + 437: + operation: search + 438: + end: 14864435 + operation: insert + start: 14800000 + 439: + end: 9857619 + operation: delete + start: 9800000 + 440: + operation: search + 441: + end: 14977510 + operation: insert + start: 14900000 + 442: + ids_end: 12500000 + ids_start: 12454126 + operation: replace + tags_end: 12445874 + tags_start: 12400000 + 443: + operation: search + 444: + end: 15054613 + operation: insert + start: 15000000 + 445: + end: 10065469 + operation: delete + start: 10000000 + 446: + ids_end: 100000 + ids_start: 88328 + operation: replace + tags_end: 11672 + tags_start: 0 + 447: + operation: search + 448: + end: 15180147 + operation: insert + start: 15100000 + 449: + ids_end: 7700000 + ids_start: 7660650 + operation: replace + tags_end: 7639350 + tags_start: 7600000 + 450: + ids_end: 12700000 + ids_start: 12656392 + operation: replace + tags_end: 12643608 + tags_start: 12600000 + 451: + operation: search + 452: + end: 15258685 + operation: insert + start: 15200000 + 453: + end: 10253427 + operation: delete + start: 10200000 + 454: + ids_end: 7600000 + ids_start: 7585571 + operation: replace + tags_end: 7514429 + tags_start: 7500000 + 455: + ids_end: 12800000 + ids_start: 12766593 + operation: replace + tags_end: 12733407 + tags_start: 12700000 + 456: + operation: search + 457: + end: 15381651 + operation: insert + start: 15300000 + 458: + ids_end: 12900000 + ids_start: 12855590 + operation: replace + tags_end: 12844410 + tags_start: 12800000 + 459: + operation: search + 460: + end: 15475179 + operation: insert + start: 15400000 + 461: + ids_end: 7300000 + ids_start: 7275401 + operation: replace + tags_end: 7224599 + tags_start: 7200000 + 462: + ids_end: 13000000 + ids_start: 12962271 + operation: replace + tags_end: 12937729 + tags_start: 12900000 + 463: + operation: search + 464: + end: 15580485 + operation: insert + start: 15500000 + 465: + end: 10557874 + operation: delete + start: 10500000 + 466: + ids_end: 7100000 + ids_start: 7059507 + operation: replace + tags_end: 7040493 + tags_start: 7000000 + 467: + ids_end: 13100000 + ids_start: 13059095 + operation: replace + tags_end: 13040905 + tags_start: 13000000 + 468: + operation: search + 469: + end: 15673410 + operation: insert + start: 15600000 + 470: + ids_end: 700000 + ids_start: 662778 + operation: replace + tags_end: 637222 + tags_start: 600000 + 471: + ids_end: 7000000 + ids_start: 6983366 + operation: replace + tags_end: 6916634 + tags_start: 6900000 + 472: + ids_end: 13200000 + ids_start: 13186142 + operation: replace + tags_end: 13113858 + tags_start: 13100000 + 473: + operation: search + 474: + end: 15771505 + operation: insert + start: 15700000 + 475: + ids_end: 6800000 + ids_start: 6766110 + operation: replace + tags_end: 6733890 + tags_start: 6700000 + 476: + ids_end: 13300000 + ids_start: 13288018 + operation: replace + tags_end: 13211982 + tags_start: 13200000 + 477: + operation: search + 478: + end: 15877558 + operation: insert + start: 15800000 + 479: + end: 10854536 + operation: delete + start: 10800000 + 480: + operation: search + 481: + end: 15979532 + operation: insert + start: 15900000 + 482: + operation: search + 483: + end: 16072284 + operation: insert + start: 16000000 + 484: + ids_end: 6400000 + ids_start: 6360197 + operation: replace + tags_end: 6339803 + tags_start: 6300000 + 485: + ids_end: 13600000 + ids_start: 13580060 + operation: replace + tags_end: 13519940 + tags_start: 13500000 + 486: + operation: search + 487: + end: 16181034 + operation: insert + start: 16100000 + 488: + end: 11186854 + operation: delete + start: 11100000 + 489: + ids_end: 1200000 + ids_start: 1166860 + operation: replace + tags_end: 1133140 + tags_start: 1100000 + 490: + ids_end: 6200000 + ids_start: 6164091 + operation: replace + tags_end: 6135909 + tags_start: 6100000 + 491: + operation: search + 492: + end: 16272255 + operation: insert + start: 16200000 + 493: + end: 11270191 + operation: delete + start: 11200000 + 494: + operation: search + 495: + end: 16360474 + operation: insert + start: 16300000 + 496: + end: 11386372 + operation: delete + start: 11300000 + 497: + operation: search + 498: + end: 16481306 + operation: insert + start: 16400000 + 499: + end: 11473000 + operation: delete + start: 11400000 + 500: + ids_end: 5600000 + ids_start: 5587709 + operation: replace + tags_end: 5512291 + tags_start: 5500000 + 501: + ids_end: 14000000 + ids_start: 13972370 + operation: replace + tags_end: 13927630 + tags_start: 13900000 + 502: + operation: search + 503: + end: 16575736 + operation: insert + start: 16500000 + 504: + end: 11579593 + operation: delete + start: 11500000 + 505: + ids_end: 5400000 + ids_start: 5388189 + operation: replace + tags_end: 5311811 + tags_start: 5300000 + 506: + ids_end: 14100000 + ids_start: 14082055 + operation: replace + tags_end: 14017945 + tags_start: 14000000 + 507: + operation: search + 508: + end: 16677276 + operation: insert + start: 16600000 + 509: + ids_end: 5200000 + ids_start: 5169046 + operation: replace + tags_end: 5130954 + tags_start: 5100000 + 510: + ids_end: 14200000 + ids_start: 14165070 + operation: replace + tags_end: 14134930 + tags_start: 14100000 + 511: + operation: search + 512: + end: 16751843 + operation: insert + start: 16700000 + 513: + end: 11786057 + operation: delete + start: 11700000 + 514: + operation: search + 515: + end: 16874433 + operation: insert + start: 16800000 + 516: + end: 11876088 + operation: delete + start: 11800000 + 517: + ids_end: 4800000 + ids_start: 4751728 + operation: replace + tags_end: 4748272 + tags_start: 4700000 + 518: + ids_end: 14400000 + ids_start: 14386580 + operation: replace + tags_end: 14313420 + tags_start: 14300000 + 519: + operation: search + 520: + end: 16964182 + operation: insert + start: 16900000 + 521: + ids_end: 2000000 + ids_start: 1976700 + operation: replace + tags_end: 1923300 + tags_start: 1900000 + 522: + ids_end: 4400000 + ids_start: 4376223 + operation: replace + tags_end: 4323777 + tags_start: 4300000 + 523: + ids_end: 14500000 + ids_start: 14485809 + operation: replace + tags_end: 14414191 + tags_start: 14400000 + 524: + operation: search + 525: + end: 17089291 + operation: insert + start: 17000000 + 526: + ids_end: 4200000 + ids_start: 4167743 + operation: replace + tags_end: 4132257 + tags_start: 4100000 + 527: + operation: search + 528: + end: 17173220 + operation: insert + start: 17100000 + 529: + end: 12154716 + operation: delete + start: 12100000 + 530: + ids_end: 14700000 + ids_start: 14665784 + operation: replace + tags_end: 14634216 + tags_start: 14600000 + 531: + operation: search + 532: + end: 17263932 + operation: insert + start: 17200000 + 533: + ids_end: 14800000 + ids_start: 14782319 + operation: replace + tags_end: 14717681 + tags_start: 14700000 + 534: + operation: search + 535: + end: 17355123 + operation: insert + start: 17300000 + 536: + ids_end: 2800000 + ids_start: 2765888 + operation: replace + tags_end: 2734112 + tags_start: 2700000 + 537: + ids_end: 3200000 + ids_start: 3167093 + operation: replace + tags_end: 3132907 + tags_start: 3100000 + 538: + ids_end: 14900000 + ids_start: 14864435 + operation: replace + tags_end: 14835565 + tags_start: 14800000 + 539: + operation: search + 540: + end: 17451837 + operation: insert + start: 17400000 + 541: + end: 12454126 + operation: delete + start: 12400000 + 542: + ids_end: 300000 + ids_start: 257370 + operation: replace + tags_end: 242630 + tags_start: 200000 + 543: + ids_end: 500000 + ids_start: 481703 + operation: replace + tags_end: 418297 + tags_start: 400000 + 544: + ids_end: 600000 + ids_start: 559269 + operation: replace + tags_end: 540731 + tags_start: 500000 + 545: + ids_end: 900000 + ids_start: 878324 + operation: replace + tags_end: 821676 + tags_start: 800000 + 546: + ids_end: 1000000 + ids_start: 973085 + operation: replace + tags_end: 926915 + tags_start: 900000 + 547: + ids_end: 2200000 + ids_start: 2171272 + operation: replace + tags_end: 2128728 + tags_start: 2100000 + 548: + operation: search + 549: + end: 17562320 + operation: insert + start: 17500000 + 550: + ids_end: 2600000 + ids_start: 2576439 + operation: replace + tags_end: 2523561 + tags_start: 2500000 + 551: + ids_end: 15100000 + ids_start: 15054613 + operation: replace + tags_end: 15045387 + tags_start: 15000000 + 552: + operation: search + 553: + end: 17655072 + operation: insert + start: 17600000 + 554: + end: 12656392 + operation: delete + start: 12600000 + 555: + operation: search + 556: + end: 17778559 + operation: insert + start: 17700000 + 557: + end: 12766593 + operation: delete + start: 12700000 + 558: + ids_end: 15300000 + ids_start: 15258685 + operation: replace + tags_end: 15241315 + tags_start: 15200000 + 559: + operation: search + 560: + end: 17865842 + operation: insert + start: 17800000 + 561: + end: 12855590 + operation: delete + start: 12800000 + 562: + ids_end: 15400000 + ids_start: 15381651 + operation: replace + tags_end: 15318349 + tags_start: 15300000 + 563: + operation: search + 564: + end: 17964901 + operation: insert + start: 17900000 + 565: + end: 12962271 + operation: delete + start: 12900000 + 566: + operation: search + 567: + end: 18050505 + operation: insert + start: 18000000 + 568: + end: 13059095 + operation: delete + start: 13000000 + 569: + operation: search + 570: + end: 18167657 + operation: insert + start: 18100000 + 571: + end: 13186142 + operation: delete + start: 13100000 + 572: + ids_end: 15700000 + ids_start: 15673410 + operation: replace + tags_end: 15626590 + tags_start: 15600000 + 573: + operation: search + 574: + end: 18256312 + operation: insert + start: 18200000 + 575: + end: 13288018 + operation: delete + start: 13200000 + 576: + ids_end: 15800000 + ids_start: 15771505 + operation: replace + tags_end: 15728495 + tags_start: 15700000 + 577: + operation: search + 578: + end: 18370329 + operation: insert + start: 18300000 + 579: + ids_end: 15900000 + ids_start: 15877558 + operation: replace + tags_end: 15822442 + tags_start: 15800000 + 580: + operation: search + 581: + end: 18478242 + operation: insert + start: 18400000 + 582: + ids_end: 3500000 + ids_start: 3452322 + operation: replace + tags_end: 3447678 + tags_start: 3400000 + 583: + ids_end: 16000000 + ids_start: 15979532 + operation: replace + tags_end: 15920468 + tags_start: 15900000 + 584: + operation: search + 585: + end: 18564831 + operation: insert + start: 18500000 + 586: + end: 13580060 + operation: delete + start: 13500000 + 587: + operation: search + 588: + end: 18656798 + operation: insert + start: 18600000 + 589: + ids_end: 16200000 + ids_start: 16181034 + operation: replace + tags_end: 16118966 + tags_start: 16100000 + 590: + operation: search + 591: + end: 18777802 + operation: insert + start: 18700000 + 592: + ids_end: 3800000 + ids_start: 3785820 + operation: replace + tags_end: 3714180 + tags_start: 3700000 + 593: + ids_end: 16300000 + ids_start: 16272255 + operation: replace + tags_end: 16227745 + tags_start: 16200000 + 594: + operation: search + 595: + end: 18877442 + operation: insert + start: 18800000 + 596: + operation: search + 597: + end: 18968030 + operation: insert + start: 18900000 + 598: + end: 13972370 + operation: delete + start: 13900000 + 599: + operation: search + 600: + end: 19062881 + operation: insert + start: 19000000 + 601: + end: 14082055 + operation: delete + start: 14000000 + 602: + ids_end: 4100000 + ids_start: 4086390 + operation: replace + tags_end: 4013610 + tags_start: 4000000 + 603: + ids_end: 16600000 + ids_start: 16575736 + operation: replace + tags_end: 16524264 + tags_start: 16500000 + 604: + operation: search + 605: + end: 19179765 + operation: insert + start: 19100000 + 606: + end: 14165070 + operation: delete + start: 14100000 + 607: + ids_end: 16700000 + ids_start: 16677276 + operation: replace + tags_end: 16622724 + tags_start: 16600000 + 608: + operation: search + 609: + end: 19274566 + operation: insert + start: 19200000 + 610: + ids_end: 4300000 + ids_start: 4271418 + operation: replace + tags_end: 4228582 + tags_start: 4200000 + 611: + operation: search + 612: + end: 19352795 + operation: insert + start: 19300000 + 613: + end: 14386580 + operation: delete + start: 14300000 + 614: + operation: search + 615: + end: 19471638 + operation: insert + start: 19400000 + 616: + end: 14485809 + operation: delete + start: 14400000 + 617: + ids_end: 17000000 + ids_start: 16964182 + operation: replace + tags_end: 16935818 + tags_start: 16900000 + 618: + operation: search + 619: + end: 19575050 + operation: insert + start: 19500000 + 620: + ids_end: 4600000 + ids_start: 4554879 + operation: replace + tags_end: 4545121 + tags_start: 4500000 + 621: + operation: search + 622: + end: 19688094 + operation: insert + start: 19600000 + 623: + end: 14665784 + operation: delete + start: 14600000 + 624: + operation: search + 625: + end: 19759248 + operation: insert + start: 19700000 + 626: + end: 14782319 + operation: delete + start: 14700000 + 627: + ids_end: 17300000 + ids_start: 17263932 + operation: replace + tags_end: 17236068 + tags_start: 17200000 + 628: + operation: search + 629: + end: 19860550 + operation: insert + start: 19800000 + 630: + end: 14864435 + operation: delete + start: 14800000 + 631: + operation: search + 632: + end: 19963746 + operation: insert + start: 19900000 + 633: + ids_end: 17500000 + ids_start: 17451837 + operation: replace + tags_end: 17448163 + tags_start: 17400000 + 634: + operation: search + 635: + end: 20060937 + operation: insert + start: 20000000 + 636: + end: 15054613 + operation: delete + start: 15000000 + 637: + operation: search + 638: + end: 20187045 + operation: insert + start: 20100000 + 639: + ids_end: 17700000 + ids_start: 17655072 + operation: replace + tags_end: 17644928 + tags_start: 17600000 + 640: + operation: search + 641: + end: 20253864 + operation: insert + start: 20200000 + 642: + end: 15258685 + operation: delete + start: 15200000 + 643: + operation: search + 644: + end: 20377597 + operation: insert + start: 20300000 + 645: + end: 15381651 + operation: delete + start: 15300000 + 646: + operation: search + 647: + end: 20452919 + operation: insert + start: 20400000 + 648: + ids_end: 18000000 + ids_start: 17964901 + operation: replace + tags_end: 17935099 + tags_start: 17900000 + 649: + operation: search + 650: + end: 20588112 + operation: insert + start: 20500000 + 651: + operation: search + 652: + end: 20668079 + operation: insert + start: 20600000 + 653: + end: 15673410 + operation: delete + start: 15600000 + 654: + ids_end: 18200000 + ids_start: 18167657 + operation: replace + tags_end: 18132343 + tags_start: 18100000 + 655: + operation: search + 656: + end: 20766974 + operation: insert + start: 20700000 + 657: + end: 15771505 + operation: delete + start: 15700000 + 658: + operation: search + 659: + end: 20874767 + operation: insert + start: 20800000 + 660: + end: 15877558 + operation: delete + start: 15800000 + 661: + operation: search + 662: + end: 20965555 + operation: insert + start: 20900000 + 663: + end: 15979532 + operation: delete + start: 15900000 + 664: + ids_end: 6000000 + ids_start: 5977997 + operation: replace + tags_end: 5922003 + tags_start: 5900000 + 665: + ids_end: 18500000 + ids_start: 18478242 + operation: replace + tags_end: 18421758 + tags_start: 18400000 + 666: + operation: search + 667: + end: 21054392 + operation: insert + start: 21000000 + 668: + ids_end: 6100000 + ids_start: 6051674 + operation: replace + tags_end: 6048326 + tags_start: 6000000 + 669: + ids_end: 18600000 + ids_start: 18564831 + operation: replace + tags_end: 18535169 + tags_start: 18500000 + 670: + operation: search + 671: + end: 21181734 + operation: insert + start: 21100000 + 672: + end: 16181034 + operation: delete + start: 16100000 + 673: + operation: search + 674: + end: 21287359 + operation: insert + start: 21200000 + 675: + end: 16272255 + operation: delete + start: 16200000 + 676: + ids_end: 18800000 + ids_start: 18777802 + operation: replace + tags_end: 18722198 + tags_start: 18700000 + 677: + operation: search + 678: + end: 21351652 + operation: insert + start: 21300000 + 679: + ids_end: 18900000 + ids_start: 18877442 + operation: replace + tags_end: 18822558 + tags_start: 18800000 + 680: + operation: search + 681: + end: 21462621 + operation: insert + start: 21400000 + 682: + ids_end: 19000000 + ids_start: 18968030 + operation: replace + tags_end: 18931970 + tags_start: 18900000 + 683: + operation: search + 684: + end: 21568194 + operation: insert + start: 21500000 + 685: + end: 16575736 + operation: delete + start: 16500000 + 686: + operation: search + 687: + end: 21652574 + operation: insert + start: 21600000 + 688: + end: 16677276 + operation: delete + start: 16600000 + 689: + ids_end: 6700000 + ids_start: 6668798 + operation: replace + tags_end: 6631202 + tags_start: 6600000 + 690: + operation: search + 691: + end: 21757718 + operation: insert + start: 21700000 + 692: + ids_end: 19300000 + ids_start: 19274566 + operation: replace + tags_end: 19225434 + tags_start: 19200000 + 693: + operation: search + 694: + end: 21880303 + operation: insert + start: 21800000 + 695: + ids_end: 19400000 + ids_start: 19352795 + operation: replace + tags_end: 19347205 + tags_start: 19300000 + 696: + operation: search + 697: + end: 21975249 + operation: insert + start: 21900000 + 698: + end: 16964182 + operation: delete + start: 16900000 + 699: + ids_end: 19500000 + ids_start: 19471638 + operation: replace + tags_end: 19428362 + tags_start: 19400000 + 700: + operation: search + 701: + end: 22074315 + operation: insert + start: 22000000 + 702: + ids_end: 19600000 + ids_start: 19575050 + operation: replace + tags_end: 19524950 + tags_start: 19500000 + 703: + operation: search + 704: + end: 22179300 + operation: insert + start: 22100000 + 705: + operation: search + 706: + end: 22262416 + operation: insert + start: 22200000 + 707: + end: 17263932 + operation: delete + start: 17200000 + 708: + ids_end: 19800000 + ids_start: 19759248 + operation: replace + tags_end: 19740752 + tags_start: 19700000 + 709: + operation: search + 710: + end: 22368756 + operation: insert + start: 22300000 + 711: + ids_end: 19900000 + ids_start: 19860550 + operation: replace + tags_end: 19839450 + tags_start: 19800000 + 712: + operation: search + 713: + end: 22455672 + operation: insert + start: 22400000 + 714: + end: 17451837 + operation: delete + start: 17400000 + 715: + ids_end: 7500000 + ids_start: 7468578 + operation: replace + tags_end: 7431422 + tags_start: 7400000 + 716: + ids_end: 20000000 + ids_start: 19963746 + operation: replace + tags_end: 19936254 + tags_start: 19900000 + 717: + operation: search + 718: + end: 22565622 + operation: insert + start: 22500000 + 719: + ids_end: 20100000 + ids_start: 20060937 + operation: replace + tags_end: 20039063 + tags_start: 20000000 + 720: + operation: search + 721: + end: 22682429 + operation: insert + start: 22600000 + 722: + end: 17655072 + operation: delete + start: 17600000 + 723: + ids_end: 20200000 + ids_start: 20187045 + operation: replace + tags_end: 20112955 + tags_start: 20100000 + 724: + operation: search + 725: + end: 22780174 + operation: insert + start: 22700000 + 726: + operation: search + 727: + end: 22879494 + operation: insert + start: 22800000 + 728: + ids_end: 20400000 + ids_start: 20377597 + operation: replace + tags_end: 20322403 + tags_start: 20300000 + 729: + operation: search + 730: + end: 22982463 + operation: insert + start: 22900000 + 731: + end: 17964901 + operation: delete + start: 17900000 + 732: + ids_end: 20500000 + ids_start: 20452919 + operation: replace + tags_end: 20447081 + tags_start: 20400000 + 733: + operation: search + 734: + end: 23071016 + operation: insert + start: 23000000 + 735: + operation: search + 736: + end: 23169134 + operation: insert + start: 23100000 + 737: + end: 18167657 + operation: delete + start: 18100000 + 738: + ids_end: 20700000 + ids_start: 20668079 + operation: replace + tags_end: 20631921 + tags_start: 20600000 + 739: + operation: search + 740: + end: 23258470 + operation: insert + start: 23200000 + 741: + ids_end: 20800000 + ids_start: 20766974 + operation: replace + tags_end: 20733026 + tags_start: 20700000 + 742: + operation: search + 743: + end: 23351776 + operation: insert + start: 23300000 + 744: + ids_end: 20900000 + ids_start: 20874767 + operation: replace + tags_end: 20825233 + tags_start: 20800000 + 745: + operation: search + 746: + end: 23483178 + operation: insert + start: 23400000 + 747: + end: 18478242 + operation: delete + start: 18400000 + 748: + ids_end: 21000000 + ids_start: 20965555 + operation: replace + tags_end: 20934445 + tags_start: 20900000 + 749: + operation: search + 750: + end: 23571886 + operation: insert + start: 23500000 + 751: + end: 18564831 + operation: delete + start: 18500000 + 752: + ids_end: 8600000 + ids_start: 8583855 + operation: replace + tags_end: 8516145 + tags_start: 8500000 + 753: + ids_end: 21100000 + ids_start: 21054392 + operation: replace + tags_end: 21045608 + tags_start: 21000000 + 754: + operation: search + 755: + end: 23654461 + operation: insert + start: 23600000 + 756: + operation: search + 757: + end: 23788210 + operation: insert + start: 23700000 + 758: + end: 18777802 + operation: delete + start: 18700000 + 759: + operation: search + 760: + end: 23856818 + operation: insert + start: 23800000 + 761: + end: 18877442 + operation: delete + start: 18800000 + 762: + operation: search + 763: + end: 23976304 + operation: insert + start: 23900000 + 764: + end: 18968030 + operation: delete + start: 18900000 + 765: + operation: search + 766: + end: 24081251 + operation: insert + start: 24000000 + 767: + operation: search + 768: + end: 24177453 + operation: insert + start: 24100000 + 769: + operation: search + 770: + end: 24280632 + operation: insert + start: 24200000 + 771: + end: 19274566 + operation: delete + start: 19200000 + 772: + ids_end: 21800000 + ids_start: 21757718 + operation: replace + tags_end: 21742282 + tags_start: 21700000 + 773: + operation: search + 774: + end: 24355445 + operation: insert + start: 24300000 + 775: + end: 19352795 + operation: delete + start: 19300000 + 776: + ids_end: 21900000 + ids_start: 21880303 + operation: replace + tags_end: 21819697 + tags_start: 21800000 + 777: + operation: search + 778: + end: 24454902 + operation: insert + start: 24400000 + 779: + end: 19471638 + operation: delete + start: 19400000 + 780: + ids_end: 9500000 + ids_start: 9461110 + operation: replace + tags_end: 9438890 + tags_start: 9400000 + 781: + ids_end: 22000000 + ids_start: 21975249 + operation: replace + tags_end: 21924751 + tags_start: 21900000 + 782: + operation: search + 783: + end: 24588806 + operation: insert + start: 24500000 + 784: + end: 19575050 + operation: delete + start: 19500000 + 785: + ids_end: 22100000 + ids_start: 22074315 + operation: replace + tags_end: 22025685 + tags_start: 22000000 + 786: + operation: search + 787: + end: 24663932 + operation: insert + start: 24600000 + 788: + ids_end: 22200000 + ids_start: 22179300 + operation: replace + tags_end: 22120700 + tags_start: 22100000 + 789: + operation: search + 790: + end: 24774610 + operation: insert + start: 24700000 + 791: + end: 19759248 + operation: delete + start: 19700000 + 792: + ids_end: 22300000 + ids_start: 22262416 + operation: replace + tags_end: 22237584 + tags_start: 22200000 + 793: + operation: search + 794: + end: 24877039 + operation: insert + start: 24800000 + 795: + end: 19860550 + operation: delete + start: 19800000 + 796: + operation: search + 797: + end: 24986405 + operation: insert + start: 24900000 + 798: + end: 19963746 + operation: delete + start: 19900000 + 799: + ids_end: 22500000 + ids_start: 22455672 + operation: replace + tags_end: 22444328 + tags_start: 22400000 + 800: + operation: search + 801: + end: 25075397 + operation: insert + start: 25000000 + 802: + end: 20060937 + operation: delete + start: 20000000 + 803: + operation: search + 804: + end: 25156238 + operation: insert + start: 25100000 + 805: + end: 20187045 + operation: delete + start: 20100000 + 806: + ids_end: 10200000 + ids_start: 10157799 + operation: replace + tags_end: 10142201 + tags_start: 10100000 + 807: + ids_end: 22700000 + ids_start: 22682429 + operation: replace + tags_end: 22617571 + tags_start: 22600000 + 808: + operation: search + 809: + end: 25255292 + operation: insert + start: 25200000 + 810: + ids_end: 22800000 + ids_start: 22780174 + operation: replace + tags_end: 22719826 + tags_start: 22700000 + 811: + operation: search + 812: + end: 25387274 + operation: insert + start: 25300000 + 813: + end: 20377597 + operation: delete + start: 20300000 + 814: + operation: search + 815: + end: 25486716 + operation: insert + start: 25400000 + 816: + end: 20452919 + operation: delete + start: 20400000 + 817: + ids_end: 10500000 + ids_start: 10470707 + operation: replace + tags_end: 10429293 + tags_start: 10400000 + 818: + ids_end: 23000000 + ids_start: 22982463 + operation: replace + tags_end: 22917537 + tags_start: 22900000 + 819: + operation: search + 820: + end: 25569459 + operation: insert + start: 25500000 + 821: + ids_end: 23100000 + ids_start: 23071016 + operation: replace + tags_end: 23028984 + tags_start: 23000000 + 822: + operation: search + 823: + end: 25661731 + operation: insert + start: 25600000 + 824: + end: 20668079 + operation: delete + start: 20600000 + 825: + operation: search + 826: + end: 25766725 + operation: insert + start: 25700000 + 827: + end: 20766974 + operation: delete + start: 20700000 + 828: + ids_end: 23300000 + ids_start: 23258470 + operation: replace + tags_end: 23241530 + tags_start: 23200000 + 829: + operation: search + 830: + end: 25879660 + operation: insert + start: 25800000 + 831: + end: 20874767 + operation: delete + start: 20800000 + 832: + operation: search + 833: + end: 25977396 + operation: insert + start: 25900000 + 834: + end: 20965555 + operation: delete + start: 20900000 + 835: + operation: search + 836: + end: 26080005 + operation: insert + start: 26000000 + 837: + end: 21054392 + operation: delete + start: 21000000 + 838: + operation: search + 839: + end: 26170741 + operation: insert + start: 26100000 + 840: + operation: search + 841: + end: 26280304 + operation: insert + start: 26200000 + 842: + ids_end: 23800000 + ids_start: 23788210 + operation: replace + tags_end: 23711790 + tags_start: 23700000 + 843: + operation: search + 844: + end: 26356990 + operation: insert + start: 26300000 + 845: + ids_end: 23900000 + ids_start: 23856818 + operation: replace + tags_end: 23843182 + tags_start: 23800000 + 846: + operation: search + 847: + end: 26451790 + operation: insert + start: 26400000 + 848: + operation: search + 849: + end: 26553427 + operation: insert + start: 26500000 + 850: + ids_end: 24100000 + ids_start: 24081251 + operation: replace + tags_end: 24018749 + tags_start: 24000000 + 851: + operation: search + 852: + end: 26667691 + operation: insert + start: 26600000 + 853: + operation: search + 854: + end: 26770240 + operation: insert + start: 26700000 + 855: + end: 21757718 + operation: delete + start: 21700000 + 856: + ids_end: 24300000 + ids_start: 24280632 + operation: replace + tags_end: 24219368 + tags_start: 24200000 + 857: + operation: search + 858: + end: 26859320 + operation: insert + start: 26800000 + 859: + end: 21880303 + operation: delete + start: 21800000 + 860: + operation: search + 861: + end: 26960969 + operation: insert + start: 26900000 + 862: + end: 21975249 + operation: delete + start: 21900000 + 863: + ids_end: 24500000 + ids_start: 24454902 + operation: replace + tags_end: 24445098 + tags_start: 24400000 + 864: + operation: search + 865: + end: 27077063 + operation: insert + start: 27000000 + 866: + end: 22074315 + operation: delete + start: 22000000 + 867: + ids_end: 24600000 + ids_start: 24588806 + operation: replace + tags_end: 24511194 + tags_start: 24500000 + 868: + operation: search + 869: + end: 27170307 + operation: insert + start: 27100000 + 870: + end: 22179300 + operation: delete + start: 22100000 + 871: + ids_end: 24700000 + ids_start: 24663932 + operation: replace + tags_end: 24636068 + tags_start: 24600000 + 872: + operation: search + 873: + end: 27277146 + operation: insert + start: 27200000 + 874: + end: 22262416 + operation: delete + start: 22200000 + 875: + ids_end: 24800000 + ids_start: 24774610 + operation: replace + tags_end: 24725390 + tags_start: 24700000 + 876: + operation: search + 877: + end: 27362254 + operation: insert + start: 27300000 + 878: + ids_end: 24900000 + ids_start: 24877039 + operation: replace + tags_end: 24822961 + tags_start: 24800000 + 879: + operation: search + 880: + end: 27465087 + operation: insert + start: 27400000 + 881: + end: 22455672 + operation: delete + start: 22400000 + 882: + operation: search + 883: + end: 27564186 + operation: insert + start: 27500000 + 884: + ids_end: 12600000 + ids_start: 12582984 + operation: replace + tags_end: 12517016 + tags_start: 12500000 + 885: + operation: search + 886: + end: 27683360 + operation: insert + start: 27600000 + 887: + end: 22682429 + operation: delete + start: 22600000 + 888: + ids_end: 25200000 + ids_start: 25156238 + operation: replace + tags_end: 25143762 + tags_start: 25100000 + 889: + operation: search + 890: + end: 27764674 + operation: insert + start: 27700000 + 891: + end: 22780174 + operation: delete + start: 22700000 + 892: + operation: search + 893: + end: 27874332 + operation: insert + start: 27800000 + 894: + operation: search + 895: + end: 27965617 + operation: insert + start: 27900000 + 896: + end: 22982463 + operation: delete + start: 22900000 + 897: + ids_end: 25500000 + ids_start: 25486716 + operation: replace + tags_end: 25413284 + tags_start: 25400000 + 898: + operation: search + 899: + end: 28074037 + operation: insert + start: 28000000 + 900: + end: 23071016 + operation: delete + start: 23000000 + 901: + ids_end: 25600000 + ids_start: 25569459 + operation: replace + tags_end: 25530541 + tags_start: 25500000 + 902: + operation: search + 903: + end: 28153772 + operation: insert + start: 28100000 + 904: + operation: search + 905: + end: 28252236 + operation: insert + start: 28200000 + 906: + end: 23258470 + operation: delete + start: 23200000 + 907: + operation: search + 908: + end: 28363030 + operation: insert + start: 28300000 + 909: + operation: search + 910: + end: 28483372 + operation: insert + start: 28400000 + 911: + ids_end: 13500000 + ids_start: 13455127 + operation: replace + tags_end: 13444873 + tags_start: 13400000 + 912: + ids_end: 26000000 + ids_start: 25977396 + operation: replace + tags_end: 25922604 + tags_start: 25900000 + 913: + operation: search + 914: + end: 28588398 + operation: insert + start: 28500000 + 915: + ids_end: 26100000 + ids_start: 26080005 + operation: replace + tags_end: 26019995 + tags_start: 26000000 + 916: + operation: search + 917: + end: 28680663 + operation: insert + start: 28600000 + 918: + operation: search + 919: + end: 28760808 + operation: insert + start: 28700000 + 920: + end: 23788210 + operation: delete + start: 23700000 + 921: + ids_end: 13800000 + ids_start: 13760287 + operation: replace + tags_end: 13739713 + tags_start: 13700000 + 922: + operation: search + 923: + end: 28861237 + operation: insert + start: 28800000 + 924: + end: 23856818 + operation: delete + start: 23800000 + 925: + ids_end: 26400000 + ids_start: 26356990 + operation: replace + tags_end: 26343010 + tags_start: 26300000 + 926: + operation: search + 927: + end: 28988645 + operation: insert + start: 28900000 + 928: + operation: search + 929: + end: 29066506 + operation: insert + start: 29000000 + 930: + end: 24081251 + operation: delete + start: 24000000 + 931: + ids_end: 26600000 + ids_start: 26553427 + operation: replace + tags_end: 26546573 + tags_start: 26500000 + 932: + operation: search + 933: + end: 29180421 + operation: insert + start: 29100000 + 934: + ids_end: 26700000 + ids_start: 26667691 + operation: replace + tags_end: 26632309 + tags_start: 26600000 + 935: + operation: search + 936: + end: 29269712 + operation: insert + start: 29200000 + 937: + end: 24280632 + operation: delete + start: 24200000 + 938: + ids_end: 14300000 + ids_start: 14258243 + operation: replace + tags_end: 14241757 + tags_start: 14200000 + 939: + ids_end: 26800000 + ids_start: 26770240 + operation: replace + tags_end: 26729760 + tags_start: 26700000 + 940: + operation: search + 941: + end: 29381230 + operation: insert + start: 29300000 + 942: + operation: search + 943: + end: 29478342 + operation: insert + start: 29400000 + 944: + end: 24454902 + operation: delete + start: 24400000 + 945: + ids_end: 27000000 + ids_start: 26960969 + operation: replace + tags_end: 26939031 + tags_start: 26900000 + 946: + operation: search + 947: + end: 29582656 + operation: insert + start: 29500000 + 948: + end: 24588806 + operation: delete + start: 24500000 + 949: + operation: search + 950: + end: 29682691 + operation: insert + start: 29600000 + 951: + end: 24663932 + operation: delete + start: 24600000 + 952: + ids_end: 27200000 + ids_start: 27170307 + operation: replace + tags_end: 27129693 + tags_start: 27100000 + 953: + operation: search + 954: + end: 29767818 + operation: insert + start: 29700000 + 955: + end: 24774610 + operation: delete + start: 24700000 + 956: + operation: search + 957: + end: 29853489 + operation: insert + start: 29800000 + 958: + end: 24877039 + operation: delete + start: 24800000 + 959: + ids_end: 27400000 + ids_start: 27362254 + operation: replace + tags_end: 27337746 + tags_start: 27300000 + 960: + operation: search + 961: + end: 29988218 + operation: insert + start: 29900000 + 962: + operation: search + 963: + end: 30069754 + operation: insert + start: 30000000 + 964: + end: 88328 + operation: delete + start: 0 + 965: + ids_end: 27600000 + ids_start: 27564186 + operation: replace + tags_end: 27535814 + tags_start: 27500000 + 966: + operation: search + 967: + end: 30187478 + operation: insert + start: 30100000 + 968: + end: 25156238 + operation: delete + start: 25100000 + 969: + operation: search + 970: + end: 30274373 + operation: insert + start: 30200000 + 971: + ids_end: 27800000 + ids_start: 27764674 + operation: replace + tags_end: 27735326 + tags_start: 27700000 + 972: + operation: search + 973: + end: 30371540 + operation: insert + start: 30300000 + 974: + ids_end: 27900000 + ids_start: 27874332 + operation: replace + tags_end: 27825668 + tags_start: 27800000 + 975: + operation: search + 976: + end: 30473860 + operation: insert + start: 30400000 + 977: + end: 25486716 + operation: delete + start: 25400000 + 978: + operation: search + 979: + end: 30566245 + operation: insert + start: 30500000 + 980: + end: 25569459 + operation: delete + start: 25500000 + 981: + ids_end: 15600000 + ids_start: 15580485 + operation: replace + tags_end: 15519515 + tags_start: 15500000 + 982: + operation: search + 983: + end: 30680560 + operation: insert + start: 30600000 + 984: + end: 662778 + operation: delete + start: 600000 + 985: + ids_end: 28200000 + ids_start: 28153772 + operation: replace + tags_end: 28146228 + tags_start: 28100000 + 986: + operation: search + 987: + end: 30769904 + operation: insert + start: 30700000 + 988: + ids_end: 28300000 + ids_start: 28252236 + operation: replace + tags_end: 28247764 + tags_start: 28200000 + 989: + operation: search + 990: + end: 30879111 + operation: insert + start: 30800000 + 991: + ids_end: 28400000 + ids_start: 28363030 + operation: replace + tags_end: 28336970 + tags_start: 28300000 + 992: + operation: search + 993: + end: 30978265 + operation: insert + start: 30900000 + 994: + end: 25977396 + operation: delete + start: 25900000 + 995: + operation: search + 996: + end: 31054348 + operation: insert + start: 31000000 + 997: + end: 26080005 + operation: delete + start: 26000000 + 998: + ids_end: 16100000 + ids_start: 16072284 + operation: replace + tags_end: 16027716 + tags_start: 16000000 + 999: + ids_end: 28600000 + ids_start: 28588398 + operation: replace + tags_end: 28511602 + tags_start: 28500000 + 1000: + operation: search + 1001: + end: 31179057 + operation: insert + start: 31100000 + 1002: + end: 1166860 + operation: delete + start: 1100000 + 1003: + ids_end: 28700000 + ids_start: 28680663 + operation: replace + tags_end: 28619337 + tags_start: 28600000 + 1004: + operation: search + 1005: + end: 31280053 + operation: insert + start: 31200000 + 1006: + operation: search + 1007: + end: 31369352 + operation: insert + start: 31300000 + 1008: + end: 26356990 + operation: delete + start: 26300000 + 1009: + ids_end: 16400000 + ids_start: 16360474 + operation: replace + tags_end: 16339526 + tags_start: 16300000 + 1010: + ids_end: 28900000 + ids_start: 28861237 + operation: replace + tags_end: 28838763 + tags_start: 28800000 + 1011: + operation: search + 1012: + end: 31481323 + operation: insert + start: 31400000 + 1013: + ids_end: 16500000 + ids_start: 16481306 + operation: replace + tags_end: 16418694 + tags_start: 16400000 + 1014: + ids_end: 29000000 + ids_start: 28988645 + operation: replace + tags_end: 28911355 + tags_start: 28900000 + 1015: + operation: search + 1016: + end: 31584741 + operation: insert + start: 31500000 + 1017: + end: 26553427 + operation: delete + start: 26500000 + 1018: + operation: search + 1019: + end: 31663896 + operation: insert + start: 31600000 + 1020: + end: 26667691 + operation: delete + start: 26600000 + 1021: + operation: search + 1022: + end: 31752028 + operation: insert + start: 31700000 + 1023: + end: 26770240 + operation: delete + start: 26700000 + 1024: + ids_end: 16800000 + ids_start: 16751843 + operation: replace + tags_end: 16748157 + tags_start: 16700000 + 1025: + operation: search + 1026: + end: 31889366 + operation: insert + start: 31800000 + 1027: + ids_end: 16900000 + ids_start: 16874433 + operation: replace + tags_end: 16825567 + tags_start: 16800000 + 1028: + operation: search + 1029: + end: 31958604 + operation: insert + start: 31900000 + 1030: + end: 1976700 + operation: delete + start: 1900000 + 1031: + end: 26960969 + operation: delete + start: 26900000 + 1032: + operation: search + 1033: + end: 32060523 + operation: insert + start: 32000000 + 1034: + ids_end: 29600000 + ids_start: 29582656 + operation: replace + tags_end: 29517344 + tags_start: 29500000 + 1035: + operation: search + 1036: + end: 32153562 + operation: insert + start: 32100000 + 1037: + end: 27170307 + operation: delete + start: 27100000 + 1038: + ids_end: 17200000 + ids_start: 17173220 + operation: replace + tags_end: 17126780 + tags_start: 17100000 + 1039: + ids_end: 29700000 + ids_start: 29682691 + operation: replace + tags_end: 29617309 + tags_start: 29600000 + 1040: + operation: search + 1041: + end: 32280233 + operation: insert + start: 32200000 + 1042: + ids_end: 29800000 + ids_start: 29767818 + operation: replace + tags_end: 29732182 + tags_start: 29700000 + 1043: + operation: search + 1044: + end: 32388817 + operation: insert + start: 32300000 + 1045: + end: 27362254 + operation: delete + start: 27300000 + 1046: + ids_end: 17400000 + ids_start: 17355123 + operation: replace + tags_end: 17344877 + tags_start: 17300000 + 1047: + ids_end: 29900000 + ids_start: 29853489 + operation: replace + tags_end: 29846511 + tags_start: 29800000 + 1048: + operation: search + 1049: + end: 32476998 + operation: insert + start: 32400000 + 1050: + operation: search + 1051: + end: 32581358 + operation: insert + start: 32500000 + 1052: + end: 2576439 + operation: delete + start: 2500000 + 1053: + end: 27564186 + operation: delete + start: 27500000 + 1054: + ids_end: 30100000 + ids_start: 30069754 + operation: replace + tags_end: 30030246 + tags_start: 30000000 + 1055: + operation: search + 1056: + end: 32670405 + operation: insert + start: 32600000 + 1057: + ids_end: 30200000 + ids_start: 30187478 + operation: replace + tags_end: 30112522 + tags_start: 30100000 + 1058: + operation: search + 1059: + end: 32770459 + operation: insert + start: 32700000 + 1060: + end: 27764674 + operation: delete + start: 27700000 + 1061: + ids_end: 17800000 + ids_start: 17778559 + operation: replace + tags_end: 17721441 + tags_start: 17700000 + 1062: + ids_end: 30300000 + ids_start: 30274373 + operation: replace + tags_end: 30225627 + tags_start: 30200000 + 1063: + ids_end: 32200000 + ids_start: 32153562 + operation: replace + tags_end: 32146438 + tags_start: 32100000 + 1064: + operation: search + 1065: + end: 32862426 + operation: insert + start: 32800000 + 1066: + end: 27874332 + operation: delete + start: 27800000 + 1067: + ids_end: 30400000 + ids_start: 30371540 + operation: replace + tags_end: 30328460 + tags_start: 30300000 + 1068: + operation: search + 1069: + end: 32963286 + operation: insert + start: 32900000 + 1070: + ids_end: 30500000 + ids_start: 30473860 + operation: replace + tags_end: 30426140 + tags_start: 30400000 + 1071: + operation: search + 1072: + end: 33079546 + operation: insert + start: 33000000 + 1073: + ids_end: 18100000 + ids_start: 18050505 + operation: replace + tags_end: 18049495 + tags_start: 18000000 + 1074: + operation: search + 1075: + end: 33189380 + operation: insert + start: 33100000 + 1076: + end: 28153772 + operation: delete + start: 28100000 + 1077: + ids_end: 30700000 + ids_start: 30680560 + operation: replace + tags_end: 30619440 + tags_start: 30600000 + 1078: + operation: search + 1079: + end: 33271438 + operation: insert + start: 33200000 + 1080: + end: 28252236 + operation: delete + start: 28200000 + 1081: + ids_end: 18300000 + ids_start: 18256312 + operation: replace + tags_end: 18243688 + tags_start: 18200000 + 1082: + ids_end: 31500000 + ids_start: 31481323 + operation: replace + tags_end: 31418677 + tags_start: 31400000 + 1083: + operation: search + 1084: + end: 33371788 + operation: insert + start: 33300000 + 1085: + end: 28363030 + operation: delete + start: 28300000 + 1086: + ids_end: 18400000 + ids_start: 18370329 + operation: replace + tags_end: 18329671 + tags_start: 18300000 + 1087: + ids_end: 30900000 + ids_start: 30879111 + operation: replace + tags_end: 30820889 + tags_start: 30800000 + 1088: + operation: search + 1089: + end: 33453834 + operation: insert + start: 33400000 + 1090: + end: 3452322 + operation: delete + start: 3400000 + 1091: + ids_end: 31000000 + ids_start: 30978265 + operation: replace + tags_end: 30921735 + tags_start: 30900000 + 1092: + ids_end: 31300000 + ids_start: 31280053 + operation: replace + tags_end: 31219947 + tags_start: 31200000 + 1093: + operation: search + 1094: + end: 33561365 + operation: insert + start: 33500000 + 1095: + end: 28588398 + operation: delete + start: 28500000 + 1096: + operation: search + 1097: + end: 33660784 + operation: insert + start: 33600000 + 1098: + end: 28680663 + operation: delete + start: 28600000 + 1099: + ids_end: 31100000 + ids_start: 31054348 + operation: replace + tags_end: 31045652 + tags_start: 31000000 + 1100: + ids_end: 31200000 + ids_start: 31179057 + operation: replace + tags_end: 31120943 + tags_start: 31100000 + 1101: + operation: search + 1102: + end: 33782141 + operation: insert + start: 33700000 + 1103: + end: 3785820 + operation: delete + start: 3700000 + 1104: + operation: search + 1105: + end: 33866335 + operation: insert + start: 33800000 + 1106: + end: 28861237 + operation: delete + start: 28800000 + 1107: + ids_end: 30800000 + ids_start: 30769904 + operation: replace + tags_end: 30730096 + tags_start: 30700000 + 1108: + ids_end: 31400000 + ids_start: 31369352 + operation: replace + tags_end: 31330648 + tags_start: 31300000 + 1109: + operation: search + 1110: + end: 33959032 + operation: insert + start: 33900000 + 1111: + end: 28988645 + operation: delete + start: 28900000 + 1112: + operation: search + 1113: + end: 34074977 + operation: insert + start: 34000000 + 1114: + end: 4086390 + operation: delete + start: 4000000 + 1115: + ids_end: 19100000 + ids_start: 19062881 + operation: replace + tags_end: 19037119 + tags_start: 19000000 + 1116: + ids_end: 31600000 + ids_start: 31584741 + operation: replace + tags_end: 31515259 + tags_start: 31500000 + 1117: + operation: search + 1118: + end: 34159967 + operation: insert + start: 34100000 + 1119: + operation: search + 1120: + end: 34251951 + operation: insert + start: 34200000 + 1121: + end: 4271418 + operation: delete + start: 4200000 + 1122: + operation: search + 1123: + end: 34376932 + operation: insert + start: 34300000 + 1124: + ids_end: 30000000 + ids_start: 29988218 + operation: replace + tags_end: 29911782 + tags_start: 29900000 + 1125: + ids_end: 31900000 + ids_start: 31889366 + operation: replace + tags_end: 31810634 + tags_start: 31800000 + 1126: + operation: search + 1127: + end: 34475063 + operation: insert + start: 34400000 + 1128: + ids_end: 32000000 + ids_start: 31958604 + operation: replace + tags_end: 31941396 + tags_start: 31900000 + 1129: + operation: search + 1130: + end: 34576634 + operation: insert + start: 34500000 + 1131: + end: 4554879 + operation: delete + start: 4500000 + 1132: + end: 29582656 + operation: delete + start: 29500000 + 1133: + ids_end: 32100000 + ids_start: 32060523 + operation: replace + tags_end: 32039477 + tags_start: 32000000 + 1134: + operation: search + 1135: + end: 34689985 + operation: insert + start: 34600000 + 1136: + end: 29682691 + operation: delete + start: 29600000 + 1137: + operation: search + 1138: + end: 34770968 + operation: insert + start: 34700000 + 1139: + end: 29767818 + operation: delete + start: 29700000 + 1140: + ids_end: 29500000 + ids_start: 29478342 + operation: replace + tags_end: 29421658 + tags_start: 29400000 + 1141: + ids_end: 32300000 + ids_start: 32280233 + operation: replace + tags_end: 32219767 + tags_start: 32200000 + 1142: + operation: search + 1143: + end: 34871209 + operation: insert + start: 34800000 + 1144: + end: 29853489 + operation: delete + start: 29800000 + 1145: + ids_end: 32400000 + ids_start: 32388817 + operation: replace + tags_end: 32311183 + tags_start: 32300000 + 1146: + operation: search + 1147: + end: 34973928 + operation: insert + start: 34900000 + 1148: + ids_end: 29200000 + ids_start: 29180421 + operation: replace + tags_end: 29119579 + tags_start: 29100000 + 1149: + ids_end: 32500000 + ids_start: 32476998 + operation: replace + tags_end: 32423002 + tags_start: 32400000 + 1150: + operation: search + max_pts: 6682767 diff --git a/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml new file mode 100644 index 00000000..f4fdcdd1 --- /dev/null +++ b/neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml @@ -0,0 +1,854 @@ +wikipedia-35M: + 1: + end: 64251 + operation: insert + start: 0 + 2: + operation: search + 3: + end: 166838 + operation: insert + start: 100000 + 4: + operation: search + 5: + end: 275755 + operation: insert + start: 200000 + 6: + operation: search + 7: + end: 360735 + operation: insert + start: 300000 + 8: + operation: search + 9: + end: 488718 + operation: insert + start: 400000 + 10: + operation: search + 11: + end: 569950 + operation: insert + start: 500000 + 12: + operation: search + 13: + end: 659553 + operation: insert + start: 600000 + 14: + operation: search + 15: + end: 780330 + operation: insert + start: 700000 + 16: + operation: search + 17: + end: 865051 + operation: insert + start: 800000 + 18: + operation: search + 19: + end: 982285 + operation: insert + start: 900000 + 20: + operation: search + 21: + end: 1069368 + operation: insert + start: 1000000 + 22: + operation: search + 23: + end: 1151523 + operation: insert + start: 1100000 + 24: + ids_end: 200000 + ids_start: 166838 + operation: replace + tags_end: 133162 + tags_start: 100000 + 25: + operation: search + 26: + end: 1253863 + operation: insert + start: 1200000 + 27: + ids_end: 300000 + ids_start: 275755 + operation: replace + tags_end: 224245 + tags_start: 200000 + 28: + operation: search + 29: + end: 1365793 + operation: insert + start: 1300000 + 30: + operation: search + 31: + end: 1454749 + operation: insert + start: 1400000 + 32: + operation: search + 33: + end: 1550796 + operation: insert + start: 1500000 + 34: + ids_end: 600000 + ids_start: 569950 + operation: replace + tags_end: 530050 + tags_start: 500000 + 35: + operation: search + 36: + end: 1662617 + operation: insert + start: 1600000 + 37: + ids_end: 700000 + ids_start: 659553 + operation: replace + tags_end: 640447 + tags_start: 600000 + 38: + operation: search + 39: + end: 1785822 + operation: insert + start: 1700000 + 40: + ids_end: 800000 + ids_start: 780330 + operation: replace + tags_end: 719670 + tags_start: 700000 + 41: + operation: search + 42: + end: 1864168 + operation: insert + start: 1800000 + 43: + ids_end: 900000 + ids_start: 865051 + operation: replace + tags_end: 834949 + tags_start: 800000 + 44: + operation: search + 45: + end: 1979356 + operation: insert + start: 1900000 + 46: + ids_end: 1000000 + ids_start: 982285 + operation: replace + tags_end: 917715 + tags_start: 900000 + 47: + operation: search + 48: + end: 2056965 + operation: insert + start: 2000000 + 49: + ids_end: 1100000 + ids_start: 1069368 + operation: replace + tags_end: 1030632 + tags_start: 1000000 + 50: + operation: search + 51: + end: 2176001 + operation: insert + start: 2100000 + 52: + ids_end: 1200000 + ids_start: 1151523 + operation: replace + tags_end: 1148477 + tags_start: 1100000 + 53: + operation: search + 54: + end: 2282950 + operation: insert + start: 2200000 + 55: + ids_end: 1300000 + ids_start: 1253863 + operation: replace + tags_end: 1246137 + tags_start: 1200000 + 56: + operation: search + 57: + end: 2354422 + operation: insert + start: 2300000 + 58: + ids_end: 1400000 + ids_start: 1365793 + operation: replace + tags_end: 1334207 + tags_start: 1300000 + 59: + operation: search + 60: + end: 2487070 + operation: insert + start: 2400000 + 61: + ids_end: 1500000 + ids_start: 1454749 + operation: replace + tags_end: 1445251 + tags_start: 1400000 + 62: + operation: search + 63: + end: 2584407 + operation: insert + start: 2500000 + 64: + ids_end: 1600000 + ids_start: 1550796 + operation: replace + tags_end: 1549204 + tags_start: 1500000 + 65: + operation: search + 66: + end: 2651920 + operation: insert + start: 2600000 + 67: + operation: search + 68: + end: 2765188 + operation: insert + start: 2700000 + 69: + ids_end: 1800000 + ids_start: 1785822 + operation: replace + tags_end: 1714178 + tags_start: 1700000 + 70: + operation: search + 71: + end: 2873489 + operation: insert + start: 2800000 + 72: + operation: search + 73: + end: 2958436 + operation: insert + start: 2900000 + 74: + ids_end: 2000000 + ids_start: 1979356 + operation: replace + tags_end: 1920644 + tags_start: 1900000 + 75: + operation: search + 76: + end: 3069005 + operation: insert + start: 3000000 + 77: + ids_end: 2100000 + ids_start: 2056965 + operation: replace + tags_end: 2043035 + tags_start: 2000000 + 78: + operation: search + 79: + end: 3167529 + operation: insert + start: 3100000 + 80: + ids_end: 2200000 + ids_start: 2176001 + operation: replace + tags_end: 2123999 + tags_start: 2100000 + 81: + operation: search + 82: + end: 3275840 + operation: insert + start: 3200000 + 83: + ids_end: 2300000 + ids_start: 2282950 + operation: replace + tags_end: 2217050 + tags_start: 2200000 + 84: + operation: search + 85: + end: 3362648 + operation: insert + start: 3300000 + 86: + ids_end: 2400000 + ids_start: 2354422 + operation: replace + tags_end: 2345578 + tags_start: 2300000 + 87: + operation: search + 88: + end: 3484397 + operation: insert + start: 3400000 + 89: + ids_end: 2500000 + ids_start: 2487070 + operation: replace + tags_end: 2412930 + tags_start: 2400000 + 90: + operation: search + 91: + end: 3564604 + operation: insert + start: 3500000 + 92: + ids_end: 2600000 + ids_start: 2584407 + operation: replace + tags_end: 2515593 + tags_start: 2500000 + 93: + operation: search + 94: + end: 3654955 + operation: insert + start: 3600000 + 95: + ids_end: 2700000 + ids_start: 2651920 + operation: replace + tags_end: 2648080 + tags_start: 2600000 + 96: + operation: search + 97: + end: 3759848 + operation: insert + start: 3700000 + 98: + ids_end: 2800000 + ids_start: 2765188 + operation: replace + tags_end: 2734812 + tags_start: 2700000 + 99: + operation: search + 100: + end: 3855441 + operation: insert + start: 3800000 + 101: + ids_end: 2900000 + ids_start: 2873489 + operation: replace + tags_end: 2826511 + tags_start: 2800000 + 102: + operation: search + 103: + end: 3950383 + operation: insert + start: 3900000 + 104: + ids_end: 500000 + ids_start: 488718 + operation: replace + tags_end: 411282 + tags_start: 400000 + 105: + ids_end: 3000000 + ids_start: 2958436 + operation: replace + tags_end: 2941564 + tags_start: 2900000 + 106: + operation: search + 107: + end: 4059507 + operation: insert + start: 4000000 + 108: + ids_end: 3100000 + ids_start: 3069005 + operation: replace + tags_end: 3030995 + tags_start: 3000000 + 109: + operation: search + 110: + end: 4174009 + operation: insert + start: 4100000 + 111: + ids_end: 3200000 + ids_start: 3167529 + operation: replace + tags_end: 3132471 + tags_start: 3100000 + 112: + operation: search + 113: + end: 4250836 + operation: insert + start: 4200000 + 114: + operation: search + 115: + end: 4366822 + operation: insert + start: 4300000 + 116: + ids_end: 3400000 + ids_start: 3362648 + operation: replace + tags_end: 3337352 + tags_start: 3300000 + 117: + operation: search + 118: + end: 4487047 + operation: insert + start: 4400000 + 119: + operation: search + 120: + end: 4562294 + operation: insert + start: 4500000 + 121: + ids_end: 3600000 + ids_start: 3564604 + operation: replace + tags_end: 3535396 + tags_start: 3500000 + 122: + operation: search + 123: + end: 4671750 + operation: insert + start: 4600000 + 124: + ids_end: 3700000 + ids_start: 3654955 + operation: replace + tags_end: 3645045 + tags_start: 3600000 + 125: + operation: search + 126: + end: 4768251 + operation: insert + start: 4700000 + 127: + ids_end: 3800000 + ids_start: 3759848 + operation: replace + tags_end: 3740152 + tags_start: 3700000 + 128: + operation: search + 129: + end: 4869053 + operation: insert + start: 4800000 + 130: + ids_end: 3900000 + ids_start: 3855441 + operation: replace + tags_end: 3844559 + tags_start: 3800000 + 131: + operation: search + 132: + end: 4954090 + operation: insert + start: 4900000 + 133: + ids_end: 4000000 + ids_start: 3950383 + operation: replace + tags_end: 3949617 + tags_start: 3900000 + 134: + operation: search + 135: + end: 5088684 + operation: insert + start: 5000000 + 136: + ids_end: 100000 + ids_start: 64251 + operation: replace + tags_end: 35749 + tags_start: 0 + 137: + ids_end: 4100000 + ids_start: 4059507 + operation: replace + tags_end: 4040493 + tags_start: 4000000 + 138: + operation: search + 139: + end: 5181875 + operation: insert + start: 5100000 + 140: + ids_end: 4200000 + ids_start: 4174009 + operation: replace + tags_end: 4125991 + tags_start: 4100000 + 141: + operation: search + 142: + end: 5255364 + operation: insert + start: 5200000 + 143: + ids_end: 4300000 + ids_start: 4250836 + operation: replace + tags_end: 4249164 + tags_start: 4200000 + 144: + operation: search + 145: + end: 5377645 + operation: insert + start: 5300000 + 146: + ids_end: 400000 + ids_start: 360735 + operation: replace + tags_end: 339265 + tags_start: 300000 + 147: + ids_end: 4400000 + ids_start: 4366822 + operation: replace + tags_end: 4333178 + tags_start: 4300000 + 148: + operation: search + 149: + end: 5464068 + operation: insert + start: 5400000 + 150: + ids_end: 4500000 + ids_start: 4487047 + operation: replace + tags_end: 4412953 + tags_start: 4400000 + 151: + operation: search + 152: + end: 5564696 + operation: insert + start: 5500000 + 153: + ids_end: 4600000 + ids_start: 4562294 + operation: replace + tags_end: 4537706 + tags_start: 4500000 + 154: + operation: search + 155: + end: 5667027 + operation: insert + start: 5600000 + 156: + ids_end: 4700000 + ids_start: 4671750 + operation: replace + tags_end: 4628250 + tags_start: 4600000 + 157: + operation: search + 158: + end: 5780902 + operation: insert + start: 5700000 + 159: + ids_end: 4800000 + ids_start: 4768251 + operation: replace + tags_end: 4731749 + tags_start: 4700000 + 160: + operation: search + 161: + end: 5867663 + operation: insert + start: 5800000 + 162: + ids_end: 4900000 + ids_start: 4869053 + operation: replace + tags_end: 4830947 + tags_start: 4800000 + 163: + operation: search + 164: + end: 5952565 + operation: insert + start: 5900000 + 165: + operation: search + 166: + end: 6057093 + operation: insert + start: 6000000 + 167: + ids_end: 5100000 + ids_start: 5088684 + operation: replace + tags_end: 5011316 + tags_start: 5000000 + 168: + operation: search + 169: + end: 6167379 + operation: insert + start: 6100000 + 170: + ids_end: 5200000 + ids_start: 5181875 + operation: replace + tags_end: 5118125 + tags_start: 5100000 + 171: + operation: search + 172: + end: 6263730 + operation: insert + start: 6200000 + 173: + ids_end: 5300000 + ids_start: 5255364 + operation: replace + tags_end: 5244636 + tags_start: 5200000 + 174: + operation: search + 175: + end: 6384728 + operation: insert + start: 6300000 + 176: + ids_end: 5400000 + ids_start: 5377645 + operation: replace + tags_end: 5322355 + tags_start: 5300000 + 177: + operation: search + 178: + end: 6479533 + operation: insert + start: 6400000 + 179: + operation: search + 180: + end: 6579086 + operation: insert + start: 6500000 + 181: + operation: search + 182: + end: 6676866 + operation: insert + start: 6600000 + 183: + ids_end: 1700000 + ids_start: 1662617 + operation: replace + tags_end: 1637383 + tags_start: 1600000 + 184: + operation: search + 185: + end: 6761472 + operation: insert + start: 6700000 + 186: + ids_end: 5800000 + ids_start: 5780902 + operation: replace + tags_end: 5719098 + tags_start: 5700000 + 187: + operation: search + 188: + end: 6883955 + operation: insert + start: 6800000 + 189: + ids_end: 1900000 + ids_start: 1864168 + operation: replace + tags_end: 1835832 + tags_start: 1800000 + 190: + operation: search + 191: + end: 6968079 + operation: insert + start: 6900000 + 192: + ids_end: 6000000 + ids_start: 5952565 + operation: replace + tags_end: 5947435 + tags_start: 5900000 + 193: + operation: search + 194: + end: 7071118 + operation: insert + start: 7000000 + 195: + operation: search + 196: + end: 7172134 + operation: insert + start: 7100000 + 197: + ids_end: 6200000 + ids_start: 6167379 + operation: replace + tags_end: 6132621 + tags_start: 6100000 + 198: + operation: search + 199: + end: 7288596 + operation: insert + start: 7200000 + 200: + ids_end: 6300000 + ids_start: 6263730 + operation: replace + tags_end: 6236270 + tags_start: 6200000 + 201: + operation: search + 202: + end: 7388989 + operation: insert + start: 7300000 + 203: + ids_end: 6400000 + ids_start: 6384728 + operation: replace + tags_end: 6315272 + tags_start: 6300000 + 204: + operation: search + 205: + end: 7455073 + operation: insert + start: 7400000 + 206: + ids_end: 6500000 + ids_start: 6479533 + operation: replace + tags_end: 6420467 + tags_start: 6400000 + 207: + operation: search + 208: + end: 7586198 + operation: insert + start: 7500000 + 209: + ids_end: 6600000 + ids_start: 6579086 + operation: replace + tags_end: 6520914 + tags_start: 6500000 + 210: + operation: search + 211: + end: 7689105 + operation: insert + start: 7600000 + 212: + ids_end: 6700000 + ids_start: 6676866 + operation: replace + tags_end: 6623134 + tags_start: 6600000 + 213: + operation: search + 214: + end: 7778634 + operation: insert + start: 7700000 + 215: + ids_end: 6800000 + ids_start: 6761472 + operation: replace + tags_end: 6738528 + tags_start: 6700000 + 216: + operation: search + 217: + end: 7882486 + operation: insert + start: 7800000 + 218: + ids_end: 6900000 + ids_start: 6883955 + operation: replace + tags_end: 6816045 + tags_start: 6800000 + 219: + operation: search + 220: + end: 7963534 + operation: insert + start: 7900000 + 221: + ids_end: 7000000 + ids_start: 6968079 + operation: replace + tags_end: 6931921 + tags_start: 6900000 + 222: + operation: search + max_pts: 5548955 diff --git a/neurips23/streaming/wikipedia-35M_expirationtime_runbook.yaml b/neurips23/runbooks/wikipedia-35M_expirationtime_runbook.yaml similarity index 99% rename from neurips23/streaming/wikipedia-35M_expirationtime_runbook.yaml rename to neurips23/runbooks/wikipedia-35M_expirationtime_runbook.yaml index 0014f8de..eceb5d5a 100644 --- a/neurips23/streaming/wikipedia-35M_expirationtime_runbook.yaml +++ b/neurips23/runbooks/wikipedia-35M_expirationtime_runbook.yaml @@ -3304,4 +3304,4 @@ wikipedia-35M: 1001: operation: search max_pts: 5200000 - gt_url: "https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expirationtime_runbook.yaml" + gt_url: https://comp21storage.z5.web.core.windows.net/wiki-cohere-35M/wikipedia-35M_expirationtime_runbook.yaml diff --git a/neurips23/streaming/README.md b/neurips23/streaming/README.md index 1b66e222..444c3f9e 100644 --- a/neurips23/streaming/README.md +++ b/neurips23/streaming/README.md @@ -15,7 +15,7 @@ Each vector is assumed to have a unique *id* which never changes throughout the ## Available Runbooks -Now that the number of runbooks has started to increase significantly, here we list the available runbooks with a brief description of each. +Now that the number of runbooks has started to increase significantly, here we list the available runbooks (found in `neurips23/runbooks`) with a brief description of each. 1. `simple_runbook.yaml`: A runbook executing a short sequences of insertions, searches, and deletions to aid with debugging and testing. 2. `simple_replace_runbook.yaml`: A runbook executing a short sequence of inserts, searches, and replaces to aid with debugging and testing. @@ -23,7 +23,12 @@ Now that the number of runbooks has started to increase significantly, here we l 4. `delete_runbook.yaml`: A runbook executing all steps in the clustered runbook, but which then deletes a fraction of each cluster. 5. `final_runbook.yaml`: The NeurIPS 2023 streaming challenge final runbook. It takes the `msturing-30M-clustered` dataset and performs several rounds of insertion and deletion in clustered order. 6. `msmarco-100M_expirationtime_runbook.yaml`: A runbook using the `msmarco-100M` dataset which inserts each point with a randomly chosen expiration time: never, in 200 steps, or in 50 steps. -7. `neurips23/streaming/wikipedia-35M_expirationtime_runbook.yaml`: A runbook using the `wikipedia-35M` dataset which inserts each point with a randomly chosen expiration time: never, in 200 steps, or in 50 steps. -8. `neurips23/streaming/msturing-10M_slidingwindow_runbook.yaml`: A runbook using the `msturing-10M` dataset which inserts half the points, then maintains the index at a consistent size using a sliding window. -9. `clustered_replace_runbook.yaml`: A replace-focused runbook which takes the `msturing-10M-clustered` dataset, inserts a fraction of the points in each cluster, then replaces some of that fraction with vector ids from the same cluster. -10. `random_replace_runbook.yaml`: A replace-focused runbook which takes the `msturing-10M-clustered` dataset, inserts a fraction of the points in each cluster, then replaces some of that fraction with vector ids from a different randomly selected cluster. +7. `msturing-10M_slidingwindow_runbook.yaml`: A runbook using the `msturing-10M` dataset which inserts half the points, then maintains the index at a consistent size using a sliding window. +8. `clustered_replace_runbook.yaml`: A replace-focused runbook which takes the `msturing-10M-clustered` dataset, inserts a fraction of the points in each cluster, then replaces some of that fraction with vector ids from the same cluster. +9. `random_replace_runbook.yaml`: A replace-focused runbook which takes the `msturing-10M-clustered` dataset, inserts a fraction of the points in each cluster, then replaces some of that fraction with vector ids from a different randomly selected cluster. +10. `wikipedia-35M_expirationtime_runbook.yaml`: A runbook using the `wikipedia-35M` dataset which inserts each point with a randomly chosen expiration time: never, in 100 steps, or in 20 steps. +11. `wikipedia-1M_expiration_time_runbook.yaml`: A version of `wikipedia-35M_expirationtime_runbook.yaml` using the smaller `wikipedia-1M` dataset. +12. `wikipedia-35M_expirationtime_replace_only_runbook.yaml`: A runbook that takes the same setup as `wikipedia-35M_expirationtime_runbook.yaml`, but replaces points instead of deleting them using an expiration time based system. +13. `wikipedia-1M_expiration_time_replace_only_runbook.yaml`: A version of `wikipedia-35M_expiration_time_replace_only_runbook.yaml` using the smaller `wikipedia-1M` dataset. +14. `wikipedia-35M_expirationtime_replace_delete_runbook.yaml`: A runbook that takes the same setup as `wikipedia-35M_expirationtime_runbook.yaml`, but replaces points instead of deleting them using an expiration time based system. +15. `wikipedia-1M_expiration_time_replace_delete_runbook.yaml`: A version of `wikipedia-35M_expiration_time_replace_delete_runbook.yaml` using the smaller `wikipedia-1M` dataset. diff --git a/neurips23/streaming/generate_msmarco100m_runbooks.py b/neurips23/streaming/generate_msmarco100m_runbooks.py deleted file mode 100644 index debf6996..00000000 --- a/neurips23/streaming/generate_msmarco100m_runbooks.py +++ /dev/null @@ -1,67 +0,0 @@ -import yaml -import os -import random - -dataset_name="msmarco-100M" - -random.seed(809) - -total_points=101070374 -max_t=1000 -# insert the points in 1000 steps - -data = {dataset_name: {}} - -max_num_points=0 -num_points=0 - -to_delete=[[] for _ in range(max_t+300)] - -t=1 -for i in range(max_t): - - data[dataset_name][t]={ - 'operation': 'insert', - 'start': i*(total_points//max_t), - 'end': (i+1)*(total_points//max_t) - } - t+=1 - - num_points+=total_points//max_t - - max_num_points=max(max_num_points,num_points) - - data_type = random.randint(0, 25) - if data_type == 0: - pass - # with probability 1/26, the inserted point always stay in the index - elif data_type >0 and data_type < 6: - to_delete[i+200].append(i) - # with probability 5/26, the inserted point always stay in the index for 200 steps - elif data_type >=6 and data_type: - to_delete[i+50].append(i) - # with probability 20/26, the inserted point always stay in the index for 50 steps - - - for x in to_delete[i]: - data[dataset_name][t]={ - 'operation': 'delete', - 'start': x*(total_points//max_t), - 'end': (x+1)*(total_points//max_t) - } - t+=1 - num_points-=total_points//max_t - - data[dataset_name][t]={ - 'operation': 'search', - } - t+=1 - - -data[dataset_name]["max_pts"]=max_num_points - -run_book_name=dataset_name+"_"+"expirationtime_runbook.yaml" - -with open(run_book_name, 'w') as outfile: - yaml.dump(data, outfile, default_flow_style=False) - diff --git a/neurips23/streaming/generate_wiki35m_runbooks.py b/neurips23/streaming/generate_wiki35m_runbooks.py deleted file mode 100644 index 41557fa1..00000000 --- a/neurips23/streaming/generate_wiki35m_runbooks.py +++ /dev/null @@ -1,65 +0,0 @@ -import yaml -import os -import random - -dataset_name="wikipedia-35M" - -random.seed(809) - -total_points=35000000 -max_t=350 - - -data = {dataset_name: {}} - -max_num_points=0 -num_points=0 - -to_delete=[[] for _ in range(max_t+100)] - -t=1 -for i in range(max_t): - - data[dataset_name][t]={ - 'operation': 'insert', - 'start': i*(total_points//max_t), - 'end': (i+1)*(total_points//max_t) - } - t+=1 - - num_points+=total_points//max_t - - max_num_points=max(max_num_points,num_points) - - data_type = random.randint(0, 18) - if data_type == 0: - pass - # with probability 1/19, the points added always stay in the index - elif data_type >0 and data_type < 4: - to_delete[i+100].append(i) - # with probability 3/19, the points added stay in the index for 100 steps - else: - to_delete[i+20].append(i) - # with probability 15/19, the points added stay in the index for 20 steps - - for x in to_delete[i]: - data[dataset_name][t]={ - 'operation': 'delete', - 'start': x*(total_points//max_t), - 'end': (x+1)*(total_points//max_t) - } - t+=1 - num_points-=total_points//max_t - - data[dataset_name][t]={ - 'operation': 'search', - } - t+=1 - - -data[dataset_name]["max_pts"]=max_num_points - -run_book_name=dataset_name+"_"+"expirationtime_runbook.yaml" - -with open(run_book_name, 'w') as outfile: - yaml.dump(data, outfile, default_flow_style=False)