Add msturing expiration time runbooks

benchmark/datasets.py

@@ -430,6 +430,30 @@ def __init__(self, nb_M=1000):
     def distance(self):
         return "euclidean"
 
+# 1M slice of MSTuring dataset, with ground truth corresponding to the 10K query set
+# this is needed for backwards compatibility with the streaming code
+class MSTuringANNSPQ(BillionScaleDatasetCompetitionFormat):
+    def __init__(self, nb_M=1):
+        self.nb_M = nb_M
+        self.nb = 10**6 * nb_M  
+        self.d = 100
+        self.nq = 10000
+        self.dtype = "float32"
+        self.ds_fn = "base1b.fbin"
+        self.qs_fn = "testQuery10K.fbin"
+        self.gt_fn = (
+            "msturing-1M-private-gt100" if self.nb_M == 1 else
+            None
+        )
+        self.base_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/"
+        self.basedir = os.path.join(BASEDIR, "MSTuringANNSPQ")
+
+        self.private_qs_url = None
+        self.private_gt_url = None
+
+    def distance(self):
+        return "euclidean"
+
 class MSTuringClustered10M(DatasetCompetitionFormat):
     def __init__(self):
         self.nb = 10**6 * 10
@@ -1285,6 +1309,8 @@ def short_name(self):
     'msturing-10M': lambda : MSTuringANNS(10),
     'msturing-1M': lambda : MSTuringANNS(1),
 
+    'msturingpq-1M': lambda : MSTuringANNSPQ(1),
+
     'msturing-10M-clustered': lambda: MSTuringClustered10M(),
     'msturing-30M-clustered': lambda: MSTuringClustered30M(),
 

data_export.py

@@ -105,6 +105,9 @@ def cleaned_run_metric(run_metrics):
                                     'neurips23/runbooks/delete_runbook.yaml',
                                     'neurips23/runbooks/final_runbook.yaml',
                                     'neurips23/runbooks/msturing-10M_slidingwindow_runbook.yaml',
+                                    'neurips23/runbooks/msturingpq-1M_expiration_time_runbook.yaml',
+                                    'neurips23/runbooks/msturingpq-1M_expiration_time_replace_only_runbook.yaml',
+                                    'neurips23/runbooks/msturingpq-1M_expiration_time_replace_delete_runbook.yaml',
                                     'neurips23/runbooks/wikipedia-35M_expirationtime_runbook.yaml',
                                     'neurips23/runbooks/wikipedia-1M_expiration_time_runbook.yaml',
                                     'neurips23/runbooks/wikipedia-35M_expiration_time_replace_only_runbook.yaml',

neurips23/runbooks/gen_expiration_time_runbook.py

@@ -11,8 +11,10 @@
 timesteps: how long to wait before deleting for each ratio
 seed: seed given to random generator
 do_replace: whether to include replace in runbook or not
+do_delete: whether to include delete in the runbook or not
+max_pts: whether to set a max_pts higher than the max that occurs in the runbook (default None)
 '''
-def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ratios, timesteps, seed = 0, do_replace = False, gt_url = None, do_delete = True):
+def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ratios, timesteps, seed = 0, do_replace = False, gt_url = None, do_delete = True, max_points = None):
     random.seed(seed)
     data = {dataset_name: {}}
 
@@ -99,7 +101,13 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra
         }
         t+=1
 
-    data[dataset_name]["max_pts"]=max_num_points
+    if max_points != None:
+        if max_points >= max_num_points:
+            data[dataset_name]["max_pts"]=max_points
+        else:
+            data[dataset_name]["max_pts"]=max_num_points
+    else:
+        data[dataset_name]["max_pts"]=max_num_points
 
     if gt_url is not None:
         data[dataset_name]["gt_url"] = gt_url
@@ -176,3 +184,36 @@ def gen_exp_time_runbook(dataset_name, dataset_size, max_t, runbook_filename, ra
 max_t = 1000
 gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, None)
 
+ratios = (0, 4, 18)
+timesteps = (0, 100, 20)
+seed = 5554
+dataset_file = 'msturingpq-1M_expiration_time_runbook.yaml'
+dataset_name = 'msturingpq-1M'
+dataset_size = 1000000
+max_t = 100
+gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_runbook.yaml"
+max_points = 450000
+gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, False, gt_url, max_points)
+
+ratios = (0, 4, 18)
+timesteps = (0, 100, 20)
+seed = 762
+dataset_file = 'msturingpq-1M_expiration_time_replace_only_runbook.yaml'
+dataset_name = 'msturing-1M'
+dataset_size = 1000000
+max_t = 100
+gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_only_runbook.yaml"
+max_points = None
+gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, False, max_points)
+
+ratios = (1, 8, 18)
+timesteps = (0, 100, 20)
+seed = 83
+dataset_file = 'msturingpq-1M_expiration_time_replace_delete_runbook.yaml'
+dataset_name = 'msturingpq-1M'
+dataset_size = 1000000
+max_t = 100
+gt_url = "https://comp21storage.z5.web.core.windows.net/comp21/MSFT-TURING-ANNS/msturingpq-1M_expiration_time_replace_delete_runbook.yaml"
+max_points = 270000
+gen_exp_time_runbook(dataset_name, dataset_size, max_t, dataset_file, ratios, timesteps, seed, True, gt_url, max_points)
+