From bc963e94907b2d3d0bbe5f466e3c331b8ab21ab0 Mon Sep 17 00:00:00 2001 From: Rishabh Ranjan Date: Fri, 5 Jul 2024 23:01:36 -0700 Subject: [PATCH 1/5] fix extract_dir --- relbench/datasets/__init__.py | 2 +- relbench/tasks/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/relbench/datasets/__init__.py b/relbench/datasets/__init__.py index bcef97cf..8c5617af 100644 --- a/relbench/datasets/__init__.py +++ b/relbench/datasets/__init__.py @@ -37,7 +37,7 @@ def get_dataset_names(): def download_dataset(name: str) -> None: DOWNLOAD_REGISTRY.fetch( f"{name}/db.zip", - processor=pooch.Unzip(extract_dir="db"), + processor=pooch.Unzip(extract_dir="."), progressbar=True, ) diff --git a/relbench/tasks/__init__.py b/relbench/tasks/__init__.py index c096ba80..683c924f 100644 --- a/relbench/tasks/__init__.py +++ b/relbench/tasks/__init__.py @@ -40,7 +40,7 @@ def get_task_names(dataset_name: str): def download_task(dataset_name: str, task_name: str) -> None: DOWNLOAD_REGISTRY.fetch( f"{dataset_name}/tasks/{task_name}.zip", - processor=pooch.Unzip(extract_dir=task_name), + processor=pooch.Unzip(extract_dir="."), progressbar=True, ) From 80a430170c2e8c5b47cadacc12831626f900cec1 Mon Sep 17 00:00:00 2001 From: Rishabh Ranjan Date: Fri, 5 Jul 2024 23:57:20 -0700 Subject: [PATCH 2/5] add num_eval_timestamps=40 equivalent to earlier max_eval-time_frames=40 for rel-f1 --- relbench/tasks/f1.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/relbench/tasks/f1.py b/relbench/tasks/f1.py index d11dd6cb..a001bd8c 100644 --- a/relbench/tasks/f1.py +++ b/relbench/tasks/f1.py @@ -17,6 +17,7 @@ class DriverPositionTask(NodeTask): target_col = "position" timedelta = pd.Timedelta(days=60) metrics = [r2, mae, rmse] + num_eval_timestamps = 40 def make_table(self, db: Database, timestamps: "pd.Series[pd.Timestamp]") -> Table: r"""Create Task object for rel-f1-position.""" @@ -73,6 +74,7 @@ class DriverDNFTask(NodeTask): target_col = "did_not_finish" timedelta = pd.Timedelta(days=30) metrics = [average_precision, accuracy, f1, roc_auc] + num_eval_timestamps = 40 def make_table(self, db: Database, timestamps: "pd.Series[pd.Timestamp]") -> Table: r"""Create Task object for rel-f1-dnf.""" @@ -134,6 +136,7 @@ class DriverTop3Task(NodeTask): target_col = "qualifying" timedelta = pd.Timedelta(days=30) metrics = [average_precision, accuracy, f1, roc_auc] + num_eval_timestamps = 40 def make_table(self, db: Database, timestamps: "pd.Series[pd.Timestamp]") -> Table: r"""Create Task object for rel-f1-qualifying.""" From 5e8807deb8d1cbd826e8abd2859ae40de1efbe5e Mon Sep 17 00:00:00 2001 From: Rishabh Ranjan Date: Sat, 6 Jul 2024 00:07:27 -0700 Subject: [PATCH 3/5] update rel-f1 dataset and task hashes --- relbench/datasets/hashes.json | 4 ++-- relbench/tasks/hashes.json | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/relbench/datasets/hashes.json b/relbench/datasets/hashes.json index e3083952..38c668d6 100644 --- a/relbench/datasets/hashes.json +++ b/relbench/datasets/hashes.json @@ -2,8 +2,8 @@ "rel-trial/db.zip": "9fb5ba14f7cbca8115f3dfe0800415f98d6ddc15561e56c35ee614da6b89552a", "rel-avito/db.zip": "274e692295027a753063b9201815a9d2dea94d4cda968be81be936f546d77058", "rel-stack/db.zip": "9e5acfcaef041059dba346b1a876ff108fbb496ede0955dc89be6349e777a380", - "rel-f1/db.zip": "18d52be70972fac1538afab71c9b1b02569e3aa084adb49b2bff20d468b8cc7e", + "rel-f1/db.zip": "a4e226a04507c94112933787e0c9ebc421ccb4273bc4df05ce6c2d0bc699abc5", "rel-hm/db.zip": "3fb2aa3c978cc388aa81f43f9404fed329b502104ff58f3ee435959ab95e7029", "rel-event/db.zip": "9e0b565bd5c9c0ef0496893f7ced59ab254490e5aca5bf47663e95f50e978551", "rel-amazon/db.zip": "2953b53cd534303ff57e993e95844e9d84e7d7983a8e643f28ab32eac09256fc" -} +} \ No newline at end of file diff --git a/relbench/tasks/hashes.json b/relbench/tasks/hashes.json index 86fcf397..b27c62a1 100644 --- a/relbench/tasks/hashes.json +++ b/relbench/tasks/hashes.json @@ -13,9 +13,9 @@ "rel-stack/tasks/user-post-comment.zip": "ba934afc8833ca14035dc4bf6632dd4c64614ae4fef49ff3327909a2656ca60a", "rel-stack/tasks/post-post-related.zip": "922457288d446c931fbb3f524675d135b759bf9747a1f239178241a5bbb5a7ba", "rel-stack/tasks/user-engagement.zip": "98141d35e6471e6a4d391461c1ccf2bfb3aa363975e1c11e992223c1b2a82b2d", - "rel-f1/tasks/driver-dnf.zip": "82b88c2b9752757494593979efb03edfeeac17abf2b6547292e26c185cd7d670", - "rel-f1/tasks/driver-position.zip": "df79d32706cebedff9db02f573eb62a462a3bb456b319bcccd0587538e74ea62", - "rel-f1/tasks/driver-top3.zip": "cbadec87bc058cfa7c9503236f841c65b925a83082d77139aec4d29dd5fb0fc3", + "rel-f1/tasks/driver-dnf.zip": "a4db4c36d2ecffad54b8173826bc4c2fd51854a53c0f876733737bdfb64398b7", + "rel-f1/tasks/driver-position.zip": "885c499859f22bdc941ddb0948ddb83a714c72e8187c552c6400257c6dcb2e11", + "rel-f1/tasks/driver-top3.zip": "37c126c7e648aea2f6e6f5abbae0416f2d0c9a1398611cb9f28d0e614c3b755e", "rel-hm/tasks/item-sales.zip": "92a2c71ebd6dc5ab67c14c33a3a45c9ccafee0e5f0c7c698871a88f74e8a0867", "rel-hm/tasks/user-churn.zip": "2ef2030e308c57b5bcb4b2df1458cc3c21b7286e0b658d222be010b4f90e9265", "rel-hm/tasks/user-item-purchase.zip": "c8a8bb98e1b94bb612cc2694676d9d53a29743f11f19b8829ded5a725b7afab9", @@ -29,4 +29,4 @@ "rel-amazon/tasks/user-item-purchase.zip": "5caabc2d2a5cd18e48188a8f634c63d74fb906e68419c345f1f25f1fd0db2965", "rel-amazon/tasks/item-ltv.zip": "90c766075976825daedd24025ce0cd78fc337a16bb70a0a46a43d22edccdb01e", "rel-amazon/tasks/user-ltv.zip": "2ce8e88f987c0a80f35aaa1f8511864924ed7d1152b3ff43bbbd47de0e0c5e71" -} +} \ No newline at end of file From 4d832a1b6faf0acc88ae3bb5c635aadfa3a3de65 Mon Sep 17 00:00:00 2001 From: Rishabh Ranjan Date: Sat, 6 Jul 2024 00:25:04 -0700 Subject: [PATCH 4/5] update rel-f1 dataset and task hashes --- relbench/datasets/hashes.json | 2 +- relbench/tasks/hashes.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/relbench/datasets/hashes.json b/relbench/datasets/hashes.json index 38c668d6..35212e77 100644 --- a/relbench/datasets/hashes.json +++ b/relbench/datasets/hashes.json @@ -2,7 +2,7 @@ "rel-trial/db.zip": "9fb5ba14f7cbca8115f3dfe0800415f98d6ddc15561e56c35ee614da6b89552a", "rel-avito/db.zip": "274e692295027a753063b9201815a9d2dea94d4cda968be81be936f546d77058", "rel-stack/db.zip": "9e5acfcaef041059dba346b1a876ff108fbb496ede0955dc89be6349e777a380", - "rel-f1/db.zip": "a4e226a04507c94112933787e0c9ebc421ccb4273bc4df05ce6c2d0bc699abc5", + "rel-f1/db.zip": "ec31a4e1bc2b2f9c36c05fcd3dfe2a40a506f335dc51ce79c3ec8bb40feb1482", "rel-hm/db.zip": "3fb2aa3c978cc388aa81f43f9404fed329b502104ff58f3ee435959ab95e7029", "rel-event/db.zip": "9e0b565bd5c9c0ef0496893f7ced59ab254490e5aca5bf47663e95f50e978551", "rel-amazon/db.zip": "2953b53cd534303ff57e993e95844e9d84e7d7983a8e643f28ab32eac09256fc" diff --git a/relbench/tasks/hashes.json b/relbench/tasks/hashes.json index b27c62a1..1fafd41e 100644 --- a/relbench/tasks/hashes.json +++ b/relbench/tasks/hashes.json @@ -13,9 +13,9 @@ "rel-stack/tasks/user-post-comment.zip": "ba934afc8833ca14035dc4bf6632dd4c64614ae4fef49ff3327909a2656ca60a", "rel-stack/tasks/post-post-related.zip": "922457288d446c931fbb3f524675d135b759bf9747a1f239178241a5bbb5a7ba", "rel-stack/tasks/user-engagement.zip": "98141d35e6471e6a4d391461c1ccf2bfb3aa363975e1c11e992223c1b2a82b2d", - "rel-f1/tasks/driver-dnf.zip": "a4db4c36d2ecffad54b8173826bc4c2fd51854a53c0f876733737bdfb64398b7", - "rel-f1/tasks/driver-position.zip": "885c499859f22bdc941ddb0948ddb83a714c72e8187c552c6400257c6dcb2e11", - "rel-f1/tasks/driver-top3.zip": "37c126c7e648aea2f6e6f5abbae0416f2d0c9a1398611cb9f28d0e614c3b755e", + "rel-f1/tasks/driver-dnf.zip": "948df149bc36537cb14d7624c46db886e75d9c793a48c35fde49ac9f41220655", + "rel-f1/tasks/driver-position.zip": "775b28a51604169539bbe712a2f0d15158c112bc6abf316cdd0995087a7ae03e", + "rel-f1/tasks/driver-top3.zip": "1a16abf993cbe58524054cf710bada8538e7c75d6f8f388c0b137ab3575a9a47", "rel-hm/tasks/item-sales.zip": "92a2c71ebd6dc5ab67c14c33a3a45c9ccafee0e5f0c7c698871a88f74e8a0867", "rel-hm/tasks/user-churn.zip": "2ef2030e308c57b5bcb4b2df1458cc3c21b7286e0b658d222be010b4f90e9265", "rel-hm/tasks/user-item-purchase.zip": "c8a8bb98e1b94bb612cc2694676d9d53a29743f11f19b8829ded5a725b7afab9", From d21ddb7a2482771acc490048e781f68ee9f68af9 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 07:31:38 +0000 Subject: [PATCH 5/5] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- relbench/datasets/hashes.json | 2 +- relbench/tasks/hashes.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/relbench/datasets/hashes.json b/relbench/datasets/hashes.json index 35212e77..b0ea9d92 100644 --- a/relbench/datasets/hashes.json +++ b/relbench/datasets/hashes.json @@ -6,4 +6,4 @@ "rel-hm/db.zip": "3fb2aa3c978cc388aa81f43f9404fed329b502104ff58f3ee435959ab95e7029", "rel-event/db.zip": "9e0b565bd5c9c0ef0496893f7ced59ab254490e5aca5bf47663e95f50e978551", "rel-amazon/db.zip": "2953b53cd534303ff57e993e95844e9d84e7d7983a8e643f28ab32eac09256fc" -} \ No newline at end of file +} diff --git a/relbench/tasks/hashes.json b/relbench/tasks/hashes.json index 1fafd41e..3d554eac 100644 --- a/relbench/tasks/hashes.json +++ b/relbench/tasks/hashes.json @@ -29,4 +29,4 @@ "rel-amazon/tasks/user-item-purchase.zip": "5caabc2d2a5cd18e48188a8f634c63d74fb906e68419c345f1f25f1fd0db2965", "rel-amazon/tasks/item-ltv.zip": "90c766075976825daedd24025ce0cd78fc337a16bb70a0a46a43d22edccdb01e", "rel-amazon/tasks/user-ltv.zip": "2ce8e88f987c0a80f35aaa1f8511864924ed7d1152b3ff43bbbd47de0e0c5e71" -} \ No newline at end of file +}