Added a recipe for rclone_mlc_llama2_config

krai · Oct 1, 2024 · 6cf79d4 · 6cf79d4
1 parent 665bd2f
commit 6cf79d4
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 8 deletions.
diff --git a/data_axs.json b/data_axs.json
@@ -63,7 +63,8 @@
         "huggingface_tool_detector": "huggingface_tool_detector",
         "model_llama3_recipe": "model_llama3_recipe",
         "model_training_llama2_recipe": "model_training_llama2_recipe",
-        "dataset_scrolls_gov_report_8k_recipe": "dataset_scrolls_gov_report_8k_recipe"
+        "dataset_scrolls_gov_report_8k_recipe": "dataset_scrolls_gov_report_8k_recipe",
+	"rclone_mlc_llama2_config": "rclone_mlc_llama2_config"
     },
     "repo_name": "axs2mlperf",
     "submodules": false

diff --git a/rclone_mlc_inference_config/data_axs.json b/rclone_mlc_inference_config/data_axs.json
@@ -1,4 +1,5 @@
 {
-    "tags": [ "rclone_config", "mlc_inference" ],
+    "tags": [ "rclone_config" ],
+    "rclone_remote_name": "mlc_inference",
     "file_name": "rclone.conf"
 }
diff --git a/rclone_mlc_inference_config/rclone.conf b/rclone_mlc_inference_config/rclone.conf
@@ -4,9 +4,3 @@ endpoint = https://c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com
 provider = Cloudflare
 access_key_id = f65ba5eef400db161ea49967de89f47b
 secret_access_key = fbea333914c292b854f14d3fe232bad6c5407bf0ab1bebf78833c2b359bdfd2b
-
-[mlc-llama2]
-type = drive
-scope = drive.readonly
-root_folder_id = 11tBZvvrh0FCm3XuR5E849K42TqftYdUF
-team_drive =
diff --git a/rclone_mlc_llama2_config/data_axs.json b/rclone_mlc_llama2_config/data_axs.json
@@ -0,0 +1,19 @@
+{
+    "_parent_entries": [ [ "^", "byname", "entry_creator" ] , [ "^", "byname", "shell" ] ],
+    "_producer_rules": [
+        [ [ "rclone_config", "rclone_remote_name=mlc_llama2" ], [[ "get", "pipeline"]] ] ],	
+
+    "rclone_remote_name": "mlc_llama2",
+    "file_name": "rclone.conf",
+    "this_entry_path": [ "^^", "get_path" ],
+
+    "newborn_name_template": [ "^^", "substitute", "rclone_config_#{rclone_remote_name}#" ],
+    "newborn_entry_tags": [ "rclone_config" ],
+    "newborn_entry_param_names": [ "rclone_remote_name" ],
+    "pipeline": [ "^^", "execute", [[
+        [ "run" ],
+	[],
+	[ "get", "stored_newborn_entry" ]
+	]] ],
+    "shell_cmd_with_subs": "cp #{this_entry_path}# #{newborn_entry_path}#"
+}
diff --git a/rclone_mlc_llama2_config/rclone.conf b/rclone_mlc_llama2_config/rclone.conf
@@ -0,0 +1,5 @@
+[mlc-llama2]
+type = drive
+scope = drive.readonly
+root_folder_id = 11tBZvvrh0FCm3XuR5E849K42TqftYdUF
+team_drive =