From 7e39d97c0084cfb092453566902b5d560b4cd30e Mon Sep 17 00:00:00 2001 From: Divyashree Sreepathihalli Date: Tue, 29 Oct 2024 22:18:00 -0700 Subject: [PATCH] minor fix to HF mirror script (#1962) * minor fix to HF mirror script * fix order * fix_name * update uploaded preset json --- tools/hf_uploaded_presets.json | 45 ++++++++++++++++++++++++---------- tools/mirror_weights_on_hf.py | 7 +++--- 2 files changed, 35 insertions(+), 17 deletions(-) diff --git a/tools/hf_uploaded_presets.json b/tools/hf_uploaded_presets.json index d71ecc53f..8b92d533f 100644 --- a/tools/hf_uploaded_presets.json +++ b/tools/hf_uploaded_presets.json @@ -3,6 +3,8 @@ "kaggle://keras/densenet/keras/densenet_121_imagenet/2", "kaggle://keras/densenet/keras/densenet_169_imagenet/2", "kaggle://keras/densenet/keras/densenet_201_imagenet/2", + "kaggle://keras/efficientnet/keras/efficientnet_b0_ra_imagenet", + "kaggle://keras/efficientnet/keras/efficientnet_b1_ft_imagenet", "kaggle://keras/mit/keras/mit_b0_ade20k_512/1", "kaggle://keras/mit/keras/mit_b1_ade20k_512/1", "kaggle://keras/mit/keras/mit_b2_ade20k_512/1", @@ -45,21 +47,31 @@ "kaggle://keras/resnetv1/keras/resnet_152_imagenet/2", "kaggle://keras/resnetv2/keras/resnet_v2_50_imagenet/2", "kaggle://keras/resnetv2/keras/resnet_v2_101_imagenet/2", + "kaggle://keras/resnet_vd/keras/resnet_vd_18_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_34_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_50_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_50_ssld_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_50_ssld_v2_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_50_ssld_v2_fix_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_101_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_101_ssld_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_152_imagenet", + "kaggle://keras/resnet_vd/keras/resnet_vd_200_imagenet", "kaggle://keras/sam/keras/sam_base_sa1b/4", "kaggle://keras/sam/keras/sam_large_sa1b/4", "kaggle://keras/sam/keras/sam_huge_sa1b/4", - "kaggle://kerashub/segformer/keras/segformer_b0_ade20k_512", - "kaggle://kerashub/segformer/keras/segformer_b1_ade20k_512", - "kaggle://kerashub/segformer/keras/segformer_b2_ade20k_512", - "kaggle://kerashub/segformer/keras/segformer_b3_ade20k_512", - "kaggle://kerashub/segformer/keras/segformer_b4_ade20k_512", - "kaggle://kerashub/segformer/keras/segformer_b5_ade20k_640", - "kaggle://kerashub/segformer/keras/segformer_b0_cityscapes_1024", - "kaggle://kerashub/segformer/keras/segformer_b1_ade20k_512", - "kaggle://kerashub/segformer/keras/segformer_b2_cityscapes_1024", - "kaggle://kerashub/segformer/keras/segformer_b3_cityscapes_1024", - "kaggle://kerashub/segformer/keras/segformer_b4_cityscapes_1024", - "kaggle://kerashub/segformer/keras/segformer_b5_cityscapes_1024", + "kaggle://keras/segformer/keras/segformer_b0_ade20k_512", + "kaggle://keras/segformer/keras/segformer_b1_ade20k_512", + "kaggle://keras/segformer/keras/segformer_b2_ade20k_512", + "kaggle://keras/segformer/keras/segformer_b3_ade20k_512", + "kaggle://keras/segformer/keras/segformer_b4_ade20k_512", + "kaggle://keras/segformer/keras/segformer_b5_ade20k_640", + "kaggle://keras/segformer/keras/segformer_b0_cityscapes_1024", + "kaggle://keras/segformer/keras/segformer_b1_ade20k_512", + "kaggle://keras/segformer/keras/segformer_b2_cityscapes_1024", + "kaggle://keras/segformer/keras/segformer_b3_cityscapes_1024", + "kaggle://keras/segformer/keras/segformer_b4_cityscapes_1024", + "kaggle://keras/segformer/keras/segformer_b5_cityscapes_1024", "kaggle://keras/vgg/keras/vgg_11_imagenet/1", "kaggle://keras/vgg/keras/vgg_13_imagenet/1", "kaggle://keras/vgg/keras/vgg_16_imagenet/1", @@ -138,12 +150,19 @@ "kaggle://keras/roberta/keras/roberta_base_en/2", "kaggle://keras/roberta/keras/roberta_large_en/2", "kaggle://keras/stablediffusion3/keras/stable_diffusion_3_medium/3", + "kaggle://keras/stablediffusion-3.5/keras/stable_diffusion_3.5_large/1", + "kaggle://keras/stablediffusion-3.5/keras/stable_diffusion_3.5_large_turbo/1", "kaggle://keras/t5/keras/t5_small_multi/2", + "kaggle://keras/t5/keras/t5_1.1_small/1", "kaggle://keras/t5/keras/t5_base_multi/2", + "kaggle://keras/t5/keras/t5_1.1_base/1", "kaggle://keras/t5/keras/t5_large_multi/2", + "kaggle://keras/t5/keras/t5_1.1_large/1", + "kaggle://keras/t5/keras/t5_1.1_xl/1", + "kaggle://keras/t5/keras/t5_1.1_xxl/1", "kaggle://keras/t5/keras/flan_small_multi/2", "kaggle://keras/t5/keras/flan_base_multi/2", "kaggle://keras/t5/keras/flan_large_multi/2", "kaggle://keras/xlm_roberta/keras/xlm_roberta_base_multi/2", - "kaggle://keras/xlm_roberta/keras/xlm_roberta_large_multi/2", + "kaggle://keras/xlm_roberta/keras/xlm_roberta_large_multi/2" ] diff --git a/tools/mirror_weights_on_hf.py b/tools/mirror_weights_on_hf.py index 87ab9ae30..17a790cdc 100644 --- a/tools/mirror_weights_on_hf.py +++ b/tools/mirror_weights_on_hf.py @@ -26,10 +26,9 @@ def download_and_upload_missing_models(missing_in_hf_uploads): errored_uploads = [] for kaggle_handle in missing_in_hf_uploads: try: - model_variant = kaggle_handle.split("/")[3] - hf_uri = f"{HF_BASE_URI}/{model_variant}" kaggle_handle_path = kaggle_handle.removeprefix("kaggle://") - + model_variant = kaggle_handle_path.split("/")[3] + hf_uri = f"{HF_BASE_URI}/{model_variant}" # Skip Gemma models if "gemma" in kaggle_handle_path: print(f"Skipping Gemma model preset: {kaggle_handle_path}") @@ -87,7 +86,7 @@ def main(): # Step 5: Update JSON file with newly uploaded handles update_hf_uploads_json( - JSON_FILE_PATH, {latest_kaggle_handles} - {errored_uploads} + JSON_FILE_PATH, list(set(latest_kaggle_handles) - set(errored_uploads)) ) print("uploads for the following models failed: ", errored_uploads) print("Rest of the models up to date on HuggingFace")