Merge pull request GATEOverflow#102 from anandhu-eng/addChecksum

Add checksum to 3d-unet model download
anandhu-eng · Aug 21, 2024 · 267d661 · 267d661
2 parents 3afa29d + f0dbfdc
commit 267d661
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 31 deletions.
diff --git a/script/extract-file/customize.py b/script/extract-file/customize.py
@@ -42,7 +42,7 @@ def preprocess(i):
     # By default remove archive after extraction
     remove_extracted = False if env.get('CM_EXTRACT_REMOVE_EXTRACTED','').lower() == 'no' else True
 
-    if filename.endswith(".zip"):
+    if filename.endswith(".zip") or filename.endswith(".pth"):
         env['CM_EXTRACT_TOOL'] = "unzip"
     elif filename.endswith(".tar.gz"):
         if windows:

diff --git a/script/get-ml-model-3d-unet-kits19/_cm.json b/script/get-ml-model-3d-unet-kits19/_cm.json
@@ -40,9 +40,22 @@
       "group": "framework"
     },
     "onnx,fp32": {
+      "deps":[
+        {
+          "tags": "download,file,download-file,_wget",
+          "extra_cache_tags": "3d-unet,medical-imaging",
+          "force-cache": true,
+          "env": {
+            "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_ML_MODEL_FILE_WITH_PATH",
+            "CM_DOWNLOAD_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_128x128x128_dynbatch.onnx?download=1",
+            "CM_DOWNLOAD_CHECKSUM":"82f0618fde78f9839e7c712274019b4a",
+            "CM_DOWNLOAD_FILENAME": "3dunet_kits19_128x128x128_dynbatch.onnx"
+          }
+        }
+      ],
       "env": {
         "CM_ML_MODEL_ACCURACY": "0.86170",
-        "CM_PACKAGE_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_128x128x128_dynbatch.onnx?download=1"
+        "CM_ML_MODEL_FILE": "3dunet_kits19_128x128x128_dynbatch.onnx" 
       }
     },
     "pytorch": {
@@ -52,17 +65,41 @@
       "group": "framework"
     },
     "pytorch,fp32": {
+      "deps":[
+        {
+          "tags": "download,file,download-file,_wget",
+          "extra_cache_tags": "3d-unet,medical-imaging",
+          "force-cache": true,
+          "env": {
+            "CM_DOWNLOAD_FINAL_ENV_NAME": "CM_ML_MODEL_FILE_WITH_PATH",
+            "CM_DOWNLOAD_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_pytorch.ptc?download=1",
+            "CM_DOWNLOAD_CHECKSUM":"2251109371f408c9f10a4320ffdcaef8",
+            "CM_DOWNLOAD_FILENAME": "3dunet_kits19_pytorch.ptc"
+          }
+        }
+      ],
       "env": {
         "CM_ML_MODEL_ACCURACY": "0.86170",
-        "CM_PACKAGE_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_pytorch.ptc?download=1"
+        "CM_ML_MODEL_FILE": "3dunet_kits19_pytorch.ptc"
       }
     },
     "pytorch,fp32,weights": {
+      "deps":[
+        {
+          "tags": "download-and-extract,_wget,_extract",
+          "extra_cache_tags": "3d-unet,medical-imaging",
+          "force-cache": true,
+          "env": {
+            "CM_DAE_FINAL_ENV_NAME": "CM_ML_MODEL_FILE_WITH_PATH",
+            "CM_DAE_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_pytorch_checkpoint.pth?download=1",
+            "CM_DOWNLOAD_CHECKSUM":"09c696e3ec13d83c628498bcd831eb5b",
+            "CM_DOWNLOAD_FILENAME": "3dunet_kits19_pytorch_checkpoint.pth"
+          }
+        }
+      ],
       "env": {
         "CM_ML_MODEL_ACCURACY": "0.86170",
-        "CM_ML_MODEL_FILE": "3dunet_kits19_pytorch_checkpoint.pth",
-        "CM_PACKAGE_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_pytorch_checkpoint.pth?download=1",
-        "CM_UNZIP": "yes"
+        "CM_ML_MODEL_FILE": "3dunet_kits19_pytorch_checkpoint.pth"
       }
     },
     "weights": {
@@ -80,11 +117,22 @@
       "alias": "tf"
     },
     "tf,fp32": {
+      "deps":[
+        {
+          "tags": "download-and-extract,_wget,_extract",
+          "extra_cache_tags": "3d-unet,medical-imaging",
+          "force-cache": true,
+          "env": {
+            "CM_DAE_FINAL_ENV_NAME": "CM_ML_MODEL_FILE_WITH_PATH",
+            "CM_DAE_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_128x128x128.tf.zip?download=1",
+            "CM_DOWNLOAD_CHECKSUM":"9497108bd0504ae8f85a764a807b76a9",
+            "CM_DOWNLOAD_FILENAME": "3dunet_kits19_128x128x128.tf.zip"
+          }
+        }
+      ],
       "env": {
         "CM_ML_MODEL_ACCURACY": "0.86170",
-        "CM_ML_MODEL_FILE": "3dunet_kits19_128x128x128.tf",
-        "CM_PACKAGE_URL": "https://zenodo.org/record/5597155/files/3dunet_kits19_128x128x128.tf.zip?download=1",
-        "CM_UNZIP": "yes"
+        "CM_ML_MODEL_FILE": "3dunet_kits19_128x128x128.tf"
       }
     }
   },

diff --git a/script/get-ml-model-3d-unet-kits19/customize.py b/script/get-ml-model-3d-unet-kits19/customize.py
@@ -11,28 +11,14 @@ def preprocess(i):
 
     cm = automation.cmind
 
-    path = os.getcwd()
-
-    url = env['CM_PACKAGE_URL']
-
-    print ('Downloading from {}'.format(url))
-
-    r = cm.access({'action':'download_file', 
-                   'automation':'utils,dc2743f8450541e3', 
-                   'url':url})
-    if r['return']>0: return r
-
-    filename = r['filename']
-
-    if env.get('CM_UNZIP') == "yes":
-        os.system("unzip "+filename)
-        filename = env['CM_ML_MODEL_FILE']
-        env['CM_ML_MODEL_FILE_WITH_PATH']=os.path.join(path, filename)
+    path = os.path.dirname(env['CM_ML_MODEL_FILE_WITH_PATH'])
+
+    if env.get("CM_DAE_EXTRACT_DOWNLOADED", " ") != " ":
+        env['CM_ML_MODEL_PATH'] = os.path.join(path, env['CM_ML_MODEL_FILE'])
+        env['CM_ML_MODEL_FILE_WITH_PATH'] = env['CM_ML_MODEL_PATH']
     else:
-        # Add to path
-        env['CM_ML_MODEL_FILE']=filename
-        env['CM_ML_MODEL_FILE_WITH_PATH']=r['path']
-
-    env['CM_ML_MODEL_PATH']=path
+        env['CM_ML_MODEL_PATH'] = path
 
+    env['CM_GET_DEPENDENT_CACHED_PATH'] =  env['CM_ML_MODEL_PATH']
+
     return {'return':0}