jenkins: publish casync releases from device (commaai#32142)

* publish in ci * overwrite * publish in ci * fix * test it * Revert "test it" This reverts commit b3de51d. * use right token * cleanup after uploading --------- Co-authored-by: Comma Device <[email protected]>
deanlee · Apr 9, 2024 · 8364cd2 · 8364cd2
1 parent 6c3a33a
commit 8364cd2
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 15 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -153,7 +153,7 @@ def build_release(String channel_name) {
       deviceStage("build casync", "tici-needs-can", [], [
         ["build ${channel_name}", "RELEASE=1 OPENPILOT_CHANNEL=${channel_name} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync $SOURCE_DIR/release/create_casync_build.sh"],
         ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/manifest.json && cat /data/manifest.json"],
-        //["upload ${channel_name}", "OPENPILOT_CHANNEL=${channel_name} $SOURCE_DIR/release/upload_casync_release.sh"],
+        ["upload and cleanup ${channel_name}", "$SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"],
       ])
     }
   )

diff --git a/release/upload_casync_release.py b/release/upload_casync_release.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+from openpilot.tools.lib.azure_container import AzureContainer
+
+
+if __name__ == "__main__":
+  del os.environ["AZURE_TOKEN"] # regerenate token for this bucket
+
+  OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
+
+  parser = argparse.ArgumentParser(description='upload casync folder to azure')
+  parser.add_argument("casync_dir", type=str, help="casync directory")
+  args = parser.parse_args()
+
+  casync_dir = pathlib.Path(args.casync_dir)
+
+  for f in casync_dir.rglob("*"):
+    if f.is_file():
+      blob_name = f.relative_to(casync_dir)
+      print(f"uploading {f} to {blob_name}")
+      OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)
diff --git a/release/upload_casync_release.sh b/release/upload_casync_release.sh
diff --git a/tools/lib/azure_container.py b/tools/lib/azure_container.py
@@ -57,18 +57,18 @@ def get_url(self, route_name: str, segment_num, log_type="rlog") -> str:
     ext = "hevc" if log_type.endswith('camera') else "bz2"
     return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}"
 
-  def upload_bytes(self, data: bytes | IO, blob_name: str) -> str:
+  def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str:
     from azure.storage.blob import BlobClient
     blob = BlobClient(
       account_url=self.ACCOUNT_URL,
       container_name=self.CONTAINER,
       blob_name=blob_name,
       credential=get_azure_credential(),
-      overwrite=False,
+      overwrite=overwrite,
     )
-    blob.upload_blob(data)
+    blob.upload_blob(data, overwrite=overwrite)
     return self.BASE_URL + blob_name
 
-  def upload_file(self, path: str | os.PathLike, blob_name: str) -> str:
+  def upload_file(self, path: str | os.PathLike, blob_name: str, overwrite=False) -> str:
     with open(path, "rb") as f:
-      return self.upload_bytes(f, blob_name)
+      return self.upload_bytes(f, blob_name, overwrite)