From 8364cd2f2cc4f4b753fc417333df4c707c11eb86 Mon Sep 17 00:00:00 2001
From: Justin Newberry <justin@comma.ai>
Date: Tue, 9 Apr 2024 11:38:18 -0700
Subject: [PATCH] jenkins: publish casync releases from device (#32142)

* publish in ci

* overwrite

* publish in ci

* fix

* test it

* Revert "test it"

This reverts commit b3de51dc693df79b980d7dffc18bcc3c88d83375.

* use right token

* cleanup after uploading

---------

Co-authored-by: Comma Device <device@comma.ai>
---
 Jenkinsfile                      |  2 +-
 release/upload_casync_release.py | 24 ++++++++++++++++++++++++
 release/upload_casync_release.sh |  9 ---------
 tools/lib/azure_container.py     | 10 +++++-----
 4 files changed, 30 insertions(+), 15 deletions(-)
 create mode 100755 release/upload_casync_release.py
 delete mode 100755 release/upload_casync_release.sh

diff --git a/Jenkinsfile b/Jenkinsfile
index 8fe0e76ec07a1d..340145eac15dda 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -153,7 +153,7 @@ def build_release(String channel_name) {
       deviceStage("build casync", "tici-needs-can", [], [
         ["build ${channel_name}", "RELEASE=1 OPENPILOT_CHANNEL=${channel_name} BUILD_DIR=/data/openpilot CASYNC_DIR=/data/casync $SOURCE_DIR/release/create_casync_build.sh"],
         ["create manifest", "$SOURCE_DIR/release/create_release_manifest.py /data/manifest.json && cat /data/manifest.json"],
-        //["upload ${channel_name}", "OPENPILOT_CHANNEL=${channel_name} $SOURCE_DIR/release/upload_casync_release.sh"],
+        ["upload and cleanup ${channel_name}", "$SOURCE_DIR/release/upload_casync_release.py /data/casync && rm -rf /data/casync"],
       ])
     }
   )
diff --git a/release/upload_casync_release.py b/release/upload_casync_release.py
new file mode 100755
index 00000000000000..0119ebfa0e1021
--- /dev/null
+++ b/release/upload_casync_release.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import pathlib
+from openpilot.tools.lib.azure_container import AzureContainer
+
+
+if __name__ == "__main__":
+  del os.environ["AZURE_TOKEN"] # regerenate token for this bucket
+
+  OPENPILOT_RELEASES_CONTAINER = AzureContainer("commadist", "openpilot-releases")
+
+  parser = argparse.ArgumentParser(description='upload casync folder to azure')
+  parser.add_argument("casync_dir", type=str, help="casync directory")
+  args = parser.parse_args()
+
+  casync_dir = pathlib.Path(args.casync_dir)
+
+  for f in casync_dir.rglob("*"):
+    if f.is_file():
+      blob_name = f.relative_to(casync_dir)
+      print(f"uploading {f} to {blob_name}")
+      OPENPILOT_RELEASES_CONTAINER.upload_file(str(f), str(blob_name), overwrite=True)
diff --git a/release/upload_casync_release.sh b/release/upload_casync_release.sh
deleted file mode 100755
index 02ced8338ea0db..00000000000000
--- a/release/upload_casync_release.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-CASYNC_DIR="${CASYNC_DIR:=/tmp/casync}"
-
-OPENPILOT_RELEASES="https://commadist.blob.core.windows.net/openpilot-releases/"
-
-SAS="$(python -c 'from tools.lib.azure_container import get_container_sas;print(get_container_sas("commadist","openpilot-releases"))')"
-
-azcopy cp "$CASYNC_DIR*" "$OPENPILOT_RELEASES?$SAS" --recursive
diff --git a/tools/lib/azure_container.py b/tools/lib/azure_container.py
index 52b2f37dbfb748..f5a3a8bfb197c1 100644
--- a/tools/lib/azure_container.py
+++ b/tools/lib/azure_container.py
@@ -57,18 +57,18 @@ def get_url(self, route_name: str, segment_num, log_type="rlog") -> str:
     ext = "hevc" if log_type.endswith('camera') else "bz2"
     return self.BASE_URL + f"{route_name.replace('|', '/')}/{segment_num}/{log_type}.{ext}"
 
-  def upload_bytes(self, data: bytes | IO, blob_name: str) -> str:
+  def upload_bytes(self, data: bytes | IO, blob_name: str, overwrite=False) -> str:
     from azure.storage.blob import BlobClient
     blob = BlobClient(
       account_url=self.ACCOUNT_URL,
       container_name=self.CONTAINER,
       blob_name=blob_name,
       credential=get_azure_credential(),
-      overwrite=False,
+      overwrite=overwrite,
     )
-    blob.upload_blob(data)
+    blob.upload_blob(data, overwrite=overwrite)
     return self.BASE_URL + blob_name
 
-  def upload_file(self, path: str | os.PathLike, blob_name: str) -> str:
+  def upload_file(self, path: str | os.PathLike, blob_name: str, overwrite=False) -> str:
     with open(path, "rb") as f:
-      return self.upload_bytes(f, blob_name)
+      return self.upload_bytes(f, blob_name, overwrite)