[ci/tuf-repo] multi-SP repos + use released SP images + add gimlet-e (#…

…3931) * First half of #3893, which adds all the SP images to our repos. We'll need to test this on dogfood. We still have two separate repos for now for the different ROT images; opened #3954 to track. * We are now using released SP images from the release assets over at https://github.com/oxidecomputer/hubris/releases. See also RFD 412. Unfortunately the current release assets do not yet update the `VERS` field of the caboose so those still display as `0.0.0-git`, which is reflected in the manifest. * Adds gimlet-e support.
oxidecomputer · Sep 5, 2023 · f58287b · f58287b
1 parent 0319d2c
commit f58287b
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 32 deletions.
diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh
@@ -5,8 +5,8 @@
 #: target = "helios-2.0"
 #: output_rules = [
 #:	"=/work/manifest*.toml",
-#:	"=/work/repo-dogfood.zip*",
-#:	"=/work/repo-pvt1.zip*",
+#:	"=/work/repo-*.zip.part*",
+#:	"=/work/repo-*.zip.sha256.txt",
 #: ]
 #: access_repos = [
 #:	"oxidecomputer/dvt-dock",
@@ -25,40 +25,42 @@
 #: job = "helios / build trampoline OS image"
 #:
 #: [[publish]]
-#: series = "dogfood"
+#: series = "rot-prod-rel"
 #: name = "repo.zip.parta"
-#: from_output = "/work/repo-dogfood.zip.parta"
+#: from_output = "/work/repo-rot-prod-rel.zip.parta"
 #:
 #: [[publish]]
-#: series = "dogfood"
+#: series = "rot-prod-rel"
 #: name = "repo.zip.partb"
-#: from_output = "/work/repo-dogfood.zip.partb"
+#: from_output = "/work/repo-rot-prod-rel.zip.partb"
 #:
 #: [[publish]]
-#: series = "dogfood"
+#: series = "rot-prod-rel"
 #: name = "repo.zip.sha256.txt"
-#: from_output = "/work/repo-dogfood.zip.sha256.txt"
+#: from_output = "/work/repo-rot-prod-rel.zip.sha256.txt"
 #:
 #: [[publish]]
-#: series = "pvt1"
+#: series = "rot-staging-dev"
 #: name = "repo.zip.parta"
-#: from_output = "/work/repo-pvt1.zip.parta"
+#: from_output = "/work/repo-rot-staging-dev.zip.parta"
 #:
 #: [[publish]]
-#: series = "pvt1"
+#: series = "rot-staging-dev"
 #: name = "repo.zip.partb"
-#: from_output = "/work/repo-pvt1.zip.partb"
+#: from_output = "/work/repo-rot-staging-dev.zip.partb"
 #:
 #: [[publish]]
-#: series = "pvt1"
+#: series = "rot-staging-dev"
 #: name = "repo.zip.sha256.txt"
-#: from_output = "/work/repo-pvt1.zip.sha256.txt"
+#: from_output = "/work/repo-rot-staging-dev.zip.sha256.txt"
 #:
 
 set -o errexit
 set -o pipefail
 set -o xtrace
 
+ALL_BOARDS=(gimlet-{c..e} psc-{b..c} sidecar-{b..c})
+
 TOP=$PWD
 VERSION=$(< /input/package/work/version.txt)
 
@@ -125,6 +127,23 @@ path = "/input/$kind/work/helios/image/output/os.tar.gz"
 EOF
 done
 
+# Fetch SP images from a Hubris release.
+mkdir /work/hubris
+pushd /work/hubris
+source "$TOP/tools/hubris_version"
+for tag in "${TAGS[@]}"; do
+    for board in "${ALL_BOARDS[@]}"; do
+        if [[ "${tag%-*}" = "${board%-*}" ]]; then
+            file=build-${board}-image-default-${tag#*-}.zip
+            curl -fLOsS "https://github.com/oxidecomputer/hubris/releases/download/$tag/$file"
+            grep -F "$file" "$TOP/tools/hubris_checksums" | shasum -a 256 -c -
+            mv "$file" "$board.zip"
+        fi
+    done
+done
+popd
+
+# Fetch ROT images from dvt-dock.
 source "$TOP/tools/dvt_dock_version"
 git init /work/dvt-dock
 (
@@ -147,27 +166,24 @@ caboose_util_rot() {
     echo "$output_a"
 }
 
+SERIES_LIST=()
 add_hubris_artifacts() {
     series="$1"
     rot_dir="$2"
     rot_version="$3"
     shift 3
 
+    SERIES_LIST+=("$series")
+
     manifest=/work/manifest-$series.toml
     cp /work/manifest.toml "$manifest"
 
-    for board_rev in "$@"; do
-        board=${board_rev%-?}
+    for board in gimlet psc sidecar; do
         tufaceous_board=${board//sidecar/switch}
-
         rot_image_a="/work/dvt-dock/${rot_dir}/${board}/build-${board}-rot-image-a-${rot_version}.zip"
         rot_image_b="/work/dvt-dock/${rot_dir}/${board}/build-${board}-rot-image-b-${rot_version}.zip"
-        sp_image="/work/dvt-dock/sp/${board}/build-${board_rev}-image-default.zip"
-
         rot_caboose_version=$(caboose_util_rot read-version "$rot_image_a" "$rot_image_b")
-        sp_caboose_version=$(/work/caboose-util read-version "$sp_image")
         rot_caboose_board=$(caboose_util_rot read-board "$rot_image_a" "$rot_image_b")
-        sp_caboose_board=$(/work/caboose-util read-board "$sp_image")
 
         cat >>"$manifest" <<EOF
 [[artifact.${tufaceous_board}_rot]]
@@ -181,6 +197,17 @@ path = "$rot_image_a"
 [artifact.${tufaceous_board}_rot.source.archive_b]
 kind = "file"
 path = "$rot_image_b"
+EOF
+    done
+
+    for board_rev in "$@"; do
+        board=${board_rev%-?}
+        tufaceous_board=${board//sidecar/switch}
+        sp_image="/work/hubris/${board_rev}.zip"
+        sp_caboose_version=$(/work/caboose-util read-version "$sp_image")
+        sp_caboose_board=$(/work/caboose-util read-board "$sp_image")
+
+        cat >>"$manifest" <<EOF
 [[artifact.${tufaceous_board}_sp]]
 name = "$sp_caboose_board"
 version = "$sp_caboose_version"
@@ -190,20 +217,20 @@ path = "$sp_image"
 EOF
     done
 }
-# usage:              SERIES   ROT_DIR      ROT_VERSION              BOARDS...
-add_hubris_artifacts  dogfood  staging/dev  cert-staging-dev-v1.0.0  gimlet-c psc-b sidecar-b
-add_hubris_artifacts  pvt1     prod/rel     cert-prod-rel-v1.0.0     gimlet-d psc-c sidecar-c
+# usage:              SERIES           ROT_DIR      ROT_VERSION              BOARDS...
+add_hubris_artifacts  rot-staging-dev  staging/dev  cert-staging-dev-v1.0.0  "${ALL_BOARDS[@]}"
+add_hubris_artifacts  rot-prod-rel     prod/rel     cert-prod-rel-v1.0.0     "${ALL_BOARDS[@]}"
 
-for series in dogfood pvt1; do
-    /work/tufaceous assemble --no-generate-key /work/manifest-$series.toml /work/repo-$series.zip
-    digest -a sha256 /work/repo-$series.zip > /work/repo-$series.zip.sha256.txt
+for series in "${SERIES_LIST[@]}"; do
+    /work/tufaceous assemble --no-generate-key /work/manifest-"$series".toml /work/repo-"$series".zip
+    digest -a sha256 /work/repo-"$series".zip > /work/repo-"$series".zip.sha256.txt
 
     #
-    # XXX: Buildomat currently does not support uploads greater than 1 GiB. This is
-    # an awful temporary hack which we need to strip out the moment it does.
+    # XXX: There are some issues downloading Buildomat artifacts > 1 GiB, see
+    # oxidecomputer/buildomat#36.
     #
-    split -a 1 -b 1024m /work/repo-$series.zip /work/repo-$series.zip.part
-    rm /work/repo-$series.zip
+    split -a 1 -b 1024m /work/repo-"$series".zip /work/repo-"$series".zip.part
+    rm /work/repo-"$series".zip
     # Ensure the build doesn't fail if the repo gets smaller than 1 GiB.
-    touch /work/repo-$series.zip.partb
+    touch /work/repo-"$series".zip.partb
 done
diff --git a/tools/hubris_checksums b/tools/hubris_checksums
@@ -0,0 +1,7 @@
+36ea4c454e78ecdad926bf321493fc28ff79e3133aa67ef4fd355b8c7f92676a  build-gimlet-c-image-default-v1.0.1.zip
+61f387726131e9ff98b8645034144f421c1d7bd1466b35584fd93901b6c5d5ae  build-gimlet-d-image-default-v1.0.1.zip
+63c986402aab5b3aef1f04e958234f395f40a65d218812c22a154695818e1303  build-gimlet-e-image-default-v1.0.1.zip
+526442e6d4466e357aa7f69fed400693eb542f239a03f4fc4d4c74d74afcbd6d  build-psc-b-image-default-v1.0.1.zip
+91d8537e4a5524e2d8a8047179b8eed11ca22fd741dea27edade921efd980aa2  build-psc-c-image-default-v1.0.1.zip
+50932dcb7da2c89b6958acfa36972f2d32159bb59489dc335028c49ae60702c5  build-sidecar-b-image-default-v1.0.1.zip
+d466495e45101aa21ad9cfb136bb5f1f2288bb0285e1e6f5b257d21ef5c53b86  build-sidecar-c-image-default-v1.0.1.zip
diff --git a/tools/hubris_version b/tools/hubris_version
@@ -0,0 +1 @@
+TAGS=(gimlet-v1.0.1 psc-v1.0.1 sidecar-v1.0.1)