ci/repo: Add deb fetcher (envoyproxy#6)

Signed-off-by: Ryan Northey <[email protected]>

.bazelrc

@@ -4,3 +4,7 @@ common:ci --noshow_progress
 common:ci --noshow_loading_progress
 common:ci --test_output=errors
 common:ci --//:aptly-custom=//:.aptly-ci-override
+
+common:debs-ci --config=ci
+common:debs-ci --//debs:excludes=//debs:custom-excludes.txt
+# common:debs-ci --//debs:token=//debs:token.txt

build-repository.sh

@@ -6,6 +6,8 @@ BOLD="\e[1m"
 UNDERLINE="\e[4m"
 NORMAL="\e[0m"
 
+EXCLUDE_FILE=debs/custom-excludes.txt
+DEBS_ROOT=/opt/build/cache/repository
 
 bold () {
  echo -n "${BOLD}${*}${NORMAL}"
@@ -23,5 +25,15 @@ import_public_key () {
  gpg --no-default-keyring --keyring trustedkeys.gpg --import envoy-maintainers-public.key
 }
 
+create_excludes () {
+ # Prevent re-downloading cached files
+ if [[ -e "${DEBS_ROOT}" ]]; then
+ ls "${DEBS_ROOT}" | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') | sort -u > "$EXCLUDE_FILE"
+ else
+ touch "$EXCLUDE_FILE"
+ fi
+}
+
 import_public_key
-bazel run --config=ci //debs:publish
+create_excludes
+bazel run --config=debs-ci //debs:publish

debs/BUILD

@@ -1,16 +1,162 @@
 load("@aspect_bazel_lib//lib:jq.bzl", "jq")
 
+MAINTAINER = "Envoy maintainers <[email protected]>"
+
+exports_files([
+ "custom-excludes.txt",
+ "token.txt",
+])
+
+jq(
+ name = "envoy_versions",
+ srcs = ["@envoy_repo//:project"],
+ out = "envoy_versions.json",
+ filter = """
+ .releases as $releases
+ | {
+ latest_releases: [
+ .stable_versions[]
+ | . as $minor
+ | {
+ version: $minor,
+ releases: (
+ $releases
+ | map(select(startswith("v" + $minor + ".")))
+ )
+ }
+ ]
+ }
+ """,
+ visibility = ["//visibility:public"],
+)
+
+jq(
+ name = "minor_versions",
+ srcs = [":envoy_versions"],
+ out = "minor_versions.txt",
+ filter = """
+ .latest_releases[] | .version
+ """,
+ args = ["-r"],
+ visibility = ["//visibility:public"],
+)
+
+jq(
+ name = "patch_versions",
+ srcs = [":envoy_versions"],
+ out = "patch_versions.txt",
+ filter = """
+ .latest_releases[] | .releases[]
+ """,
+ args = ["-r"],
+ visibility = ["//visibility:public"],
+)
+
+jq(
+ name = "deb_checksum_downloads",
+ srcs = [":envoy_versions"],
+ out = "deb_checksum_downloads.txt",
+ filter = """
+ reduce .latest_releases[].releases[] as $item ({};
+ .[$item] = {"signature": "%s"})
+ | with_entries(
+ {"key": "https://github.com/envoyproxy/envoy/releases/download/\\(.key)/checksums.txt.asc",
+ "value": .value})
+ """ % MAINTAINER,
+ args = ["-r"],
+ visibility = ["//visibility:public"],
+)
+
+genrule(
+ name = "published_checksums",
+ outs = ["published_checksums.txt"],
+ cmd = """
+ $(location //tools/fetch) $(location :deb_checksum_downloads) --output=json > $@
+ """,
+ tools = [
+ "//tools/fetch",
+ ":deb_checksum_downloads",
+ ],
+)
+
+jq(
+ name = "debs_downloads",
+ srcs = [":published_checksums"],
+ out = "debs_downloads.json",
+ filter = """
+ with_entries(
+ .key as $key
+ | .value as $value
+ | ($key | capture("v(?<version>[0-9.]+)") | .version) as $version
+ | {key: ("https://github.com/envoyproxy/envoy/releases/download/v\\($version)/debs.tar.gz"),
+ value: {
+ "path": "v\\($version)",
+ "checksum": (
+ $value
+ | split("\n")
+ | map(select(endswith("debs.tar.gz")))
+ | first
+ | split(" ")
+ | .[0]
+ )}})
+ """,
+ visibility = ["//visibility:public"],
+)
+
+genrule(
+ name = "empty",
+ outs = ["empty.txt"],
+ cmd = """
+ touch $@
+ """,
+)
+
+label_flag(
+ name = "excludes",
+ build_setting_default = ":empty",
+)
+
+label_flag(
+ name = "token",
+ build_setting_default = ":empty",
+)
+
+genrule(
+ name = "debs",
+ outs = ["debs.tar.gz"],
+ cmd = """
+ $(location //tools/fetch) $(location :debs_downloads) \
+ --concurrency 4 \
+ --excludes=$(location :excludes) \
+ --token-path=$(location :token) \
+ --extract-downloads \
+ --output-path=$@
+ if [[ ! -e $@ ]]; then
+ touch $@
+ fi
+ """,
+ tools = [
+ "//tools/fetch",
+ ":debs_downloads",
+ ":excludes",
+ ":token",
+ ],
+)
+
 sh_binary(
  name = "publish",
  srcs = ["publish.sh"],
  env = {
  "APTLY_BIN": "$(location @aptly)",
  "MAINTAINER_KEY": "$(location //:envoy-maintainers-public.key)",
  "APTLY_CONF": "$(location //:aptly-config)",
+ "DEBS": "$(location :debs)",
+ "DEBS_ROOT_DEFAULT": "/opt/build/cache/repository",
  },
  data = [
  "@aptly",
  "//:aptly-config",
  "//:envoy-maintainers-public.key",
+ ":debs"
  ],
 )

debs/publish.sh

@@ -6,18 +6,37 @@ APTLY_BIN="$APTLY_BIN"
 APTLY_CONF="${APTLY_CONF:-${APTLY_CONF}}"
 APTLY=("$APTLY_BIN" -config="${APTLY_CONF}")
 
-publish_root () {
+DEBS_ROOT="${DEBS_ROOT:-${DEBS_ROOT_DEFAULT}}"
+
+publish_dir () {
  "${APTLY[@]}" config show \
  | jq -r '.FileSystemPublishEndpoints.public.rootDir'
 }
 
-publish_repository () {
- PUBLIC_DIR="$(publish_root)"
+create_dirs () {
+ PUBLIC_DIR="$(publish_dir)"
  mkdir -p "${PUBLIC_DIR}"
+ mkdir -p "${DEBS_ROOT}"
+}
+
+unpack_debs () {
+ if [[ -s "$DEBS" ]]; then
+ tar xf "$DEBS" -C "$DEBS_ROOT"
+ fi
+}
+
+publish_repository () {
+ PUBLIC_DIR="$(publish_dir)"
  KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key"
  cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key"
  echo "<h1>COMING SOON: ${DEPLOY_PRIME_URL}</h1>" > "${PUBLIC_DIR}/index.html"
  echo "<div>Signing key: <a href=\"${KEY_URL}\">${KEY_URL}</div>" >> "${PUBLIC_DIR}/index.html"
 }
 
-publish_repository
+publish () {
+ create_dirs
+ unpack_debs
+ publish_repository
+}
+
+publish

tools/requirements.in

@@ -1 +1 @@
-envoy.base.utils>=0.5.4
+envoy.base.utils>=0.5.5

tools/requirements.txt

@@ -220,9 +220,9 @@ cryptography==43.0.0 \
  --hash=sha256:ee0c405832ade84d4de74b9029bedb7b31200600fa524d218fc29bfa371e97f5 \
  --hash=sha256:fdcb265de28585de5b859ae13e3846a8e805268a823a12a4da2597f1f5afc9f0
  # via pyjwt
-envoy-base-utils==0.5.4 \
- --hash=sha256:90882337c2ce509b19fc21f5d66395dad44a706e4d616baabd1ffaea256ee358 \
- --hash=sha256:c1167f567994596a3071027d57ca895ac4a8bd8969d4de846b0f2cda499fa5ad
+envoy-base-utils==0.5.5 \
+ --hash=sha256:2341b7618c92969e3ac5fc43dd16fcc39d132981ffff3f0ddbabf08a336a6fa1 \
+ --hash=sha256:a72af1df5680d4fcdeb91f06d19b7495d740f8b9a8a1549c012272e10fdd59ea
  # via -r requirements.in
 frozendict==2.4.4 \
  --hash=sha256:07c3a5dee8bbb84cba770e273cdbf2c87c8e035903af8f781292d72583416801 \