ci/repo: Add apt publishing

Signed-off-by: Ryan Northey <[email protected]>

.aptly-ci.yaml

@@ -1,5 +1,5 @@
 rootDir: /opt/build/cache/aptly
 FileSystemPublishEndpoints:
  public:
- rootDir: /opt/build/repo/repository
+ rootDir: /opt/build/cache/html
  linkMethod: symlink

.bazelrc

@@ -8,3 +8,8 @@ common:ci --//:aptly-custom=//:.aptly-ci-override
 common:debs-ci --config=ci
 common:debs-ci --//debs:excludes=//debs:custom-excludes.txt
 # common:debs-ci --//debs:token=//debs:token.txt
+
+common:publish-ci --config=debs-ci
+common:publish-ci --//tools/tarball:target=//:html
+common:publish-ci --//tools/tarball:overwrite=//tools/tarball:true
+common:publish-ci --//debs:signing-token=//debs:signing-token.txt

BUILD

@@ -91,3 +91,27 @@ jq(
  .[0] * .[1]
  """,
 )
+
+genrule(
+ name = "html",
+ outs = ["tar.gz"],
+ cmd = """
+ export APTLY_BIN="$(location @aptly)"
+ export MAINTAINER_KEY="$(location //:envoy-maintainers-public.key)"
+ export APTLY_CONF="$(location //:aptly-config)"
+ export DEBS="$(location //debs)"
+ export DEBS_ROOT_DEFAULT="/opt/build/cache/repository"
+ export SIGNING_KEY_DEFAULT="[email protected]"
+ export SIGNING_TOKEN="$(location //debs:signing-token)"
+ $(location //debs:publish)
+ tar hcf $@ -C /opt/build/cache/html .
+ """,
+ tools = [
+ "@aptly",
+ "//:aptly-config",
+ "//:envoy-maintainers-public.key",
+ "//debs",
+ "//debs:publish",
+ "//debs:signing-token",
+ ]
+)

build-repository.sh

@@ -8,6 +8,7 @@ NORMAL="\e[0m"
 
 EXCLUDE_FILE=debs/custom-excludes.txt
 DEBS_ROOT=/opt/build/cache/repository
+SIGNING_KEY_PASSPHRASE="${SIGNING_KEY_PASSPHRASE:-HACKME}"
 
 bold () {
  echo -n "${BOLD}${*}${NORMAL}"
@@ -34,6 +35,55 @@ create_excludes () {
  fi
 }
 
-import_public_key
-create_excludes
-bazel run --config=debs-ci //debs:publish
+generate_private_key () {
+ echo -e "$(underline $(bold "Import private signing key (apt): repository signing"))"
+ gpg --batch --pinentry-mode loopback --passphrase "" --gen-key <<EOF
+%echo Generating a basic OpenPGP key
+Key-Type: 1
+Key-Length: 4096
+Subkey-Type: 1
+Subkey-Length: 4096
+Name-Real: Envoy CI
+Name-Email: [email protected]
+Expire-Date: 0
+Passphrase: ${SIGNING_KEY_PASSPHRASE}
+%commit
+%echo done
+EOF
+}
+
+import_private_key () {
+ echo -e "$(underline $(bold "Import private signing key: maintainers"))"
+ echo "${SIGNING_KEY_0}${SIGNING_KEY_1}${SIGNING_KEY_2}${SIGNING_KEY_3}" | base64 -d > signing.key
+ gpg --batch --pinentry-mode loopback --import signing.key || {
+ rm -rf signing.key
+ echo "Key import failed" >&2
+ exit 1
+ }
+ rm -rf signing.key
+}
+
+create_excludes () {
+ if [[ -e "${DEBS_ROOT}" ]]; then
+ ls "${DEBS_ROOT}" \
+ | (grep -E '^v[0-9]+\.[0-9]+\.[0-9]+' || echo '') \
+ | sort -u > debs/custom-excludes.txt
+ else
+ touch debs/custom-excludes.txt
+ fi
+}
+
+build () {
+ import_public_key
+ create_excludes
+ echo "${SIGNING_KEY_PASSPHRASE}" > debs/signing-token.txt
+ if [[ "$CONTEXT" == "deploy-preview" ]]; then
+ import_private_key
+ # generate_private_key
+ else
+ import_private_key
+ fi
+ bazel run --config=publish-ci //tools/tarball:unpack /opt/build/repo/html
+}
+
+build

debs/BUILD

@@ -5,6 +5,7 @@ MAINTAINER = "Envoy maintainers <[email protected]>"
 exports_files([
  "custom-excludes.txt",
  "token.txt",
+ "signing-token.txt",
 ])
 
 jq(
@@ -114,11 +115,19 @@ genrule(
 label_flag(
  name = "excludes",
  build_setting_default = ":empty",
+ visibility = ["//visibility:public"],
 )
 
 label_flag(
  name = "token",
  build_setting_default = ":empty",
+ visibility = ["//visibility:public"],
+)
+
+label_flag(
+ name = "signing-token",
+ build_setting_default = ":empty",
+ visibility = ["//visibility:public"],
 )
 
 genrule(
@@ -141,6 +150,7 @@ genrule(
  ":excludes",
  ":token",
  ],
+ visibility = ["//visibility:public"],
 )
 
 sh_binary(
@@ -152,11 +162,15 @@ sh_binary(
  "APTLY_CONF": "$(location //:aptly-config)",
  "DEBS": "$(location :debs)",
  "DEBS_ROOT_DEFAULT": "/opt/build/cache/repository",
+ "SIGNING_KEY_DEFAULT": "[email protected]",
+ "SIGNING_TOKEN": "$(location :signing-token)",
  },
  data = [
  "@aptly",
  "//:aptly-config",
  "//:envoy-maintainers-public.key",
- ":debs"
+ ":debs",
+ ":signing-token",
  ],
+ visibility = ["//visibility:public"],
 )

debs/publish.sh

@@ -2,14 +2,29 @@
 
 set -e -o pipefail
 
+
 APTLY_BIN="$APTLY_BIN"
 APTLY_CONF="${APTLY_CONF:-${APTLY_CONF}}"
 APTLY=("$APTLY_BIN" -config="${APTLY_CONF}")
 
 DEBS_ROOT="${DEBS_ROOT:-${DEBS_ROOT_DEFAULT}}"
+REPOS=(focal jammy bullseye bookworm)
+SIGNING_KEY="${SIGNING_KEY:-${SIGNING_KEY_DEFAULT}}"
+SIGNING_TOKEN="${SIGNING_TOKEN:-${SIGNING_TOKEN}}"
+
+
+_aptly () {
+ "${APTLY[@]}" -- "${@}"
+}
+
+uid_generate() {
+ local length=${1:-7}
+ < /dev/urandom tr -dc 'A-Za-z0-9' | head -c "${length}"
+ echo
+}
 
 publish_dir () {
- "${APTLY[@]}" config show \
+ _aptly config show \
  | jq -r '.FileSystemPublishEndpoints.public.rootDir'
 }
 
@@ -25,18 +40,78 @@ unpack_debs () {
  fi
 }
 
+create_repos () {
+ existing_repos=$(_aptly repo list -json | jq -r '.[] | .Name')
+
+ for repo in "${REPOS[@]}"; do
+ if ! echo "$existing_repos" | tr ' ' '\n' | grep -q "^${repo}$"; then
+ _aptly repo create "$repo"
+ fi
+ done
+}
+
+list_current_changes () {
+ for repo in "${REPOS[@]}"; do
+ while read -r package; do
+ echo "${package}.${repo}.changes"
+ done < <(_aptly repo show -with-packages -json "${repo}" | jq -r '.Packages[]')
+ done
+}
+
+include_debs () {
+ declare -A imported
+ while read -r package; do
+ imported["$package"]=1
+ done < <(list_current_changes)
+ while read -r file; do
+ filename="$(basename "$file")"
+ if [[ "${imported[$filename]}" ]]; then
+ continue
+ fi
+ _aptly repo include -no-remove-files "$file"
+ done < <(find "${DEBS_ROOT}" -name "*.changes")
+}
+
 publish_repository () {
- PUBLIC_DIR="$(publish_dir)"
- KEY_URL="${DEPLOY_PRIME_URL}/envoy-maintainer-public.key"
- cat "$MAINTAINER_KEY" > "${PUBLIC_DIR}/envoy-maintainer-public.key"
- echo "<h1>COMING SOON: ${DEPLOY_PRIME_URL}</h1>" > "${PUBLIC_DIR}/index.html"
- echo "<div>Signing key: <a href=\"${KEY_URL}\">${KEY_URL}</div>" >> "${PUBLIC_DIR}/index.html"
+ local repo uid skip snapshot current result key
+ key=$(gpg --list-secret-keys --keyid-format LONG "$SIGNING_KEY" \
+ | grep 'sec' \
+ | awk '{print $2}' \
+ | cut -d'/' -f2)
+ if [[ -e "$SIGNING_TOKEN" ]]; then
+ SIGNING_KEY_PASSPHRASE="$(cat "$SIGNING_TOKEN")"
+ fi
+ for repo in "${REPOS[@]}"; do
+ uid=$(uid_generate)
+ skip=
+ snapshot="${repo}-${uid}"
+ _aptly snapshot create "$snapshot" from repo "$repo"
+ current=$(_aptly publish list -json \
+ | jq -r --arg dist "$repo" \
+ '.[] | select(.Distribution == $dist) | .Sources[] | select(.Component == "main") | .Name')
+ if [[ -n "$current" ]]; then
+ result=$(_aptly snapshot diff "$current" "${snapshot}")
+ if [[ "$result" == "Snapshots are identical." ]]; then
+ _aptly publish drop "${repo}" "filesystem:public:"
+ # skip=1
+ else
+ _aptly publish drop "${repo}" "filesystem:public:"
+ fi
+ fi
+ if [[ -z "$skip" ]]; then
+ _aptly publish snapshot -batch -passphrase "${SIGNING_KEY_PASSPHRASE}" -gpg-key="${key}" -distribution "${repo}" "${snapshot}" "filesystem:public:"
+ fi
+ done
 }
 
-publish () {
+main () {
  create_dirs
+ create_repos
  unpack_debs
+ include_debs
  publish_repository
 }
 
-publish
+if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+ main
+fi

netlify.toml

@@ -1,5 +1,5 @@
 [build]
-publish = "repository"
+publish = "html"
 command = "./build-repository.sh"
 
 [build.environment]

versions.bzl

@@ -40,18 +40,18 @@ VERSIONS = {
  "envoy": {
  "type": "github_archive",
  "repo": "envoyproxy/envoy",
- "version": "fea66c359069991e88bdfa4e0f2883c90cc39aef",
- "sha256": "96294f4b491c676b650ddeb07c7986ec7e48b5ae5a75c4adebbb9a58741a2fb1",
+ "version": "4108a96e215897d47cfeb3578486f0578333c1bc",
+ "sha256": "62606a0dbe73d32edfde176faaf1325f5f7a1531d1d40e12f4d6d85b75bc6b2b",
  "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
  "strip_prefix": "envoy-{version}",
  },
  "envoy_toolshed": {
  "type": "github_archive",
  "repo": "envoyproxy/toolshed",
- "version": "0.1.4",
- "sha256": "7ddfd251a89518b97c4eb8064a7d37454bbd998bf29e4cd3ad8f44227b5ca7b3",
- "urls": ["https://github.com/{repo}/archive/bazel-v{version}.tar.gz"],
- "strip_prefix": "toolshed-bazel-v{version}/bazel",
+ "version": "20c6067d87f1eb2e5a4d80f815d2aa3a10ed37ad",
+ "sha256": "9578a70d01585cdea680b1fddc86813f8282500565b1b9ceeb600940b122de43",
+ "urls": ["https://github.com/{repo}/archive/{version}.tar.gz"],
+ "strip_prefix": "toolshed-{version}/bazel",
  },
  "io_bazel_rules_go": {
  "type": "github_archive",