infra!: migrate to hetzner and sign packages

BREAKING CHANGE: Packages are hosted on cdn.archpkgs.net, because GitHub Pages has a hard size limit of 100MB. BREAKING CHANGE: Packages are signed in GitHub Actions. To migrate delete the [archpkgs] section in /etc/pacman.conf and run the new setup script as shown in the readme.
dadevel · May 18, 2024 · 3ad0a9b · 3ad0a9b
1 parent 3c7757b
commit 3ad0a9b
Show file tree

Hide file tree

Showing 9 changed files with 306 additions and 440 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,3 @@
-/repo/
+/public/
+/rclone.conf
 *.pkg.tar.zst
diff --git a/README.md b/README.md
@@ -1,52 +1,29 @@
 # archpkgs
 
-> Red teaming and pentesting tools packaged for Arch Linux.
+Red teaming and pentesting tools packaged for Arch Linux.
 
 Remarks:
 
 - packages are installed under `/opt/archpkgs`
 - Python packages are isolated in their own virtual environments
 - packages are rebuild weekly
-- for now packages are unsigned
 
 ## Setup
 
-To add the repository to your system edit `/etc/pacman.conf` and insert the `[archpkgs]` section between `[core]` and `[extra]`.
-
-~~~ ini
-...
-[core]
-Include = /etc/pacman.d/mirrorlist
-
-[archpkgs]
-SigLevel = Optional TrustAll
-Server = https://dadevel.github.io/archpkgs
-
-[extra]
-Include = /etc/pacman.d/mirrorlist
-...
-~~~
-
-Prepend `/opt/archpkgs/bin` to the `$PATH`.
-For Bash append the following line to your shell profile in `~/.bashrc`:
+Run the following command to add the repo.
 
 ~~~ bash
-export "PATH=/opt/archpkgs/bin:$PATH"
+curl -sSfL https://github.com/dadevel/archpkgs/raw/main/setup.sh | sudo bash
 ~~~
 
-Then add `/opt/archpkgs/bin` to the `secure_path` option of `sudo`.
-`/etc/sudoers` should contain a line similar to this:
-
-~~~
-Defaults secure_path="/opt/archpkgs/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
-~~~
-
-Finally run the following commands and start installing packages.
+List all packages provided by the repo.
 
 ~~~ bash
-sudo pacman -Sy && sudo pacman -Sl archpkgs
+sudo pacman -Sl archpkgs
 ~~~
 
+> **Note:** Breaking changes that require manual interaction are marked in the [commit history](https://github.com/dadevel/archpkgs/commits/main/) with an `!`.
+
 ## Development
 
 1. Clone the repo.

diff --git a/build-container.sh b/build-container.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-set -eu
+set -euo pipefail
 
 docker login ghcr.io --username "$GITHUB_USER" --password-stdin <<< "$GITHUB_TOKEN"
 trap 'docker logout ghcr.io' EXIT

diff --git a/build-repo.sh b/build-repo.sh
@@ -1,15 +1,13 @@
-#!/bin/sh
-exec docker run -i --rm -v "$PWD:/build" ghcr.io/dadevel/archpkgs-builder:latest bash << EOF
-set -euv
-shopt -s nullglob
-
+#!/usr/bin/env bash
+exec docker run -i --rm -v "$PWD:/build" -e SIGNING_KEY ghcr.io/dadevel/archpkgs-builder:latest bash << EOF
+set -euo pipefail
 cd /build
-rm -rf ./public
+echo "$SIGNING_KEY" | gpg --import -
 mkdir ./public
-cp ./index.html ./public
-ls -lA ./artifacts
-mv ./artifacts/*/*.pkg.tar.zst ./artifacts/*/*.pkg.tar.zst.sig ./public
-cd ./public
-ls -lA
-repo-add ./archpkgs.db.tar.gz ./*.pkg.tar.zst
+mv ./artifacts/package-*/*.pkg.tar.zst ./public
+for pkg in ./public/*.pkg.tar.zst; do
+    gpg --detach-sign --output "\${pkg}.sig" --sign "\${pkg}"
+done
+repo-add ./public/archpkgs.db.tar.zst ./public/*.pkg.tar.zst
+echo 'Options +Indexes' > ./public/.htaccess
 EOF
diff --git a/generate-workflow.py b/generate-workflow.py
@@ -4,6 +4,8 @@
 
 import yaml
 
+RUNNER_OS = 'ubuntu-24.04'
+
 
 def main() -> None:
     packages = [p.parent.name for p in sorted(Path.cwd().glob('*/PKGBUILD'))]
@@ -29,29 +31,26 @@ def generate_workflow(packages: list[str]) -> dict[str, Any]:
                 {'cron': '0 3 * * *'},
             ],
         },
-        'jobs': {'build-container': generate_setup_job(), 'deploy-pages': generate_deploy_job(packages)} | {f'package-{package}': generate_package_job(package) for package in packages},
+        'jobs': {'build-container': generate_setup_job(), 'deploy-repo': generate_deploy_job(packages)} | {f'package-{package}': generate_package_job(package) for package in packages},
     }
 
 
 def generate_package_job(package: str) -> dict[str, Any]:
     return {
-        'runs-on': 'ubuntu-20.04',
+        'runs-on': RUNNER_OS,
         'needs': ['build-container'],
         'steps': [
             {
                 'name': 'Checkout',
-                'uses': 'actions/checkout@v3',
-                'with': {
-                    'fetch-depth': 0,
-                }
+                'uses': 'actions/checkout@v4',
             },
             {
                 'name': 'Build package',
                 'run': f'./build-package.sh {package}',
             },
             {
                 'name': 'Upload package',
-                'uses': 'actions/upload-artifact@v3',
+                'uses': 'actions/upload-artifact@v4',
                 'with': {
                     'name': f'package-{package}',
                     'path': f'./{package}/*.pkg.tar.zst',
@@ -65,14 +64,11 @@ def generate_package_job(package: str) -> dict[str, Any]:
 
 def generate_setup_job() -> dict[str, Any]:
     return {
-        'runs-on': 'ubuntu-20.04',
+        'runs-on': RUNNER_OS,
         'steps': [
             {
                 'name': 'Checkout',
-                'uses': 'actions/checkout@v3',
-                'with': {
-                    'fetch-depth': 0,
-                }
+                'uses': 'actions/checkout@v4',
             },
             {
                 'name': 'Build and push container',
@@ -88,38 +84,38 @@ def generate_setup_job() -> dict[str, Any]:
 
 def generate_deploy_job(packages: list[str]) -> dict[str, Any]:
     return {
-        'runs-on': 'ubuntu-20.04',
+        'runs-on': RUNNER_OS,
         'needs': [f'package-{name}' for name in packages],
         'concurrency': 'ci-${{ github.ref }}',
         'steps': [
             {
                 'name': 'Checkout',
-                'uses': 'actions/checkout@v3',
-                'with': {
-                    'fetch-depth': 0,
-                }
+                'uses': 'actions/checkout@v4',
             },
             {
                 'name': 'Download artifacts',
-                'uses': 'actions/download-artifact@v3',
+                'uses': 'actions/download-artifact@v4',
                 'with': {
                     'path': './artifacts',
                 }
             },
             {
                 'name': 'Create repository',
                 'run': './build-repo.sh',
+                'env': {
+                    'SIGNING_KEY': '${{ secrets.SIGNING_KEY }}',
+                },
             },
             {
-                'name': 'Deploy pages',
-                'uses': 'peaceiris/actions-gh-pages@v3',
-                'with': {
-                    'github_token': '${{ secrets.GITHUB_TOKEN }}',
-                    'publish_dir': './public',
-                    'force_orphan': True,  # dont keep history in gh-pages branch
-                    'user_name': 'github-actions[bot]',  # commit author
-                    'user_email': 'github-actions[bot]@users.noreply.github.com',
-                }
+                'name': 'Upload repository',
+                'run': 'sudo apt-get install --no-install-recommends -y rclone && rclone --copy-links sync ./public hetzner:',
+                'env': {
+                    'RCLONE_CONFIG_HETZNER_TYPE': 'ftp',
+                    'RCLONE_CONFIG_HETZNER_HOST': '${{ secrets.HETZNER_HOSTNAME }}',
+                    'RCLONE_CONFIG_HETZNER_USER': '${{ secrets.HETZNER_USERNAME }}',
+                    'RCLONE_CONFIG_HETZNER_PASS': '${{ secrets.HETZNER_PASSWORD }}',  # echo password | rclone obscure
+                    'RCLONE_CONFIG_HETZNER_EXPLICIT_TLS': 'true',
+                },
             },
         ],
     }

diff --git a/index.html b/index.html
diff --git a/setup.sh b/setup.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# add repo
+if ! grep -q '[archpkgs]' /etc/pacman.conf 2> /dev/null; then
+    echo >> /etc/pacman.conf
+    echo '[archpkgs]' >> /etc/pacman.conf
+    echo 'Server = https://cdn.archpkgs.net' >> /etc/pacman.conf
+fi
+
+# trust signing key
+curl -sSf https://raw.githubusercontent.com/dadevel/archpkgs/main/signature.gpg | pacman-key --add -
+pacman-key --lsign-key [email protected]
+
+# refresh cache
+pacman -Sy
+
+# configure path
+echo 'export "PATH=/opt/archpkgs/bin:$PATH"' > /etc/profile.d/archpkgs.sh
+
+# configure sudo
+if grep -q secure_path= /etc/sudoers; then
+    if ! grep -q /opt/archpkgs/bin /etc/sudoers; then
+        echo 'WARNING: Please prepend "/opt/archpkgs/bin" to the "secure_path" option in /etc/sudoers'
+    fi
+fi
diff --git a/signature.gpg b/signature.gpg
@@ -0,0 +1,13 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mDMEZkhmBBYJKwYBBAHaRw8BAQdAIUFwge55FV7U8oeCFw415tIz2cWMI8y1KTrP
+Pvj8HVa0J2FkbWluQGFyY2hwa2dzLm5ldCA8YWRtaW5AYXJjaHBrZ3MubmV0PoiT
+BBMWCgA7FiEEmyTnmdrlqW3/E8eHOCT7Q0MIigsFAmZIZgQCGwMFCwkIBwICIgIG
+FQoJCAsCBBYCAwECHgcCF4AACgkQOCT7Q0MIigvLZgD+MFi1wAjxoRlhmC/qqX72
+6zPMR06IORspwcQ8J+sa6XIA/1oOc9VWdLFtqQQsRAAS7mMFR6wOaM1YtSpn65Os
+5G4OuDgEZkhmBBIKKwYBBAGXVQEFAQEHQIM2L0v2Zlcjm0PBu3zPcXzOtbOw9JZL
+ETR15wsKcRskAwEIB4h4BBgWCgAgFiEEmyTnmdrlqW3/E8eHOCT7Q0MIigsFAmZI
+ZgQCGwwACgkQOCT7Q0MIigs4PwD/bVgJuDJse2PxpWH20S3VGLK1deA33Yz8+bXS
+nbFskWEA/AxC76CZXv99KWfQ9IEGNXI1CtjjP6E3srhMzD72Sj4K
+=SbJd
+-----END PGP PUBLIC KEY BLOCK-----