hosts/hil0: nuc down, setup alterative hydra

lib/data/data.json

@@ -212,8 +212,8 @@
  "proxy": true
  },
  "hydra": {
- "on": "nuc",
- "proxy": false
+ "on": "hil0",
+ "proxy": true
  },
  "influxdb": {
  "on": "fsn0",

nixos/hosts/hil0/default.nix

@@ -31,6 +31,10 @@ in {
  services.mastodon
  services.maddy
  services.well-known
+ services.hydra
+ nix.hydra-builder-server
+ nix.hydra-builder-client
+ nix.access-tokens
  networking.as198764
  ])
  ++ [
@@ -117,5 +121,14 @@ in {
  restartUnits = ["systemd-networkd.service"];
  };
  })
+
+ # hydra extra configurations
+ {
+ services.hydra.buildMachinesFiles = [
+ "/etc/nix-build-machines/hydra-builder/machines"
+ ];
+ # limit cpu quota of nix builds
+ systemd.services.nix-daemon.serviceConfig.CPUQuota = "300%";
+ }
  ];
 }

nixos/profiles/nix/hydra-builder-client/default.nix

@@ -9,7 +9,8 @@ in {
  '';
  };
  environment.etc.${machineFile}.text = ''
- hydra-builder@nuc x86_64-linux,i686-linux /etc/${keyFile} 8 1 kvm,nixos-test,benchmark,big-parallel
+ # hydra-builder@nuc x86_64-linux,i686-linux /etc/${keyFile} 8 1 kvm,nixos-test,benchmark,big-parallel
+ hydra-builder@hil0 x86_64-linux,i686-linux /etc/${keyFile} 8 1 benchmark,big-parallel
  hydra-builder@fsn0 aarch64-linux /etc/${keyFile} 8 1 benchmark,big-parallel
  '';
  sops.secrets."hydra_builder_private_key" = {

nixos/profiles/services/hydra/_cache.nix

@@ -0,0 +1,116 @@
+{
+ config,
+ pkgs,
+ lib,
+ ...
+}: let
+ cacheS3Url = config.lib.self.data.cache_s3_url;
+ cacheBucketName = config.lib.self.data.cache_bucket_name;
+ hydraRootsDir = config.services.hydra.gcRootsDir;
+in {
+ systemd.services."copy-cache-li7g-com@" = {
+ script = ''
+ export AWS_ACCESS_KEY_ID=$(cat "$CREDENTIALS_DIRECTORY/cache-key-id")
+ export AWS_SECRET_ACCESS_KEY=$(cat "$CREDENTIALS_DIRECTORY/cache-access-key")
+ root="$1"
+ echo "root = $root"
+
+ (
+ echo "wait for lock"
+ flock 200
+ echo "enter critical section"
+
+ nix store sign "$root" --recursive --key-file "$CREDENTIALS_DIRECTORY/signing-key"
+ echo "push cache to cahche.li7g.com for hydra gcroot: $root"
+ # use multipart-upload to avoid cloudflare limit
+ nix copy --to "s3://${cacheBucketName}?endpoint=cache-overlay.li7g.com&multipart-upload=true&parallel-compression=true" "$root" --verbose
+ ) 200>/var/lib/cache-li7g-com/lock
+ '';
+ scriptArgs = "%I";
+ path = with pkgs; [
+ config.nix.package
+ fd
+ proxychains
+ util-linux
+ ];
+ serviceConfig = {
+ User = "hydra";
+ Group = "hydra";
+ Type = "oneshot";
+ StateDirectory = "cache-li7g-com";
+ LoadCredential = [
+ "cache-key-id:${config.sops.secrets."cache_key_id".path}"
+ "cache-access-key:${config.sops.secrets."cache_access_key".path}"
+ "signing-key:${config.sops.secrets."cache-li7g-com/key".path}"
+ ];
+ CPUQuota = "200%"; # limit cpu usage for parallel-compression
+ };
+ environment = lib.mkMerge [
+ {
+ HOME = "/var/lib/cache-li7g-com";
+ }
+ (lib.mkIf (config.networking.fw-proxy.enable)
+ config.networking.fw-proxy.environment)
+ ];
+ };
+ systemd.services."gc-cache-li7g-com" = {
+ script = ''
+ export AWS_ACCESS_KEY_ID=$(cat "$CREDENTIALS_DIRECTORY/cache-key-id")
+ export AWS_SECRET_ACCESS_KEY=$(cat "$CREDENTIALS_DIRECTORY/cache-access-key")
+ export B2_APPLICATION_KEY_ID=$(cat "$CREDENTIALS_DIRECTORY/cache-key-id")
+ export B2_APPLICATION_KEY=$(cat "$CREDENTIALS_DIRECTORY/cache-access-key")
+
+ (
+ echo "wait for lock"
+ flock 200
+ echo "enter critical section"
+
+ echo "canceling all unfinished multipart uploads..."
+ backblaze-b2 cancel-all-unfinished-large-files "${cacheBucketName}"
+
+ echo "removing narinfo cache..."
+ rm -rf /var/lib/cache-li7g-com/.cache
+
+ echo "performing gc..."
+ nix-gc-s3 "${cacheBucketName}" --endpoint "${cacheS3Url}" --roots "${hydraRootsDir}" --jobs 10
+ ) 200>/var/lib/cache-li7g-com/lock
+ '';
+ path = with pkgs; [
+ nix-gc-s3
+ config.nix.package
+ util-linux
+ backblaze-b2
+ ];
+ serviceConfig = {
+ Restart = "on-failure";
+ User = "hydra";
+ Group = "hydra";
+ Type = "oneshot";
+ StateDirectory = "cache-li7g-com";
+ LoadCredential = [
+ "cache-key-id:${config.sops.secrets."cache_key_id".path}"
+ "cache-access-key:${config.sops.secrets."cache_access_key".path}"
+ ];
+ };
+ environment =
+ lib.mkIf (config.networking.fw-proxy.enable)
+ config.networking.fw-proxy.environment;
+ requiredBy = ["hydra-update-gc-roots.service"];
+ after = ["hydra-update-gc-roots.service"];
+ };
+
+ sops.secrets."cache_key_id" = {
+ sopsFile = config.sops-file.terraform;
+ };
+ sops.secrets."cache_access_key" = {
+ sopsFile = config.sops-file.terraform;
+ };
+ sops.secrets."cache-li7g-com/key" = {
+ sopsFile = config.sops-file.host;
+ };
+
+ services.notify-failure.services = [
+ "copy-cache-li7g-com"
+ "gc-cache-li7g-com"
+ ];
+}

nixos/profiles/services/hydra/_dotfiles-channel-update.nix

@@ -0,0 +1,93 @@
+{
+ config,
+ lib,
+ pkgs,
+ ...
+}: {
+ systemd.services."dotfiles-channel-update@" = {
+ script = ''
+ cd "$STATE_DIRECTORY"
+
+ update_file="$1"
+ echo "update_file = $update_file"
+
+ host=$(jq -r '.host' "$update_file")
+ echo "host = $host"
+ commit=$(jq -r '.commit' "$update_file")
+ echo "commit = $commit"
+ out=$(jq -r '.out' "$update_file")
+ echo "out = $out"
+
+ target_branch="nixos-tested-$host"
+ echo "target_branch = $target_branch"
+
+ (
+ echo "wait for lock"
+ flock 200
+ echo "enter critical section"
+
+ systemctl start "copy-cache-li7g-com@$(systemd-escape "$out").service"
+
+ # update channel
+ if [ ! -d dotfiles ]; then
+ git clone https://github.com/linyinfeng/dotfiles.git
+ pushd dotfiles
+ token=$(cat "$CREDENTIALS_DIRECTORY/github-token")
+ git remote set-url origin "https://littlenano:[email protected]/linyinfeng/dotfiles.git"
+ popd
+ fi
+ cd dotfiles
+ git checkout "$target_branch" || git checkout -b "$target_branch"
+ git pull origin "$target_branch" || true
+ git fetch
+ git merge --ff-only "$commit"
+ git push --set-upstream origin "$target_branch"
+
+ ${config.programs.tg-send.wrapped} <<EOF
+ dotfiles/$target_branch
+
+ $(git show HEAD --no-patch)
+ EOF
+ ) 200>/var/lib/dotfiles-channel-update/lock
+ '';
+ scriptArgs = "%I";
+ path = with pkgs; [
+ git
+ jq
+ config.nix.package
+ util-linux
+ ];
+ serviceConfig = {
+ User = "hydra";
+ Group = "hydra";
+ Type = "oneshot";
+ SupplementaryGroups = [
+ config.users.groups.tg-send.name
+ ];
+ StateDirectory = "dotfiles-channel-update";
+ Restart = "on-failure";
+ LoadCredential = [
+ "github-token:${config.sops.secrets."nano/github-token".path}"
+ ];
+ };
+ environment =
+ lib.mkIf (config.networking.fw-proxy.enable)
+ config.networking.fw-proxy.environment;
+ };
+ sops.secrets."nano/github-token" = {
+ sopsFile = config.sops-file.get "common.yaml";
+ restartUnits = ["[email protected]"];
+ };
+
+ services.notify-failure.services = ["dotfiles-channel-update@"];
+
+ security.polkit.extraConfig = ''
+ polkit.addRule(function(action, subject) {
+ if (action.id == "org.freedesktop.systemd1.manage-units" &&
+ RegExp('dotfiles-channel-update@.+\.service|copy-cache-li7g-com@.+\.service').test(action.lookup("unit")) === true &&
+ subject.isInGroup("hydra")) {
+ return polkit.Result.YES;
+ }
+ });
+ '';
+}

nixos/profiles/services/hydra/default.nix

@@ -0,0 +1,110 @@
+{
+ config,
+ lib,
+ pkgs,
+ profiles,
+ ...
+}: let
+ cfg = config.hosts.nuc;
+ hydra-hook = pkgs.substituteAll {
+ src = ./hook.sh;
+ isExecutable = true;
+ inherit (pkgs.stdenvNoCC) shell;
+ inherit (pkgs) jq systemd postgresql;
+ };
+in {
+ imports = [
+ ./_dotfiles-channel-update.nix
+ ./_cache.nix
+ ];
+
+ config = lib.mkMerge [
+ {
+ services.nginx.virtualHosts."hydra.*" = {
+ forceSSL = true;
+ useACMEHost = "main";
+ locations."/" = {
+ proxyPass = "http://127.0.0.1:${toString config.ports.hydra}";
+ };
+ };
+
+ services.hydra = {
+ enable = true;
+ package = pkgs.hydra-master;
+ listenHost = "127.0.0.1";
+ port = config.ports.hydra;
+ hydraURL = "https://hydra.li7g.com";
+ notificationSender = "[email protected]";
+ useSubstitutes = true;
+ extraEnv = lib.mkIf (config.networking.fw-proxy.enable) config.networking.fw-proxy.environment;
+ extraConfig = ''
+ # use secret-key-files option in nix.conf instead
+ # store-uri = file:///nix/store?secret-key=${config.sops.secrets."cache-li7g-com/key".path}
+
+ Include "${config.sops.templates."hydra-extra-config".path}"
+
+ <githubstatus>
+ jobs = .*
+ excludeBuildFromContext = 1
+ </githubstatus>
+ <runcommand>
+ command = "${hydra-hook}"
+ </runcommand>
+ '';
+ };
+ # allow evaluator and queue-runner to access nix-access-tokens
+ systemd.services.hydra-evaluator.serviceConfig.SupplementaryGroups = [config.users.groups.nix-access-tokens.name];
+ systemd.services.hydra-queue-runner.serviceConfig.SupplementaryGroups = [
+ config.users.groups.nix-access-tokens.name
+ config.users.groups.hydra-builder-client.name
+ ];
+ sops.templates."hydra-extra-config" = {
+ group = "hydra";
+ mode = "440";
+ content = ''
+ <github_authorization>
+ linyinfeng = Bearer ${config.sops.placeholder."nano/github-token"}
+ littlenano = Bearer ${config.sops.placeholder."nano/github-token"}
+ </github_authorization>
+ '';
+ };
+ nix.settings.secret-key-files = [
+ "${config.sops.secrets."cache-li7g-com/key".path}"
+ ];
+ nix.settings.allowed-uris = [
+ "https://github.com/" # for nix-index-database
+ "https://gitlab.com/" # for home-manager nmd source
+ "https://git.sr.ht/" # for home-manager nmd source
+ ];
+ sops.secrets."nano/github-token" = {
+ sopsFile = config.sops-file.get "common.yaml";
+ restartUnits = ["hydra.service"];
+ };
+ sops.secrets."cache-li7g-com/key" = {
+ sopsFile = config.sops-file.host;
+ restartUnits = ["nix-daemon.service"];
+ };
+ nix.settings.trusted-users = ["@hydra"];
+ }
+
+ {
+ # email notifications
+ services.hydra.extraConfig = ''
+ email_notification = 1
+ '';
+ systemd.services.hydra-notify.serviceConfig.EnvironmentFile = config.sops.templates."hydra-email".path;
+ sops.templates."hydra-email".content = ''
+ EMAIL_SENDER_TRANSPORT=SMTP
+ [email protected]
+ EMAIL_SENDER_TRANSPORT_sasl_password=${config.sops.placeholder."mail_password"}
+ EMAIL_SENDER_TRANSPORT_host=smtp.ts.li7g.com
+ EMAIL_SENDER_TRANSPORT_port=${toString config.ports.smtp-starttls}
+ EMAIL_SENDER_TRANSPORT_ssl=starttls
+ '';
+ sops.secrets."mail_password" = {
+ sopsFile = config.sops-file.get "terraform/common.yaml";
+ restartUnits = ["hydra-notify.service"];
+ };
+ }
+ ];
+}