From 239491aa4d846276fea3ec586f615a46633860f0 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Mon, 23 Sep 2024 08:58:00 +0200 Subject: [PATCH 1/8] update-nixpkgs: fix NixOSVersion.upstream_branch --- release/update-nixpkgs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/update-nixpkgs.py b/release/update-nixpkgs.py index 43afd3113..5af5ff796 100755 --- a/release/update-nixpkgs.py +++ b/release/update-nixpkgs.py @@ -36,7 +36,7 @@ def upstream_branch(self) -> str: if self == NixOSVersion.NIXOS_UNSTABLE: return "nixos-unstable" - return self.value() + return str(self) def run_on_hydra(*args): From 15584dbc25d1f2a75c67b5ed0abeba90a72123d8 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Mon, 23 Sep 2024 09:40:04 +0200 Subject: [PATCH 2/8] update-nixpkgs: fix rebasing to origin and improve status output --- release/update-nixpkgs.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/release/update-nixpkgs.py b/release/update-nixpkgs.py index 5af5ff796..1cb09a333 100755 --- a/release/update-nixpkgs.py +++ b/release/update-nixpkgs.py @@ -90,24 +90,31 @@ def format_as_msg(self): def rebase_nixpkgs(nixpkgs_repo: Repo, nixos_version: NixOSVersion): - print("Fetching origin remote...") + print("nixpkgs: fetching origin remote...") nixpkgs_repo.git.fetch("origin") origin_ref_id = f"origin/{nixos_version}" origin_ref = nixpkgs_repo.refs[origin_ref_id] + print("nixpkgs status:") + print(nixpkgs_repo.git.status()) + if nixpkgs_repo.head.commit != origin_ref.commit: - do_reset = confirm( - f"local HEAD differs from {origin_ref_id}, hard-reset to origin?", - default=True, + prompt = ( + f"WARNING: local branch ({nixpkgs_repo.head.commit}) differs " + f"from {origin_ref_id}." + f"\nHard-reset to origin ({origin_ref.commit})?" ) + do_reset = confirm(prompt, default=True) if do_reset: - nixpkgs_repo.git.reset(hard=True) + nixpkgs_repo.head.reset(origin_ref.commit, working_tree=True) + + print(nixpkgs_repo.git.status()) - print("Fetching upstream remote...") + print("nixpkgs: fetching upstream remote...") nixpkgs_repo.git.fetch("upstream") old_rev = str(nixpkgs_repo.head.ref.commit) upstream_ref = f"upstream/{nixos_version.upstream_branch}" - print(f"Using upstream ref {upstream_ref}") + print(f"nixpkgs: using upstream ref {upstream_ref}") nixpkgs_repo.git.rebase(upstream_ref) new_rev = str(nixpkgs_repo.head.ref.commit) version_range = f"{old_rev}..{new_rev}" From 9408cd43aaa7f7bed1e38d2c6f3e73e2c47ceae3 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Mon, 23 Sep 2024 19:19:18 +0200 Subject: [PATCH 3/8] update-nixpkgs: run_on_hydra call error handling --- release/update-nixpkgs.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/release/update-nixpkgs.py b/release/update-nixpkgs.py index 1cb09a333..a43d86e5a 100755 --- a/release/update-nixpkgs.py +++ b/release/update-nixpkgs.py @@ -9,7 +9,7 @@ from git import Repo from rich import print -from typer import Argument, Option, Typer, confirm +from typer import Argument, Exit, Option, Typer, confirm PKG_UPDATE_RE = re.compile( r"(?P.+): " @@ -41,7 +41,17 @@ def upstream_branch(self) -> str: def run_on_hydra(*args): cmd = ["ssh", "hydra01"] + list(args) - proc = subprocess.run(cmd, check=True, text=True, capture_output=True) + try: + proc = subprocess.run(cmd, check=True, text=True, capture_output=True) + except subprocess.CalledProcessError as e: + print(e) + if e.stdout.strip(): + print("stdout:") + print(e.stdout) + if e.stderr.strip(): + print("stderr:") + print(e.stderr) + raise Exit(2) return proc From ebefcf9b0f0b075fc73c9f39ef270f75308d43fc Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Mon, 23 Sep 2024 21:46:27 +0200 Subject: [PATCH 4/8] important packages: add more k3s and go versions PL-133043 --- release/important_packages.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/release/important_packages.json b/release/important_packages.json index cb808f925..21953007f 100644 --- a/release/important_packages.json +++ b/release/important_packages.json @@ -58,6 +58,8 @@ "go", "go_1_19", "go_1_20", + "go_1_21", + "go_1_22", "grafana", "grub2", "haproxy", @@ -75,7 +77,10 @@ "jre", "k3s", "k3s_1_27", + "k3s_1_28", + "k3s_1_29", "k3s_1_30", + "k3s_1_31", "keycloak", "kubernetes-helm", "libffi", From cd4234b591740bf007d13d8744eeddfeb72daf26 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Mon, 23 Sep 2024 22:08:48 +0200 Subject: [PATCH 5/8] Update nixpkgs (2024-09-23) Pull upstream NixOS changes, security fixes and package updates: - asterisk: 20.9.2 -> 20.9.3 - cacert: 3.101 -> 3.104 - calibre: add patches for CVE-2024-6781, CVE-2024-6782, CVE-2024-7008, CVE-2024-7009 - clamav: 1.3.1 -> 1.3.2 - curl: apply patch for CVE-2024-8096 - k3s_1_28: 1.28.12+k3s1 -> 1.28.13+k3s1 - k3s_1_29: 1.29.7+k3s2 -> 1.29.8+k3s1 - k3s_1_30: 1.30.3+k3s1 -> 1.30.4+k3s1 - k3s_1_31: init 1.31.0+k3s1 - linux_5_15: 5.15.164 -> 5.15.167 - python312: 3.12.4 -> 3.12.5 - python3Packages.urllib3: 2.2.1 -> 2.2.2 - ruby: 3.3.4 -> 3.3.5 - runc: 1.1.12 -> 1.1.14 - slurm: 23.11.9.1 -> 23.11.10.1 - strace: 6.10 -> 6.11 - tcpdump: 4.99.4 -> 4.99.5 - unifi7: mark insecure due to CVE-2024-42025 - unifi8: 8.1.127 -> 8.4.62 - vim: 9.1.0377 -> 9.1.0707 Additional package update by us: - gitlab: 17.2.7 -> 17.2.8 PL-133043 --- flake.lock | 6 +- release/package-versions.json | 131 ++++++++++++++++++++-------------- release/versions.json | 4 +- 3 files changed, 83 insertions(+), 58 deletions(-) diff --git a/flake.lock b/flake.lock index c1b0988ae..6c7ea6607 100644 --- a/flake.lock +++ b/flake.lock @@ -410,11 +410,11 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1726668836, - "narHash": "sha256-k/m92YGpRzjB48X2po7jtNycdY40JhweOfeGysmwhjM=", + "lastModified": 1727335565, + "narHash": "sha256-D7sIMls9rUl9xkT/U/R0hBE9R+sbbz1bwy7YX7jBHJg=", "owner": "flyingcircusio", "repo": "nixpkgs", - "rev": "ecb04ae94077cca3595752f8c3adce8a5e445b34", + "rev": "8d3f4935a5ba572241685c6814258df8a93d6731", "type": "github" }, "original": { diff --git a/release/package-versions.json b/release/package-versions.json index ad35d26ec..15cba1c3e 100644 --- a/release/package-versions.json +++ b/release/package-versions.json @@ -10,9 +10,9 @@ "version": "2.4.62" }, "asterisk": { - "name": "asterisk-20.9.2", + "name": "asterisk-20.9.3", "pname": "asterisk", - "version": "20.9.2" + "version": "20.9.3" }, "auditbeat7-oss": { "name": "auditbeat-oss-7.17.16", @@ -55,9 +55,9 @@ "version": "2.5.9" }, "cacert": { - "name": "nss-cacert-3.101", + "name": "nss-cacert-3.104", "pname": "nss-cacert", - "version": "3.101" + "version": "3.104" }, "calibre": { "name": "calibre-7.10.0", @@ -70,14 +70,14 @@ "version": "18.2.4" }, "chromedriver": { - "name": "chromedriver-128.0.6613.119", + "name": "chromedriver-129.0.6668.58", "pname": "chromedriver", - "version": "128.0.6613.119" + "version": "129.0.6668.58" }, "chromium": { - "name": "chromium-128.0.6613.119", + "name": "chromium-129.0.6668.58", "pname": "chromium", - "version": "128.0.6613.119" + "version": "129.0.6668.58" }, "cifs-utils": { "name": "cifs-utils-7.0", @@ -85,9 +85,9 @@ "version": "7.0" }, "clamav": { - "name": "clamav-1.3.1", + "name": "clamav-1.3.2", "pname": "clamav", - "version": "1.3.1" + "version": "1.3.2" }, "cmake": { "name": "cmake-3.29.2", @@ -155,9 +155,9 @@ "version": "2.3.21.1" }, "element-web": { - "name": "element-web-1.11.76", + "name": "element-web-1.11.77", "pname": "element-web", - "version": "1.11.76" + "version": "1.11.77" }, "erlang": { "name": "erlang-25.3.2.12", @@ -175,9 +175,9 @@ "version": "6.4.38" }, "ffmpeg": { - "name": "ffmpeg-6.1.1", + "name": "ffmpeg-6.1.2", "pname": "ffmpeg", - "version": "6.1.1" + "version": "6.1.2" }, "file": { "name": "file-5.45", @@ -190,9 +190,9 @@ "version": "7.17.16" }, "firefox": { - "name": "firefox-130.0", + "name": "firefox-130.0.1", "pname": "firefox", - "version": "130.0" + "version": "130.0.1" }, "gcc": { "name": "gcc-wrapper-13.2.0", @@ -220,9 +220,9 @@ "version": "2.44.1" }, "gitaly": { - "name": "gitaly-17.2.7", + "name": "gitaly-17.2.8", "pname": "gitaly", - "version": "17.2.7" + "version": "17.2.8" }, "github-runner": { "name": "github-runner-2.319.1", @@ -230,9 +230,9 @@ "version": "2.319.1" }, "gitlab": { - "name": "gitlab-17.2.7", + "name": "gitlab-17.2.8", "pname": "gitlab", - "version": "17.2.7" + "version": "17.2.8" }, "gitlab-container-registry": { "name": "gitlab-container-registry-4.9.0", @@ -240,14 +240,14 @@ "version": "4.9.0" }, "gitlab-ee": { - "name": "gitlab-ee-17.2.7", + "name": "gitlab-ee-17.2.8", "pname": "gitlab-ee", - "version": "17.2.7" + "version": "17.2.8" }, "gitlab-pages": { - "name": "gitlab-pages-17.2.7", + "name": "gitlab-pages-17.2.8", "pname": "gitlab-pages", - "version": "17.2.7" + "version": "17.2.8" }, "gitlab-runner": { "name": "gitlab-runner-17.1.0", @@ -255,9 +255,9 @@ "version": "17.1.0" }, "gitlab-workhorse": { - "name": "gitlab-workhorse-17.2.7", + "name": "gitlab-workhorse-17.2.8", "pname": "gitlab-workhorse", - "version": "17.2.7" + "version": "17.2.8" }, "glibc": { "name": "glibc-2.39-52", @@ -281,6 +281,16 @@ }, "go_1_19": {}, "go_1_20": {}, + "go_1_21": { + "name": "go-1.21.13", + "pname": "go", + "version": "1.21.13" + }, + "go_1_22": { + "name": "go-1.22.6", + "pname": "go", + "version": "1.22.6" + }, "grafana": { "name": "grafana-10.4.8", "pname": "grafana", @@ -357,19 +367,34 @@ "version": "21.0.3+9" }, "k3s": { - "name": "k3s-1.30.3+k3s1", + "name": "k3s-1.30.4+k3s1", "pname": "k3s", - "version": "1.30.3+k3s1" + "version": "1.30.4+k3s1" }, "k3s_1_27": { "name": "k3s-1.27.14+k3s1", "pname": "k3s", "version": "1.27.14+k3s1" }, + "k3s_1_28": { + "name": "k3s-1.28.13+k3s1", + "pname": "k3s", + "version": "1.28.13+k3s1" + }, + "k3s_1_29": { + "name": "k3s-1.29.8+k3s1", + "pname": "k3s", + "version": "1.29.8+k3s1" + }, "k3s_1_30": { - "name": "k3s-1.30.3+k3s1", + "name": "k3s-1.30.4+k3s1", + "pname": "k3s", + "version": "1.30.4+k3s1" + }, + "k3s_1_31": { + "name": "k3s-1.31.0+k3s1", "pname": "k3s", - "version": "1.30.3+k3s1" + "version": "1.31.0+k3s1" }, "keycloak": { "name": "keycloak-24.0.5", @@ -437,9 +462,9 @@ "version": "0.2.5" }, "linux_5_15": { - "name": "linux-5.15.164", + "name": "linux-5.15.167", "pname": "linux", - "version": "5.15.164" + "version": "5.15.167" }, "logrotate": { "name": "logrotate-3.21.0", @@ -827,9 +852,9 @@ "version": "3.11.9" }, "python312": { - "name": "python3-3.12.4", + "name": "python3-3.12.5", "pname": "python3", - "version": "3.12.4" + "version": "3.12.5" }, "python38": {}, "python39": { @@ -908,9 +933,9 @@ "version": "235" }, "python3Packages.urllib3": { - "name": "python3.11-urllib3-2.2.1", + "name": "python3.11-urllib3-2.2.2", "pname": "urllib3", - "version": "2.2.1" + "version": "2.2.2" }, "qemu": { "name": "qemu-8.2.6", @@ -927,6 +952,11 @@ "pname": "rabbitmq-server", "version": "3.12.13" }, + "rclone": { + "name": "rclone-1.66.0", + "pname": "rclone", + "version": "1.66.0" + }, "re2c": { "name": "re2c-3.1", "pname": "re2c", @@ -964,9 +994,9 @@ "version": "3.2.4" }, "runc": { - "name": "runc-1.1.12", + "name": "runc-1.1.14", "pname": "runc", - "version": "1.1.12" + "version": "1.1.14" }, "screen": { "name": "screen-4.9.1", @@ -974,9 +1004,9 @@ "version": "4.9.1" }, "slurm": { - "name": "slurm-23.11.9.1", + "name": "slurm-23.11.10.1", "pname": "slurm", - "version": "23.11.9.1" + "version": "23.11.10.1" }, "solr": { "name": "solr-8.11.2", @@ -984,9 +1014,9 @@ "version": "8.11.2" }, "strace": { - "name": "strace-6.10", + "name": "strace-6.11", "pname": "strace", - "version": "6.10" + "version": "6.11" }, "strongswan": { "name": "strongswan-5.9.14", @@ -1014,9 +1044,9 @@ "version": "255.9" }, "tcpdump": { - "name": "tcpdump-4.99.4", + "name": "tcpdump-4.99.5", "pname": "tcpdump", - "version": "4.99.4" + "version": "4.99.5" }, "telegraf": { "name": "telegraf-1.30.3", @@ -1038,15 +1068,10 @@ "pname": "apache-tomcat", "version": "9.0.88" }, - "unifi7": { - "name": "unifi-controller-7.5.187", - "pname": "unifi-controller", - "version": "7.5.187" - }, "unifi8": { - "name": "unifi-controller-8.1.127", + "name": "unifi-controller-8.4.62", "pname": "unifi-controller", - "version": "8.1.127" + "version": "8.4.62" }, "unzip": { "name": "unzip-6.0", @@ -1064,9 +1089,9 @@ "version": "7.4.3" }, "vim": { - "name": "vim-9.1.0377", + "name": "vim-9.1.0707", "pname": "vim", - "version": "9.1.0377" + "version": "9.1.0707" }, "webkitgtk": { "name": "webkitgtk-2.44.3+abi=4.0", diff --git a/release/versions.json b/release/versions.json index 074377bf3..c4236fc08 100644 --- a/release/versions.json +++ b/release/versions.json @@ -8,9 +8,9 @@ "url": "https://gitlab.flyingcircus.io/flyingcircus/nixos-mailserver.git/" }, "nixpkgs": { - "hash": "sha256-k/m92YGpRzjB48X2po7jtNycdY40JhweOfeGysmwhjM=", + "hash": "sha256-D7sIMls9rUl9xkT/U/R0hBE9R+sbbz1bwy7YX7jBHJg=", "owner": "flyingcircusio", "repo": "nixpkgs", - "rev": "ecb04ae94077cca3595752f8c3adce8a5e445b34" + "rev": "8d3f4935a5ba572241685c6814258df8a93d6731" } } From 8d20c90d208afc952241abf4761e0b49f6654ea9 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Mon, 23 Sep 2024 22:12:46 +0200 Subject: [PATCH 6/8] important packages: remove obsolete go versions They have been removed from nixpkgs but we still tried to track their versions. PL-133043 --- release/important_packages.json | 2 -- release/package-versions.json | 2 -- 2 files changed, 4 deletions(-) diff --git a/release/important_packages.json b/release/important_packages.json index 21953007f..70e638a75 100644 --- a/release/important_packages.json +++ b/release/important_packages.json @@ -56,8 +56,6 @@ "gnumake", "gnupg", "go", - "go_1_19", - "go_1_20", "go_1_21", "go_1_22", "grafana", diff --git a/release/package-versions.json b/release/package-versions.json index 15cba1c3e..4be3f41ac 100644 --- a/release/package-versions.json +++ b/release/package-versions.json @@ -279,8 +279,6 @@ "pname": "go", "version": "1.22.6" }, - "go_1_19": {}, - "go_1_20": {}, "go_1_21": { "name": "go-1.21.13", "pname": "go", From e040f33ff3495acc5b06c7ef9039788f7adf8788 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Wed, 25 Sep 2024 15:38:55 +0200 Subject: [PATCH 7/8] important packages: remove obsolete unifi7 PL-133043 --- release/important_packages.json | 1 - 1 file changed, 1 deletion(-) diff --git a/release/important_packages.json b/release/important_packages.json index 70e638a75..7fb72e4fc 100644 --- a/release/important_packages.json +++ b/release/important_packages.json @@ -216,7 +216,6 @@ "tomcat10", "tomcat9", "unzip", - "unifi7", "unifi8", "util-linux", "varnish", From b7475201f067404d972ee640ffb3fe50a1989862 Mon Sep 17 00:00:00 2001 From: Tobias Stenzel Date: Thu, 26 Sep 2024 14:50:13 +0200 Subject: [PATCH 8/8] kernel: pin stable version to 5.15.164 We are trying out a 6.11 kernel in non-prod right now, for (most) prod systems we like to keep the 5.15.164 kernel we have been using for some time now. Before, we used a revert in our nixpkgs fork to get the desired version but we have multiple kernel updates from upstream nixpkgs now and we want to pin it here to avoid any unwanted updates and confusion. PL-133043 --- nixos/platform/kernel.nix | 2 +- pkgs/overlay.nix | 18 ++++++++++++++++++ release/important_packages.json | 3 ++- release/package-versions.json | 11 ++++++++--- tests/kernelversions.nix | 13 ++++++------- 5 files changed, 35 insertions(+), 12 deletions(-) diff --git a/nixos/platform/kernel.nix b/nixos/platform/kernel.nix index ca7de545b..2b2b48ea1 100644 --- a/nixos/platform/kernel.nix +++ b/nixos/platform/kernel.nix @@ -29,7 +29,7 @@ in { boot.kernelPackages = if config.flyingcircus.useVerificationKernel then pkgs.linuxPackagesFor pkgs.linuxKernelVerify - else pkgs.linuxKernel.packages.linux_5_15; + else pkgs.linuxPackagesFor pkgs.linuxKernelStable; # Use this spelling if you need to try out custom kernels, try out patches # or otherwise deviate from our nixpkgs upstream. diff --git a/pkgs/overlay.nix b/pkgs/overlay.nix index 77bf3faf3..ec5435db1 100644 --- a/pkgs/overlay.nix +++ b/pkgs/overlay.nix @@ -120,6 +120,24 @@ builtins.mapAttrs (_: patchPhps phpLogPermissionPatch) { }; }; + linuxKernelStable = + let + kernelPackage = super.linux_5_15; + version = "5.15.164"; + in + kernelPackage.override { + argsOverride = { + src = super.fetchurl { + url = "https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-${version}.tar.xz"; + hash = "sha256-7GCY+u1kuKR7oXcugSputEQ4X3qjxg0+RzmrL9OykYY="; + }; + modDirVersion = version; + inherit version; + }; + }; + + + matomo-beta = super.matomo-beta.overrideAttrs (oldAttrs: { installPhase = '' runHook preInstall diff --git a/release/important_packages.json b/release/important_packages.json index 7fb72e4fc..8a5e2aab8 100644 --- a/release/important_packages.json +++ b/release/important_packages.json @@ -92,7 +92,8 @@ "libxml2", "libxslt", "libyaml", - "linux_5_15", + "linuxKernelStable", + "linuxKernelVerify", "logrotate", "lz4", "mailutils", diff --git a/release/package-versions.json b/release/package-versions.json index 4be3f41ac..e176f5dac 100644 --- a/release/package-versions.json +++ b/release/package-versions.json @@ -459,10 +459,15 @@ "pname": "libyaml", "version": "0.2.5" }, - "linux_5_15": { - "name": "linux-5.15.167", + "linuxKernelStable": { + "name": "linux-5.15.164", "pname": "linux", - "version": "5.15.167" + "version": "5.15.164" + }, + "linuxKernelVerify": { + "name": "linux-6.11", + "pname": "linux", + "version": "6.11" }, "logrotate": { "name": "logrotate-3.21.0", diff --git a/tests/kernelversions.nix b/tests/kernelversions.nix index b91488937..a5716ed2a 100644 --- a/tests/kernelversions.nix +++ b/tests/kernelversions.nix @@ -185,13 +185,12 @@ import ./make-test-python.nix ({ ... }: vm.wait_for_unit('memcached.service') vm.wait_for_open_port(11211) - foundKernel = vm.execute("uname -r")[1].strip() - if foundKernel != expected: - print(f"Expected kernel {expected!r}") - print(f"Found kernel {foundKernel!r}") - full = vm.execute("uname -a")[1] - print(f"Machine: {full}") - raise AssertionError("Unexpected kernel version") + found = vm.execute("uname -r")[1].strip() + if found != expected: + uname_a = vm.execute("uname -a")[1] + raise AssertionError( + f"Expected: {expected}, found: {found}. uname -a: {uname_a}" + ) assertKernelVersion(verifyKernel, "6.11.0") assertKernelVersion(prodKernel, "5.15.164")