From e10a48decda6d874c9eb4c374cce3d72d37ebe3d Mon Sep 17 00:00:00 2001 From: Mohamed Chiheb Ben Jemaa Date: Fri, 11 Oct 2024 16:05:13 +0200 Subject: [PATCH 1/7] try flatcar --- images/capi/Makefile | 4 +++- images/capi/packer/proxmox/flatcar.json | 26 +++++++++++++++++++++ images/capi/packer/proxmox/packer.json.tmpl | 8 +++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 images/capi/packer/proxmox/flatcar.json diff --git a/images/capi/Makefile b/images/capi/Makefile index ecfec48e98..cb009e44c9 100644 --- a/images/capi/Makefile +++ b/images/capi/Makefile @@ -389,7 +389,7 @@ NUTANIX_BUILD_NAMES ?= nutanix-ubuntu-2004 nutanix-ubuntu-2204 nutanix-ubuntu-24 HCLOUD_BUILD_NAMES ?= hcloud-ubuntu-2004 hcloud-ubuntu-2204 hcloud-ubuntu-2404 hcloud-centos-7 hcloud-rockylinux-8 hcloud-rockylinux-9 hcloud-flatcar -PROXMOX_BUILD_NAMES ?= proxmox-ubuntu-2204 proxmox-ubuntu-2404 proxmox-rockylinux-9 +PROXMOX_BUILD_NAMES ?= proxmox-ubuntu-2204 proxmox-ubuntu-2404 proxmox-rockylinux-9 proxmox-flatcar VULTR_BUILD_NAMES ?= vultr-ubuntu-2204 @@ -866,6 +866,7 @@ build-hcloud-rockylinux-9: ## Builds the Hetzner Cloud Rocky Linux 9 image build-hcloud-flatcar: ## Builds the Hetzner Cloud Flatcar image build-hcloud-all: $(HCLOUD_BUILD_TARGETS) ## Builds all Hetzner Cloud image +build-proxmox-flatcar: ## Builds Flatcar QEMU image build-proxmox-ubuntu-2204: ## Builds the Proxmox ubuntu-2204 image build-proxmox-ubuntu-2404: ## Builds the Proxmox ubuntu-2404 image build-proxmox-rockylinux-9: ## Builds the Proxmox rockylinux-9 image @@ -1047,6 +1048,7 @@ validate-hcloud-flatcar: ## Validates the Hetzner Cloud Flatcar Packer config validate-hcloud-all: $(HCLOUD_VALIDATE_TARGETS) ## Validates all Hetzner Cloud Packer config validate-proxmox-ubuntu-2204: ## Validates Ubuntu 22.04 Proxmox Packer config +validate-proxmox-flatcar: ## Validates Flatcar Proxmox Packer config validate-vultr-ubuntu-2204: ## Validates Ubuntu 22.04 Vultr Snapshot Packer config diff --git a/images/capi/packer/proxmox/flatcar.json b/images/capi/packer/proxmox/flatcar.json new file mode 100644 index 0000000000..786c25d2ef --- /dev/null +++ b/images/capi/packer/proxmox/flatcar.json @@ -0,0 +1,26 @@ +{ + "ansible_extra_vars": "ansible_python_interpreter=/opt/bin/python oem_id={{user `oem_id`}}", + "boot_command_prefix": "sudo systemctl mask sshd.socket --nowcurl -sLo /tmp/ignition.json ", + "boot_command_suffix": "/bootstrap-pass-auth.jsonsed -i \"s|BUILDERPASSWORDHASH|$(mkpasswd -5 {{user `ssh_password`}})|\" /tmp/ignition.jsonsudo flatcar-install -d /dev/sda -C {{user `channel_name`}} -V {{user `release_version`}} -i /tmp/ignition.json && sudo reboot", + "boot_wait": "180s", + "build_name": "flatcar-{{env `FLATCAR_CHANNEL`}}-{{env `FLATCAR_VERSION`}}", + "channel_name": "{{env `FLATCAR_CHANNEL`}}", + "crictl_source_type": "http", + "distribution_version": "{{env `FLATCAR_CHANNEL`}}-{{env `FLATCAR_VERSION`}}", + "distro_name": "flatcar", + "guest_os_type": "linux-64", + "http_directory": "./packer/files/flatcar/ignition/", + "iso_checksum": "https://{{env `FLATCAR_CHANNEL`}}.release.flatcar-linux.net/amd64-usr/{{env `FLATCAR_VERSION`}}/flatcar_production_iso_image.iso.DIGESTS.asc", + "iso_checksum_type": "file", + "iso_url": "https://{{env `FLATCAR_CHANNEL`}}.release.flatcar-linux.net/amd64-usr/{{env `FLATCAR_VERSION`}}/flatcar_production_iso_image.iso", + "kubernetes_cni_source_type": "http", + "kubernetes_source_type": "http", + "oem_id": "{{env `OEM_ID`}}", + "os_display_name": "Flatcar Container Linux ({{env `FLATCAR_CHANNEL`}} channel release {{env `FLATCAR_VERSION`}})", + "python_path": "/opt/bin/builder-env/site-packages", + "release_version": "{{env `FLATCAR_VERSION`}}", + "shutdown_command": "shutdown -P now", + "systemd_prefix": "/etc/systemd", + "sysusr_prefix": "/opt", + "sysusrlocal_prefix": "/opt" +} diff --git a/images/capi/packer/proxmox/packer.json.tmpl b/images/capi/packer/proxmox/packer.json.tmpl index bb0bdc226c..ee6661416f 100644 --- a/images/capi/packer/proxmox/packer.json.tmpl +++ b/images/capi/packer/proxmox/packer.json.tmpl @@ -72,6 +72,14 @@ "inline_shebang": "/bin/bash -e", "type": "shell" }, + { + "environment_vars": [ + "PYPY_HTTP_SOURCE={{user `pypy_http_source`}}" + ], + "execute_command": "BUILD_NAME={{user `build_name`}}; if [[ \"${BUILD_NAME}\" == *\"flatcar\"* ]]; then sudo {{.Vars}} -S -E bash '{{.Path}}'; fi", + "script": "./packer/files/flatcar/scripts/bootstrap-flatcar.sh", + "type": "shell" + }, { "ansible_env_vars": [ "ANSIBLE_SSH_ARGS='{{user `existing_ansible_ssh_args`}} {{user `ansible_common_ssh_args`}}'", From 56f3b73a9d873c84f4c69b27f718b2a8448c7111 Mon Sep 17 00:00:00 2001 From: Mohamed Chiheb Ben Jemaa Date: Fri, 11 Oct 2024 17:04:33 +0200 Subject: [PATCH 2/7] try flatcar -2 --- images/capi/ansible/roles/providers/tasks/proxmox.yml | 2 ++ images/capi/packer/proxmox/flatcar.json | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/images/capi/ansible/roles/providers/tasks/proxmox.yml b/images/capi/ansible/roles/providers/tasks/proxmox.yml index c40d54d336..be3e8f15f3 100644 --- a/images/capi/ansible/roles/providers/tasks/proxmox.yml +++ b/images/capi/ansible/roles/providers/tasks/proxmox.yml @@ -60,8 +60,10 @@ ansible.builtin.systemd: name: networkd-dispatcher state: started + when: ansible_os_family == "Debian" - name: Ensure networkd-dispatcher is enabled ansible.builtin.systemd: name: networkd-dispatcher enabled: true + when: ansible_os_family == "Debian" diff --git a/images/capi/packer/proxmox/flatcar.json b/images/capi/packer/proxmox/flatcar.json index 786c25d2ef..c630f12692 100644 --- a/images/capi/packer/proxmox/flatcar.json +++ b/images/capi/packer/proxmox/flatcar.json @@ -22,5 +22,6 @@ "shutdown_command": "shutdown -P now", "systemd_prefix": "/etc/systemd", "sysusr_prefix": "/opt", - "sysusrlocal_prefix": "/opt" + "sysusrlocal_prefix": "/opt", + "boot_media_path": "http://{{ .HTTPIP }}:{{ .HTTPPort }}" } From 4047d7272d50c5e688ca71c43969399824d6b97c Mon Sep 17 00:00:00 2001 From: Mohamed Chiheb Ben Jemaa Date: Fri, 11 Oct 2024 17:20:42 +0200 Subject: [PATCH 3/7] fix comment --- images/capi/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/capi/Makefile b/images/capi/Makefile index cb009e44c9..1e68960dd1 100644 --- a/images/capi/Makefile +++ b/images/capi/Makefile @@ -866,7 +866,7 @@ build-hcloud-rockylinux-9: ## Builds the Hetzner Cloud Rocky Linux 9 image build-hcloud-flatcar: ## Builds the Hetzner Cloud Flatcar image build-hcloud-all: $(HCLOUD_BUILD_TARGETS) ## Builds all Hetzner Cloud image -build-proxmox-flatcar: ## Builds Flatcar QEMU image +build-proxmox-flatcar: ## Builds Flatcar Proxmox image build-proxmox-ubuntu-2204: ## Builds the Proxmox ubuntu-2204 image build-proxmox-ubuntu-2404: ## Builds the Proxmox ubuntu-2404 image build-proxmox-rockylinux-9: ## Builds the Proxmox rockylinux-9 image From f924299335224f8c1af5a71d73dab1a7e305398d Mon Sep 17 00:00:00 2001 From: Mohamed Chiheb Ben Jemaa Date: Fri, 11 Oct 2024 17:21:14 +0200 Subject: [PATCH 4/7] sort jsonm --- images/capi/packer/proxmox/flatcar.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/capi/packer/proxmox/flatcar.json b/images/capi/packer/proxmox/flatcar.json index c630f12692..306cfcebd1 100644 --- a/images/capi/packer/proxmox/flatcar.json +++ b/images/capi/packer/proxmox/flatcar.json @@ -2,6 +2,7 @@ "ansible_extra_vars": "ansible_python_interpreter=/opt/bin/python oem_id={{user `oem_id`}}", "boot_command_prefix": "sudo systemctl mask sshd.socket --nowcurl -sLo /tmp/ignition.json ", "boot_command_suffix": "/bootstrap-pass-auth.jsonsed -i \"s|BUILDERPASSWORDHASH|$(mkpasswd -5 {{user `ssh_password`}})|\" /tmp/ignition.jsonsudo flatcar-install -d /dev/sda -C {{user `channel_name`}} -V {{user `release_version`}} -i /tmp/ignition.json && sudo reboot", + "boot_media_path": "http://{{ .HTTPIP }}:{{ .HTTPPort }}", "boot_wait": "180s", "build_name": "flatcar-{{env `FLATCAR_CHANNEL`}}-{{env `FLATCAR_VERSION`}}", "channel_name": "{{env `FLATCAR_CHANNEL`}}", @@ -22,6 +23,5 @@ "shutdown_command": "shutdown -P now", "systemd_prefix": "/etc/systemd", "sysusr_prefix": "/opt", - "sysusrlocal_prefix": "/opt", - "boot_media_path": "http://{{ .HTTPIP }}:{{ .HTTPPort }}" + "sysusrlocal_prefix": "/opt" } From 21fff9d4d957132fe2262d01e52932c62f276802 Mon Sep 17 00:00:00 2001 From: Mohamed Chiheb Ben Jemaa Date: Mon, 21 Oct 2024 19:18:14 +0200 Subject: [PATCH 5/7] Build Proxmox flatcar with openstack OEM --- .../ignition/proxmox-bootstrap-pass-auth.json | 46 +++++++++++++++++++ images/capi/packer/proxmox/flatcar.json | 5 +- 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json diff --git a/images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json b/images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json new file mode 100644 index 0000000000..8e09b18a1c --- /dev/null +++ b/images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json @@ -0,0 +1,46 @@ +{ + "ignition": { + "config": {}, + "security": { + "tls": {} + }, + "timeouts": {}, + "version": "2.3.0" + }, + "networkd": {}, + "passwd": { + "users": [ + { + "groups": [ + "wheel", + "sudo", + "docker" + ], + "name": "builder", + "passwordHash": "BUILDERPASSWORDHASH" + } + ] + }, + "storage": {}, + "systemd": { + "units": [ + { + "enable": true, + "name": "docker.service" + }, + { + "mask": true, + "name": "update-engine.service" + }, + { + "mask": true, + "name": "locksmithd.service" + }, + { + "name": "flatcar-openstack-hostname.service", + "enabled": false, + "mask": true + } + ] + } +} diff --git a/images/capi/packer/proxmox/flatcar.json b/images/capi/packer/proxmox/flatcar.json index 306cfcebd1..4f91f16ac8 100644 --- a/images/capi/packer/proxmox/flatcar.json +++ b/images/capi/packer/proxmox/flatcar.json @@ -1,7 +1,7 @@ { "ansible_extra_vars": "ansible_python_interpreter=/opt/bin/python oem_id={{user `oem_id`}}", "boot_command_prefix": "sudo systemctl mask sshd.socket --nowcurl -sLo /tmp/ignition.json ", - "boot_command_suffix": "/bootstrap-pass-auth.jsonsed -i \"s|BUILDERPASSWORDHASH|$(mkpasswd -5 {{user `ssh_password`}})|\" /tmp/ignition.jsonsudo flatcar-install -d /dev/sda -C {{user `channel_name`}} -V {{user `release_version`}} -i /tmp/ignition.json && sudo reboot", + "boot_command_suffix": "/proxmox-bootstrap-pass-auth.jsonsed -i \"s|BUILDERPASSWORDHASH|$(mkpasswd -5 {{user `ssh_password`}})|\" /tmp/ignition.jsonsudo flatcar-install -d /dev/sda -C {{user `channel_name`}} -V {{user `release_version`}} -i /tmp/ignition.json && sudo reboot", "boot_media_path": "http://{{ .HTTPIP }}:{{ .HTTPPort }}", "boot_wait": "180s", "build_name": "flatcar-{{env `FLATCAR_CHANNEL`}}-{{env `FLATCAR_VERSION`}}", @@ -23,5 +23,6 @@ "shutdown_command": "shutdown -P now", "systemd_prefix": "/etc/systemd", "sysusr_prefix": "/opt", - "sysusrlocal_prefix": "/opt" + "sysusrlocal_prefix": "/opt", + "unmount_iso": "true" } From b23c70e9fce3f78a81f8cb580915902aa3573619 Mon Sep 17 00:00:00 2001 From: Mohamed Chiheb Ben Jemaa Date: Wed, 23 Oct 2024 13:36:35 +0200 Subject: [PATCH 6/7] Add scsi controller and write docs --- .../ignition/proxmox-bootstrap-pass-auth.json | 46 ------------------- images/capi/packer/proxmox/README.md | 37 ++++++++++++++- images/capi/packer/proxmox/flatcar.json | 2 +- images/capi/packer/proxmox/packer.json.tmpl | 4 +- 4 files changed, 39 insertions(+), 50 deletions(-) delete mode 100644 images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json diff --git a/images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json b/images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json deleted file mode 100644 index 8e09b18a1c..0000000000 --- a/images/capi/packer/files/flatcar/ignition/proxmox-bootstrap-pass-auth.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "ignition": { - "config": {}, - "security": { - "tls": {} - }, - "timeouts": {}, - "version": "2.3.0" - }, - "networkd": {}, - "passwd": { - "users": [ - { - "groups": [ - "wheel", - "sudo", - "docker" - ], - "name": "builder", - "passwordHash": "BUILDERPASSWORDHASH" - } - ] - }, - "storage": {}, - "systemd": { - "units": [ - { - "enable": true, - "name": "docker.service" - }, - { - "mask": true, - "name": "update-engine.service" - }, - { - "mask": true, - "name": "locksmithd.service" - }, - { - "name": "flatcar-openstack-hostname.service", - "enabled": false, - "mask": true - } - ] - } -} diff --git a/images/capi/packer/proxmox/README.md b/images/capi/packer/proxmox/README.md index 96f45bb15d..fb6141b4a2 100644 --- a/images/capi/packer/proxmox/README.md +++ b/images/capi/packer/proxmox/README.md @@ -1,11 +1,12 @@ +## Custom Kubernetes version + To build an image using a specific version of Kubernetes use the "PACKER_FLAGS" env var like in the example below: ``` PACKER_FLAGS="--var 'kubernetes_rpm_version=1.28.3' --var 'kubernetes_semver=v1.28.3' --var 'kubernetes_series=v1.28' --var 'kubernetes_deb_version=1.28.3-1.1'" make build-proxmox-ubuntu-2204 ``` - -# ISO files +## ISO files To use existing ISO files, set the `ISO_FILE` environment variable to the path of the ISO file. For example, to use a local ISO file, set the `ISO_FILE` environment variable like this: @@ -13,3 +14,35 @@ For example, to use a local ISO file, set the `ISO_FILE` environment variable li ``` export ISO_FILE="local:iso/ubuntu-24.04.1-live-server-amd64.iso" ``` + +## Flatcar for Proxmox + +Currently, Proxmox doesn't support ignition and it's currently in-development. +* https://github.com/coreos/fedora-coreos-tracker/issues/1652 +* https://github.com/flatcar/scripts/pull/1783 + +But we do a trick to make it working on Proxmox, until the support is already released. + +We use OEM_ID `nutanix` which is an openstack provider that loads ignition from device with label `config-2`: +https://github.com/coreos/ignition/blob/main/internal/providers/nutanix/nutanix.go#L51 + +Therefore, we build an image with `OEM_ID=nutanix` so that we can provide an ISO that contain the ignition file in `/openstack/latest/user_data` +https://github.com/coreos/ignition/blob/main/internal/providers/nutanix/nutanix.go#L40C29-L40C56 + +**To build a Proxmox template for flatcar** + +```shell +export PROXMOX_URL="https://example.net:8006/api2/json" +export PROXMOX_USERNAME='root@pam!proxmox' +export PROXMOX_TOKEN="xxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx" +export PROXMOX_NODE="pve1" +export PROXMOX_ISO_POOL="local" +export PROXMOX_BRIDGE="vmbr1" +export PROXMOX_STORAGE_POOL="ceph_pool" + +## flatcar version +export FLATCAR_VERSION=4081.1.0 +export FLATCAR_CHANNEL=beta + +export OEM_ID=nutanix # make sure to choose OEM_ID=nutanix +``` diff --git a/images/capi/packer/proxmox/flatcar.json b/images/capi/packer/proxmox/flatcar.json index 4f91f16ac8..64bd2df5ff 100644 --- a/images/capi/packer/proxmox/flatcar.json +++ b/images/capi/packer/proxmox/flatcar.json @@ -1,7 +1,7 @@ { "ansible_extra_vars": "ansible_python_interpreter=/opt/bin/python oem_id={{user `oem_id`}}", "boot_command_prefix": "sudo systemctl mask sshd.socket --nowcurl -sLo /tmp/ignition.json ", - "boot_command_suffix": "/proxmox-bootstrap-pass-auth.jsonsed -i \"s|BUILDERPASSWORDHASH|$(mkpasswd -5 {{user `ssh_password`}})|\" /tmp/ignition.jsonsudo flatcar-install -d /dev/sda -C {{user `channel_name`}} -V {{user `release_version`}} -i /tmp/ignition.json && sudo reboot", + "boot_command_suffix": "/bootstrap-pass-auth.jsonsed -i \"s|BUILDERPASSWORDHASH|$(mkpasswd -5 {{user `ssh_password`}})|\" /tmp/ignition.jsonsudo flatcar-install -d /dev/sda -C {{user `channel_name`}} -V {{user `release_version`}} -i /tmp/ignition.json && sudo reboot", "boot_media_path": "http://{{ .HTTPIP }}:{{ .HTTPPort }}", "boot_wait": "180s", "build_name": "flatcar-{{env `FLATCAR_CHANNEL`}}-{{env `FLATCAR_VERSION`}}", diff --git a/images/capi/packer/proxmox/packer.json.tmpl b/images/capi/packer/proxmox/packer.json.tmpl index ee6661416f..be037a95d8 100644 --- a/images/capi/packer/proxmox/packer.json.tmpl +++ b/images/capi/packer/proxmox/packer.json.tmpl @@ -19,6 +19,7 @@ "type": "scsi" } ], + "scsi_controller": "{{user `scsi_controller`}}", "http_directory": "{{user `http_directory`}}", "insecure_skip_tls_verify": true, "iso_checksum": "{{user `iso_checksum_type`}}:{{user `iso_checksum`}}", @@ -215,6 +216,7 @@ "token": "{{env `PROXMOX_TOKEN`}}", "username": "{{env `PROXMOX_USERNAME`}}", "vlan_tag": "{{env `PROXMOX_VLAN`}}", - "vmid": "" + "vmid": "", + "scsi_controller": "virtio-scsi-pci" } } From b79e31abcf0825e5701db4e50fb332027ae4c4da Mon Sep 17 00:00:00 2001 From: Mohamed Chiheb Ben Jemaa Date: Thu, 14 Nov 2024 10:26:46 +0100 Subject: [PATCH 7/7] Flatcar for Proxmox VE --- images/capi/packer/proxmox/README.md | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/images/capi/packer/proxmox/README.md b/images/capi/packer/proxmox/README.md index fb6141b4a2..e3319b2aee 100644 --- a/images/capi/packer/proxmox/README.md +++ b/images/capi/packer/proxmox/README.md @@ -17,17 +17,11 @@ export ISO_FILE="local:iso/ubuntu-24.04.1-live-server-amd64.iso" ## Flatcar for Proxmox -Currently, Proxmox doesn't support ignition and it's currently in-development. +Currently, Proxmox support is only available in the `alpha` channel of Flatcar. +* https://www.flatcar.org/releases#alpha-release * https://github.com/coreos/fedora-coreos-tracker/issues/1652 -* https://github.com/flatcar/scripts/pull/1783 -But we do a trick to make it working on Proxmox, until the support is already released. - -We use OEM_ID `nutanix` which is an openstack provider that loads ignition from device with label `config-2`: -https://github.com/coreos/ignition/blob/main/internal/providers/nutanix/nutanix.go#L51 - -Therefore, we build an image with `OEM_ID=nutanix` so that we can provide an ISO that contain the ignition file in `/openstack/latest/user_data` -https://github.com/coreos/ignition/blob/main/internal/providers/nutanix/nutanix.go#L40C29-L40C56 +Therefore, we need to choose the right channel and version for flatcar along with `OEM_ID=proxmoxve`. **To build a Proxmox template for flatcar** @@ -41,8 +35,8 @@ export PROXMOX_BRIDGE="vmbr1" export PROXMOX_STORAGE_POOL="ceph_pool" ## flatcar version -export FLATCAR_VERSION=4081.1.0 -export FLATCAR_CHANNEL=beta +export FLATCAR_VERSION=4152.0.0 +export FLATCAR_CHANNEL=alpha -export OEM_ID=nutanix # make sure to choose OEM_ID=nutanix +export OEM_ID=proxmoxve # make sure to choose OEM_ID=proxmoxve ```