From 83d90da13e2c8c3d6400eee464353b2170e17546 Mon Sep 17 00:00:00 2001 From: Steven Kreitzer Date: Sat, 16 Nov 2024 06:59:47 -0600 Subject: [PATCH] feat(talos): adding new disk --- README.md | 22 +++---- .../cloudnative-pg/cluster/cluster.yaml | 18 +++--- kubernetes/apps/databases/kustomization.yaml | 6 +- kubernetes/apps/home/kustomization.yaml | 16 ++--- kubernetes/apps/media/kustomization.yaml | 28 ++++----- kubernetes/apps/monitoring/kustomization.yaml | 20 +++---- .../openebs/app/helmrelease.yaml | 3 + .../rook-ceph/cluster/helmrelease.yaml | 29 +--------- kubernetes/apps/security/kustomization.yaml | 4 +- talos/README.md | 11 ++++ talos/talconfig.yaml | 58 +++++++++++++------ 11 files changed, 111 insertions(+), 104 deletions(-) create mode 100644 talos/README.md diff --git a/README.md b/README.md index bf00de3116..b846b4e94e 100644 --- a/README.md +++ b/README.md @@ -127,17 +127,17 @@ I have two instances of `external-dns` running in my cluster. The private DNS in rack -| Device | Count | OS Disk Size | Data Disk Size | Ram | Operating System | Purpose | -|---------------------------|-------|-----------------|-----------------------------|------|------------------|-----------------| -| MS-01 (i9-13900H) | 3 | 1.92TB M.2 NVMe | 3.84TB U.2 NVMe (rook-ceph) | 96GB | Talos | Kubernetes | -| USW Pro Max 24 PoE | 1 | - | - | - | UniFi OS | 2.5G PoE Switch | -| USW Pro Aggregation | 1 | - | - | - | UniFi OS | 10G/25G Switch | -| USP PDU Pro | 1 | - | - | - | UniFi OS | PDU | -| UDM Pro Max | 1 | - | 2x16TB HDD | - | UniFi OS | Router & NVR | -| Synology NAS RS1221+ | 1 | - | 8x22TB HDD | 32GB | - | NFS | -| APC SMT15000RM2UNC | 1 | - | - | - | - | UPS | -| TESmart 8 Port KVM Switch | 1 | - | - | - | - | KVM | -| PiKVM (RasPi 4) | 1 | 64GB (SD) | - | 4GB | PiKVM (Arch) | KVM | +| Device | Count | OS Disk Size | Data Disk Size | Ram | Operating System | Purpose | +|---------------------------|-------|-----------------|------------------------------|------|------------------|-------------------------| +| MS-01 (i9-13900H) | 3 | 1.92TB M.2 NVMe | 3.84TB U.2 & 1.92TB M.2 NVMe | 96GB | Talos | Kubernetes | +| Synology NAS RS1221+ | 1 | - | 8x22TB HDD | 32GB | - | NFS | +| PiKVM (RasPi 4) | 1 | 64GB (SD) | - | 4GB | PiKVM | KVM | +| TESmart 8 Port KVM Switch | 1 | - | - | - | - | Network KVM (for PiKVM) | +| UniFi UDM Pro Max | 1 | - | 2x16TB HDD | - | UniFi OS | Router & NVR | +| UniFi USW Pro Aggregation | 1 | - | - | - | UniFi OS | 10G/25Gb Core Switch | +| UniFi USW Pro Max 24 PoE | 1 | - | - | - | UniFi OS | 2.5Gb PoE Switch | +| UniFi USP PDU Pro | 1 | - | - | - | UniFi OS | PDU | +| APC SMT15000RM2UNC | 1 | - | - | - | - | UPS | --- ## ⭐ Stargazers diff --git a/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml b/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml index 9ca1449bfd..e3e654efc6 100644 --- a/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml +++ b/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml @@ -42,7 +42,7 @@ spec: endpointURL: https://ba20be062ff7623f9d887afa37183c8f.r2.cloudflarestorage.com # Note: serverName version needs to be inclemented # when recovering from an existing cnpg cluster - serverName: postgres-v7 + serverName: postgres-v8 s3Credentials: accessKeyId: name: cloudnative-pg-secret @@ -52,12 +52,12 @@ spec: key: AWS_SECRET_ACCESS_KEY # # Note: previousCluster needs to be set to the name of the previous # # cluster when recovering from an existing cnpg cluster - # bootstrap: - # recovery: - # source: &previousCluster postgres-v7 + bootstrap: + recovery: + source: &previousCluster postgres-v7 # # Note: externalClusters is needed when recovering from an existing cnpg cluster - # externalClusters: - # - name: *previousCluster - # barmanObjectStore: - # <<: *barmanObjectStore - # serverName: *previousCluster + externalClusters: + - name: *previousCluster + barmanObjectStore: + <<: *barmanObjectStore + serverName: *previousCluster diff --git a/kubernetes/apps/databases/kustomization.yaml b/kubernetes/apps/databases/kustomization.yaml index afd6621451..964fe42223 100644 --- a/kubernetes/apps/databases/kustomization.yaml +++ b/kubernetes/apps/databases/kustomization.yaml @@ -3,6 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./cloudnative-pg/ks.yaml - - ./dragonfly/ks.yaml - - ./emqx/ks.yaml + # - ./cloudnative-pg/ks.yaml + # - ./dragonfly/ks.yaml + # - ./emqx/ks.yaml diff --git a/kubernetes/apps/home/kustomization.yaml b/kubernetes/apps/home/kustomization.yaml index 2a9938a680..456870616d 100644 --- a/kubernetes/apps/home/kustomization.yaml +++ b/kubernetes/apps/home/kustomization.yaml @@ -3,11 +3,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./atuin/ks.yaml - - ./go2rtc/ks.yaml - - ./hajimari/ks.yaml - - ./home-assistant/ks.yaml - - ./miniflux/ks.yaml - - ./node-red/ks.yaml - - ./thelounge/ks.yaml - - ./zigbee2mqtt/ks.yaml + # - ./atuin/ks.yaml + # - ./go2rtc/ks.yaml + # - ./hajimari/ks.yaml + # - ./home-assistant/ks.yaml + # - ./miniflux/ks.yaml + # - ./node-red/ks.yaml + # - ./thelounge/ks.yaml + # - ./zigbee2mqtt/ks.yaml diff --git a/kubernetes/apps/media/kustomization.yaml b/kubernetes/apps/media/kustomization.yaml index 0875dd8342..5cd8cd6f5f 100644 --- a/kubernetes/apps/media/kustomization.yaml +++ b/kubernetes/apps/media/kustomization.yaml @@ -3,17 +3,17 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./autobrr/ks.yaml - - ./bazarr/ks.yaml - - ./cross-seed/ks.yaml - - ./omegabrr/ks.yaml - - ./overseerr/ks.yaml - - ./plex/ks.yaml - - ./prowlarr/ks.yaml - - ./qbittorrent/ks.yaml - - ./radarr/ks.yaml - - ./recyclarr/ks.yaml - - ./sabnzbd/ks.yaml - - ./sonarr/ks.yaml - - ./tautulli/ks.yaml - - ./unpackerr/ks.yaml + # - ./autobrr/ks.yaml + # - ./bazarr/ks.yaml + # - ./cross-seed/ks.yaml + # - ./omegabrr/ks.yaml + # - ./overseerr/ks.yaml + # - ./plex/ks.yaml + # - ./prowlarr/ks.yaml + # - ./qbittorrent/ks.yaml + # - ./radarr/ks.yaml + # - ./recyclarr/ks.yaml + # - ./sabnzbd/ks.yaml + # - ./sonarr/ks.yaml + # - ./tautulli/ks.yaml + # - ./unpackerr/ks.yaml diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml index e495bc9596..767af52c4a 100644 --- a/kubernetes/apps/monitoring/kustomization.yaml +++ b/kubernetes/apps/monitoring/kustomization.yaml @@ -3,13 +3,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./gatus/ks.yaml - - ./grafana/ks.yaml - - ./karma/ks.yaml - - ./kromgo/ks.yaml - - ./loki/ks.yaml - - ./thanos/ks.yaml - - ./unpoller/ks.yaml - - ./kube-prometheus-stack/ks.yaml - - ./vector/ks.yaml - - ./exporters + # - ./gatus/ks.yaml + # - ./grafana/ks.yaml + # - ./karma/ks.yaml + # - ./kromgo/ks.yaml + # - ./loki/ks.yaml + # - ./thanos/ks.yaml + # - ./unpoller/ks.yaml + # - ./kube-prometheus-stack/ks.yaml + # - ./vector/ks.yaml + # - ./exporters diff --git a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml index a2a514c2d1..dd0e510799 100644 --- a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml +++ b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml @@ -27,7 +27,10 @@ spec: image: registry: quay.io/ hostpathClass: + enabled: true name: openebs-hostpath + isDefaultClass: false + basePath: /var/mnt/extra/openebs/local helperPod: image: registry: quay.io/ diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index 161ae8981a..e97bfb27ce 100644 --- a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -76,36 +76,9 @@ spec: operator: Exists mon: <<: *placement - resources: - mgr: - requests: - cpu: 100m - memory: 512Mi - limits: - memory: 2Gi - mon: - requests: - cpu: 50m - memory: 512Mi - limits: - memory: 1Gi - osd: - requests: - cpu: 500m - memory: 2Gi - limits: - memory: 8Gi - mgr-sidecar: - requests: - cpu: 50m - memory: 128Mi - limits: - memory: 256Mi storage: useAllNodes: true useAllDevices: false - config: - osdsPerDevice: "1" devicePathFilter: /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_.* cephBlockPools: - name: ceph-blockpool @@ -121,7 +94,7 @@ spec: allowVolumeExpansion: true parameters: imageFormat: "2" - imageFeatures: layering + imageFeatures: layering,striping,exclusive-lock,object-map,fast-diff,deep-flatten csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner diff --git a/kubernetes/apps/security/kustomization.yaml b/kubernetes/apps/security/kustomization.yaml index 65f2123360..2c8b675574 100644 --- a/kubernetes/apps/security/kustomization.yaml +++ b/kubernetes/apps/security/kustomization.yaml @@ -3,7 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./namespace.yaml - - ./authelia/ks.yaml + # - ./authelia/ks.yaml - ./external-secrets/ks.yaml - - ./glauth/ks.yaml + # - ./glauth/ks.yaml - ./onepassword-connect/ks.yaml diff --git a/talos/README.md b/talos/README.md new file mode 100644 index 0000000000..c97b6b3f79 --- /dev/null +++ b/talos/README.md @@ -0,0 +1,11 @@ +# MS-01 Configuration + +MS-01 Barebones i9-13900H\ +Crucial 96GB DDR5-5600\ +Google Coral TPU M.2 A+E Key + +# MS-01 Storage Configuration + +Samsung PM9A3 3.84TB U.2 (PCIe 4.0 x4) [rook-ceph]\ +Samsung PM9A3 1.92TB M.2 (PCIe 3.0 x4) [openebs]\ +Samsung PM9A3 1.92TB M.2 (PCIe 3.0 x2) [talos] diff --git a/talos/talconfig.yaml b/talos/talconfig.yaml index 021312ff34..25a1d260c0 100644 --- a/talos/talconfig.yaml +++ b/talos/talconfig.yaml @@ -26,9 +26,13 @@ cniConfig: nodes: - hostname: m0.k8s.internal ipAddress: 192.168.10.10 - installDiskSelector: - serial: 052410221DA7 controlPlane: true + installDiskSelector: + serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2) + machineDisks: + - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4) + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -77,12 +81,18 @@ nodes: routes: - network: 169.254.255.12/32 metric: 2048 + nodeAnnotations: + network.rook.io/mon-ip: 169.254.255.10 - hostname: m1.k8s.internal ipAddress: 192.168.10.11 - installDiskSelector: - serial: 052410221DA8 controlPlane: true + installDiskSelector: + serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2) + machineDisks: + - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4) + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -131,12 +141,18 @@ nodes: routes: - network: 169.254.255.12/32 metric: 2048 + nodeAnnotations: + network.rook.io/mon-ip: 169.254.255.11 - hostname: m2.k8s.internal ipAddress: 192.168.10.12 - installDiskSelector: - serial: 052410221B9A controlPlane: true + installDiskSelector: + serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2) + machineDisks: + - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4) + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -185,6 +201,8 @@ nodes: routes: - network: 169.254.255.11/32 metric: 2048 + nodeAnnotations: + network.rook.io/mon-ip: 169.254.255.12 controlPlane: nodeLabels: @@ -194,6 +212,7 @@ controlPlane: schematic: customization: extraKernelArgs: + - intel_idle.max_cstate=0 - intel_iommu=on - iommu=pt - mitigations=off @@ -255,9 +274,11 @@ controlPlane: kubelet: extraArgs: rotate-server-certificates: true + extraConfig: + maxPods: 150 extraMounts: - - destination: /var/openebs/local - source: /var/openebs/local + - destination: /var/mnt/extra + source: /var/mnt/extra type: bind options: - bind @@ -300,9 +321,8 @@ controlPlane: content: | [ NFSMount_Global_Options ] nfsvers=4.1 - async=True hard=True - nconnect=8 + nconnect=16 noatime=True # Configure NTP @@ -344,6 +364,7 @@ controlPlane: machine: kernel: modules: + - name: nbd - name: thunderbolt - name: thunderbolt_net @@ -365,11 +386,19 @@ controlPlane: - |- cluster: allowSchedulingOnMasters: true + apiServer: + extraArgs: + enable-aggregator-routing: "true" controllerManager: extraArgs: bind-address: 0.0.0.0 coreDNS: disabled: true + etcd: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 + advertisedSubnets: + - 169.254.255.0/24 proxy: disabled: true scheduler: @@ -389,15 +418,6 @@ controlPlane: topologyKey: kubernetes.io/hostname whenUnsatisfiable: ScheduleAnyway - # ETCD configuration - - |- - cluster: - etcd: - extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 - advertisedSubnets: - - 192.168.10.0/24 - # Disable default API server admission plugins. - |- - op: remove