From fc4d4ab08227de73c5362d4c5bc0d2cc0ba3e9e0 Mon Sep 17 00:00:00 2001 From: Steven Kreitzer Date: Sat, 16 Nov 2024 06:59:47 -0600 Subject: [PATCH] feat(talos): adding new disk --- README.md | 22 +++---- .../cloudnative-pg/cluster/cluster.yaml | 18 +++--- .../openebs/app/helmrelease.yaml | 3 + .../rook-ceph/cluster/helmrelease.yaml | 29 +--------- talos/README.md | 11 ++++ talos/talconfig.yaml | 58 +++++++++++++------ 6 files changed, 74 insertions(+), 67 deletions(-) create mode 100644 talos/README.md diff --git a/README.md b/README.md index bf00de311..b846b4e94 100644 --- a/README.md +++ b/README.md @@ -127,17 +127,17 @@ I have two instances of `external-dns` running in my cluster. The private DNS in rack -| Device | Count | OS Disk Size | Data Disk Size | Ram | Operating System | Purpose | -|---------------------------|-------|-----------------|-----------------------------|------|------------------|-----------------| -| MS-01 (i9-13900H) | 3 | 1.92TB M.2 NVMe | 3.84TB U.2 NVMe (rook-ceph) | 96GB | Talos | Kubernetes | -| USW Pro Max 24 PoE | 1 | - | - | - | UniFi OS | 2.5G PoE Switch | -| USW Pro Aggregation | 1 | - | - | - | UniFi OS | 10G/25G Switch | -| USP PDU Pro | 1 | - | - | - | UniFi OS | PDU | -| UDM Pro Max | 1 | - | 2x16TB HDD | - | UniFi OS | Router & NVR | -| Synology NAS RS1221+ | 1 | - | 8x22TB HDD | 32GB | - | NFS | -| APC SMT15000RM2UNC | 1 | - | - | - | - | UPS | -| TESmart 8 Port KVM Switch | 1 | - | - | - | - | KVM | -| PiKVM (RasPi 4) | 1 | 64GB (SD) | - | 4GB | PiKVM (Arch) | KVM | +| Device | Count | OS Disk Size | Data Disk Size | Ram | Operating System | Purpose | +|---------------------------|-------|-----------------|------------------------------|------|------------------|-------------------------| +| MS-01 (i9-13900H) | 3 | 1.92TB M.2 NVMe | 3.84TB U.2 & 1.92TB M.2 NVMe | 96GB | Talos | Kubernetes | +| Synology NAS RS1221+ | 1 | - | 8x22TB HDD | 32GB | - | NFS | +| PiKVM (RasPi 4) | 1 | 64GB (SD) | - | 4GB | PiKVM | KVM | +| TESmart 8 Port KVM Switch | 1 | - | - | - | - | Network KVM (for PiKVM) | +| UniFi UDM Pro Max | 1 | - | 2x16TB HDD | - | UniFi OS | Router & NVR | +| UniFi USW Pro Aggregation | 1 | - | - | - | UniFi OS | 10G/25Gb Core Switch | +| UniFi USW Pro Max 24 PoE | 1 | - | - | - | UniFi OS | 2.5Gb PoE Switch | +| UniFi USP PDU Pro | 1 | - | - | - | UniFi OS | PDU | +| APC SMT15000RM2UNC | 1 | - | - | - | - | UPS | --- ## ⭐ Stargazers diff --git a/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml b/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml index 9ca1449bf..e3e654efc 100644 --- a/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml +++ b/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml @@ -42,7 +42,7 @@ spec: endpointURL: https://ba20be062ff7623f9d887afa37183c8f.r2.cloudflarestorage.com # Note: serverName version needs to be inclemented # when recovering from an existing cnpg cluster - serverName: postgres-v7 + serverName: postgres-v8 s3Credentials: accessKeyId: name: cloudnative-pg-secret @@ -52,12 +52,12 @@ spec: key: AWS_SECRET_ACCESS_KEY # # Note: previousCluster needs to be set to the name of the previous # # cluster when recovering from an existing cnpg cluster - # bootstrap: - # recovery: - # source: &previousCluster postgres-v7 + bootstrap: + recovery: + source: &previousCluster postgres-v7 # # Note: externalClusters is needed when recovering from an existing cnpg cluster - # externalClusters: - # - name: *previousCluster - # barmanObjectStore: - # <<: *barmanObjectStore - # serverName: *previousCluster + externalClusters: + - name: *previousCluster + barmanObjectStore: + <<: *barmanObjectStore + serverName: *previousCluster diff --git a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml index a2a514c2d..dd0e51079 100644 --- a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml +++ b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml @@ -27,7 +27,10 @@ spec: image: registry: quay.io/ hostpathClass: + enabled: true name: openebs-hostpath + isDefaultClass: false + basePath: /var/mnt/extra/openebs/local helperPod: image: registry: quay.io/ diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml index 161ae8981..e97bfb27c 100644 --- a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml +++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -76,36 +76,9 @@ spec: operator: Exists mon: <<: *placement - resources: - mgr: - requests: - cpu: 100m - memory: 512Mi - limits: - memory: 2Gi - mon: - requests: - cpu: 50m - memory: 512Mi - limits: - memory: 1Gi - osd: - requests: - cpu: 500m - memory: 2Gi - limits: - memory: 8Gi - mgr-sidecar: - requests: - cpu: 50m - memory: 128Mi - limits: - memory: 256Mi storage: useAllNodes: true useAllDevices: false - config: - osdsPerDevice: "1" devicePathFilter: /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_.* cephBlockPools: - name: ceph-blockpool @@ -121,7 +94,7 @@ spec: allowVolumeExpansion: true parameters: imageFormat: "2" - imageFeatures: layering + imageFeatures: layering,striping,exclusive-lock,object-map,fast-diff,deep-flatten csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner diff --git a/talos/README.md b/talos/README.md new file mode 100644 index 000000000..c97b6b3f7 --- /dev/null +++ b/talos/README.md @@ -0,0 +1,11 @@ +# MS-01 Configuration + +MS-01 Barebones i9-13900H\ +Crucial 96GB DDR5-5600\ +Google Coral TPU M.2 A+E Key + +# MS-01 Storage Configuration + +Samsung PM9A3 3.84TB U.2 (PCIe 4.0 x4) [rook-ceph]\ +Samsung PM9A3 1.92TB M.2 (PCIe 3.0 x4) [openebs]\ +Samsung PM9A3 1.92TB M.2 (PCIe 3.0 x2) [talos] diff --git a/talos/talconfig.yaml b/talos/talconfig.yaml index 021312ff3..25a1d260c 100644 --- a/talos/talconfig.yaml +++ b/talos/talconfig.yaml @@ -26,9 +26,13 @@ cniConfig: nodes: - hostname: m0.k8s.internal ipAddress: 192.168.10.10 - installDiskSelector: - serial: 052410221DA7 controlPlane: true + installDiskSelector: + serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2) + machineDisks: + - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4) + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -77,12 +81,18 @@ nodes: routes: - network: 169.254.255.12/32 metric: 2048 + nodeAnnotations: + network.rook.io/mon-ip: 169.254.255.10 - hostname: m1.k8s.internal ipAddress: 192.168.10.11 - installDiskSelector: - serial: 052410221DA8 controlPlane: true + installDiskSelector: + serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2) + machineDisks: + - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4) + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -131,12 +141,18 @@ nodes: routes: - network: 169.254.255.12/32 metric: 2048 + nodeAnnotations: + network.rook.io/mon-ip: 169.254.255.11 - hostname: m2.k8s.internal ipAddress: 192.168.10.12 - installDiskSelector: - serial: 052410221B9A controlPlane: true + installDiskSelector: + serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2) + machineDisks: + - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4) + partitions: + - mountpoint: /var/mnt/extra networkInterfaces: - interface: bond0 bond: @@ -185,6 +201,8 @@ nodes: routes: - network: 169.254.255.11/32 metric: 2048 + nodeAnnotations: + network.rook.io/mon-ip: 169.254.255.12 controlPlane: nodeLabels: @@ -194,6 +212,7 @@ controlPlane: schematic: customization: extraKernelArgs: + - intel_idle.max_cstate=0 - intel_iommu=on - iommu=pt - mitigations=off @@ -255,9 +274,11 @@ controlPlane: kubelet: extraArgs: rotate-server-certificates: true + extraConfig: + maxPods: 150 extraMounts: - - destination: /var/openebs/local - source: /var/openebs/local + - destination: /var/mnt/extra + source: /var/mnt/extra type: bind options: - bind @@ -300,9 +321,8 @@ controlPlane: content: | [ NFSMount_Global_Options ] nfsvers=4.1 - async=True hard=True - nconnect=8 + nconnect=16 noatime=True # Configure NTP @@ -344,6 +364,7 @@ controlPlane: machine: kernel: modules: + - name: nbd - name: thunderbolt - name: thunderbolt_net @@ -365,11 +386,19 @@ controlPlane: - |- cluster: allowSchedulingOnMasters: true + apiServer: + extraArgs: + enable-aggregator-routing: "true" controllerManager: extraArgs: bind-address: 0.0.0.0 coreDNS: disabled: true + etcd: + extraArgs: + listen-metrics-urls: http://0.0.0.0:2381 + advertisedSubnets: + - 169.254.255.0/24 proxy: disabled: true scheduler: @@ -389,15 +418,6 @@ controlPlane: topologyKey: kubernetes.io/hostname whenUnsatisfiable: ScheduleAnyway - # ETCD configuration - - |- - cluster: - etcd: - extraArgs: - listen-metrics-urls: http://0.0.0.0:2381 - advertisedSubnets: - - 192.168.10.0/24 - # Disable default API server admission plugins. - |- - op: remove