feat(talos): adding new disk

buroa · Nov 18, 2024 · 83d90da · 83d90da
1 parent 48cc031
commit 83d90da
Show file tree

Hide file tree

Showing 11 changed files with 111 additions and 104 deletions.
diff --git a/README.md b/README.md
@@ -127,17 +127,17 @@ I have two instances of `external-dns` running in my cluster. The private DNS in
   <img src="https://github.com/user-attachments/assets/43bd0ca8-a1a8-49d5-9b9a-04fbdcecdd3f" align="center" alt="rack"/>
 </details>
 
-| Device                    | Count | OS Disk Size    | Data Disk Size              | Ram  | Operating System | Purpose         |
-|---------------------------|-------|-----------------|-----------------------------|------|------------------|-----------------|
-| MS-01 (i9-13900H)         | 3     | 1.92TB M.2 NVMe | 3.84TB U.2 NVMe (rook-ceph) | 96GB | Talos            | Kubernetes      |
-| USW Pro Max 24 PoE        | 1     | -               | -                           | -    | UniFi OS         | 2.5G PoE Switch |
-| USW Pro Aggregation       | 1     | -               | -                           | -    | UniFi OS         | 10G/25G Switch  |
-| USP PDU Pro               | 1     | -               | -                           | -    | UniFi OS         | PDU             |
-| UDM Pro Max               | 1     | -               | 2x16TB HDD                  | -    | UniFi OS         | Router & NVR    |
-| Synology NAS RS1221+      | 1     | -               | 8x22TB HDD                  | 32GB | -                | NFS             |
-| APC SMT15000RM2UNC        | 1     | -               | -                           | -    | -                | UPS             |
-| TESmart 8 Port KVM Switch | 1     | -               | -                           | -    | -                | KVM             |
-| PiKVM (RasPi 4)           | 1     | 64GB (SD)       | -                           | 4GB  | PiKVM (Arch)     | KVM             |
+| Device                    | Count | OS Disk Size    | Data Disk Size               | Ram  | Operating System | Purpose                 |
+|---------------------------|-------|-----------------|------------------------------|------|------------------|-------------------------|
+| MS-01 (i9-13900H)         | 3     | 1.92TB M.2 NVMe | 3.84TB U.2 & 1.92TB M.2 NVMe | 96GB | Talos            | Kubernetes              |
+| Synology NAS RS1221+      | 1     | -               | 8x22TB HDD                   | 32GB | -                | NFS                     |
+| PiKVM (RasPi 4)           | 1     | 64GB (SD)       | -                            | 4GB  | PiKVM            | KVM                     |
+| TESmart 8 Port KVM Switch | 1     | -               | -                            | -    | -                | Network KVM (for PiKVM) |
+| UniFi UDM Pro Max         | 1     | -               | 2x16TB HDD                   | -    | UniFi OS         | Router & NVR            |
+| UniFi USW Pro Aggregation | 1     | -               | -                            | -    | UniFi OS         | 10G/25Gb Core Switch    |
+| UniFi USW Pro Max 24 PoE  | 1     | -               | -                            | -    | UniFi OS         | 2.5Gb PoE Switch        |
+| UniFi USP PDU Pro         | 1     | -               | -                            | -    | UniFi OS         | PDU                     |
+| APC SMT15000RM2UNC        | 1     | -               | -                            | -    | -                | UPS                     |
 ---
 
 ## ⭐ Stargazers

diff --git a/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml b/kubernetes/apps/databases/cloudnative-pg/cluster/cluster.yaml
@@ -42,7 +42,7 @@ spec:
       endpointURL: https://ba20be062ff7623f9d887afa37183c8f.r2.cloudflarestorage.com
       # Note: serverName version needs to be inclemented
       # when recovering from an existing cnpg cluster
-      serverName: postgres-v7
+      serverName: postgres-v8
       s3Credentials:
         accessKeyId:
           name: cloudnative-pg-secret
@@ -52,12 +52,12 @@ spec:
           key: AWS_SECRET_ACCESS_KEY
   # # Note: previousCluster needs to be set to the name of the previous
   # # cluster when recovering from an existing cnpg cluster
-  # bootstrap:
-  #   recovery:
-  #     source: &previousCluster postgres-v7
+  bootstrap:
+    recovery:
+      source: &previousCluster postgres-v7
   # # Note: externalClusters is needed when recovering from an existing cnpg cluster
-  # externalClusters:
-  #   - name: *previousCluster
-  #     barmanObjectStore:
-  #       <<: *barmanObjectStore
-  #       serverName: *previousCluster
+  externalClusters:
+    - name: *previousCluster
+      barmanObjectStore:
+        <<: *barmanObjectStore
+        serverName: *previousCluster
diff --git a/kubernetes/apps/databases/kustomization.yaml b/kubernetes/apps/databases/kustomization.yaml
@@ -3,6 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./cloudnative-pg/ks.yaml
-  - ./dragonfly/ks.yaml
-  - ./emqx/ks.yaml
+  # - ./cloudnative-pg/ks.yaml
+  # - ./dragonfly/ks.yaml
+  # - ./emqx/ks.yaml
diff --git a/kubernetes/apps/home/kustomization.yaml b/kubernetes/apps/home/kustomization.yaml
@@ -3,11 +3,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./atuin/ks.yaml
-  - ./go2rtc/ks.yaml
-  - ./hajimari/ks.yaml
-  - ./home-assistant/ks.yaml
-  - ./miniflux/ks.yaml
-  - ./node-red/ks.yaml
-  - ./thelounge/ks.yaml
-  - ./zigbee2mqtt/ks.yaml
+  # - ./atuin/ks.yaml
+  # - ./go2rtc/ks.yaml
+  # - ./hajimari/ks.yaml
+  # - ./home-assistant/ks.yaml
+  # - ./miniflux/ks.yaml
+  # - ./node-red/ks.yaml
+  # - ./thelounge/ks.yaml
+  # - ./zigbee2mqtt/ks.yaml
diff --git a/kubernetes/apps/media/kustomization.yaml b/kubernetes/apps/media/kustomization.yaml
@@ -3,17 +3,17 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./autobrr/ks.yaml
-  - ./bazarr/ks.yaml
-  - ./cross-seed/ks.yaml
-  - ./omegabrr/ks.yaml
-  - ./overseerr/ks.yaml
-  - ./plex/ks.yaml
-  - ./prowlarr/ks.yaml
-  - ./qbittorrent/ks.yaml
-  - ./radarr/ks.yaml
-  - ./recyclarr/ks.yaml
-  - ./sabnzbd/ks.yaml
-  - ./sonarr/ks.yaml
-  - ./tautulli/ks.yaml
-  - ./unpackerr/ks.yaml
+  # - ./autobrr/ks.yaml
+  # - ./bazarr/ks.yaml
+  # - ./cross-seed/ks.yaml
+  # - ./omegabrr/ks.yaml
+  # - ./overseerr/ks.yaml
+  # - ./plex/ks.yaml
+  # - ./prowlarr/ks.yaml
+  # - ./qbittorrent/ks.yaml
+  # - ./radarr/ks.yaml
+  # - ./recyclarr/ks.yaml
+  # - ./sabnzbd/ks.yaml
+  # - ./sonarr/ks.yaml
+  # - ./tautulli/ks.yaml
+  # - ./unpackerr/ks.yaml
diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml
@@ -3,13 +3,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./gatus/ks.yaml
-  - ./grafana/ks.yaml
-  - ./karma/ks.yaml
-  - ./kromgo/ks.yaml
-  - ./loki/ks.yaml
-  - ./thanos/ks.yaml
-  - ./unpoller/ks.yaml
-  - ./kube-prometheus-stack/ks.yaml
-  - ./vector/ks.yaml
-  - ./exporters
+  # - ./gatus/ks.yaml
+  # - ./grafana/ks.yaml
+  # - ./karma/ks.yaml
+  # - ./kromgo/ks.yaml
+  # - ./loki/ks.yaml
+  # - ./thanos/ks.yaml
+  # - ./unpoller/ks.yaml
+  # - ./kube-prometheus-stack/ks.yaml
+  # - ./vector/ks.yaml
+  # - ./exporters
diff --git a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml
@@ -27,7 +27,10 @@ spec:
         image:
           registry: quay.io/
       hostpathClass:
+        enabled: true
         name: openebs-hostpath
+        isDefaultClass: false
+        basePath: /var/mnt/extra/openebs/local
       helperPod:
         image:
           registry: quay.io/

diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml
@@ -76,36 +76,9 @@ spec:
                       operator: Exists
         mon:
           <<: *placement
-      resources:
-        mgr:
-          requests:
-            cpu: 100m
-            memory: 512Mi
-          limits:
-            memory: 2Gi
-        mon:
-          requests:
-            cpu: 50m
-            memory: 512Mi
-          limits:
-            memory: 1Gi
-        osd:
-          requests:
-            cpu: 500m
-            memory: 2Gi
-          limits:
-            memory: 8Gi
-        mgr-sidecar:
-          requests:
-            cpu: 50m
-            memory: 128Mi
-          limits:
-            memory: 256Mi
       storage:
         useAllNodes: true
         useAllDevices: false
-        config:
-          osdsPerDevice: "1"
         devicePathFilter: /dev/disk/by-id/nvme-SAMSUNG_MZQL23T8HCLS-00A07_.*
     cephBlockPools:
       - name: ceph-blockpool
@@ -121,7 +94,7 @@ spec:
           allowVolumeExpansion: true
           parameters:
             imageFormat: "2"
-            imageFeatures: layering
+            imageFeatures: layering,striping,exclusive-lock,object-map,fast-diff,deep-flatten
             csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
             csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
             csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner

diff --git a/kubernetes/apps/security/kustomization.yaml b/kubernetes/apps/security/kustomization.yaml
@@ -3,7 +3,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./namespace.yaml
-  - ./authelia/ks.yaml
+  # - ./authelia/ks.yaml
   - ./external-secrets/ks.yaml
-  - ./glauth/ks.yaml
+  # - ./glauth/ks.yaml
   - ./onepassword-connect/ks.yaml
diff --git a/talos/README.md b/talos/README.md
@@ -0,0 +1,11 @@
+# MS-01 Configuration
+
+MS-01 Barebones i9-13900H\
+Crucial 96GB DDR5-5600\
+Google Coral TPU M.2 A+E Key
+
+# MS-01 Storage Configuration
+
+Samsung PM9A3 3.84TB U.2 (PCIe 4.0 x4) [rook-ceph]\
+Samsung PM9A3 1.92TB M.2 (PCIe 3.0 x4) [openebs]\
+Samsung PM9A3 1.92TB M.2 (PCIe 3.0 x2) [talos]
diff --git a/talos/talconfig.yaml b/talos/talconfig.yaml
@@ -26,9 +26,13 @@ cniConfig:
 nodes:
   - hostname: m0.k8s.internal
     ipAddress: 192.168.10.10
-    installDiskSelector:
-      serial: 052410221DA7
     controlPlane: true
+    installDiskSelector:
+      serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2)
+    machineDisks:
+      - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4)
+        partitions:
+          - mountpoint: /var/mnt/extra
     networkInterfaces:
       - interface: bond0
         bond:
@@ -77,12 +81,18 @@ nodes:
         routes:
           - network: 169.254.255.12/32
             metric: 2048
+    nodeAnnotations:
+      network.rook.io/mon-ip: 169.254.255.10
 
   - hostname: m1.k8s.internal
     ipAddress: 192.168.10.11
-    installDiskSelector:
-      serial: 052410221DA8
     controlPlane: true
+    installDiskSelector:
+      serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2)
+    machineDisks:
+      - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4)
+        partitions:
+          - mountpoint: /var/mnt/extra
     networkInterfaces:
       - interface: bond0
         bond:
@@ -131,12 +141,18 @@ nodes:
         routes:
           - network: 169.254.255.12/32
             metric: 2048
+    nodeAnnotations:
+      network.rook.io/mon-ip: 169.254.255.11
 
   - hostname: m2.k8s.internal
     ipAddress: 192.168.10.12
-    installDiskSelector:
-      serial: 052410221B9A
     controlPlane: true
+    installDiskSelector:
+      serial: XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x2)
+    machineDisks:
+      - device: /dev/disk/by-id/nvme-SAMSUNG_MZ1L21T9HCLS-00A07_XYZ # TODO: Samsung PM9A3 1.92TB NVMe PCIe 4.0x4 M.2 22110 (PCIe 3.0 x4)
+        partitions:
+          - mountpoint: /var/mnt/extra
     networkInterfaces:
       - interface: bond0
         bond:
@@ -185,6 +201,8 @@ nodes:
         routes:
           - network: 169.254.255.11/32
             metric: 2048
+    nodeAnnotations:
+      network.rook.io/mon-ip: 169.254.255.12
 
 controlPlane:
   nodeLabels:
@@ -194,6 +212,7 @@ controlPlane:
   schematic:
     customization:
       extraKernelArgs:
+        - intel_idle.max_cstate=0
         - intel_iommu=on
         - iommu=pt
         - mitigations=off
@@ -255,9 +274,11 @@ controlPlane:
         kubelet:
           extraArgs:
             rotate-server-certificates: true
+          extraConfig:
+            maxPods: 150
           extraMounts:
-            - destination: /var/openebs/local
-              source: /var/openebs/local
+            - destination: /var/mnt/extra
+              source: /var/mnt/extra
               type: bind
               options:
                 - bind
@@ -300,9 +321,8 @@ controlPlane:
             content: |
               [ NFSMount_Global_Options ]
               nfsvers=4.1
-              async=True
               hard=True
-              nconnect=8
+              nconnect=16
               noatime=True
 
     # Configure NTP
@@ -344,6 +364,7 @@ controlPlane:
       machine:
         kernel:
           modules:
+            - name: nbd
             - name: thunderbolt
             - name: thunderbolt_net
 
@@ -365,11 +386,19 @@ controlPlane:
     - |-
       cluster:
         allowSchedulingOnMasters: true
+        apiServer:
+          extraArgs:
+            enable-aggregator-routing: "true"
         controllerManager:
           extraArgs:
             bind-address: 0.0.0.0
         coreDNS:
           disabled: true
+        etcd:
+          extraArgs:
+            listen-metrics-urls: http://0.0.0.0:2381
+          advertisedSubnets:
+            - 169.254.255.0/24
         proxy:
           disabled: true
         scheduler:
@@ -389,15 +418,6 @@ controlPlane:
                           topologyKey: kubernetes.io/hostname
                           whenUnsatisfiable: ScheduleAnyway
 
-    # ETCD configuration
-    - |-
-      cluster:
-        etcd:
-          extraArgs:
-            listen-metrics-urls: http://0.0.0.0:2381
-          advertisedSubnets:
-            - 192.168.10.0/24
-
     # Disable default API server admission plugins.
     - |-
       - op: remove