Merge pull request #501 from ricsanfre/flux

Migration to FluxCD
ricsanfre · Oct 7, 2024 · 6c7e9f3 · 6c7e9f3
2 parents 7ff5ad6 + 8e083b5
commit 6c7e9f3
Show file tree

Hide file tree

Showing 866 changed files with 23,235 additions and 4,094 deletions.
diff --git a/README.md b/README.md
@@ -10,21 +10,21 @@
     <td><img src="docs/assets/img/pi-cluster-3.0.png" width="600" /></td>
 </table>
 
-**K3S Kubernetes Cluster at home automated with Ansible and ArgoCD**
+**K3S Kubernetes Cluster at home automated with Ansible and FluxCD**
 
 
 This is an educational project to build a hybrid x86/ARM Kubernetes cluster at home, using Raspberry Pi and refurbished x86 mini PCs, learn to deploy basic kubernetes services and automate its deployment and configuration applying IaC (infrastructure as a code) and GitOps methodologies.
 
 
 The entire process for creating this cluster at home, from cluster design and architecture to step-by-step manual configuration guides, has been documented and it is published in the project website: https://picluster.ricsanfre.com.
 
-This repository contains all source code used to automate all manual tasks described in the documentation: Cloud-init's configuration files, Ansible's source code (playbooks/roles), and packaged Kubernetes applications (helm and kustomize) to be deployed using ArgoCD. 
+This repository contains all source code used to automate all manual tasks described in the documentation: Cloud-init's configuration files, Ansible's source code (playbooks/roles), and packaged Kubernetes applications (helm and kustomize) to be deployed using FluxCD. 
 
 Since its deployment is completely automated, the cluster can be re-deployed in minutes as many times as needed for testing new cluster configurations, new software versions or just take you out of any mesh you could cause playing with the cluster.
 
 ## Scope
 
-The scope of this project is to build a hybrid x86/ARM kubernetes cluster at home, using low cost Raspeberry PIs and old refurbished mini PCs, and automate its deployment and configuration applying **IaC (infrastructure as a code)** and **GitOps** methodologies with tools like [Ansible](https://docs.ansible.com/), [cloud-init](https://cloudinit.readthedocs.io/en/latest/) and [Argo CD](https://argo-cd.readthedocs.io/en/stable/).
+The scope of this project is to build a hybrid x86/ARM kubernetes cluster at home, using low cost Raspeberry PIs and old refurbished mini PCs, and automate its deployment and configuration applying **IaC (infrastructure as a code)** and **GitOps** methodologies with tools like [Ansible](https://docs.ansible.com/), [cloud-init](https://cloudinit.readthedocs.io/en/latest/) and [Flux CD](https://fluxcd.io/).
 
 As part of the project, the goal is to use a lightweight Kubernetes flavor based on [K3S](https://k3s.io/) and deploy cluster basic services such as:
 - Distributed block storage for POD's persistent volumes, [LongHorn](https://longhorn.io/).
@@ -46,7 +46,7 @@ Also deployment of services for building a cloud-native microservices architectu
 
 ## Technology Stack
 
-The following picture shows the set of opensource solutions used so far in the cluster, which installation process has been documented and its deployment has been automated with Ansible/ArgoCD:
+The following picture shows the set of opensource solutions used so far in the cluster, which installation process has been documented and its deployment has been automated with Ansible/FluxCD:
 
 <p align="center">
   <img src="docs/assets/img/pi-cluster-icons.png" width="500"/>
@@ -65,8 +65,8 @@ The following picture shows the set of opensource solutions used so far in the c
         <td>Automate OS configuration, external services installation and k3s installation and bootstrapping</td>
     </tr>
     <tr>
-        <td><img width="32" src="docs/assets/img/logos/argocd.svg"></td>
-        <td><a href="https://argoproj.github.io/cd">ArgoCD</a></td>
+        <td><img width="32" src="docs/assets/img/logos/flux-cd.png" alt="fluxcd logo"></td>
+        <td><a href="https://fluxcd.io/">FluxCD</a></td>
         <td>GitOps tool for deploying applications to Kubernetes</td>
     </tr>
     <tr>
@@ -115,7 +115,7 @@ The following picture shows the set of opensource solutions used so far in the c
         <td>Kubernetes Ingress Controller</td>
     </tr> 
     <tr>
-        <td><img width="32" src="/assets/img/logos/istio-icon-color.svg" alt="istio logo"></td>
+        <td><img width="32" src="docs/assets/img/logos/istio-icon-color.svg" alt="istio logo"></td>
         <td><a href="https://istio.io/">Istio</a></td>
         <td>Kubernetes Service Mesh</td>
     </tr>

diff --git a/ansible-runner/Dockerfile b/ansible-runner/Dockerfile
@@ -1,8 +1,11 @@
+FROM ghcr.io/helmfile/helmfile:v0.167.1 AS helmfile
+
+
 FROM python:slim
 ARG ANSIBLE_GALAXY_CLI_COLLECTION_OPTS=--ignore-certs
 ARG ANSIBLE_GALAXY_CLI_ROLE_OPTS=--ignore-certs
 RUN apt-get update -qq && \
-    apt-get install sudo git apt-utils python3-pip pwgen gnupg -y && \
+    apt-get install sudo git apt-utils python3-pip pwgen gnupg curl -y && \
     apt-get clean && \
     rm -rf /usr/share/doc/* /usr/share/man/* /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -23,6 +26,8 @@ RUN ANSIBLE_GALAXY_DISABLE_GPG_VERIFY=1 ansible-galaxy collection install $ANSIB
 # Configure ansible-runner
 RUN ansible-playbook ansible_runner_setup.yml
 
+# Copy helmfile
+COPY --from=helmfile /usr/local/bin/helmfile /usr/local/bin/helmfile
 
 ENV USER runner
 ENV FOLDER /home/runner
@@ -39,6 +44,9 @@ RUN for dir in \
 
 USER $USER
 
+# Install helmfile helm plugins
+RUN helmfile init --force
+
 RUN echo "export GPG_TTY=\$(tty)" >> /home/runner/.bashrc
 
 WORKDIR /runner
diff --git a/ansible/create_vault_credentials.yml b/ansible/create_vault_credentials.yml
@@ -12,6 +12,18 @@
       private: true
 
   pre_tasks:
+    - name: Ask for GIT PAT credentials
+      when: git_private_repo
+      block:
+        - name: Ask for GitHub PAT credentials
+          pause:
+            prompt: "Enter Github PAT: "
+            echo: false
+          register: prompt
+        - name: Set gitHub pat password variable
+          set_fact:
+            github_pat: "{{ prompt.user_input }}"
+          no_log: true
     - name: Ask for SAN centralized credentials
       when: centralized_san
       block:
@@ -33,7 +45,6 @@
           set_fact:
             san_iscsi_mutual_pass: "{{ prompt.user_input }}"
           no_log: true
-
   tasks:
 
     - name: Create random passwords

diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml
@@ -11,6 +11,10 @@ ansible_ssh_private_key_file: ~/.ssh/id_rsa
 # Whether to use centralized SAN architecture or not
 centralized_san: false
 
+# GitOps Repo
+# Whether the repo is private or not
+git_private_repo: false
+
 #######################
 # DNS configuration
 #######################

diff --git a/ansible/k3s_bootstrap.yml b/ansible/k3s_bootstrap.yml
@@ -55,17 +55,24 @@
         overlay: "prod"
       when: overlay is undefined
 
-    - name: Install CRDs
-      include_tasks: tasks/crds_install.yml
+    - name: Install cilium and coreDNS
+      become: false
+      shell: |
+        set -o pipefail
+        helmfile --quiet --file ../kubernetes/clusters/bootstrap/helmfile.yaml apply --skip-diff-on-install --suppress-diff
+      args:
+        executable: /bin/bash
+      register: output
+      changed_when: true
 
-    - name: Install Cilium CNI
-      include_tasks: tasks/cilium_install.yml
+    - name: Config Cilium CNI
+      include_tasks: tasks/cilium_config.yml
 
     - name: Configure Vault integration
       include_tasks: tasks/configure_vault_integration.yml
 
-    - name: ArgoCD bootstrap
-      include_tasks: tasks/argocd_bootstrap.yml
+    - name: Flux bootstrap
+      include_tasks: tasks/fluxcd_bootstrap.yml
 
     - name: Install cli utils.
       include_tasks: tasks/install_cli_utils.yml
diff --git a/ansible/tasks/cilium_config.yml b/ansible/tasks/cilium_config.yml
@@ -0,0 +1,34 @@
+---
+
+- name: "Wait for K3S worker nodes to be ready. Nodes Ready status when Cilium has been installed"
+  command:
+    cmd: "kubectl get nodes {{ item }}"
+  register: nodes
+  until:
+    - '" Ready "  in nodes.stdout'
+  retries: 10
+  delay: 60
+  with_items: "{{ groups['k3s_worker'] }}"
+
+- name: Wait for Cilium CRDs to be ready
+  become: false
+  shell: |
+    set -o pipefail
+    while ! kubectl wait --for condition=established --timeout=60s crd/ciliuml2announcementpolicies.cilium.io crd/ciliuml2announcementpolicies.cilium.io
+    do
+    sleep 10
+    done
+  args:
+    executable: /bin/bash
+  changed_when: false
+
+- name: Cilium configuration
+  become: false
+  shell: |
+    set -o pipefail
+    kubectl kustomize --enable-helm --load-restrictor=LoadRestrictionsNone \
+     ../kubernetes/platform/cilium/config/overlays/"{{overlay}}" | kubectl apply -f -
+  args:
+    executable: /bin/bash
+  register: output
+  changed_when: true
diff --git a/ansible/tasks/configure_vault_integration.yml b/ansible/tasks/configure_vault_integration.yml
@@ -5,7 +5,7 @@
   shell: |
     set -o pipefail
     kubectl kustomize --enable-helm --load-restrictor=LoadRestrictionsNone \
-     ../kubernetes/bootstrap/vault/overlays/"{{overlay}}" | kubectl apply -f -
+     ../kubernetes/clusters/bootstrap/vault/overlays/"{{overlay}}" | kubectl apply -f -
   args:
     executable: /bin/bash
   register: output

diff --git a/ansible/tasks/fluxcd_bootstrap.yml b/ansible/tasks/fluxcd_bootstrap.yml
@@ -0,0 +1,25 @@
+---
+
+- name: Deploy Flux CD
+  shell: |
+    kubectl kustomize  \
+      ../kubernetes/clusters/bootstrap/flux | kubectl apply -f -
+  args:
+    executable: /bin/bash
+
+- name: Create Github secret
+  shell: |
+    kubectl create secret generic flux-system -n flux-system \
+    --from-literal=username=git \
+    --from-literal=password="{{vault.flux.github.pat}}"
+  when: git_private_repo
+  args:
+    executable: /bin/bash
+  changed_when: false
+
+- name: Bootstrap applications
+  shell: |
+    kubectl kustomize  \
+      ../kubernetes/clusters/"{{overlay}}"/config | kubectl apply -f -
+  args:
+    executable: /bin/bash
diff --git a/ansible/tasks/install_cli_utils.yml b/ansible/tasks/install_cli_utils.yml
@@ -8,4 +8,3 @@
   loop:
     - longhorn-util
     - velero-cli
-    - linkerd-cli
diff --git a/ansible/vars/picluster.yml b/ansible/vars/picluster.yml
@@ -31,9 +31,11 @@ k3s_kubelet_config: |
 # --flannel-backend=none
 # --disable-kube-proxy
 # --disable-network-policy
+# --disable-helm-controller
 # --disable 'servicelb'
 # --disable 'traefik'
 # --disable 'local-storage'
+# --disable 'coredns'
 # --node-taint 'node-role.kubernetes.io/control-plane:NoSchedule'
 # --kube-controller-manager-arg 'bind-address=0.0.0.0'
 # --kube-proxy-arg 'metrics-bind-address=0.0.0.0'
@@ -45,14 +47,19 @@ k3s_server_config:
   # Disable Flannel CNI
   flannel-backend: none
   disable-network-policy: true
+  # Disable Helm Controller
+  disable-helm-controller: true
   # Disable kube-proxy (using cilium kube-proxy replacement)
   disable-kube-proxy: true
   tls-san:
     - "{{ k3s_api_vip }}"  # IP to HAProxy
+  # Disable K3s addons: coredns, local path, servicelb, traefik and metric-server
   disable:
+    - coredns
     - local-storage
     - servicelb
     - traefik
+    - metrics-server
   write-kubeconfig-mode: 644
   node-taint:
     - 'node-role.kubernetes.io/control-plane:NoSchedule'

diff --git a/ansible/vars/vault.yml.j2 b/ansible/vars/vault.yml.j2
@@ -10,6 +10,10 @@ vault:
   cluster:
     k3s:
      token: {{ k3s_token }}
+  # Flux secret
+  flux:
+    github:
+      pat: {{ github_pat | default("") }}
   # Ingress secrets
   ingress:
     admin: