diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2a7a9e7..9a35950 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,10 +1,13 @@ name: Pull Request on: pull_request: - branches: - - '*' - tags-ignore: - - '*' + types: + - opened + - synchronize + - reopened + - ready_for_review +env: + GO_VERSION: 1.22.5 jobs: helm-template: name: 'Test Helm Template' @@ -12,9 +15,18 @@ jobs: steps: - name: Checkout uses: actions/checkout@v1 + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + cache: true - name: Install Helm uses: azure/setup-helm@v3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Generate Documentation + run: make docs + - name: Generated Code Checked In + run: '[[ -z $(git status --porcelain) ]]' - name: Run Test run: make test diff --git a/Makefile b/Makefile index 22dcc15..33d639e 100644 --- a/Makefile +++ b/Makefile @@ -50,3 +50,8 @@ test: helm lint --strict charts/$${chart}; \ helm template charts/$${chart} > /dev/null; \ done + +.PHONY: docs +docs: + @go install github.com/unikorn-cloud/core/hack/docs-generate@main + docs-generate diff --git a/README.md b/README.md index 096de67..a137ce1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ + + # Helm Charts to Deploy Cluster API ## Why? @@ -14,49 +16,7 @@ When we encounter one of the annoying environment variables, we replace it with This chart requires the following to be installed on the target cluster first: -### Cert-Manager - -
-Helm - -```shell -helm repo add jetstack https://charts.jetstack.io -helm repo update -helm install cert-manager jetstack/cert-manager --version v1.15.1 --namespace cert-manager --create-namespace --set crds.enabled=true -``` -
- -
-ArgoCD - -```yaml ---- -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - generateName: cert-manager- - namespace: argocd -spec: - project: default - source: - chart: cert-manager - repoURL: https://charts.jetstack.io - targetRevision: v1.15.1 - helm: - releaseName: cert-manager - parameters: - - name: installCRDs - value: "true" - destination: - server: https://kubernetes.default.svc - namespace: cert-manager - syncPolicy: - automated: - selfHeal: true - syncOptions: - - CreateNamespace=true -``` -
+* [Jetstack cert-manager](https://cert-manager.io/) ## Deploying One-Shot @@ -68,7 +28,7 @@ There is a top level chart-of-charts that will just install everything as a big ```shell helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api helm repo update -helm install cluster-api unikorn-cloud-capi/cluster-api --version v0.2.0 +helm upgrade --install cluster-api unikorn-cloud-capi/cluster-api -n cluster-api --create-namespace --version v0.2.0 ``` @@ -90,7 +50,7 @@ spec: targetRevision: v0.2.0 destination: server: https://kubernetes.default.svc - namespace: foo + namespace: cluster-api ignoreDifferences: # Aggregated roles are mangically updated by the API. - group: rbac.authorization.k8s.io @@ -113,6 +73,7 @@ spec: selfHeal: true syncOptions: - RespectIgnoreDifferences=true + - CreateNamespace=true ``` @@ -128,7 +89,7 @@ You may want to be a little less gung-ho and deploy the pieces as separate appli ```shell helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api helm repo update -helm install cluster-api-core unikorn-cloud-capi/cluster-api-core --version v0.2.0 +helm upgrade --install cluster-api-core unikorn-cloud-capi/cluster-api-core -n cluster-api --create-namespace --version v0.2.0 ``` @@ -150,7 +111,7 @@ spec: targetRevision: v0.2.0 destination: server: https://kubernetes.default.svc - namespace: foo + namespace: cluster-api ignoreDifferences: # Aggregated roles are mangically updated by the API. - group: rbac.authorization.k8s.io @@ -168,6 +129,7 @@ spec: selfHeal: true syncOptions: - RespectIgnoreDifferences=true + - CreateNamespace=true ``` @@ -179,7 +141,7 @@ spec: ```shell helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api helm repo update -helm install cluster-api-bootstrap-kubeadm unikorn-cloud-capi/cluster-api-bootstrap-kubeadm --version v0.2.0 +helm upgrade --install cluster-api-bootstrap-kubeadm unikorn-cloud-capi/cluster-api-bootstrap-kubeadm -n cluster-api --create-namespace --version v0.2.0 ``` @@ -201,7 +163,7 @@ spec: targetRevision: v0.2.0 destination: server: https://kubernetes.default.svc - namespace: foo + namespace: cluster-api ignoreDifferences: - group: apiextensions.k8s.io jsonPointers: @@ -212,6 +174,7 @@ spec: selfHeal: true syncOptions: - RespectIgnoreDifferences=true + - CreateNamespace=true ``` @@ -223,7 +186,7 @@ spec: ```shell helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api helm repo update -helm install cluster-api-control-plane-kubeadm unikorn-cloud-capi/cluster-api-control-plane-kubeadm --version v0.2.0 +helm upgrade --install cluster-api-control-plane-kubeadm unikorn-cloud-capi/cluster-api-control-plane-kubeadm -n cluster-api --create-namespace --version v0.2.0 ``` @@ -245,7 +208,7 @@ spec: targetRevision: v0.2.0 destination: server: https://kubernetes.default.svc - namespace: foo + namespace: cluster-api ignoreDifferences: - group: rbac.authorization.k8s.io jsonPointers: @@ -261,6 +224,7 @@ spec: selfHeal: true syncOptions: - RespectIgnoreDifferences=true + - CreateNamespace=true ``` diff --git a/README.md.tmpl b/README.md.tmpl new file mode 100644 index 0000000..e33495a --- /dev/null +++ b/README.md.tmpl @@ -0,0 +1,244 @@ +# Helm Charts to Deploy Cluster API + +## Why? + +`clusterctl` is very opinionated, it will pull down some kustomize generated manifests, then do some environment substitution on them. +This isn't compatible with ArgoCD for example, hence this project. + +## How + +In simple terms, we run `kubectl kustomize`, chop up the manifests and auto generate templates. +When we encounter one of the annoying environment variables, we replace it with Go templating, then add the replacement into `values.yaml`. + +## Deploying Prerequisites + +This chart requires the following to be installed on the target cluster first: + +* [Jetstack cert-manager](https://cert-manager.io/) + +## Deploying One-Shot + +There is a top level chart-of-charts that will just install everything as a big bang operation. + +
+Helm + +```shell +helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api +helm repo update +helm upgrade --install cluster-api unikorn-cloud-capi/cluster-api -n cluster-api --create-namespace --version {{ .CAPI_CHART_VERSION }} +``` +
+ +
+ArgoCD + +```yaml +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: cluster-api + namespace: argocd +spec: + project: default + source: + repoURL: https://unikorn-cloud.github.io/helm-cluster-api + chart: cluster-api + targetRevision: {{ .CAPI_CHART_VERSION }} + destination: + server: https://kubernetes.default.svc + namespace: cluster-api + ignoreDifferences: + # Aggregated roles are mangically updated by the API. + - group: rbac.authorization.k8s.io + kind: ClusterRole + name: capi-aggregated-manager-role + jsonPointers: + - /rules + - group: rbac.authorization.k8s.io + kind: ClusterRole + name: capi-kubeadm-control-plane-aggregated-manager-role + jsonPointers: + - /rules + # CA certs are injected by cert-manager mutation + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + jsonPointers: + - /spec/conversion/webhook/clientConfig/caBundle + syncPolicy: + automated: + selfHeal: true + syncOptions: + - RespectIgnoreDifferences=true + - CreateNamespace=true +``` +
+ +## Deploying Main Components + +You may want to be a little less gung-ho and deploy the pieces as separate applications. + +### Core + +
+Helm + +```shell +helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api +helm repo update +helm upgrade --install cluster-api-core unikorn-cloud-capi/cluster-api-core -n cluster-api --create-namespace --version {{ .CAPI_CORE_CHART_VERSION }} +``` +
+ +
+ArgoCD + +```yaml +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + generateName: cluster-api-core- + namespace: argocd +spec: + project: default + source: + repoURL: https://unikorn-cloud.github.io/helm-cluster-api + chart: cluster-api-core + targetRevision: {{ .CAPI_CORE_CHART_VERSION }} + destination: + server: https://kubernetes.default.svc + namespace: cluster-api + ignoreDifferences: + # Aggregated roles are mangically updated by the API. + - group: rbac.authorization.k8s.io + kind: ClusterRole + name: capi-aggregated-manager-role + jsonPointers: + - /rules + # CA certs are injected by cert-manager mutation + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + jsonPointers: + - /spec/conversion/webhook/clientConfig/caBundle + syncPolicy: + automated: + selfHeal: true + syncOptions: + - RespectIgnoreDifferences=true + - CreateNamespace=true +``` +
+ +### Bootstrap + +
+Helm + +```shell +helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api +helm repo update +helm upgrade --install cluster-api-bootstrap-kubeadm unikorn-cloud-capi/cluster-api-bootstrap-kubeadm -n cluster-api --create-namespace --version {{ .CAPI_BOOTSTRAP_KUBEADM_CHART_VERISON }} +``` +
+ +
+ArgoCD + +```yaml +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + generateName: cluster-api-bootstrap-kubeadm- + namespace: argocd +spec: + project: default + source: + repoURL: https://unikorn-cloud.github.io/helm-cluster-api + chart: cluster-api-bootstrap-kubeadm + targetRevision: {{ .CAPI_BOOTSTRAP_KUBEADM_CHART_VERISON }} + destination: + server: https://kubernetes.default.svc + namespace: cluster-api + ignoreDifferences: + - group: apiextensions.k8s.io + jsonPointers: + - /spec/conversion/webhook/clientConfig/caBundle + kind: CustomResourceDefinition + syncPolicy: + automated: + selfHeal: true + syncOptions: + - RespectIgnoreDifferences=true + - CreateNamespace=true +``` +
+ +### Control Plane + +
+Helm + +```shell +helm repo add unikorn-cloud-capi https://unikorn-cloud.github.io/helm-cluster-api +helm repo update +helm upgrade --install cluster-api-control-plane-kubeadm unikorn-cloud-capi/cluster-api-control-plane-kubeadm -n cluster-api --create-namespace --version {{ .CAPI_CONTROLPLANE_KUBEADM_CHART_VERSION }} +``` +
+ +
+ArgoCD + +```yaml +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + generateName: cluster-api-control-plane-kubeadm- + namespace: argocd +spec: + project: default + source: + repoURL: https://unikorn-cloud.github.io/helm-cluster-api + chart: cluster-api-control-plane-kubeadm + targetRevision: {{ .CAPI_CONTROLPLANE_KUBEADM_CHART_VERSION }} + destination: + server: https://kubernetes.default.svc + namespace: cluster-api + ignoreDifferences: + - group: rbac.authorization.k8s.io + jsonPointers: + - /rules + kind: ClusterRole + name: capi-kubeadm-control-plane-aggregated-manager-role + - group: apiextensions.k8s.io + jsonPointers: + - /spec/conversion/webhook/clientConfig/caBundle + kind: CustomResourceDefinition + syncPolicy: + automated: + selfHeal: true + syncOptions: + - RespectIgnoreDifferences=true + - CreateNamespace=true +``` +
+ +## Deploying Infrastructure Providers and Clusters + +Add providers to allow CAPI to talk to various cloud providers. + +### OpenStack + +* [Install the provider](charts/cluster-api-provider-openstack/README.md) +* [Install a cluster](charts/cluster-api-cluster-openstack/README.md) + +## Developers + +It's a simple as: + +* Bump the versions in `Makefile` and `charts/cluster-api/Chart.yaml` +* Run `make` +* Commit and merge. diff --git a/charts/cluster-api-cluster-openstack/README.md b/charts/cluster-api-cluster-openstack/README.md index e17639a..1e90acf 100644 --- a/charts/cluster-api-cluster-openstack/README.md +++ b/charts/cluster-api-cluster-openstack/README.md @@ -1,3 +1,5 @@ + + # Installing an OpenStack Cluster ... is quite involved! diff --git a/charts/cluster-api-cluster-openstack/README.md.tmpl b/charts/cluster-api-cluster-openstack/README.md.tmpl new file mode 100644 index 0000000..2e5d5e7 --- /dev/null +++ b/charts/cluster-api-cluster-openstack/README.md.tmpl @@ -0,0 +1,130 @@ +# Installing an OpenStack Cluster + +... is quite involved! + +## Configuration Variables + +Please consult the [`values.yaml`](values.yaml) file for some basic examples. +The [`values.schema.json`](values.schema.json) file documents structure, types and required fields further. + +## Helm + +When using Helm directly, deprovisioning will delete the identity secret used to access OpenStack immediately and result in a deadlock. +Don't use this :smile: + +## ArgoCD + +Unlike Helm, ArgoCD can provision and deprovision in "waves", thus we can keep the identity secret alive for the duration of deprovisioning. +This is the only supported method of operation. + +Here's an example application: + +```yaml +--- +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: foo + namespace: argocd +spec: + destination: + server: https://kubernetes.default.svc + namespace: foo + project: default + source: + repoURL: https://unikorn-cloud.github.io/helm-cluster-api + chart: cluster-api-cluster-openstack + targetRevision: {{ .CLUSTER_OPENSTACK_CHART_VERSION }} + helm: + releaseName: foo + # Remove the default work queue. + parameters: + - name: workload.default + value: null + values: |- + openstack: + cloud: REDACTED + cloudsYAML: REDACTED + ca: REDACTED + sshKeyName: REDACTED + region: en-west-1 + failureDomain: eu-west-1a + externalNetworkID: dadfef54-d1c5-447a-8933-f515eeadd822 + api: + allowList: + - 123.45.67.89 + certificateSANs: + - kubernetes.my-domain.com + controlPlane: + version: v1.30.2 + replicas: 3 + skipKubeProxy: false + machine: + imageID: 7a517603-aa70-47a9-a6f3-c102d30e67c0 + flavorID: 061f0cf2-2503-4005-89ed-ff1dc217874f + diskSize: 40 + workloadPools: + general-purpose: + replicas: 3 + version: v1.30.2 + machine: + imageID: 7a517603-aa70-47a9-a6f3-c102d30e67c0 + flavorID: 061f0cf2-2503-4005-89ed-ff1dc217874f + diskSize: 100 + autoscaling: + limits: + minReplicas: 3 + maxReplicas: 10 + scheduler: + cpu: 4 + memory: 16G + gpu: + version: v1.30.2 + replicas: 3 + machine: + imageID: 7a517603-aa70-47a9-a6f3-c102d30e67c0 + flavorID: 061f0cf2-2503-4005-89ed-ff1dc217874f + diskSize: 100 + autoscaling: + limits: + minReplicas: 3 + maxReplicas: 10 + scheduler: + cpu: 4 + memory: 32G + gpu: + type: nvidia.com/gpu + count: 1 + network: + nodeCIDR: 192.168.0.0/12 + serviceCIDRs: + - 172.16.0.0/12 + podCIDRs: + - 10.0.0.0/8 + dnsNameservers: + - 1.1.1.1 + - 8.8.8.8 + syncPolicy: + automated: + selfHeal: true + syncOptions: + - CreateNamespace=true +``` + +This by itself will not actually provision a working cluster. +See below for more details. + +### Getting Working Cluster + +To achieve a working cluster that is correctly scaled and works, you will also need to concurrently install: + +* A CNI +* [The Openstack cloud provider](https://github.com/kubernetes/cloud-provider-openstack) + +To do this, grab the kubeconfig file, subsituting the correct namespace and release name: + +```shell +kubectl -n foo foo-kubeconfig -o 'jsonpath={.data.value}' | base64 -d +``` + +Then use Helm of similar to provision against that kubeconfig. diff --git a/docs-generate.yaml b/docs-generate.yaml new file mode 100644 index 0000000..e1ef28c --- /dev/null +++ b/docs-generate.yaml @@ -0,0 +1,30 @@ +variables: +- name: CAPI_CHART_VERSION + yaml: + file: charts/cluster-api/Chart.yaml + pointer: /version +- name: CAPI_BOOTSTRAP_KUBEADM_CHART_VERISON + yaml: + file: charts/cluster-api-bootstrap-kubeadm/Chart.yaml + pointer: /version +- name: CAPI_CONTROLPLANE_KUBEADM_CHART_VERSION + yaml: + file: charts/cluster-api-control-plane-kubeadm/Chart.yaml + pointer: /version +- name: CAPI_CORE_CHART_VERSION + yaml: + file: charts/cluster-api-core/Chart.yaml + pointer: /version +- name: CAPO_CHART_VERSION + yaml: + file: charts/cluster-api-cluster-openstack/Chart.yaml + pointer: /version +- name: CLUSTER_OPENSTACK_CHART_VERSION + yaml: + file: charts/cluster-api-cluster-openstack/Chart.yaml + pointer: /version +files: +- in: README.md.tmpl + out: README.md +- in: charts/cluster-api-cluster-openstack/README.md.tmpl + out: charts/cluster-api-cluster-openstack/README.md diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..cd20d0c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/unikorn-cloud/helm-cluster-api + +go 1.22.5 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e69de29