Update the cluster to update the nodes

terraform/main.tf

@@ -109,8 +109,9 @@ module "azure_aks_pr" {
     module.azuread_applications.identity_2
   ]
 
-  azure_container_registry_id      = module.acr_proxy.id
-  azure_container_registry_enpoint = module.acr_proxy.endpoint
+  azure_container_registry_username = module.acr_proxy.username
+  azure_container_registry_password = module.acr_proxy.password
+  azure_container_registry_enpoint  = module.acr_proxy.endpoint
 
   tags = local.tags
 }
@@ -135,8 +136,9 @@ module "azure_aks_nightly" {
     module.azuread_applications.identity_2
   ]
 
-  azure_container_registry_id      = module.acr_proxy.id
-  azure_container_registry_enpoint = module.acr_proxy.endpoint
+  azure_container_registry_username = module.acr_proxy.username
+  azure_container_registry_password = module.acr_proxy.password
+  azure_container_registry_enpoint  = module.acr_proxy.endpoint
 
   tags = local.tags
 }

terraform/modules/azure/aks/main.tf

@@ -1,3 +1,11 @@
+terraform {
+  required_providers {
+    kubectl = {
+      source = "gavinbunney/kubectl"
+    }
+  }
+}
+
 provider "azurerm" {
   features {}
   skip_provider_registration = true
@@ -31,8 +39,6 @@ resource "azurerm_kubernetes_cluster" "aks" {
   oidc_issuer_enabled = true
   node_resource_group = var.node_resource_group_name
 
-  node_os_channel_upgrade = "Unmanaged"
-
   monitor_metrics {}
 
   default_node_pool {
@@ -62,14 +68,6 @@ resource "azurerm_kubernetes_cluster" "aks" {
   }
 }
 
-## ACR Permissions
-
-resource "azurerm_role_assignment" "kubweb_to_acr" {
-  scope                = var.azure_container_registry_id
-  role_definition_name = "AcrPull"
-  principal_id         = azurerm_kubernetes_cluster.aks.kubelet_identity[0].object_id
-}
-
 ## Workload Identity Federation
 
 resource "azurerm_federated_identity_credential" "msi_federation" {
@@ -388,22 +386,104 @@ provider "helm" {
   }
 }
 
-resource "helm_release" "spegel" {
-  name             = "spegel"
-  namespace        = "spegel"
-  repository       = "oci://ghcr.io/spegel-org/helm-charts"
-  chart            = "spegel"
-  create_namespace = true
-  version          = "v0.0.27"
-
-  values = [
-    <<EOF
-spegel:
-  additionalMirrorRegistries: 
-    - ${var.azure_container_registry_enpoint}
-  appendMirrors: false
-  registries:
-  - https://docker.io
+provider "kubectl" {
+  host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
+  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
+  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
+  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
+}
+
+
+resource "kubectl_manifest" "namespace" {
+  yaml_body = <<YAML
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: node-installer
+YAML
+}
+
+resource "kubectl_manifest" "configuration" {
+  yaml_body = <<YAML
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: installer-config
+  namespace: node-installer
+data:
+  install.sh: |
+    #!/bin/bash
+
+    # Update docker.io mirror
+    mkdir -p /etc/containerd/certs.d/docker.io
+    cat <<EOF > /etc/containerd/certs.d/docker.io/hosts.toml
+    server = 'https://registry-1.docker.io'
+
+    [host.'https://${var.azure_container_registry_enpoint}/v2']
+    capabilities = ['pull', 'resolve']
+    override_path = true
     EOF
-  ]
+
+    # Update credentials
+    if grep "${var.azure_container_registry_enpoint}" /etc/containerd/config.toml; 
+    then 
+      echo "credentials already set, ignorning"
+    else 
+      cat <<EOF >> /etc/containerd/config.toml
+    [plugins."io.containerd.grpc.v1.cri".registry.configs."${var.azure_container_registry_enpoint}".auth]
+      username = "${var.azure_container_registry_username}"
+      password = "${var.azure_container_registry_password}"
+    EOF
+      # Restart containerd
+      systemctl restart containerd
+    fi
+YAML
+
+  depends_on = [kubectl_manifest.namespace]
+}
+
+resource "kubectl_manifest" "daemonset" {
+  yaml_body = <<YAML
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: installer
+  namespace: node-installer
+spec:
+  selector:
+    matchLabels:
+      job: installer
+  template:
+    metadata:
+      labels:
+        job: installer
+    spec:
+      hostPID: true
+      restartPolicy: Always
+      containers:
+      - image: patnaikshekhar/node-installer:1.3
+        name: installer
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: install-script
+          mountPath: /tmp
+        - name: host-mount
+          mountPath: /host
+        - name: containerd
+          mountPath: /etc/containerd
+      volumes:
+      - name: install-script
+        configMap:
+          name: installer-config
+      - name: containerd
+        hostPath:
+          path: /etc/containerd
+          type: Directory
+      - name: host-mount
+        hostPath:
+          path: /tmp/install
+YAML
+
+  depends_on = [kubectl_manifest.configuration]
 }

terraform/modules/azure/aks/vars.tf

@@ -36,14 +36,19 @@ variable "node_resource_group_name" {
   description = "AKS nodes resource group name"
 }
 
-variable "azure_container_registry_id" {
+variable "azure_container_registry_enpoint" {
   type        = string
-  description = "Azure Container Registry ID"
+  description = "Azure Container Registry Endpoint"
 }
 
-variable "azure_container_registry_enpoint" {
+variable "azure_container_registry_username" {
   type        = string
-  description = "Azure Container Registry Endpoint"
+  description = "Azure Container Registry Username"
+}
+
+variable "azure_container_registry_password" {
+  type        = string
+  description = "Azure Container Registry Password"
 }
 
 variable "default_node_pool_count" {

terraform/modules/azure/registry/main.tf

@@ -5,6 +5,7 @@ provider "azurerm" {
 
 locals {
   registry_name = "${var.unique_project_name}proxy"
+  username      = "e2e-cluster-puller"
 }
 
 data "azurerm_resource_group" "rg" {
@@ -19,6 +20,28 @@ resource "azurerm_container_registry" "acr" {
   tags                = var.tags
 }
 
+resource "azurerm_container_registry_scope_map" "acr_policy" {
+  name                    = "e2e-cluster-puller"
+  container_registry_name = azurerm_container_registry.acr.name
+  resource_group_name     = data.azurerm_resource_group.rg.name
+  actions = [
+    "content/read",
+  ]
+}
+
+resource "azurerm_container_registry_token" "acr_user" {
+  name                    = local.username
+  container_registry_name = azurerm_container_registry.acr.name
+  resource_group_name     = data.azurerm_resource_group.rg.name
+  scope_map_id            = azurerm_container_registry_scope_map.acr_policy.id
+}
+
+resource "azurerm_container_registry_token_password" "acr_token" {
+  container_registry_token_id = azurerm_container_registry_token.acr_user.id
+
+  password1 {}
+}
+
 resource "azurerm_container_registry_cache_rule" "docker-io" {
   name                  = "docker-io"
   container_registry_id = azurerm_container_registry.acr.id

terraform/modules/azure/registry/outputs.tf

@@ -1,8 +1,12 @@
-output "id" {
-  value = azurerm_container_registry.acr.id
-}
-
 output "endpoint" {
   value = azurerm_container_registry.acr.login_server
 }
 
+output "username" {
+  value = local.username
+}
+
+output "password" {
+  value = azurerm_container_registry_token_password.acr_token.password1[0].value
+}
+

terraform/terraform.tf

@@ -30,6 +30,10 @@ terraform {
       source  = "hashicorp/tls"
       version = "=4.0.5"
     }
+    kubectl = {
+      source  = "gavinbunney/kubectl"
+      version = "1.14.0"
+    }
   }
 }