diff --git a/docs/src/guide/cluster_addons.md b/docs/src/guide/cluster_addons.md index 3d093827..8dd5fbb0 100644 --- a/docs/src/guide/cluster_addons.md +++ b/docs/src/guide/cluster_addons.md @@ -1,6 +1,6 @@ # Cluster Add-ons -With this module to manage both essential and optional add-ons on enhanced OKE clusters. +With this module to manage both essential and optional add-ons on **enhanced** OKE clusters. This module provides the option to remove [Essential addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-essential-addons) and to manage, both essential & [optional addons](https://docs.oracle.com/en-us/iaas/Content/ContEng/Tasks/contengintroducingclusteraddons.htm#contengintroducingclusteraddons__section-optional-addons). diff --git a/docs/src/guide/extensions_networking.md b/docs/src/guide/extensions_networking.md index 892d2563..d97e49a7 100644 --- a/docs/src/guide/extensions_networking.md +++ b/docs/src/guide/extensions_networking.md @@ -23,6 +23,91 @@ {{#include ../../../examples/extensions/vars-extensions-cilium.auto.tfvars:4:}} ``` +Cillium is a eBPF based CNI for Kubernetes that can be configured on OKE clusters. + +The OKE cluster should be initially configured to run **flannel**. + +On **enhanced** clusters we can use the cluster-addons module to remove flannel extension and kube-proxy (Optional) at cluster creation. + + +``` +cluster_addons_to_remove = { + Flannel = { + remove_k8s_resources = true + }, + KubeProxy = { + remove_k8s_resources = true + } +} +``` + +If you want to use cilium as [kube-proxy replacement](https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/), you can use the following helm_values: + +``` +cilium_helm_values = { + kubeProxyReplacement = true +} +``` + +For the basic clusters you can add the following label to the worker nodes to prevent flannel pods from being scheduled: + +``` +oci.oraclecloud.com/custom-k8s-networking=true +``` + +If you want to override and of the default values(listed below) you can use the `cilium_helm_values` variable: + +``` +"annotateK8sNode": true +"cluster": + "id": 1 + "name": "oke-${var.state_id}" +"clustermesh": + "apiserver": + "kvstoremesh": + "enabled": false + "useAPIServer": false +"cni": + "exclusive": true + "install": true +"hubble": + "metrics": + "dashboards": + "enabled": false + "relay": + "enabled": true + "ui": + "enabled": true +"installNoConntrackIptablesRules": false +"ipam": + "mode": "kubernetes" +"k8s": + "requireIPv4PodCIDR": true +"k8sServiceHost": "${var.cluster_private_endpoint}" +"k8sServicePort": "6443" +"kubeProxyReplacement": false +"operator": + "prometheus": + "enabled": false +"pmtuDiscovery": + "enabled": true +"rollOutCiliumPods": true +"tunnelProtocol": "vxlan" +``` + + +**Notes:** +1. Tested with OKE version `v1.29.1` and the worker nodes running: `Oracle-Linux-8.9-2024.05.29-0-OKE-1.29.1-707`. + +2. In case the `hubble-relay` and `hubble-ui` pods fail to start, run the following commands: + +``` +kubectl delete pod --namespace kube-system -l k8s-app=kube-dns +kubectl delete pod --namespace kube-system -l k8s-app=hubble-relay +kubectl delete pod --namespace kube-system -l k8s-app=hubble-ui +kubectl delete pod --namespace kube-system -l k8s-app=kube-dns-autoscaler +``` + ### References * [cilium.io](https://cilium.io) diff --git a/examples/extensions/vars-extensions-cilium.auto.tfvars b/examples/extensions/vars-extensions-cilium.auto.tfvars index 6a52f9c7..6966e7c9 100644 --- a/examples/extensions/vars-extensions-cilium.auto.tfvars +++ b/examples/extensions/vars-extensions-cilium.auto.tfvars @@ -1,9 +1,9 @@ -# Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. +# Copyright (c) 2017, 2024 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl cilium_install = true cilium_reapply = false -cilium_namespace = "network" -cilium_helm_version = "45.2.0" +cilium_namespace = "kube-system" +cilium_helm_version = "1.16.3" cilium_helm_values = {} cilium_helm_values_files = [] diff --git a/module-extensions.tf b/module-extensions.tf index 83bda7d7..6e668538 100644 --- a/module-extensions.tf +++ b/module-extensions.tf @@ -1,6 +1,13 @@ # Copyright (c) 2017, 2023 Oracle Corporation and/or its affiliates. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl +locals { + cluster_private_endpoint = ( var.create_cluster ? + coalesce(split(":", lookup(one(module.cluster[*].endpoints), "private_endpoint", ""))...) : + coalesce(split(":", lookup(local.existing_cluster_endpoints, "private_endpoint", ""))...) + ) +} + module "extensions" { source = "./modules/extensions" depends_on = [ module.network ] @@ -12,6 +19,7 @@ module "extensions" { kubernetes_version = var.kubernetes_version expected_node_count = local.worker_count_expected worker_pools = one(module.workers[*].worker_pools) + cluster_private_endpoint = local.cluster_private_endpoint # Bastion/operator connection ssh_private_key = sensitive(local.ssh_private_key) diff --git a/module-workers.tf b/module-workers.tf index 2adc9c6c..ae996693 100644 --- a/module-workers.tf +++ b/module-workers.tf @@ -58,7 +58,7 @@ module "workers" { image_type = var.worker_image_type kubeproxy_mode = var.kubeproxy_mode max_pods_per_node = var.max_pods_per_node - node_labels = var.worker_node_labels + node_labels = alltrue([var.cluster_type == "basic", var.cilium_install == true]) ? merge(var.worker_node_labels, {"oci.oraclecloud.com/custom-k8s-networking" = true}) : var.worker_node_labels node_metadata = var.worker_node_metadata agent_config = var.agent_config platform_config = var.platform_config diff --git a/modules/extensions/cilium.tf b/modules/extensions/cilium.tf index c096b907..335c5c3a 100644 --- a/modules/extensions/cilium.tf +++ b/modules/extensions/cilium.tf @@ -2,45 +2,47 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl locals { - cilium_helm_crds_file = join("/", [local.yaml_manifest_path, "cilium.crds.yaml"]) - cilium_helm_manifest_file = join("/", [local.yaml_manifest_path, "cilium.manifest.yaml"]) - cilium_helm_values_file = join("/", [local.yaml_manifest_path, "cilium.values.yaml"]) - cilium_net_attach_def_file = join("/", [local.yaml_manifest_path, "cilium.net_attach_def.yaml"]) - cilium_veth_config_map_file = join("/", [local.yaml_manifest_path, "cilium.cni_config_map.yaml"]) - - cilium_helm_crds = one(data.helm_template.cilium[*].crds) - cilium_helm_manifest = one(data.helm_template.cilium[*].manifest) + cilium_helm_crds_file = join("/", [local.yaml_manifest_path, "cilium.crds.yaml"]) + cilium_helm_manifest_file = join("/", [local.yaml_manifest_path, "cilium.manifest.yaml"]) + cilium_helm_values_file = join("/", [local.yaml_manifest_path, "cilium.values.yaml"]) + cilium_helm_values_override_file = join("/", [local.yaml_manifest_path, "cilium.values-override.yaml"]) + cilium_net_attach_def_file = join("/", [local.yaml_manifest_path, "cilium.net_attach_def.yaml"]) + cilium_veth_config_map_file = join("/", [local.yaml_manifest_path, "cilium.cni_config_map.yaml"]) + + cilium_helm_crds = one(data.helm_template.cilium[*].crds) + cilium_helm_values_override = one(data.helm_template.cilium[*].values) + + cilium_helm_repository = "https://helm.cilium.io" cilium_vxlan_cni = { install = true - chainingMode = "none" exclusive = true # !var.multus_install } - # TODO Support Flannel w/ generic-veth & tunnel disabled - cilium_tunnel = "vxlan" # var.cni_type == "flannel" ? "disabled" : "vxlan" - cilium_flannel_cni = { - install = true - chainingMode = "generic-veth" - configMap = "cni-configuration" - customConf = var.cni_type == "flannel" - exclusive = !var.multus_install - } - cilium_helm_values = { annotateK8sNode = true - cluster = { name = "oke-${var.state_id}" } - clustermesh = { useAPIServer = true } + cluster = { + name = "oke-${var.state_id}" + id = 1 + } + clustermesh = { + useAPIServer = false + apiserver = { + kvstoremesh = { + enabled = false + } + } + } cni = local.cilium_vxlan_cni - containerRuntime = { integration = "crio" } - installIptablesRules = true installNoConntrackIptablesRules = false ipam = { mode = "kubernetes" } - ipv4NativeRoutingCIDR = element(var.vcn_cidrs, 0) - kubeProxyReplacement = "disabled" + kubeProxyReplacement = false + k8sServiceHost = var.cluster_private_endpoint + k8sServicePort = "6443" pmtuDiscovery = { enabled = true } - tunnel = local.cilium_tunnel - + rollOutCiliumPods = true + tunnelProtocol = local.cilium_tunnel + hubble = { metrics = { dashboards = { enabled = var.prometheus_install } @@ -52,19 +54,9 @@ locals { k8s = { requireIPv4PodCIDR = true # wait for Kubernetes to provide the PodCIDR (ipam kubernetes) - enableIPv4Masquerade = true # var.cni_type != "flannel" # masquerade IPv4 traffic leaving the node from endpoints } # Prometheus metrics - metrics = { - dashboards = { enabled = var.prometheus_install } - # # serviceMonitor = { enabled = var.prometheus_enabled } - } - - prometheus = { - enabled = var.prometheus_install - # serviceMonitor = { enabled = var.prometheus_enabled } - } operator = { prometheus = { @@ -74,6 +66,17 @@ locals { } } + # TODO Support Flannel w/ generic-veth & tunnel disabled + cilium_tunnel = "vxlan" # var.cni_type == "flannel" ? "disabled" : "vxlan" + + cilium_flannel_cni = { + install = true + chainingMode = "generic-veth" + configMap = "cni-configuration" + customConf = var.cni_type == "flannel" + exclusive = !var.multus_install + } + cilium_net_attach_def_conf = { cniVersion = "0.3.1" name = "cilium" @@ -126,15 +129,16 @@ locals { data = { "cni-config" = jsonencode(local.cilium_veth_conf) } } - cilium_net_attach_def_yaml = yamlencode(local.cilium_net_attach_def) - cilium_veth_config_map_yaml = yamlencode(local.cilium_veth_config_map) - cilium_helm_values_yaml = yamlencode(local.cilium_helm_values) + cilium_net_attach_def_yaml = yamlencode(local.cilium_net_attach_def) + cilium_veth_config_map_yaml = yamlencode(local.cilium_veth_config_map) + cilium_helm_values_yaml = yamlencode(merge(local.cilium_helm_values, var.cilium_helm_values)) + cilium_helm_values_override_yaml = local.cilium_helm_values_override != null ? join("\n", local.cilium_helm_values_override) : "" } data "helm_template" "cilium" { count = var.cilium_install ? 1 : 0 chart = "cilium" - repository = "https://helm.cilium.io" + repository = local.cilium_helm_repository version = var.cilium_helm_version kube_version = var.kubernetes_version @@ -165,7 +169,7 @@ resource "null_resource" "cilium" { triggers = { helm_version = var.cilium_helm_version crds_md5 = try(md5(join("\n", local.cilium_helm_crds)), null) - manifest_md5 = try(md5(local.cilium_helm_manifest), null) + manifest_md5 = try(md5(local.cilium_helm_values_override_yaml), null) reapply = var.cilium_reapply ? uuid() : null } @@ -190,24 +194,19 @@ resource "null_resource" "cilium" { } provisioner "file" { - content = local.cilium_helm_manifest - destination = local.cilium_helm_manifest_file - } - - provisioner "file" { - content = local.cilium_helm_values_yaml - destination = local.cilium_helm_values_file + content = local.cilium_helm_values_override_yaml + destination = local.cilium_helm_values_override_file } - provisioner "file" { - content = local.cilium_net_attach_def_yaml - destination = local.cilium_net_attach_def_file - } + # provisioner "file" { + # content = local.cilium_net_attach_def_yaml + # destination = local.cilium_net_attach_def_file + # } - provisioner "file" { - content = local.cilium_veth_config_map_yaml - destination = local.cilium_veth_config_map_file - } + # provisioner "file" { + # content = local.cilium_veth_config_map_yaml + # destination = local.cilium_veth_config_map_file + # } provisioner "remote-exec" { inline = [for c in compact([ @@ -219,7 +218,7 @@ resource "null_resource" "cilium" { format(local.kubectl_apply_server_ns_file, var.cilium_namespace, local.cilium_helm_crds_file), # Install full manifest - format(local.kubectl_apply_ns_file, var.cilium_namespace, local.cilium_helm_manifest_file), + format(local.helm_upgrade_install, "cilium", "cilium", local.cilium_helm_repository, var.cilium_helm_version, var.cilium_namespace, local.cilium_helm_values_override_file), # Install Network Attachment Definition when Multus is enabled # var.multus_install ? format(local.kubectl_apply_file, local.cilium_net_attach_def_file) : null, diff --git a/modules/extensions/locals.tf b/modules/extensions/locals.tf index 555823fe..397f44a9 100644 --- a/modules/extensions/locals.tf +++ b/modules/extensions/locals.tf @@ -4,6 +4,7 @@ locals { yaml_manifest_path = "/home/${var.operator_user}/yaml" kubectl = "set -o pipefail; kubectl" + helm = "set -o pipefail; helm" kubectl_apply_ns_file = "${local.kubectl} apply -n %s -f %s" kubectl_apply_file = "${local.kubectl} apply -f %s" kubectl_apply_server_file = "${local.kubectl} apply --force-conflicts=true --server-side -f %s" @@ -11,4 +12,5 @@ locals { kubectl_create_missing_ns = "${local.kubectl} create ns %s --dry-run=client -o yaml | kubectl apply -f -" selector_linux = { "kubernetes.io/os" = "linux" } output_log = "bash -c \"%s | tee >(systemd-cat -t %s -p info)\"" + helm_upgrade_install = "${local.helm} upgrade --install %s %s --repo %s --version %s --namespace %s --create-namespace --skip-crds -f %s" } diff --git a/modules/extensions/variables.tf b/modules/extensions/variables.tf index a89aa0b5..ebb69901 100644 --- a/modules/extensions/variables.tf +++ b/modules/extensions/variables.tf @@ -7,6 +7,7 @@ variable "state_id" { type = string } variable "worker_pools" { type = any } variable "kubernetes_version" { type = string } variable "expected_node_count" { type = number } +variable "cluster_private_endpoint" { type = string } # Connection variable "bastion_host" { type = string } @@ -25,7 +26,7 @@ variable "cilium_install" { type = bool } variable "cilium_reapply" { type = bool } variable "cilium_namespace" { type = string } variable "cilium_helm_version" { type = string } -variable "cilium_helm_values" { type = map(string) } +variable "cilium_helm_values" { type = any } variable "cilium_helm_values_files" { type = list(string) } # CNI: Multus diff --git a/modules/workers/locals.tf b/modules/workers/locals.tf index 37be8a12..35633c33 100644 --- a/modules/workers/locals.tf +++ b/modules/workers/locals.tf @@ -157,7 +157,6 @@ locals { { "oke.oraclecloud.com/tf.module" = "terraform-oci-oke" "oke.oraclecloud.com/tf.state_id" = var.state_id - # "oke.oraclecloud.com/tf.workspace" = terraform.workspace "oke.oraclecloud.com/pool.name" = pool_name "oke.oraclecloud.com/pool.mode" = pool.mode "oke.oraclecloud.com/cluster_autoscaler" = pool.allow_autoscaler ? "allowed" : "disabled" diff --git a/variables-extensions.tf b/variables-extensions.tf index a7706d0f..f2e2520b 100644 --- a/variables-extensions.tf +++ b/variables-extensions.tf @@ -22,7 +22,7 @@ variable "cilium_namespace" { } variable "cilium_helm_version" { - default = "1.14.4" + default = "1.16.3" description = "Version of the Helm chart to install. List available releases using `helm search repo [keyword] --versions`." type = string } @@ -30,7 +30,7 @@ variable "cilium_helm_version" { variable "cilium_helm_values" { default = {} description = "Map of individual Helm chart values. See https://registry.terraform.io/providers/hashicorp/helm/latest/docs/data-sources/template." - type = map(string) + type = any } variable "cilium_helm_values_files" {