diff --git a/master.tf b/master.tf index 7b8afc6..e9d0f7c 100644 --- a/master.tf +++ b/master.tf @@ -146,6 +146,20 @@ data "ignition_file" "kube-apiserver" { } } +data "template_file" "audit-policy" { + template = "${file("${path.module}/resources/audit-policy.yaml")}" +} + +data "ignition_file" "audit-policy" { + mode = 0644 + filesystem = "root" + path = "/etc/kubernetes/config/audit-policy.yaml" + + content { + content = "${data.template_file.audit-policy.rendered}" + } +} + data "template_file" "kube-controller-manager" { template = "${file("${path.module}/resources/kube-controller-manager.yaml")}" @@ -226,6 +240,7 @@ locals { data "ignition_config" "master" { files = ["${concat( list( + data.ignition_file.audit-policy.id, data.ignition_file.cfssl.id, data.ignition_file.cfssljson.id, data.ignition_file.cfssl-client-config.id, diff --git a/resources/audit-policy.yaml b/resources/audit-policy.yaml new file mode 100644 index 0000000..8ef8b24 --- /dev/null +++ b/resources/audit-policy.yaml @@ -0,0 +1,148 @@ +# Based on https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/gci/configure-helper.sh#L758 +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +rules: + # The following requests were manually identified as high-volume and low-risk, so drop them. + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core + resources: ["endpoints", "services", "services/status"] + - level: None + # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port. + users: ["system:unsecured"] + namespaces: ["kube-system"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["configmaps"] + - level: None + users: ["kubelet"] # legacy kubelet identity + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + userGroups: ["system:nodes"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["nodes", "nodes/status"] + - level: None + users: + - system:kube-controller-manager + - system:kube-scheduler + - system:serviceaccount:kube-system:endpoint-controller + verbs: ["get", "update"] + namespaces: ["kube-system"] + resources: + - group: "" # core + resources: ["endpoints"] + - level: None + users: ["system:apiserver"] + verbs: ["get"] + resources: + - group: "" # core + resources: ["namespaces", "namespaces/status", "namespaces/finalize"] + # Don't log HPA fetching metrics. + - level: None + users: + - system:kube-controller-manager + verbs: ["get", "list"] + resources: + - group: "metrics.k8s.io" + # Don't log these read-only URLs. + - level: None + nonResourceURLs: + - /healthz* + - /version + - /swagger* + # Don't log events requests. + - level: None + resources: + - group: "" # core + resources: ["events"] + # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes + - level: Request + users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + - level: Request + userGroups: ["system:nodes"] + verbs: ["update","patch"] + resources: + - group: "" # core + resources: ["nodes/status", "pods/status"] + omitStages: + - "RequestReceived" + # deletecollection calls can be large, don't log responses for expected namespace deletions + - level: Request + users: ["system:serviceaccount:kube-system:namespace-controller"] + verbs: ["deletecollection"] + omitStages: + - "RequestReceived" + # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, so only log at the Metadata level. + - level: Metadata + resources: + - group: "" # core + resources: ["secrets", "configmaps"] + - group: authentication.k8s.io + resources: ["tokenreviews"] + omitStages: + - "RequestReceived" + # Get responses can be large; skip them. + - level: Request + verbs: ["get", "list", "watch"] + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for known APIs + - level: RequestResponse + resources: + - group: "" # core + - group: "admissionregistration.k8s.io" + - group: "apiextensions.k8s.io" + - group: "apiregistration.k8s.io" + - group: "apps" + - group: "authentication.k8s.io" + - group: "authorization.k8s.io" + - group: "autoscaling" + - group: "batch" + - group: "certificates.k8s.io" + - group: "extensions" + - group: "metrics.k8s.io" + - group: "networking.k8s.io" + - group: "policy" + - group: "rbac.authorization.k8s.io" + - group: "scheduling.k8s.io" + - group: "settings.k8s.io" + - group: "storage.k8s.io" + omitStages: + - "RequestReceived" + # Default level for all other requests. + - level: Metadata + omitStages: + - "RequestReceived" diff --git a/resources/kube-apiserver.yaml b/resources/kube-apiserver.yaml index 816ef5f..27a8ecc 100644 --- a/resources/kube-apiserver.yaml +++ b/resources/kube-apiserver.yaml @@ -33,6 +33,7 @@ spec: - --oidc-client-id=${oidc_client_id} - --authorization-mode=Node,RBAC - --apiserver-count=${master_instance_count} + - --audit-policy-file=/etc/kubernetes/config/audit-policy.yaml - --audit-log-path=/var/log/kube-api-server/audit - --audit-log-maxsize=100 - --audit-log-maxbackup=20 @@ -65,6 +66,9 @@ spec: - mountPath: /etc/kubernetes/ssl name: ssl-certs-kubernetes readOnly: true + - mountPath: /etc/kubernetes/config + name: kubernetes-configurations + readOnly: true - mountPath: /etc/ssl/certs name: ssl-certs-host readOnly: true @@ -74,6 +78,9 @@ spec: - hostPath: path: /etc/kubernetes/ssl name: ssl-certs-kubernetes + - hostPath: + path: /etc/kubernetes/config + name: kubernetes-configurations - hostPath: path: /usr/share/ca-certificates name: ssl-certs-host diff --git a/variables.tf b/variables.tf index a64a975..e443377 100644 --- a/variables.tf +++ b/variables.tf @@ -185,7 +185,6 @@ variable "feature_gates" { type = "map" default = { - "AdvancedAuditing" = "false" "ExpandPersistentVolumes" = "true" "PodShareProcessNamespace" = "true" }