Enable advanced auditing (#55)

utilitywarehouse · Oct 10, 2018 · 782e9a0 · 782e9a0
1 parent 89b2e96
commit 782e9a0
Show file tree

Hide file tree

Showing 4 changed files with 170 additions and 1 deletion.
diff --git a/master.tf b/master.tf
@@ -146,6 +146,20 @@ data "ignition_file" "kube-apiserver" {
   }
 }
 
+data "template_file" "audit-policy" {
+  template = "${file("${path.module}/resources/audit-policy.yaml")}"
+}
+
+data "ignition_file" "audit-policy" {
+  mode       = 0644
+  filesystem = "root"
+  path       = "/etc/kubernetes/config/audit-policy.yaml"
+
+  content {
+    content = "${data.template_file.audit-policy.rendered}"
+  }
+}
+
 data "template_file" "kube-controller-manager" {
   template = "${file("${path.module}/resources/kube-controller-manager.yaml")}"
 
@@ -226,6 +240,7 @@ locals {
 data "ignition_config" "master" {
   files = ["${concat(
     list(
+        data.ignition_file.audit-policy.id,
         data.ignition_file.cfssl.id,
         data.ignition_file.cfssljson.id,
         data.ignition_file.cfssl-client-config.id,

diff --git a/resources/audit-policy.yaml b/resources/audit-policy.yaml
@@ -0,0 +1,148 @@
+# Based on https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/gci/configure-helper.sh#L758
+apiVersion: audit.k8s.io/v1beta1
+kind: Policy
+rules:
+  # The following requests were manually identified as high-volume and low-risk, so drop them.
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints", "services", "services/status"]
+  - level: None
+    # Ingress controller reads 'configmaps/ingress-uid' through the unsecured port.
+    users: ["system:unsecured"]
+    namespaces: ["kube-system"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["configmaps"]
+  - level: None
+    users: ["kubelet"] # legacy kubelet identity
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    userGroups: ["system:nodes"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["nodes", "nodes/status"]
+  - level: None
+    users:
+      - system:kube-controller-manager
+      - system:kube-scheduler
+      - system:serviceaccount:kube-system:endpoint-controller
+    verbs: ["get", "update"]
+    namespaces: ["kube-system"]
+    resources:
+      - group: "" # core
+        resources: ["endpoints"]
+  - level: None
+    users: ["system:apiserver"]
+    verbs: ["get"]
+    resources:
+      - group: "" # core
+        resources: ["namespaces", "namespaces/status", "namespaces/finalize"]
+  # Don't log HPA fetching metrics.
+  - level: None
+    users:
+      - system:kube-controller-manager
+    verbs: ["get", "list"]
+    resources:
+      - group: "metrics.k8s.io"
+  # Don't log these read-only URLs.
+  - level: None
+    nonResourceURLs:
+      - /healthz*
+      - /version
+      - /swagger*
+  # Don't log events requests.
+  - level: None
+    resources:
+      - group: "" # core
+        resources: ["events"]
+  # node and pod status calls from nodes are high-volume and can be large, don't log responses for expected updates from nodes
+  - level: Request
+    users: ["kubelet", "system:node-problem-detector", "system:serviceaccount:kube-system:node-problem-detector"]
+    verbs: ["update","patch"]
+    resources:
+      - group: "" # core
+        resources: ["nodes/status", "pods/status"]
+    omitStages:
+      - "RequestReceived"
+  - level: Request
+    userGroups: ["system:nodes"]
+    verbs: ["update","patch"]
+    resources:
+      - group: "" # core
+        resources: ["nodes/status", "pods/status"]
+    omitStages:
+      - "RequestReceived"
+  # deletecollection calls can be large, don't log responses for expected namespace deletions
+  - level: Request
+    users: ["system:serviceaccount:kube-system:namespace-controller"]
+    verbs: ["deletecollection"]
+    omitStages:
+      - "RequestReceived"
+  # Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data, so only log at the Metadata level.
+  - level: Metadata
+    resources:
+      - group: "" # core
+        resources: ["secrets", "configmaps"]
+      - group: authentication.k8s.io
+        resources: ["tokenreviews"]
+    omitStages:
+      - "RequestReceived"
+  # Get responses can be large; skip them.
+  - level: Request
+    verbs: ["get", "list", "watch"]
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "scheduling.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for known APIs
+  - level: RequestResponse
+    resources:
+      - group: "" # core
+      - group: "admissionregistration.k8s.io"
+      - group: "apiextensions.k8s.io"
+      - group: "apiregistration.k8s.io"
+      - group: "apps"
+      - group: "authentication.k8s.io"
+      - group: "authorization.k8s.io"
+      - group: "autoscaling"
+      - group: "batch"
+      - group: "certificates.k8s.io"
+      - group: "extensions"
+      - group: "metrics.k8s.io"
+      - group: "networking.k8s.io"
+      - group: "policy"
+      - group: "rbac.authorization.k8s.io"
+      - group: "scheduling.k8s.io"
+      - group: "settings.k8s.io"
+      - group: "storage.k8s.io"
+    omitStages:
+      - "RequestReceived"
+  # Default level for all other requests.
+  - level: Metadata
+    omitStages:
+      - "RequestReceived"
diff --git a/resources/kube-apiserver.yaml b/resources/kube-apiserver.yaml
@@ -33,6 +33,7 @@ spec:
     - --oidc-client-id=${oidc_client_id}
     - --authorization-mode=Node,RBAC
     - --apiserver-count=${master_instance_count}
+    - --audit-policy-file=/etc/kubernetes/config/audit-policy.yaml
     - --audit-log-path=/var/log/kube-api-server/audit
     - --audit-log-maxsize=100
     - --audit-log-maxbackup=20
@@ -65,6 +66,9 @@ spec:
     - mountPath: /etc/kubernetes/ssl
       name: ssl-certs-kubernetes
       readOnly: true
+    - mountPath: /etc/kubernetes/config
+      name: kubernetes-configurations
+      readOnly: true
     - mountPath: /etc/ssl/certs
       name: ssl-certs-host
       readOnly: true
@@ -74,6 +78,9 @@ spec:
   - hostPath:
       path: /etc/kubernetes/ssl
     name: ssl-certs-kubernetes
+  - hostPath:
+      path: /etc/kubernetes/config
+    name: kubernetes-configurations
   - hostPath:
       path: /usr/share/ca-certificates
     name: ssl-certs-host

diff --git a/variables.tf b/variables.tf
@@ -185,7 +185,6 @@ variable "feature_gates" {
   type        = "map"
 
   default = {
-    "AdvancedAuditing"         = "false"
     "ExpandPersistentVolumes"  = "true"
     "PodShareProcessNamespace" = "true"
   }