KubeauditConfig feature (#193)

* define schema for config file and add cobra flag to specify config

* initialize root logic for kubeauditConfig

* push logic to PodOverride, prior to testing

* pre-testing build

* add support for custom capabilities

* finalize tests and update readme

* cleanup the mess

* add override for netpols

* add netpol feature to auditconfig override and add suggestion

* add from config to file names

* set nill auditConfig for test

* reset auditConfig flag after use in testing suite

* reset back

* get rid extra newline

* temp commit to change filenames to lowecase

* revert back

* oops forgot about the README

* increase test coverage

* add headsup for unmarshling unreachability

README.md

@@ -16,6 +16,7 @@ privileged, ... You get the gist of it and more on that later. Just know:
 - [Autofix](#autofix)
 - [Audits](#audits)
 - [Override Labels](#labels)
+- [Audit Configuration](#audit-configuration)
 - [Contribute!](#contribute)
 
 <a name="installation" />
@@ -76,6 +77,10 @@ or
 1. if run with `-j/--json` it will log output json formatted so that its output
  can be used by other programs easily
 
+`kubeaudit` supports using manual audit configuration provided by the user, use the command
+`kubeaudit -f/--manifest /path/to/manifest.yml -k/--auditConfig /path/to/config.yml`
+For more details on audit config check out [Audit Configuration](#audit-configuration).
+
 `kubeaudit` has four different log levels `INFO, WARN, ERROR` controlled by
 `-v/--verbose LEVEL` and for those who counted and want to work on `kubeaudit`
 `DEBUG`
@@ -311,7 +316,7 @@ WARN[0000] Memory limit exceeded, it is set to 512Mi but it must not exceed 125M
 
 ## Audit AppArmor
 
-It checks that AppArmor is enabled for all containers by making sure the following annotation exists on the pod. 
+It checks that AppArmor is enabled for all containers by making sure the following annotation exists on the pod.
 There must be an annotation for each container in the pod:
 
 ```
@@ -337,8 +342,8 @@ ERRO[0000] AppArmor disabled. Annotation=container.apparmor.security.beta.kubern
 
 ## Audit Seccomp
 
-It checks that Seccomp is enabled for all containers by making sure one or both of the following annotations exists 
-on the pod. If no pod annotation is used, then there must be an annotation for each container. Container annotations 
+It checks that Seccomp is enabled for all containers by making sure one or both of the following annotations exists
+on the pod. If no pod annotation is used, then there must be an annotation for each container. Container annotations
 override the pod annotation:
 
 ```
@@ -349,7 +354,7 @@ seccomp.security.alpha.kubernetes.io/pod: <profile>
 container.seccomp.security.alpha.kubernetes.io/<container name>: <profile>
 ```
 
-where profile can be "runtime/default" or start with "localhost/" to be considered valid. "docker/default" is 
+where profile can be "runtime/default" or start with "localhost/" to be considered valid. "docker/default" is
 deprecated and will show a warning. It should be replaced with "runtime/default".
 
 If the Seccomp annotation is missing:
@@ -576,7 +581,46 @@ capabilitiesToBeDropped:
  - SETFCAP #Set file capabilities.
 ```
 
-This can be overridden by using `-d` flag and providing your own defaults in the yaml format as shown above.
+This can be overridden by using `-k` flag and providing your own defaults in the yaml format as shown below.
+
+<a name="audit-configuration" />
+
+## Audit Configuration
+
+Allows configuring your own audit settings for kubeaudit. By default following configuration is used:
+
+```
+apiVersion: v1
+kind: kubeauditConfig
+audit: true # Set to false if you want kubeaudit to not audit your k8s manifests
+spec:
+ capabilities: # List of all supported capabilities
+ NET_ADMIN: drop # Set to `keep` to keep capability
+ SETPCAP: drop # Set to `keep` to keep capability
+ MKNOD: drop # Set to `keep` to keep capability
+ AUDIT_WRITE: drop # Set to `keep` to keep capability
+ CHOWN: drop # Set to `keep` to keep capability
+ NET_RAW: drop # Set to `keep` to keep capability
+ DAC_OVERRIDE: drop # Set to `keep` to keep capability
+ FOWNER: drop # Set to `keep` to keep capability
+ FSETID: drop # Set to `keep` to keep capability
+ KILL: drop # Set to `keep` to keep capability
+ SETGID: drop # Set to `keep` to keep capability
+ SETUID: drop # Set to `keep` to keep capability
+ NET_BIND_SERVICE: drop # Set to `keep` to keep capability
+ SYS_CHROOT: drop # Set to `keep` to keep capability
+ SETFCAP: drop # Set to `keep` to keep capability
+ overrides: # List of all supported overrides
+ privilege-escalation: deny # Set to `allow` to skip auditing potential vulnerability
+ privileged: deny # Set to `allow` to skip auditing potential vulnerability
+ run-as-root: deny # Set to `allow` to skip auditing potential vulnerability
+ automount-service-account-token: deny # Set to `allow` to skip auditing potential vulnerability
+ read-only-root-filesystem-false: deny # Set to `allow` to skip auditing potential vulnerability
+ non-default-deny-ingress-network-policy: deny # Set to `allow` to skip auditing potential vulnerability
+ non-default-deny-egress-network-policy: deny # Set to `allow` to skip auditing potential vulnerability
+```
+
+<a name="contribute" />
 
 ## Contributing
 

cmd/allowPrivilegeEscalation_test.go

@@ -51,3 +51,12 @@ func TestAllowPrivilegeEscalationMultipleAllowMultipleContainers(t *testing.T) {
 func TestAllowPrivilegeEscalationSingleAllowMultipleContainers(t *testing.T) {
  runAuditTest(t, "allow_privilege_escalation_true_single_allowed_multiple_containers_v1beta.yml", auditAllowPrivilegeEscalation, []int{ErrorAllowPrivilegeEscalationTrue, ErrorAllowPrivilegeEscalationTrueAllowed})
 }
+
+func TestAllowPrivilegeEscalationFromConfig(t *testing.T) {
+ rootConfig.auditConfig = "../configs/allow_privilege_escalation_from_config.yml"
+ runAuditTest(t, "security_context_nil_v1.yml", auditAllowPrivilegeEscalation, []int{ErrorAllowPrivilegeEscalationTrueAllowed})
+ runAuditTest(t, "allow_privilege_escalation_nil_v1.yml", auditAllowPrivilegeEscalation, []int{ErrorAllowPrivilegeEscalationTrueAllowed})
+ runAuditTest(t, "allow_privilege_escalation_true_v1.yml", auditAllowPrivilegeEscalation, []int{ErrorAllowPrivilegeEscalationTrueAllowed})
+ runAuditTest(t, "allow_privilege_escalation_true_single_allowed_multiple_containers_v1beta.yml", auditAllowPrivilegeEscalation, []int{ErrorAllowPrivilegeEscalationTrueAllowed})
+ rootConfig.auditConfig = ""
+}

cmd/automountServiceAccountToken_test.go

@@ -25,3 +25,11 @@ func TestServiceAccountTokenMisconfiguredAllowV1(t *testing.T) {
 func TestServiceAccountTokenTrueAndDefaultNameV1(t *testing.T) {
  runAuditTest(t, "service_account_token_true_and_default_name_v1.yml", auditAutomountServiceAccountToken, []int{ErrorAutomountServiceAccountTokenTrueAndNoName})
 }
+
+func TestAutomountServiceAccountTokenFromConfig(t *testing.T) {
+ rootConfig.auditConfig = "../configs/allow_automount_service_account_token_from_config.yml"
+ runAuditTest(t, "service_account_token_deprecated_v1.yml", auditAutomountServiceAccountToken, []int{ErrorServiceAccountTokenDeprecated})
+ runAuditTest(t, "service_account_token_true_and_no_name_v1.yml", auditAutomountServiceAccountToken, []int{ErrorAutomountServiceAccountTokenTrueAllowed})
+ runAuditTest(t, "service_account_token_nil_and_no_name_v1.yml", auditAutomountServiceAccountToken, []int{ErrorMisconfiguredKubeauditAllow})
+ rootConfig.auditConfig = ""
+}

cmd/capabilities.go

@@ -2,19 +2,15 @@ package cmd
 
 import (
  "fmt"
- "io/ioutil"
- "os"
  "strings"
 
+ "io/ioutil"
+
  "github.com/Shopify/yaml"
  log "github.com/sirupsen/logrus"
  "github.com/spf13/cobra"
 )
 
-type capsDropList struct {
- Drop []string `yaml:"capabilitiesToBeDropped"`
-}
-
 const defaultDropCapConfig = `
 # SANE DEFAULTS:
 capabilitiesToBeDropped:
@@ -35,25 +31,44 @@ capabilitiesToBeDropped:
  - SETFCAP #Set file capabilities.
 `
 
+var defaultCapList = &KubeauditConfigCapabilities{
+ // SANE DEFAULTS:
+ NetAdmin: "drop",
+ SetPCAP: "drop",
+ MKNOD: "drop",
+ AuditWrite: "drop",
+ Chown: "drop",
+ NetRaw: "drop",
+ DacOverride: "drop",
+ FOWNER: "drop",
+ FSetID: "drop",
+ Kill: "drop",
+ SetGID: "drop",
+ SetUID: "drop",
+ NetBindService: "drop",
+ SYSChroot: "drop",
+ SetFCAP: "drop",
+}
+
 func recommendedCapabilitiesToBeDropped() (dropCapSet CapSet, err error) {
- yamlFile := []byte(defaultDropCapConfig)
- if rootConfig.dropCapConfig != "" {
- if _, err = os.Stat(rootConfig.dropCapConfig); err != nil {
- return
- }
- yamlFile, err = ioutil.ReadFile(rootConfig.dropCapConfig)
+ var kubeauditConfig = &KubeauditConfig{}
+ if rootConfig.auditConfig != "" {
+ data, err := ioutil.ReadFile(rootConfig.auditConfig)
  if err != nil {
- return
+ log.Println(err)
+ return dropCapSet, err
  }
- }
- caps := capsDropList{}
- err = yaml.Unmarshal(yamlFile, &caps)
- if err != nil {
- return
- }
- dropCapSet = make(CapSet)
- for _, drop := range caps.Drop {
- dropCapSet[CapabilityV1(drop)] = true
+
+ // err check for unmarshalling is not useful as Root Init crashes the program if Config is not well formed
+ yaml.Unmarshal(data, kubeauditConfig)
+
+ if kubeauditConfig != nil && kubeauditConfig.Spec != nil && kubeauditConfig.Spec.Capabilities != nil {
+ dropCapSet = dropCapFromConfigList(kubeauditConfig.Spec.Capabilities)
+ } else {
+ dropCapSet = dropCapFromConfigList(defaultCapList)
+ }
+ } else {
+ dropCapSet = dropCapFromConfigList(defaultCapList)
  }
  return
 }
@@ -172,8 +187,7 @@ An ERROR log is generated when a pod has a capability which is on the drop list.
 A WARN log is generated when a pod has a capability allowed which is on the drop list.
 
 Example usage:
-kubeaudit caps
-kubeaudit caps -d drop_v1.yml`, defaultDropCapConfig),
+kubeaudit caps`, defaultDropCapConfig),
  Run: runAudit(auditCapabilities),
 }
 

cmd/capabilities_test.go

@@ -33,6 +33,11 @@ func TestCapabilitiesSomeDroppedV1Beta2(t *testing.T) {
  runAuditTest(t, "capabilities_some_dropped_v1beta2.yml", auditCapabilities, []int{ErrorCapabilityNotDropped})
 }
 
+func TestAllowAuditCapabilitiesSomeDroppedFromConfigV1Beta2(t *testing.T) {
+ rootConfig.auditConfig = "../configs/allow_audit_from_config.yml"
+ runAuditTest(t, "capabilities_some_dropped_v1beta2.yml", auditCapabilities, []int{ErrorCapabilityNotDropped})
+}
+
 func TestCapabilitiesMisconfiguredAllowV1Beta2(t *testing.T) {
  runAuditTest(t, "capabilities_misconfigured_allow_v1beta2.yml", auditCapabilities, []int{ErrorMisconfiguredKubeauditAllow})
 }
@@ -53,12 +58,26 @@ func TestCapabilitiesSomeAllowedMultiContainersMixLabelsV1Beta2(t *testing.T) {
  runAuditTest(t, "capabilities_some_allowed_multi_containers_mix_labels_v1beta2.yml", auditCapabilities, []int{ErrorCapabilityAllowed, ErrorCapabilityAllowed})
 }
 
-func TestCapabilitiesManualConfigV1(t *testing.T) {
- rootConfig.dropCapConfig = "../configs/capSetConfig.yaml"
- runAuditTest(t, "capabilities_some_dropped_v1beta2.yml", auditCapabilities, []int{})
-}
-
 func TestCapabilitiesManualConfigV2(t *testing.T) {
- rootConfig.dropCapConfig = "../fake/file/path"
+ rootConfig.auditConfig = "../fake/file/path"
  runAuditTest(t, "capabilities_some_dropped_v1beta2.yml", auditCapabilities, []int{KubeauditInternalError})
+ rootConfig.auditConfig = ""
+}
+
+func TestCustomCapabilitiesToBeDroppedV1(t *testing.T) {
+ assert := assert.New(t)
+ rootConfig.auditConfig = "../configs/custom_capabilities_to_be_dropped_v1.yml"
+ capabilities, err := recommendedCapabilitiesToBeDropped()
+ assert.Nil(err)
+ assert.Equal(NewCapSetFromArray([]CapabilityV1{"MKNOD", "CHOWN", "DAC_OVERRIDE", "FSETID", "SETGID", "NET_BIND_SERVICE", "SETFCAP"}), capabilities, "")
+ rootConfig.auditConfig = ""
+}
+
+func TestCustomCapabilitiesToBeDroppedV2(t *testing.T) {
+ assert := assert.New(t)
+ rootConfig.auditConfig = "../configs/custom_capabilities_to_be_dropped_v1.yml"
+ capabilities, err := recommendedCapabilitiesToBeDropped()
+ assert.Nil(err)
+ assert.NotEqual(NewCapSetFromArray([]CapabilityV1{"MKNOD", "SYS_CHROOT", "KILL", "CHOWN", "DAC_OVERRIDE", "FSETID", "SETGID", "NET_BIND_SERVICE", "SETFCAP"}), capabilities, "")
+ rootConfig.auditConfig = ""
 }

cmd/capabilities_util.go

@@ -0,0 +1,81 @@
+package cmd
+
+import "reflect"
+
+func dropCapFromConfigList(capList *KubeauditConfigCapabilities) (dropCapSet CapSet) {
+ var configCapabilityValue reflect.Value
+ var r reflect.Value
+ dropCapSet = make(CapSet)
+ r = reflect.ValueOf(capList)
+ configCapabilityValue = reflect.Indirect(r).FieldByName("SetPCAP")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("SETPCAP")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("MKNOD")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("MKNOD")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("AuditWrite")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("AUDIT_WRITE")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("Chown")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("CHOWN")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("NetRaw")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("NET_RAW")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("DacOverride")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("DAC_OVERRIDE")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("FOWNER")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("FOWNER")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("FSetID")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("FSETID")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("Kill")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("KILL")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("SetGID")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("SETGID")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("SetUID")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("SETUID")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("NetBindService")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("NET_BIND_SERVICE")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("SYSChroot")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("SYS_CHROOT")] = true
+ }
+
+ configCapabilityValue = reflect.Indirect(r).FieldByName("SetFCAP")
+ if configCapabilityValue.String() == "drop" {
+ dropCapSet[CapabilityV1("SETFCAP")] = true
+ }
+
+ return dropCapSet
+}