Merge pull request #37 from segmentio/yolken-improve-validation

Clean up validation, add OPA support

Dockerfile

@@ -34,7 +34,6 @@ RUN pip3 install awscli
 
 COPY --from=builder \
     /usr/local/bin/helm \
-    /usr/local/bin/kubeval \
     /usr/local/bin/kubectl \
     /usr/local/bin/kubeapply \
     /usr/local/bin/

Dockerfile.lambda

@@ -34,7 +34,6 @@ RUN pip3 install awscli
 COPY --from=builder \
     /usr/local/bin/aws-iam-authenticator \
     /usr/local/bin/helm \
-    /usr/local/bin/kubeval \
     /usr/local/bin/kubectl \
     /usr/local/bin/kubeapply \
     /usr/local/bin/

README.md

@@ -58,7 +58,6 @@ environments. We welcome feedback and collaboration to make `kubeapply` useful t
 
 - [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/): v1.16 or newer
 - [`helm`](https://helm.sh/docs/intro/install/): v3.5.0 or newer (only needed if using helm charts)
-- [`kubeval`](https://kubeval.instrumenta.dev/installation/): v0.15.0 or newer
 
 Make sure that they're installed locally and available in your path.
 
@@ -218,9 +217,14 @@ other source types use custom code in the `kubeapply` binary.
 
 #### Validate
 
-`kubeapply validate [path to cluster config]`
+`kubeapply validate [path to cluster config] --policy=[path to OPA policy in rego format]`
 
-This validates all of the expanded configs for the cluster by wrapping `kubeval`.
+This validates all of the expanded configs for the cluster using the
+[`kubeconform`](https://github.com/yannh/kubeconform) library. It also, optionally, supports
+validating configs using one or more [OPA](https://www.openpolicyagent.org/) policies in
+rego format. The latter allows checking that configs satisfy organization-specific standards,
+e.g. that resource labels are in the correct format, that images are only pulled from the
+expected registries, etc.
 
 #### Diff
 

cmd/kubeapply/subcmd/check.go

@@ -30,9 +30,6 @@ func checkRun(cmd *cobra.Command, args []string) error {
 	if err := checkDep("kubectl", "version"); err != nil {
 		return err
 	}
-	if err := checkDep("kubeval", "--version"); err != nil {
-		return err
-	}
 
 	return nil
 }

cmd/kubeapply/subcmd/validate.go

@@ -15,14 +15,21 @@ import (
 
 var validateCmd = &cobra.Command{
 	Use:   "validate [cluster configs]",
-	Short: "validate checks the cluster configs using kubeval",
+	Short: "validate checks the cluster configs using kubeconform and (optionally) opa policies",
 	Args:  cobra.MinimumNArgs(1),
 	RunE:  validateRun,
 }
 
 type validateFlags struct {
 	// Expand before validating.
 	expand bool
+
+	// Number of worker goroutines to use for validation.
+	numWorkers int
+
+	// Paths to OPA policy rego files that will be run against kube resources.
+	// See https://www.openpolicyagent.org/ for more details.
+	policies []string
 }
 
 var validateFlagValues validateFlags
@@ -34,6 +41,18 @@ func init() {
 		false,
 		"Expand before validating",
 	)
+	validateCmd.Flags().IntVar(
+		&validateFlagValues.numWorkers,
+		"num-workers",
+		4,
+		"Number of workers to use for validation",
+	)
+	validateCmd.Flags().StringArrayVar(
+		&validateFlagValues.policies,
+		"policy",
+		[]string{},
+		"Paths to OPA policies",
+	)
 
 	RootCmd.AddCommand(validateCmd)
 }
@@ -88,43 +107,106 @@ func validateClusterPath(ctx context.Context, path string) error {
 func execValidation(ctx context.Context, clusterConfig *config.ClusterConfig) error {
 	log.Infof("Validating cluster %s", clusterConfig.DescriptiveName())
 
-	kubeValidator := validation.NewKubeValidator()
+	kubeconformChecker, err := validation.NewKubeconformChecker()
+	if err != nil {
+		return err
+	}
 
-	log.Infof(
-		"Checking that expanded configs for %s are valid YAML",
-		clusterConfig.DescriptiveName(),
+	policies, err := validation.DefaultPoliciesFromGlobs(
+		ctx,
+		validateFlagValues.policies,
+		map[string]interface{}{
+			// TODO: Add more parameters here (or entire config)?
+			"cluster": clusterConfig.Cluster,
+			"region":  clusterConfig.Region,
+			"env":     clusterConfig.Env,
+		},
 	)
-	err := kubeValidator.CheckYAML(clusterConfig.AbsSubpaths())
 	if err != nil {
 		return err
 	}
 
-	log.Infof("Running kubeval on configs in %+v", clusterConfig.AbsSubpaths())
-	results, err := kubeValidator.RunKubeval(ctx, clusterConfig.AbsSubpaths()[0])
+	checkers := []validation.Checker{kubeconformChecker}
+	for _, policy := range policies {
+		checkers = append(checkers, policy)
+	}
+
+	validator := validation.NewKubeValidator(
+		validation.KubeValidatorConfig{
+			NumWorkers: validateFlagValues.numWorkers,
+			Checkers:   checkers,
+		},
+	)
+
+	log.Infof("Running validator on configs in %+v", clusterConfig.AbsSubpaths())
+	results, err := validator.RunChecks(ctx, clusterConfig.AbsSubpaths()[0])
 	if err != nil {
 		return err
 	}
 
-	numInvalidFiles := 0
+	numInvalidResourceChecks := 0
+	numValidResourceChecks := 0
+	numSkippedResourceChecks := 0
 
 	for _, result := range results {
-		switch result.Status {
-		case "valid":
-			log.Infof("File %s OK", result.Filename)
-		case "skipped":
-			log.Debugf("File %s skipped", result.Filename)
-		case "invalid":
-			numInvalidFiles++
-			log.Errorf("File %s is invalid; errors: %+v", result.Filename, result.Errors)
-		default:
-			log.Infof("Unrecognized result type: %+v", result)
+		for _, checkResult := range result.CheckResults {
+			switch checkResult.Status {
+			case validation.StatusValid:
+				numValidResourceChecks++
+				log.Debugf(
+					"Resource %s in file %s OK according to check %s",
+					result.Resource.PrettyName(),
+					result.Resource.Path,
+					checkResult.CheckName,
+				)
+			case validation.StatusSkipped:
+				numSkippedResourceChecks++
+				log.Debugf(
+					"Resource %s in file %s was skipped by check %s",
+					result.Resource.PrettyName(),
+					result.Resource.Path,
+					checkResult.CheckName,
+				)
+			case validation.StatusError:
+				numInvalidResourceChecks++
+				log.Errorf(
+					"Resource %s in file %s could not be processed by check %s: %s",
+					result.Resource.PrettyName(),
+					result.Resource.Path,
+					checkResult.CheckName,
+					checkResult.Message,
+				)
+			case validation.StatusInvalid:
+				numInvalidResourceChecks++
+				log.Errorf(
+					"Resource %s in file %s is invalid according to check %s: %s",
+					result.Resource.PrettyName(),
+					result.Resource.Path,
+					checkResult.CheckName,
+					checkResult.Message,
+				)
+			case validation.StatusEmpty:
+			default:
+				log.Infof("Unrecognized result type: %+v", result)
+			}
 		}
 	}
 
-	if numInvalidFiles > 0 {
-		return fmt.Errorf("Validation failed for %d files", numInvalidFiles)
+	if numInvalidResourceChecks > 0 {
+		return fmt.Errorf(
+			"Validation failed for %d resources in cluster %s (%d checks valid, %d skipped)",
+			numInvalidResourceChecks,
+			clusterConfig.DescriptiveName(),
+			numValidResourceChecks,
+			numSkippedResourceChecks,
+		)
 	}
 
-	log.Infof("Validation of cluster %s passed", clusterConfig.DescriptiveName())
+	log.Infof(
+		"Validation of cluster %s passed (%d checks valid, %d skipped)",
+		clusterConfig.DescriptiveName(),
+		numValidResourceChecks,
+		numSkippedResourceChecks,
+	)
 	return nil
 }

examples/kubeapply-test-cluster/README.md

@@ -52,7 +52,7 @@ config or profile files.
 
 ##### (3) `make validate`
 
-Runs `kubeval` over the expanded configs to validate that they are legitimate Kubernetes
+Runs `kubeconform` over the expanded configs to validate that they are legitimate Kubernetes
 configs before continuing.
 
 ##### (4) `make diff`

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/aws/aws-lambda-go v1.15.0
 	github.com/aws/aws-sdk-go v1.29.16
 	github.com/briandowns/spinner v1.11.1
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/fatih/color v1.7.0
 	github.com/ghodss/yaml v1.0.0
@@ -22,6 +21,7 @@ require (
 	github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/olekukonko/tablewriter v0.0.4
+	github.com/open-policy-agent/opa v0.27.1
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/segmentio/conf v1.2.0
 	github.com/segmentio/encoding v0.2.7
@@ -32,6 +32,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/stripe/skycfg v0.0.0-20200303020846-4f599970a3e6
 	github.com/x-cray/logrus-prefixed-formatter v0.5.2
+	github.com/yannh/kubeconform v0.4.6
 	github.com/zorkian/go-datadog-api v2.28.0+incompatible // indirect
 	go.starlark.net v0.0.0-20201204201740-42d4f566359b
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d