feat(kubernetes): support label selectors in deploy manifest stages

via a new labelSelectors pipeline configuration property.  The syntax is the same as what's currently implemented in delete manifest stages.  For example:

{
  "labelSelectors": {
    "selectors": [
      {
        "kind": "EQUALS",
        "key": "my-label-key",
        "values": [
          "my-value"
        ],
      }
    ]
  }
}

See https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ and
KubernetesSelector for more.  Multiple selectors combine with AND (i.e. must all be
satisfied).

Note that kubectl replace doesn't support label selectors, so
KubernetesDeployManifestOperation throws an exception if a deploy manifest stage that
specifies (non-empty) label selectors has a manifest with a strategy.spinnaker.io/replace:
"true" annotation.  Although it's possible to implement the label selector logic in
clouddriver, this PR explicitly avoids that, and leaves the label selector logic to
kubectl.

It's possible that none of the manifests may satisfy the label selectors.  In that case, a
new pipeline configuration property named allowNothingSelected determines the behavior.
If false (the default), KubernetesDeployManifestOperation throws an exception.  If true,
the operation succeeds even though nothing was deployed.

closes spinnaker/spinnaker#3695.

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/description/manifest/KubernetesDeployManifestDescription.java

@@ -18,6 +18,7 @@
 package com.netflix.spinnaker.clouddriver.kubernetes.description.manifest;
 
 import com.netflix.spinnaker.clouddriver.kubernetes.description.KubernetesAtomicOperationDescription;
+import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesSelectorList;
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
 import com.netflix.spinnaker.moniker.Moniker;
 import java.util.List;
@@ -41,6 +42,14 @@ public class KubernetesDeployManifestDescription extends KubernetesAtomicOperati
   private boolean enableTraffic = true;
   private List<String> services;
   private Strategy strategy;
+  private KubernetesSelectorList labelSelectors = new KubernetesSelectorList();
+
+  /**
+   * If false, and using (non-empty) label selectors, fail if a deploy manifest operation doesn't
+   * deploy anything. If a particular deploy manifest stage intentionally specifies label selectors
+   * that none of the resources satisfy, set this to true to allow the stage to succeed.
+   */
+  private boolean allowNothingSelected = false;
 
   public boolean isBlueGreen() {
     return Strategy.RED_BLACK.equals(this.strategy) || Strategy.BLUE_GREEN.equals(this.strategy);

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/op/handler/CanDeploy.java

@@ -35,11 +35,12 @@ default OperationResult deploy(
       KubernetesManifestStrategy.DeployStrategy deployStrategy,
       KubernetesManifestStrategy.ServerSideApplyStrategy serverSideApplyStrategy,
       Task task,
-      String opName) {
+      String opName,
+      KubernetesSelectorList labelSelectors) {
     // If the manifest has a generateName, we must apply with kubectl create as all other operations
     // require looking up a manifest by name, which will fail.
     if (manifest.hasGenerateName()) {
-      KubernetesManifest result = credentials.create(manifest, task, opName);
+      KubernetesManifest result = credentials.create(manifest, task, opName, labelSelectors);
       return new OperationResult().addManifest(result);
     }
 
@@ -51,13 +52,13 @@ default OperationResult deploy(
               manifest.getKind(),
               manifest.getNamespace(),
               manifest.getName(),
-              new KubernetesSelectorList(),
+              labelSelectors,
               new V1DeleteOptions(),
               task,
               opName);
         } catch (KubectlJobExecutor.KubectlException ignored) {
         }
-        deployedManifest = credentials.deploy(manifest, task, opName);
+        deployedManifest = credentials.deploy(manifest, task, opName, labelSelectors);
         break;
       case REPLACE:
         deployedManifest = credentials.createOrReplace(manifest, task, opName);
@@ -70,14 +71,23 @@ default OperationResult deploy(
           cmdArgs.add("--force-conflicts=true");
         }
         deployedManifest =
-            credentials.deploy(manifest, task, opName, cmdArgs.toArray(new String[cmdArgs.size()]));
+            credentials.deploy(
+                manifest,
+                task,
+                opName,
+                labelSelectors,
+                cmdArgs.toArray(new String[cmdArgs.size()]));
         break;
       case APPLY:
-        deployedManifest = credentials.deploy(manifest, task, opName);
+        deployedManifest = credentials.deploy(manifest, task, opName, labelSelectors);
         break;
       default:
         throw new AssertionError(String.format("Unknown deploy strategy: %s", deployStrategy));
     }
-    return new OperationResult().addManifest(deployedManifest);
+    OperationResult operationResult = new OperationResult();
+    if (deployedManifest != null) {
+      operationResult.addManifest(deployedManifest);
+    }
+    return operationResult;
   }
 }

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/op/job/KubectlJobExecutor.java

@@ -66,6 +66,11 @@
 public class KubectlJobExecutor {
   private static final Logger log = LoggerFactory.getLogger(KubectlJobExecutor.class);
   private static final String NOT_FOUND_STRING = "(NotFound)";
+  private static final String NO_OBJECTS_PASSED_TO_STRING = "error: no objects passed to";
+  private static final String NO_OBJECTS_PASSED_TO_APPLY_STRING =
+      NO_OBJECTS_PASSED_TO_STRING + " apply";
+  private static final String NO_OBJECTS_PASSED_TO_CREATE_STRING =
+      NO_OBJECTS_PASSED_TO_STRING + " create";
   private static final String KUBECTL_COMMAND_OPTION_TOKEN = "--token=";
   private static final String KUBECTL_COMMAND_OPTION_KUBECONFIG = "--kubeconfig=";
   private static final String KUBECTL_COMMAND_OPTION_CONTEXT = "--context=";
@@ -522,11 +527,23 @@ public ImmutableList<KubernetesManifest> list(
     return status.getOutput();
   }
 
+  /**
+   * Invoke kubectl apply with the given manifest and (if present) label selectors.
+   *
+   * @param credentials k8s account credentials
+   * @param manifest the manifest to apply
+   * @param task the task performing this kubectl invocation
+   * @param opName the name of the operation performing this kubectl invocation
+   * @param labelSelectors label selectors
+   * @return the manifest parsed from stdout of the kubectl invocation, or null if a label selector
+   *     is present and kubectl returned "no objects passed to apply"
+   */
   public KubernetesManifest deploy(
       KubernetesCredentials credentials,
       KubernetesManifest manifest,
       Task task,
       String opName,
+      KubernetesSelectorList labelSelectors,
       String... cmdArgs) {
     log.info("Deploying manifest {}", manifest.getFullResourceName());
     List<String> command = kubectlAuthPrefix(credentials);
@@ -538,12 +555,22 @@ public KubernetesManifest deploy(
     command.add("json");
     command.add("-f");
     command.add("-");
+    addLabelSelectors(command, labelSelectors);
 
     JobResult<String> status = executeKubectlCommand(credentials, command, Optional.of(manifest));
 
     persistKubectlJobOutput(credentials, status, manifest.getFullResourceName(), task, opName);
 
     if (status.getResult() != JobResult.Result.SUCCESS) {
+      // If the caller provided a label selector, kubectl returns "no objects
+      // passed to apply" if none of the given objects satisfy the selector.
+      // Instead of throwing an exception, leave it to higher level logic to
+      // decide how to behave.
+      if (labelSelectors.isNotEmpty()
+          && status.getError().contains(NO_OBJECTS_PASSED_TO_APPLY_STRING)) {
+        return null;
+      }
+
       throw new KubectlException(
           "Deploy failed for manifest: "
               + manifest.getFullResourceName()
@@ -554,6 +581,16 @@ public KubernetesManifest deploy(
     return getKubernetesManifestFromJobResult(status, manifest);
   }
 
+  /**
+   * Invoke kubectl replace with the given manifest. Note that kubectl replace doesn't support label
+   * selectors.
+   *
+   * @param credentials k8s account credentials
+   * @param manifest the manifest to replace
+   * @param task the task performing this kubectl invocation
+   * @param opName the name of the operation performing this kubectl invocation
+   * @return the manifest parsed from stdout of the kubectl invocation
+   */
   public KubernetesManifest replace(
       KubernetesCredentials credentials, KubernetesManifest manifest, Task task, String opName) {
     log.info("Replacing manifest {}", manifest.getFullResourceName());
@@ -588,8 +625,23 @@ public KubernetesManifest replace(
     return getKubernetesManifestFromJobResult(status, manifest);
   }
 
+  /**
+   * Invoke kubectl create with the given manifest and (if present) label selectors.
+   *
+   * @param credentials k8s account credentials
+   * @param manifest the manifest to create
+   * @param task the task performing this kubectl invocation
+   * @param opName the name of the operation performing this kubectl invocation
+   * @param labelSelectors label selectors
+   * @return the manifest parsed from stdout of the kubectl invocation, or null if a label selector
+   *     is present and kubectl returned "no objects passed to create"
+   */
   public KubernetesManifest create(
-      KubernetesCredentials credentials, KubernetesManifest manifest, Task task, String opName) {
+      KubernetesCredentials credentials,
+      KubernetesManifest manifest,
+      Task task,
+      String opName,
+      KubernetesSelectorList labelSelectors) {
     log.info("Creating manifest {}", manifest.getFullResourceName());
     List<String> command = kubectlAuthPrefix(credentials);
 
@@ -599,12 +651,22 @@ public KubernetesManifest create(
     command.add("json");
     command.add("-f");
     command.add("-");
+    addLabelSelectors(command, labelSelectors);
 
     JobResult<String> status = executeKubectlCommand(credentials, command, Optional.of(manifest));
 
     persistKubectlJobOutput(credentials, status, manifest.getFullResourceName(), task, opName);
 
     if (status.getResult() != JobResult.Result.SUCCESS) {
+      // If the caller provided a label selector, kubectl returns "no objects
+      // passed to create" if none of the given objects satisfy the selector.
+      // Instead of throwing an exception, leave it to higher level logic to
+      // decide how to behave.
+      if (labelSelectors.isNotEmpty()
+          && status.getError().contains(NO_OBJECTS_PASSED_TO_CREATE_STRING)) {
+        return null;
+      }
+
       throw new KubectlException(
           "Create failed for manifest: "
               + manifest.getFullResourceName()
@@ -676,10 +738,7 @@ private List<String> kubectlLookupInfo(
     } else {
       command.add(kind.toString());
     }
-
-    if (labelSelectors != null && !labelSelectors.isEmpty()) {
-      command.add("-l=" + labelSelectors);
-    }
+    addLabelSelectors(command, labelSelectors);
 
     return command;
   }
@@ -1068,6 +1127,12 @@ private void persistKubectlJobOutput(
     }
   }
 
+  private void addLabelSelectors(List<String> command, KubernetesSelectorList labelSelectors) {
+    if (labelSelectors != null && !labelSelectors.isEmpty()) {
+      command.add("-l=" + labelSelectors);
+    }
+  }
+
   public static class KubectlException extends RuntimeException {
     public KubectlException(String message) {
       super(message);

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/op/manifest/KubernetesDeployManifestOperation.java

@@ -32,6 +32,7 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.op.OperationResult;
 import com.netflix.spinnaker.clouddriver.kubernetes.op.handler.*;
 import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesCredentials;
+import com.netflix.spinnaker.clouddriver.kubernetes.security.KubernetesSelectorList;
 import com.netflix.spinnaker.clouddriver.orchestration.AtomicOperation;
 import com.netflix.spinnaker.kork.artifacts.model.Artifact;
 import com.netflix.spinnaker.moniker.Moniker;
@@ -145,6 +146,19 @@ public OperationResult operate(List<OperationResult> _unused) {
 
     checkIfArtifactsBound(result);
 
+    KubernetesSelectorList labelSelectors = this.description.getLabelSelectors();
+
+    // kubectl replace doesn't support selectors, so fail if any manifest uses
+    // the replace strategy
+    if (labelSelectors.isNotEmpty()
+        && toDeploy.stream()
+            .map((holder) -> holder.getStrategy().getDeployStrategy())
+            .anyMatch(
+                (strategy) -> strategy == KubernetesManifestStrategy.DeployStrategy.REPLACE)) {
+      throw new IllegalArgumentException(
+          "label selectors not supported with replace strategy, not deploying");
+    }
+
     toDeploy.forEach(
         holder -> {
           KubernetesResourceProperties properties = findResourceProperties(holder.manifest);
@@ -163,7 +177,8 @@ public OperationResult operate(List<OperationResult> _unused) {
                   strategy.getDeployStrategy(),
                   strategy.getServerSideApplyStrategy(),
                   getTask(),
-                  OP_NAME));
+                  OP_NAME,
+                  labelSelectors));
 
           result.getCreatedArtifacts().add(holder.artifact);
           getTask()
@@ -175,6 +190,17 @@ public OperationResult operate(List<OperationResult> _unused) {
                       + accountName);
         });
 
+    // If a label selector was specified and nothing has been deployed, throw an
+    // exception to fail the task if configured to do so.
+    if (!description.isAllowNothingSelected()
+        && labelSelectors.isNotEmpty()
+        && result.getManifests().isEmpty()) {
+      throw new IllegalStateException(
+          "nothing deployed to account "
+              + accountName
+              + " with label selector(s) "
+              + labelSelectors.toString());
+    }
     result.removeSensitiveKeys(credentials.getResourcePropertyRegistry());
 
     getTask()

clouddriver-kubernetes/src/main/java/com/netflix/spinnaker/clouddriver/kubernetes/security/KubernetesCredentials.java

@@ -562,12 +562,16 @@ public Collection<KubernetesPodMetric> topPod(KubernetesCoordinates coords) {
   }
 
   public KubernetesManifest deploy(
-      KubernetesManifest manifest, Task task, String opName, String... cmdArgs) {
+      KubernetesManifest manifest,
+      Task task,
+      String opName,
+      KubernetesSelectorList selectorList,
+      String... cmdArgs) {
     return runAndRecordMetrics(
         "deploy",
         manifest.getKind(),
         manifest.getNamespace(),
-        () -> jobExecutor.deploy(this, manifest, task, opName, cmdArgs));
+        () -> jobExecutor.deploy(this, manifest, task, opName, selectorList, cmdArgs));
   }
 
   private KubernetesManifest replace(KubernetesManifest manifest, Task task, String opName) {
@@ -582,16 +586,20 @@ public KubernetesManifest createOrReplace(KubernetesManifest manifest, Task task
     try {
       return replace(manifest, task, opName);
     } catch (KubectlNotFoundException e) {
-      return create(manifest, task, opName);
+      // Although create supports label selectors, replace doesn't.  Assume that
+      // some higher-level logic prevents this operation in combination with
+      // label selectors.
+      return create(manifest, task, opName, new KubernetesSelectorList());
     }
   }
 
-  public KubernetesManifest create(KubernetesManifest manifest, Task task, String opName) {
+  public KubernetesManifest create(
+      KubernetesManifest manifest, Task task, String opName, KubernetesSelectorList selectorList) {
     return runAndRecordMetrics(
         "create",
         manifest.getKind(),
         manifest.getNamespace(),
-        () -> jobExecutor.create(this, manifest, task, opName));
+        () -> jobExecutor.create(this, manifest, task, opName, selectorList));
   }
 
   public List<Integer> historyRollout(KubernetesKind kind, String namespace, String name) {