Merge branch 'kubeflow:master' into ocean-spark-v2

.github/workflows/release.yaml

@@ -65,12 +65,54 @@ jobs:
             echo "Tag '${VERSION}' does not exist."
           fi
 
-  build_images:
+  release_sparkctl:
     needs:
       - check-release
 
     runs-on: ubuntu-latest
 
+    strategy:
+      fail-fast: true
+      matrix:
+        os:
+          - linux
+          - darwin
+        arch:
+          - amd64
+          - arm64
+
+    env:
+      GOOS: ${{ matrix.os }}
+      GOARCH: ${{ matrix.arch }}
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v4
+
+      - name: Read version from VERSION file
+        run: |
+          VERSION=$(cat VERSION | sed "s/^v//")
+          echo "VERSION=${VERSION}" >> $GITHUB_ENV
+
+      - name: Build sparkctl binary
+        run: |
+          make build-sparkctl
+          tar -czvf sparkctl-${VERSION}-${GOOS}-${GOARCH}.tgz -C bin sparkctl
+
+      - name: Upload sparkctl binary
+        uses: actions/upload-artifact@v4
+        with:
+          name: sparkctl-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}
+          path: sparkctl-${{ env.VERSION }}-${{ env.GOOS }}-${{ env.GOARCH }}.tgz
+          if-no-files-found: error
+          retention-days: 1
+
+  build_images:
+    needs:
+      - release_sparkctl
+
+    runs-on: ubuntu-latest
+
     strategy:
       fail-fast: false
       matrix:
@@ -90,10 +132,6 @@ jobs:
       - name: Read version from VERSION file
         run: |
           VERSION=$(cat VERSION)
-          if [[ ! ${VERSION} =~ ${{ env.SEMVER_PATTERN }} ]]; then
-              echo "Version '${VERSION}' does not match semver pattern."
-              exit 1
-          fi
           echo "VERSION=${VERSION}" >> $GITHUB_ENV
 
       - name: Docker meta
@@ -250,6 +288,11 @@ jobs:
             helm package charts/${chart}
           done
 
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: sparkctl-*
+
       - name: Release
         id: release
         uses: softprops/action-gh-release@v2
@@ -262,3 +305,4 @@ jobs:
           draft: true
           files: |
             *.tgz
+            sparkctl-*/sparkctl-*.tgz

Dockerfile

@@ -26,7 +26,9 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \
     go mod download
 
 COPY . .
+
 ENV GOCACHE=/root/.cache/go-build
+
 ARG TARGETARCH
 
 RUN --mount=type=cache,target=/go/pkg/mod/ \
@@ -35,6 +37,10 @@ RUN --mount=type=cache,target=/go/pkg/mod/ \
 
 FROM ${SPARK_IMAGE}
 
+ARG SPARK_UID=185
+
+ARG SPARK_GID=185
+
 USER root
 
 RUN apt-get update \
@@ -45,7 +51,7 @@ RUN mkdir -p /etc/k8s-webhook-server/serving-certs /home/spark && \
     chmod -R g+rw /etc/k8s-webhook-server/serving-certs && \
     chown -R spark /etc/k8s-webhook-server/serving-certs /home/spark
 
-USER spark
+USER ${SPARK_UID}:${SPARK_GID}
 
 COPY --from=builder /workspace/bin/spark-operator /usr/bin/spark-operator
 

Makefile

@@ -109,7 +109,7 @@ print-%: ; @echo $*=$($*)
 
 .PHONY: manifests
 manifests: controller-gen ## Generate CustomResourceDefinition, RBAC and WebhookConfiguration manifests.
-	$(CONTROLLER_GEN) crd rbac:roleName=spark-operator-controller webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) crd:generateEmbeddedObjectMeta=true rbac:roleName=spark-operator-controller webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.

api/v1beta2/sparkapplication_types.go

@@ -409,6 +409,14 @@ type Dependencies struct {
 // SparkPodSpec defines common things that can be customized for a Spark driver or executor pod.
 // TODO: investigate if we should use v1.PodSpec and limit what can be set instead.
 type SparkPodSpec struct {
+	// Template is a pod template that can be used to define the driver or executor pod configurations that Spark configurations do not support.
+	// Spark version >= 3.0.0 is required.
+	// Ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template.
+	// +optional
+	// +kubebuilder:validation:Schemaless
+	// +kubebuilder:validation:Type:=object
+	// +kubebuilder:pruning:PreserveUnknownFields
+	Template *corev1.PodTemplateSpec `json:"template,omitempty"`
 	// Cores maps to `spark.driver.cores` or `spark.executor.cores` for the driver and executors, respectively.
 	// +optional
 	// +kubebuilder:validation:Minimum=1

charts/spark-operator-chart/README.md

@@ -106,13 +106,13 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | controller.affinity | object | `{}` | Affinity for controller pods. |
 | controller.tolerations | list | `[]` | List of node taints to tolerate for controller pods. |
 | controller.priorityClassName | string | `""` | Priority class for controller pods. |
-| controller.podSecurityContext | object | `{}` | Security context for controller pods. |
+| controller.podSecurityContext | object | `{"fsGroup":185}` | Security context for controller pods. |
 | controller.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). The labelSelector field in topology spread constraint will be set to the selector labels for controller pods if not specified. |
 | controller.env | list | `[]` | Environment variables for controller containers. |
 | controller.envFrom | list | `[]` | Environment variable sources for controller containers. |
 | controller.volumeMounts | list | `[]` | Volume mounts for controller containers. |
 | controller.resources | object | `{}` | Pod resource requests and limits for controller containers. Note, that each job submission will spawn a JVM within the controller pods using "/usr/local/openjdk-11/bin/java -Xmx128m". Kubernetes may kill these Java processes at will to enforce resource limits. When that happens, you will see the following error: 'failed to run spark-submit for SparkApplication [...]: signal: killed' - when this happens, you may want to increase memory limits. |
-| controller.securityContext | object | `{}` | Security context for controller containers. |
+| controller.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"runAsNonRoot":true}` | Security context for controller containers. |
 | controller.sidecars | list | `[]` | Sidecar containers for controller pods. |
 | controller.podDisruptionBudget.enable | bool | `false` | Specifies whether to create pod disruption budget for controller. Ref: [Specifying a Disruption Budget for your Application](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) |
 | controller.podDisruptionBudget.minAvailable | int | `1` | The number of pods that must be available. Require `controller.replicas` to be greater than 1 |
@@ -144,13 +144,13 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | webhook.affinity | object | `{}` | Affinity for webhook pods. |
 | webhook.tolerations | list | `[]` | List of node taints to tolerate for webhook pods. |
 | webhook.priorityClassName | string | `""` | Priority class for webhook pods. |
-| webhook.podSecurityContext | object | `{}` | Security context for webhook pods. |
+| webhook.podSecurityContext | object | `{"fsGroup":185}` | Security context for webhook pods. |
 | webhook.topologySpreadConstraints | list | `[]` | Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in. Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/). The labelSelector field in topology spread constraint will be set to the selector labels for webhook pods if not specified. |
 | webhook.env | list | `[]` | Environment variables for webhook containers. |
 | webhook.envFrom | list | `[]` | Environment variable sources for webhook containers. |
 | webhook.volumeMounts | list | `[]` | Volume mounts for webhook containers. |
 | webhook.resources | object | `{}` | Pod resource requests and limits for webhook pods. |
-| webhook.securityContext | object | `{}` | Security context for webhook containers. |
+| webhook.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"runAsNonRoot":true}` | Security context for webhook containers. |
 | webhook.podDisruptionBudget.enable | bool | `false` | Specifies whether to create pod disruption budget for webhook. Ref: [Specifying a Disruption Budget for your Application](https://kubernetes.io/docs/tasks/run-application/configure-pdb/) |
 | webhook.podDisruptionBudget.minAvailable | int | `1` | The number of pods that must be available. Require `webhook.replicas` to be greater than 1 |
 | spark.jobNamespaces | list | `["default"]` | List of namespaces where to run spark jobs. If empty string is included, all namespaces will be allowed. Make sure the namespaces have already existed. |

charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -4755,6 +4755,13 @@ spec:
                           - name
                           type: object
                         type: array
+                      template:
+                        description: |-
+                          Template is a pod template that can be used to define the driver or executor pod configurations that Spark configurations do not support.
+                          Spark version >= 3.0.0 is required.
+                          Ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template.
+                        type: object
+                        x-kubernetes-preserve-unknown-fields: true
                       terminationGracePeriodSeconds:
                         description: Termination grace period seconds for the pod
                         format: int64
@@ -9512,6 +9519,13 @@ spec:
                           - name
                           type: object
                         type: array
+                      template:
+                        description: |-
+                          Template is a pod template that can be used to define the driver or executor pod configurations that Spark configurations do not support.
+                          Spark version >= 3.0.0 is required.
+                          Ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template.
+                        type: object
+                        x-kubernetes-preserve-unknown-fields: true
                       terminationGracePeriodSeconds:
                         description: Termination grace period seconds for the pod
                         format: int64
@@ -10351,6 +10365,23 @@ spec:
                                     May contain labels and annotations that will be copied into the PVC
                                     when creating it. No other fields are allowed and will be rejected during
                                     validation.
+                                  properties:
+                                    annotations:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                    finalizers:
+                                      items:
+                                        type: string
+                                      type: array
+                                    labels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                    name:
+                                      type: string
+                                    namespace:
+                                      type: string
                                   type: object
                                 spec:
                                   description: |-

charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml

@@ -4694,6 +4694,13 @@ spec:
                       - name
                       type: object
                     type: array
+                  template:
+                    description: |-
+                      Template is a pod template that can be used to define the driver or executor pod configurations that Spark configurations do not support.
+                      Spark version >= 3.0.0 is required.
+                      Ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
                   terminationGracePeriodSeconds:
                     description: Termination grace period seconds for the pod
                     format: int64
@@ -9421,6 +9428,13 @@ spec:
                       - name
                       type: object
                     type: array
+                  template:
+                    description: |-
+                      Template is a pod template that can be used to define the driver or executor pod configurations that Spark configurations do not support.
+                      Spark version >= 3.0.0 is required.
+                      Ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
                   terminationGracePeriodSeconds:
                     description: Termination grace period seconds for the pod
                     format: int64
@@ -10257,6 +10271,23 @@ spec:
                                 May contain labels and annotations that will be copied into the PVC
                                 when creating it. No other fields are allowed and will be rejected during
                                 validation.
+                              properties:
+                                annotations:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                                finalizers:
+                                  items:
+                                    type: string
+                                  type: array
+                                labels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                                name:
+                                  type: string
+                                namespace:
+                                  type: string
                               type: object
                             spec:
                               description: |-

charts/spark-operator-chart/values.yaml

@@ -120,10 +120,8 @@ controller:
   priorityClassName: ""
 
   # -- Security context for controller pods.
-  podSecurityContext: {}
-    # runAsUser: 1000
-    # runAsGroup: 2000
-    # fsGroup: 3000
+  podSecurityContext:
+    fsGroup: 185
 
   # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in.
   # Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/).
@@ -158,10 +156,13 @@ controller:
     #   memory: 300Mi
 
   # -- Security context for controller containers.
-  securityContext: {}
-    # runAsUser: 1000
-    # runAsGroup: 2000
-    # fsGroup: 3000
+  securityContext:
+    privileged: false
+    allowPrivilegeEscalation: false
+    runAsNonRoot: true
+    capabilities:
+      drop:
+      - ALL
 
   # -- Sidecar containers for controller pods.
   sidecars: []
@@ -266,10 +267,8 @@ webhook:
   priorityClassName: ""
 
   # -- Security context for webhook pods.
-  podSecurityContext: {}
-    # runAsUser: 1000
-    # runAsGroup: 2000
-    # fsGroup: 3000
+  podSecurityContext:
+    fsGroup: 185
 
   # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in.
   # Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/).
@@ -301,10 +300,13 @@ webhook:
     #   memory: 300Mi
 
   # -- Security context for webhook containers.
-  securityContext: {}
-    # runAsUser: 1000
-    # runAsGroup: 2000
-    # fsGroup: 3000
+  securityContext:
+    privileged: false
+    allowPrivilegeEscalation: false
+    runAsNonRoot: true
+    capabilities:
+      drop:
+      - ALL
 
   # Pod disruption budget for webhook to avoid service degradation.
   podDisruptionBudget:

config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -4755,6 +4755,13 @@ spec:
                           - name
                           type: object
                         type: array
+                      template:
+                        description: |-
+                          Template is a pod template that can be used to define the driver or executor pod configurations that Spark configurations do not support.
+                          Spark version >= 3.0.0 is required.
+                          Ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template.
+                        type: object
+                        x-kubernetes-preserve-unknown-fields: true
                       terminationGracePeriodSeconds:
                         description: Termination grace period seconds for the pod
                         format: int64
@@ -9512,6 +9519,13 @@ spec:
                           - name
                           type: object
                         type: array
+                      template:
+                        description: |-
+                          Template is a pod template that can be used to define the driver or executor pod configurations that Spark configurations do not support.
+                          Spark version >= 3.0.0 is required.
+                          Ref: https://spark.apache.org/docs/latest/running-on-kubernetes.html#pod-template.
+                        type: object
+                        x-kubernetes-preserve-unknown-fields: true
                       terminationGracePeriodSeconds:
                         description: Termination grace period seconds for the pod
                         format: int64
@@ -10351,6 +10365,23 @@ spec:
                                     May contain labels and annotations that will be copied into the PVC
                                     when creating it. No other fields are allowed and will be rejected during
                                     validation.
+                                  properties:
+                                    annotations:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                    finalizers:
+                                      items:
+                                        type: string
+                                      type: array
+                                    labels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                    name:
+                                      type: string
+                                    namespace:
+                                      type: string
                                   type: object
                                 spec:
                                   description: |-