Improvements for K8S deployment (especially ITs)

.github/workflows/build_container_image.yaml

@@ -10,8 +10,67 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
+  ghcrbuild:
+    name: Build container image for GHCR
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      # https://stackoverflow.com/questions/59810838/how-to-get-the-short-sha-for-the-github-workflow
+      - name: Set outputs
+        id: commit
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+      - name: Set branch name
+        id: branch
+        run: |
+          if [[ "$GITHUB_REF" == "refs/tags/"* ]]; then
+            echo "name=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" == "refs/heads/dev" ]]; then
+            echo "name=dev" >> $GITHUB_OUTPUT
+          elif [[ "$GITHUB_REF" == "refs/heads/release_"* ]]; then
+            echo "name=${GITHUB_REF#refs/heads/release_}-auto" >> $GITHUB_OUTPUT
+          fi
+        shell: bash
+      - name: Extract metadata for container image
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=raw,value=${{steps.branch.outputs.name}}
+      - name: Build args
+        id: buildargs
+        run: |
+            echo "gitcommit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT 
+            echo "builddate=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          platforms: linux/amd64
+
+      - name: Login to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push container image to ghcr
+        uses: docker/build-push-action@v4
+        with:
+          build-args: |
+              GIT_COMMIT=${{ steps.buildargs.outputs.gitcommit }}
+              BUILD_DATE=${{ steps.buildargs.outputs.builddate }}
+              IMAGE_TAG=${{ steps.branch.outputs.name }}
+          file: .k8s_ci.Dockerfile
+          push: true
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64
+
   build:
-    name: Build container image
+    name: Build container image for Galaxy repos
     runs-on: ubuntu-latest
     if: github.repository_owner == 'galaxyproject'
     steps:
@@ -58,3 +117,4 @@ jobs:
         uses: actions-hub/docker@master
         with:
           args: push galaxy/galaxy-min:${{ steps.branch.outputs.name }}
+

lib/galaxy/dependencies/conditional-requirements.txt

@@ -36,7 +36,7 @@ custos-sdk
 chronos-python==1.2.1
 
 # Kubernetes job runner
-pykube-ng==21.3.0
+pykube-ng==23.6.0
 
 # Synnefo / Pithos+ object store client
 kamaki

lib/galaxy/jobs/runners/kubernetes.py

@@ -20,6 +20,7 @@
 )
 from galaxy.jobs.runners.util.pykube_util import (
     deduplicate_entries,
+    DEFAULT_INGRESS_API_VERSION,
     DEFAULT_JOB_API_VERSION,
     delete_ingress,
     delete_job,
@@ -109,6 +110,8 @@ def __init__(self, app, nworkers, **kwargs):
             k8s_interactivetools_use_ssl=dict(map=bool, default=False),
             k8s_interactivetools_ingress_annotations=dict(map=str),
             k8s_interactivetools_ingress_class=dict(map=str, default=None),
+            k8s_interactivetools_tls_secret=dict(map=str, default=None),
+            k8s_ingress_api_version=dict(map=str, default=DEFAULT_INGRESS_API_VERSION),
         )
 
         if "runner_param_specs" not in kwargs:
@@ -249,6 +252,7 @@ def __configure_port_routing(self, ajs):
         service = Service(self._pykube_api, k8s_service_obj)
         service.create()
         ingress = Ingress(self._pykube_api, k8s_ingress_obj)
+        ingress.version = self.runner_params.get("k8s_ingress_api_version")
         ingress.create()
 
     def __get_overridable_params(self, job_wrapper, param_key):
@@ -429,6 +433,54 @@ def __get_k8s_service_spec(self, ajs):
         }
         return k8s_spec_template
 
+    def __get_k8s_ingress_rules_spec(self, ajs, entry_points):
+        """This represents the template for the "rules" portion of the Ingress spec."""
+        if "v1beta1" in self.runner_params.get("k8s_ingress_api_version"):
+            rules_spec = [
+                {
+                    "host": ep["domain"],
+                    "http": {
+                        "paths": [
+                            {
+                                "backend": {
+                                    "serviceName": self.__get_k8s_job_name(
+                                        self.__produce_k8s_job_prefix(), ajs.job_wrapper
+                                    ),
+                                    "servicePort": int(ep["tool_port"]),
+                                },
+                                "path": ep.get("entry_path", "/"),
+                                "pathType": "Prefix",
+                            }
+                        ]
+                    },
+                }
+                for ep in entry_points
+            ]
+        else:
+            rules_spec = [
+                {
+                    "host": ep["domain"],
+                    "http": {
+                        "paths": [
+                            {
+                                "backend": {
+                                    "service": {
+                                        "name": self.__get_k8s_job_name(
+                                            self.__produce_k8s_job_prefix(), ajs.job_wrapper
+                                        ),
+                                        "port": {"number": int(ep["tool_port"])},
+                                    }
+                                },
+                                "path": ep.get("entry_path", "/"),
+                                "pathType": "ImplementationSpecific",
+                            }
+                        ]
+                    },
+                }
+                for ep in entry_points
+            ]
+        return rules_spec
+
     def __get_k8s_ingress_spec(self, ajs):
         """The k8s spec template is nothing but a Ingress spec, except that it is nested and does not have an apiversion
         nor kind."""
@@ -466,39 +518,22 @@ def __get_k8s_ingress_spec(self, ajs):
                 },
                 "annotations": {"app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id},
             },
-            "spec": {
-                "rules": [
-                    {
-                        "host": ep["domain"],
-                        "http": {
-                            "paths": [
-                                {
-                                    "backend": {
-                                        "service": {
-                                            "name": self.__get_k8s_job_name(
-                                                self.__produce_k8s_job_prefix(), ajs.job_wrapper
-                                            ),
-                                            "port": {"number": int(ep["tool_port"])},
-                                        }
-                                    },
-                                    "path": ep.get("entry_path", "/"),
-                                    "pathType": "Prefix",
-                                }
-                            ]
-                        },
-                    }
-                    for ep in entry_points
-                ],
-            },
+            "spec": {"rules": self.__get_k8s_ingress_rules_spec(ajs, entry_points)},
         }
         default_ingress_class = self.runner_params.get("k8s_interactivetools_ingress_class")
         if default_ingress_class:
             k8s_spec_template["spec"]["ingressClassName"] = default_ingress_class
         if self.runner_params.get("k8s_interactivetools_use_ssl"):
             domains = list({e["domain"] for e in entry_points})
-            k8s_spec_template["spec"]["tls"] = [
-                {"hosts": [domain], "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains
-            ]
+            override_secret = self.runner_params.get("k8s_interactivetools_tls_secret")
+            if override_secret:
+                k8s_spec_template["spec"]["tls"] = [
+                    {"hosts": [domain], "secretName": override_secret} for domain in domains
+                ]
+            else:
+                k8s_spec_template["spec"]["tls"] = [
+                    {"hosts": [domain], "secretName": re.sub("[^a-z0-9-]", "-", domain)} for domain in domains
+                ]
         if self.runner_params.get("k8s_interactivetools_ingress_annotations"):
             new_ann = yaml.safe_load(self.runner_params.get("k8s_interactivetools_ingress_annotations"))
             k8s_spec_template["metadata"]["annotations"].update(new_ann)