Merge branch 'main' into data-folder

Qiskit · Oct 29, 2024 · f3dfbec · f3dfbec
2 parents feec25b + 6f7a5f1
commit f3dfbec
Show file tree

Hide file tree

Showing 55 changed files with 1,712 additions and 1,491 deletions.
diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml
@@ -10,8 +10,6 @@ jobs:
     timeout-minutes: 45
     steps:
       - uses: actions/checkout@v4
-      - name: Build the function
-        run: docker build -t test_function:latest --build-arg TARGETARCH="amd64" -f ./tests/basic/function/Sample-Docker ./tests/basic
       - name: Build the containers
         run: docker compose -f docker-compose-dev.yaml build
       - name: Run the jupyter profile
@@ -29,6 +27,7 @@ jobs:
         shell: bash
         run: |
           cd tests/basic
+          rm ./06_function.py
           for f in *.py; do echo "$f" && python "$f" &>> basic.log; done
           done=$(cat basic.log | grep -c "DONE")
           if [[ $done == 4 ]]

diff --git a/.github/workflows/kubernetes-deploy.yaml b/.github/workflows/kubernetes-deploy.yaml
@@ -24,6 +24,8 @@ jobs:
         with:
           k8s-version: 1.29.x
           kind-worker-count: 0
+      - name: Label nodes
+        run: kubectl label node kind-control-plane has-gpu=gpu has-cpu=cpu
       - name: Build and load gateway
         run: |
           docker build -t gateway:test -f ./gateway/Dockerfile .

diff --git a/.github/workflows/notebook-local-verify.yaml b/.github/workflows/notebook-local-verify.yaml
@@ -22,6 +22,8 @@ jobs:
           for f in tests/basic/*.py; do sed -i "s/import ServerlessClient/import LocalClient/;s/= ServerlessClient(/= LocalClient(/;/token=os\.environ\.get/d;/host=os\.environ\.get/d" "$f"; done
           for f in tests/experimental/*.py; do sed -i "s/import ServerlessClient/import LocalClient/;s/= ServerlessClient(/= LocalClient(/;/token=os\.environ\.get/d;/host=os\.environ\.get/d" "$f"; done
           rm tests/basic/06_function.py
+          rm tests/experimental/file_download.py
+          rm tests/experimental/manage_data_directory.py
       - name: install dependencies
         shell: bash
         run: pip install client/

diff --git a/.github/workflows/update-component-versions.yaml b/.github/workflows/update-component-versions.yaml
@@ -44,6 +44,8 @@ jobs:
           sed -i "s/ray-node:${OLDNUM}/ray-node:${NEWNUM}/" charts/qiskit-serverless/values.yaml
           sed -i "s/version: ${OLDNUM}/version: ${NEWNUM}/" charts/qiskit-serverless/values.yaml
           sed -i "s/qiskit-serverless\/ray-node:${OLDNUM}/qiskit-serverless\/ray-node:${NEWNUM}/" docs/deployment/custom_function/Sample-Dockerfile
+          sed -i "s/FROM icr.io\/quantum-public\/qiskit-serverless\/ray-node${OLDNUM}/FROM icr.io\/quantum-public\/qiskit-serverless\/ray-node${NEWNUM}/" docs/deployment/deploying_custom_image_function.rst
+          sed -i "s/FROM icr.io\/quantum-public\/qiskit-serverless\/ray-node${OLDNUM}/FROM icr.io\/quantum-public\/qiskit-serverless\/ray-node${NEWNUM}/" docs/deployment/example_custom_image_function.rst
           sed -i "s/qiskit-serverless\/ray-node:${OLDNUM}/qiskit-serverless\/ray-node:${NEWNUM}/" gateway/main/settings.py
           cd charts/qiskit-serverless
           helm dependency update

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -146,7 +146,7 @@ instead of forking it.
 ### Development environment
 
 This repository contains several projects with different technologies. Depending on the project that you selected (eg. gateway), from the project directory you will run:
-- `pip install -r requirements.txt requirements-dev.txt` for python projects (strongly consider using a [virtual environment](https://docs.python.org/3/library/venv.html)!).
+- `pip install -r requirements.txt -r requirements-dev.txt` for python projects (strongly consider using a [virtual environment](https://docs.python.org/3/library/venv.html)!).
 - `helm dependency build` for helm (Before running this command, make sure to check for helm configuration instructions specific to your selected project charts).
 -  `terraform init` for terraform.
 

diff --git a/charts/qiskit-serverless/Chart.lock b/charts/qiskit-serverless/Chart.lock
@@ -1,7 +1,7 @@
 dependencies:
 - name: gateway
   repository: ""
-  version: 0.17.0
+  version: 0.17.1
 - name: nginx-ingress-controller
   repository: https://charts.bitnami.com/bitnami
   version: 9.11.0
@@ -11,5 +11,5 @@ dependencies:
 - name: kuberay-operator
   repository: https://ray-project.github.io/kuberay-helm
   version: 1.1.1
-digest: sha256:0eaad7053682fb99f4c801a5f224dcf756968ebe9e5d3580f75eb94ceb7f8f21
-generated: "2024-09-26T14:09:39.57006234Z"
+digest: sha256:e550274e4089d7b3e2926e0a1049b088a407875053a9ad071b288ef50e993076
+generated: "2024-10-09T14:47:34.054577191Z"
diff --git a/charts/qiskit-serverless/Chart.yaml b/charts/qiskit-serverless/Chart.yaml
@@ -4,13 +4,13 @@ description: Qiskit-Serverless helm chart that contains different dependencies.
 
 type: application
 
-version: 0.17.0
-appVersion: "0.17.0"
+version: 0.17.1
+appVersion: "0.17.1"
 
 dependencies:
   - name: gateway
     condition: gatewayEnable
-    version: 0.17.0
+    version: 0.17.1
   - name: nginx-ingress-controller
     condition: nginxIngressControllerEnable
     version: 9.11.0

diff --git a/charts/qiskit-serverless/charts/gateway/Chart.yaml b/charts/qiskit-serverless/charts/gateway/Chart.yaml
@@ -15,10 +15,10 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.17.0
+version: 0.17.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "0.17.0"
+appVersion: "0.17.1"
diff --git a/charts/qiskit-serverless/charts/gateway/templates/deployment.yaml b/charts/qiskit-serverless/charts/gateway/templates/deployment.yaml
@@ -119,8 +119,14 @@ spec:
               value: {{ .Values.application.ray.minReplicas | quote }}
             - name: RAY_CLUSTER_WORKER_MAX_REPLICAS
               value: {{ .Values.application.ray.maxReplicas | quote }}
+            - name: RAY_CLUSTER_CPU_NODE_SELECTOR_LABEL
+              value: {{ .Values.application.nodeSelector.cpu | quote }}
+            - name: RAY_CLUSTER_GPU_NODE_SELECTOR_LABEL
+              value: {{ .Values.application.nodeSelector.gpu | quote }}
             - name: LIMITS_CPU_PER_TASK
               value: {{ .Values.application.ray.cpu | quote }}
+            - name: LIMITS_GPU_PER_TASK
+              value: {{ .Values.application.ray.gpu | quote }}
             - name: LIMITS_MEMORY_PER_TASK
               value: {{ .Values.application.ray.memory | quote }}
 {{- if .Values.application.superuser.enable }}
@@ -233,6 +239,9 @@ spec:
         - name: ray-cluster-template
           configMap:
             name: rayclustertemplate
+        - name: gpu-jobs
+          configMap:
+            name: gpujobs
       serviceAccountName: {{ include "gateway.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
@@ -271,6 +280,8 @@ spec:
               name: gateway-pv-storage
             - mountPath: "/tmp/templates/"
               name: ray-cluster-template
+            - mountPath: "/tmp/gpujobs/"
+              name: gpu-jobs
           resources:
             {{- toYaml .Values.scheduler.resources | nindent 12 }}
           env:
@@ -310,10 +321,18 @@ spec:
               value: {{ .Release.Namespace }}
             - name: RAY_NODE_IMAGE
               value: {{ .Values.application.ray.nodeImage | quote }}
+            - name: RAY_CLUSTER_CPU_NODE_SELECTOR_LABEL
+              value: {{ .Values.application.nodeSelector.cpu | quote }}
+            - name: RAY_CLUSTER_GPU_NODE_SELECTOR_LABEL
+              value: {{ .Values.application.nodeSelector.gpu | quote }}
             - name: LIMITS_JOBS_PER_USER
               value: {{ .Values.application.limits.maxJobsPerUser | quote }}
             - name: LIMITS_MAX_CLUSTERS
               value: {{ .Values.application.limits.maxComputeResources | quote }}
+            - name: LIMITS_GPU_CLUSTERS
+              value: {{ .Values.application.limits.maxGpuResources | quote }}
+            - name: GATEWAY_GPU_JOBS_CONFIG
+              value: {{ .Values.application.ray.gpuJobsConfig | quote }}
 {{- if .Values.application.limits.keepClusterOnComplete }}
             - name: RAY_CLUSTER_NO_DELETE_ON_COMPLETE
               value: "True"

diff --git a/charts/qiskit-serverless/charts/gateway/templates/gpu-jobs.yaml b/charts/qiskit-serverless/charts/gateway/templates/gpu-jobs.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: gpujobs
+data:
+  gpu-jobs.json: |
+    {
+        "gpu-functions": {}
+    }
diff --git a/charts/qiskit-serverless/charts/gateway/templates/rayclustertemplate.yaml b/charts/qiskit-serverless/charts/gateway/templates/rayclustertemplate.yaml
@@ -98,9 +98,11 @@ data:
                 protocol: TCP
               resources:
                 limits:
+                  nvidia.com/gpu: {{`{{gpu_request}}`}}
                   cpu: {{ .Values.application.ray.cpu }}
                   memory: {{ .Values.application.ray.memory }}Gi
                 requests:
+                  nvidia.com/gpu: {{`{{gpu_request}}`}}
                   cpu: {{ .Values.application.ray.cpu }}
                   memory: {{ .Values.application.ray.memory }}Gi
               securityContext:
@@ -230,6 +232,7 @@ data:
             serviceAccount: ray-cluster-sa
 {{- end }}
             nodeSelector:
+              {{`{{node_selector_label}}`}}
             tolerations: []
             securityContext:
               fsGroup: 1000

diff --git a/charts/qiskit-serverless/charts/gateway/values.yaml b/charts/qiskit-serverless/charts/gateway/values.yaml
@@ -18,15 +18,16 @@ application:
   superuser:
     enable: true
   ray:
-    nodeImage: "icr.io/quantum-public/qiskit-serverless/ray-node:0.17.0"
+    nodeImage: "icr.io/quantum-public/qiskit-serverless/ray-node:0.17.1"
     cpu: 2
     memory: 2
+    gpu: 1
     replicas: 1
     minReplicas: 1
     maxReplicas: 4
     opensslImage: registry.access.redhat.com/ubi8/openssl:8.8-9
     kubectlImage: alpine/k8s:1.29.2@sha256:a51aa37f0a34ff827c7f2f9cb7f6fbb8f0e290fa625341be14c2fcc4b1880f60
-    proxyImage: "icr.io/quantum-public/qiskit-serverless/proxy:0.17.0"
+    proxyImage: "icr.io/quantum-public/qiskit-serverless/proxy:0.17.1"
     scrapeWithPrometheus: true
     openTelemetry: false
     openTelemetryCollector:
@@ -36,6 +37,7 @@ application:
       port: 4317
       insecure: 0
     useTLS: true
+    gpuJobsConfig: "api/v1/gpu-jobs.json"
   proxy:
     enabled: true
     cpu: 1
@@ -44,6 +46,7 @@ application:
   limits:
     maxJobsPerUser: 2
     maxComputeResources: 4
+    maxGpuResources: 1
     keepClusterOnComplete: False
     programTimeoutDays: 14
   qiskitRuntime:

diff --git a/charts/qiskit-serverless/values.yaml b/charts/qiskit-serverless/values.yaml
@@ -2,7 +2,7 @@
 # Qiskit Serverless Info
 # ===================
 global:
-  version: 0.17.0
+  version: 0.17.1
 
 # ===================
 # Qiskit Serverless configs
@@ -47,7 +47,7 @@ gateway:
   image:
     repository: "icr.io/quantum-public/qiskit-serverless/gateway"
     pullPolicy: IfNotPresent
-    tag: "0.17.0"
+    tag: "0.17.1"
   application:
     siteHost: "http://gateway:8000"
     rayHost: "http://kuberay-head-svc:8265"
@@ -59,12 +59,16 @@ gateway:
       type: ClusterIP
       port: 8000
     ray:
-      nodeImage: "icr.io/quantum-public/qiskit-serverless/ray-node:0.17.0"
+      nodeImage: "icr.io/quantum-public/qiskit-serverless/ray-node:0.17.1"
       opensslImage: registry.access.redhat.com/ubi8/openssl:8.8-9
       kubectlImage: alpine/k8s:1.29.2@sha256:a51aa37f0a34ff827c7f2f9cb7f6fbb8f0e290fa625341be14c2fcc4b1880f60
+      gpuJobsConfig: "/tmp/gpujobs/gpu-jobs.json"
     limits:
       maxJobsPerUser: 2
       maxComputeResources: 4
+    nodeSelector:
+      cpu: "has-cpu: cpu"
+      gpu: "has-gpu: gpu"
   cos:
     claimName: gateway-claim
 

diff --git a/client/qiskit_serverless/VERSION.txt b/client/qiskit_serverless/VERSION.txt
@@ -1 +1 @@
-0.17.0
+0.17.1
diff --git a/client/qiskit_serverless/__init__.py b/client/qiskit_serverless/__init__.py
@@ -22,20 +22,15 @@
 from importlib_metadata import version as metadata_version, PackageNotFoundError
 
 from .core import (
-    BaseProvider,
     BaseClient,
     distribute_task,
     distribute_qiskit_function,
     get,
     put,
     get_refs_by_status,
-    ServerlessProvider,
     ServerlessClient,
-    IBMServerlessProvider,
     IBMServerlessClient,
-    RayProvider,
     RayClient,
-    LocalProvider,
     LocalClient,
     save_result,
     Configuration,

diff --git a/client/qiskit_serverless/core/__init__.py b/client/qiskit_serverless/core/__init__.py
@@ -31,11 +31,7 @@
     BaseClient
     RayClient
     LocalClient
-    ComputeResource
     Job
-    GatewayJobClient
-    BaseJobClient
-    RayJobClient
     save_result
     QiskitPattern
     QiskitFunction
@@ -51,25 +47,13 @@
 
 """
 
-from .client import (
-    BaseProvider,
-    BaseClient,
-    ComputeResource,
-    ServerlessProvider,
-    ServerlessClient,
-    IBMServerlessProvider,
-    IBMServerlessClient,
-    LocalProvider,
-    LocalClient,
-    RayProvider,
-    RayClient,
-)
+from .client import BaseClient
+
+from .clients.local_client import LocalClient
+from .clients.ray_client import RayClient
+from .clients.serverless_client import ServerlessClient, IBMServerlessClient
 
 from .job import (
-    BaseJobClient,
-    RayJobClient,
-    GatewayJobClient,
-    LocalJobClient,
     Job,
     save_result,
     Configuration,