diff --git a/.github/workflows/e2e-test-kfpv1.yaml b/.github/workflows/e2e-test-kfpv1.yaml new file mode 100644 index 00000000000..52807f7cbaf --- /dev/null +++ b/.github/workflows/e2e-test-kfpv1.yaml @@ -0,0 +1,45 @@ +name: E2E Test with kubeflow pipelines v1 + +on: + pull_request: + paths-ignore: + - "pkg/new-ui/v1beta1/frontend/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + +jobs: + e2e: + runs-on: ubuntu-20.04 + timeout-minutes: 120 + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Setup Test Env + uses: ./.github/workflows/template-setup-e2e-test + with: + kubernetes-version: ${{ matrix.kubernetes-version }} + python-version: "3.10" + + - name: Run e2e test with ${{ matrix.experiments }} experiments + uses: ./.github/workflows/template-e2e-test + with: + experiments: ${{ matrix.experiments }} + training-operator: true + # Comma Delimited + trial-images: kfpv1-metrics-collector + install-kfp: 1.8.1 + experiment-namespace: kubeflow + + strategy: + fail-fast: false + matrix: + kubernetes-version: ["v1.23.13", "v1.24.7", "v1.25.3"] + # Comma Delimited + experiments: + - "katib-kfp-example-e2e-v1" diff --git a/.github/workflows/publish-core-images.yaml b/.github/workflows/publish-core-images.yaml index 750ab03c99e..5708e9ce9ac 100644 --- a/.github/workflows/publish-core-images.yaml +++ b/.github/workflows/publish-core-images.yaml @@ -32,3 +32,5 @@ jobs: dockerfile: cmd/metricscollector/v1beta1/file-metricscollector/Dockerfile - component-name: tfevent-metrics-collector dockerfile: cmd/metricscollector/v1beta1/tfevent-metricscollector/Dockerfile + - component-name: kfpv1-metrics-collector + dockerfile: cmd/metricscollector/v1beta1/kfp-metricscollector/v1/Dockerfile diff --git a/.github/workflows/template-e2e-test/action.yaml b/.github/workflows/template-e2e-test/action.yaml index ef1ca26064d..6337c8215bf 100644 --- a/.github/workflows/template-e2e-test/action.yaml +++ b/.github/workflows/template-e2e-test/action.yaml @@ -21,6 +21,15 @@ inputs: required: false description: mysql or postgres default: mysql + install-kfp: + required: false + description: whether kubeflow pipelines is required + as a dependency. If so provide version as string (eg 1.8.1) + default: false + experiment-namespace: + required: false + description: namespace to execute test experiment in + default: default runs: using: composite @@ -31,8 +40,8 @@ runs: - name: Setup Katib shell: bash - run: ./test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh ${{ inputs.katib-ui }} ${{ inputs.training-operator }} ${{ inputs.database-type }} + run: ./test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh ${{ inputs.katib-ui }} ${{ inputs.training-operator }} ${{ inputs.database-type }} ${{ inputs.install-kfp }} - name: Run E2E Experiment shell: bash - run: ./test/e2e/v1beta1/scripts/gh-actions/run-e2e-experiment.sh ${{ inputs.experiments }} + run: ./test/e2e/v1beta1/scripts/gh-actions/run-e2e-experiment.sh ${{ inputs.experiments }} ${{ inputs.experiment-namespace }} diff --git a/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/Dockerfile b/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/Dockerfile new file mode 100644 index 00000000000..9d7722e5f30 --- /dev/null +++ b/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.10-slim + +ARG TARGETARCH +ENV TARGET_DIR /opt/katib +ENV METRICS_COLLECTOR_DIR cmd/metricscollector/v1beta1/kfp-metricscollector/v1 +ENV PYTHONPATH ${TARGET_DIR}:${TARGET_DIR}/pkg/apis/manager/v1beta1/python:${TARGET_DIR}/pkg/metricscollector/v1beta1/kfp-metricscollector/v1::${TARGET_DIR}/pkg/metricscollector/v1beta1/common/ + +ADD ./pkg/ ${TARGET_DIR}/pkg/ +ADD ./${METRICS_COLLECTOR_DIR}/ ${TARGET_DIR}/${METRICS_COLLECTOR_DIR}/ + +WORKDIR ${TARGET_DIR}/${METRICS_COLLECTOR_DIR} + +RUN if [ "${TARGETARCH}" = "arm64" ]; then \ + apt-get -y update && \ + apt-get -y install gfortran libpcre3 libpcre3-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/*; \ + fi + +RUN pip install --no-cache-dir -r requirements.txt +RUN chgrp -R 0 ${TARGET_DIR} \ + && chmod -R g+rwX ${TARGET_DIR} + +ENTRYPOINT ["python", "main.py"] diff --git a/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/main.py b/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/main.py new file mode 100644 index 00000000000..333e70553eb --- /dev/null +++ b/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/main.py @@ -0,0 +1,101 @@ +# Copyright 2023 The Kubeflow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +from logging import INFO, StreamHandler, getLogger + +import api_pb2 +import const +import grpc +from metrics_loader import MetricsCollector +from pns import WaitMainProcesses + +timeout_in_seconds = 60 + + +def parse_options(): + parser = argparse.ArgumentParser( + description="KFP V1 MetricsCollector", add_help=True + ) + + # TODO (andreyvelich): Add early stopping flags. + parser.add_argument("-s-db", "--db_manager_server_addr", type=str, default="") + parser.add_argument("-t", "--pod_name", type=str, default="") + parser.add_argument( + "-path", + "--metrics_file_dir", + type=str, + default=const.DEFAULT_METRICS_FILE_KFPV1_DIR, + ) + parser.add_argument("-m", "--metric_names", type=str, default="") + parser.add_argument("-o-type", "--objective_type", type=str, default="") + parser.add_argument("-f", "--metric_filters", type=str, default="") + parser.add_argument( + "-p", "--poll_interval", type=int, default=const.DEFAULT_POLL_INTERVAL + ) + parser.add_argument( + "-timeout", "--timeout", type=int, default=const.DEFAULT_TIMEOUT + ) + parser.add_argument( + "-w", "--wait_all_processes", type=str, default=const.DEFAULT_WAIT_ALL_PROCESSES + ) + opt = parser.parse_args() + return opt + + +if __name__ == "__main__": + logger = getLogger(__name__) + handler = StreamHandler() + handler.setLevel(INFO) + logger.setLevel(INFO) + logger.addHandler(handler) + logger.propagate = False + opt = parse_options() + wait_all_processes = opt.wait_all_processes.lower() == "true" + db_manager_server = opt.db_manager_server_addr.split(":") + trial_name = "-".join(opt.pod_name.split("-")[:-1]) + if len(db_manager_server) != 2: + raise Exception( + "Invalid Katib DB manager service address: %s" % opt.db_manager_server_addr + ) + + WaitMainProcesses( + pool_interval=opt.poll_interval, + timout=opt.timeout, + wait_all=wait_all_processes, + completed_marked_dir=None, + ) + + mc = MetricsCollector(opt.metric_names.split(";")) + observation_log = mc.parse_file(opt.metrics_file_dir) + + channel = grpc.beta.implementations.insecure_channel( + db_manager_server[0], int(db_manager_server[1]) + ) + + with api_pb2.beta_create_DBManager_stub(channel) as client: + logger.info( + "In " + + trial_name + + " " + + str(len(observation_log.metric_logs)) + + " metrics will be reported." + ) + client.ReportObservationLog( + api_pb2.ReportObservationLogRequest( + trial_name=trial_name, observation_log=observation_log + ), + timeout=timeout_in_seconds, + ) diff --git a/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/requirements.txt b/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/requirements.txt new file mode 100644 index 00000000000..b73a43f3fba --- /dev/null +++ b/cmd/metricscollector/v1beta1/kfp-metricscollector/v1/requirements.txt @@ -0,0 +1,5 @@ +psutil==5.9.4 +rfc3339>=6.2 +grpcio==1.41.1 +googleapis-common-protos==1.6.0 +protobuf==3.20.0 diff --git a/examples/v1beta1/kubeflow-pipelines/README.md b/examples/v1beta1/kubeflow-pipelines/README.md index df1e2bf0041..0c2ff8b6956 100644 --- a/examples/v1beta1/kubeflow-pipelines/README.md +++ b/examples/v1beta1/kubeflow-pipelines/README.md @@ -3,6 +3,10 @@ The following examples show how to use Katib with [Kubeflow Pipelines](https://github.com/kubeflow/pipelines). +Two different aspects are illustrated here: +A) How to orchestrate Katib experiments from Kubeflow pipelines using the Katib Kubeflow Component (Example 1 & 2) +B) How to use Katib to tune parameters of Kubeflow pipelines + You can find the Katib Component source code for the Kubeflow Pipelines [here](https://github.com/kubeflow/pipelines/tree/master/components/kubeflow/katib-launcher). @@ -13,6 +17,8 @@ You have to install the following Python SDK to run these examples: - [`kfp`](https://pypi.org/project/kfp/) >= 1.8.12 - [`kubeflow-katib`](https://pypi.org/project/kubeflow-katib/) >= 0.13.0 +In order to run parameter tuning over Kubeflow pipelines, additionally Katib needs to be setup to run with Argo workflow tasks. The setup is described within the example notebook (3). + ## Multi-User Pipelines Setup The Notebooks examples run Pipelines in multi-user mode and your Kubeflow Notebook @@ -25,10 +31,12 @@ to give an access Kubeflow Notebook to run Kubeflow Pipelines. The following Pipelines are deployed from Kubeflow Notebook: -- [Kubeflow E2E MNIST](kubeflow-e2e-mnist.ipynb) +1) [Kubeflow E2E MNIST](kubeflow-e2e-mnist.ipynb) + +2) [Katib Experiment with Early Stopping](early-stopping.ipynb) -- [Katib Experiment with Early Stopping](early-stopping.ipynb) +3) [Tune parameters of a `MNIST` kubeflow pipeline with Katib](kubeflow-kfpv1-opt-mnist.ipynb) -The following Pipelines have to be compiled and uploaded to the Kubeflow Pipelines UI: +The following Pipelines have to be compiled and uploaded to the Kubeflow Pipelines UI for examples 1 & 2: - [MPIJob Horovod](mpi-job-horovod.py) diff --git a/examples/v1beta1/kubeflow-pipelines/katib-kfp-example-e2e-v1.yaml b/examples/v1beta1/kubeflow-pipelines/katib-kfp-example-e2e-v1.yaml new file mode 100644 index 00000000000..18d6683825c --- /dev/null +++ b/examples/v1beta1/kubeflow-pipelines/katib-kfp-example-e2e-v1.yaml @@ -0,0 +1,374 @@ +apiVersion: kubeflow.org/v1beta1 +kind: Experiment +metadata: + name: katib-e2e-2023-07-20-22h-37m-57s + namespace: kubeflow +spec: + algorithm: + algorithmName: random + maxFailedTrialCount: 2 + maxTrialCount: 5 + metricsCollectorSpec: + collector: + customCollector: + args: + - -m + - val-accuracy;accuracy + - -s + - katib-db-manager.kubeflow:6789 + - -t + - $(PodName) + - -path + - /tmp/outputs/mlpipeline_metrics + env: + - name: PodName + valueFrom: + fieldRef: + fieldPath: metadata.name + image: docker.io/votti/kfpv1-metricscollector:v0.0.10 + imagePullPolicy: Always + name: custom-metrics-logger-and-collector + kind: Custom + source: + fileSystemPath: + kind: File + path: /tmp/outputs/mlpipeline_metrics/data + objective: + additionalMetricNames: + - accuracy + goal: 0.9 + objectiveMetricName: val-accuracy + type: maximize + parallelTrialCount: 5 + parameters: + - feasibleSpace: + max: '0.001' + min: '0.00001' + name: learning_rate + parameterType: double + - feasibleSpace: + max: '64' + min: '16' + name: batch_size + parameterType: int + - feasibleSpace: + list: + - '0' + - '1' + name: histogram_norm + parameterType: discrete + trialTemplate: + failureCondition: status.[@this].#(phase=="Failed")# + primaryContainerName: main + primaryPodLabels: + katib.kubeflow.org/model-training: 'true' + retain: false + successCondition: status.[@this].#(phase=="Succeeded")# + trialParameters: + - description: Learning rate for the training model + name: learningRate + reference: learning_rate + - description: Batch size for NN training + name: batchSize + reference: batch_size + - description: Histogram normalization of image on? + name: histogramNorm + reference: histogram_norm + trialSpec: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: + annotations: + pipelines.kubeflow.org/kfp_sdk_version: 1.8.12 + pipelines.kubeflow.org/pipeline_compilation_time: '2023-07-20T22:37:57.355215' + pipelines.kubeflow.org/pipeline_spec: '{"inputs": [{"default": "0.0001", + "name": "lr", "optional": true, "type": "Float"}, {"default": "Adam", + "name": "optimizer", "optional": true, "type": "String"}, {"default": + "categorical_crossentropy", "name": "loss", "optional": true, "type": + "String"}, {"default": "3", "name": "epochs", "optional": true, "type": + "Integer"}, {"default": "5", "name": "batch_size", "optional": true, "type": + "Integer"}, {"default": "False", "name": "histogram_norm", "optional": + true, "type": "Boolean"}, {"default": "${trialParameters.learningRate}", + "name": "lr"}, {"default": "${trialParameters.batchSize}", "name": "batch_size"}, + {"default": "${trialParameters.histogramNorm}", "name": "histogram_norm"}], + "name": "Minimal KFP1 pipeline for e2e testing"}' + generateName: minimal-kfp1-pipeline-for-e2e-testing- + labels: + pipelines.kubeflow.org/kfp_sdk_version: 1.8.12 + spec: + arguments: + parameters: + - name: lr + value: ${trialParameters.learningRate} + - name: optimizer + value: Adam + - name: loss + value: categorical_crossentropy + - name: epochs + value: '3' + - name: batch_size + value: ${trialParameters.batchSize} + - name: histogram_norm + value: ${trialParameters.histogramNorm} + entrypoint: minimal-kfp1-pipeline-for-e2e-testing + serviceAccountName: pipeline-runner + templates: + - dag: + tasks: + - arguments: + parameters: + - name: histogram_norm + value: '{{inputs.parameters.histogram_norm}}' + name: prep-e2e + template: prep-e2e + - arguments: + artifacts: + - from: '{{tasks.prep-e2e.outputs.artifacts.prep-e2e-output_nr}}' + name: prep-e2e-output_nr + parameters: + - name: batch_size + value: '{{inputs.parameters.batch_size}}' + - name: epochs + value: '{{inputs.parameters.epochs}}' + - name: loss + value: '{{inputs.parameters.loss}}' + - name: lr + value: '{{inputs.parameters.lr}}' + - name: optimizer + value: '{{inputs.parameters.optimizer}}' + dependencies: + - prep-e2e + name: train-e2e + template: train-e2e + inputs: + parameters: + - name: batch_size + - name: epochs + - name: histogram_norm + - name: loss + - name: lr + - name: optimizer + name: minimal-kfp1-pipeline-for-e2e-testing + - container: + args: + - --histogram-norm + - '{{inputs.parameters.histogram_norm}}' + - --output-nr + - /tmp/outputs/output_nr/data + command: + - sh + - -ec + - 'program_path=$(mktemp) + + printf "%s" "$0" > "$program_path" + + python3 -u "$program_path" "$@" + + ' + - "def _make_parent_dirs_and_return_path(file_path: str):\n import\ + \ os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n \ + \ return file_path\n\ndef prep_e2e(\n output_nr_path, # type:\ + \ ignore # noqa: F821\n histogram_norm = True,\n):\n with open(output_nr_path,\ + \ 'w') as writer:\n writer.write(str(int(histogram_norm)))\n\n\ + def _deserialize_bool(s) -> bool:\n from distutils.util import strtobool\n\ + \ return strtobool(s) == 1\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Prep\ + \ e2e', description='')\n_parser.add_argument(\"--histogram-norm\",\ + \ dest=\"histogram_norm\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\n\ + _parser.add_argument(\"--output-nr\", dest=\"output_nr_path\", type=_make_parent_dirs_and_return_path,\ + \ required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\ + \n_outputs = prep_e2e(**_parsed_args)\n" + image: python:3.7 + inputs: + parameters: + - name: histogram_norm + metadata: + annotations: + pipelines.kubeflow.org/arguments.parameters: '{"histogram_norm": "{{inputs.parameters.histogram_norm}}"}' + pipelines.kubeflow.org/component_ref: '{}' + pipelines.kubeflow.org/component_spec: '{"implementation": {"container": + {"args": [{"if": {"cond": {"isPresent": "histogram_norm"}, "then": + ["--histogram-norm", {"inputValue": "histogram_norm"}]}}, "--output-nr", + {"outputPath": "output_nr"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf + \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", + "def _make_parent_dirs_and_return_path(file_path: str):\n import + os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return + file_path\n\ndef prep_e2e(\n output_nr_path, # type: ignore # + noqa: F821\n histogram_norm = True,\n):\n with open(output_nr_path, + ''w'') as writer:\n writer.write(str(int(histogram_norm)))\n\ndef + _deserialize_bool(s) -> bool:\n from distutils.util import strtobool\n return + strtobool(s) == 1\n\nimport argparse\n_parser = argparse.ArgumentParser(prog=''Prep + e2e'', description='''')\n_parser.add_argument(\"--histogram-norm\", + dest=\"histogram_norm\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--output-nr\", + dest=\"output_nr_path\", type=_make_parent_dirs_and_return_path, required=True, + default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs + = prep_e2e(**_parsed_args)\n"], "image": "python:3.7"}}, "inputs": + [{"default": "True", "name": "histogram_norm", "optional": true, "type": + "Boolean"}], "name": "Prep e2e", "outputs": [{"name": "output_nr", + "type": "Integer"}]}' + pipelines.kubeflow.org/task_display_name: Prepare a dummy output that + should be cached + labels: + pipelines.kubeflow.org/cache_enabled: 'true' + pipelines.kubeflow.org/enable_caching: 'true' + pipelines.kubeflow.org/kfp_sdk_version: 1.8.12 + pipelines.kubeflow.org/pipeline-sdk-type: kfp + name: prep-e2e + outputs: + artifacts: + - name: prep-e2e-output_nr + path: /tmp/outputs/output_nr/data + - container: + args: + - --input-nr + - /tmp/inputs/input_nr/data + - --lr + - '{{inputs.parameters.lr}}' + - --optimizer + - '{{inputs.parameters.optimizer}}' + - --loss + - '{{inputs.parameters.loss}}' + - --epochs + - '{{inputs.parameters.epochs}}' + - --batch-size + - '{{inputs.parameters.batch_size}}' + - --mlpipeline-metrics + - /tmp/outputs/mlpipeline_metrics/data + command: + - sh + - -ec + - 'program_path=$(mktemp) + + printf "%s" "$0" > "$program_path" + + python3 -u "$program_path" "$@" + + ' + - "def _make_parent_dirs_and_return_path(file_path: str):\n import\ + \ os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n \ + \ return file_path\n\ndef train_e2e(\n input_nr_path, # type:\ + \ ignore # noqa: F821\n mlpipeline_metrics_path, # type: ignore\ + \ # noqa: F821\n lr = 1e-4,\n optimizer = \"Adam\",\n loss\ + \ = \"categorical_crossentropy\",\n epochs = 1,\n batch_size =\ + \ 32,\n):\n \"\"\"\n This is the simulated train part of our ML\ + \ pipeline where training is performed\n \"\"\"\n import json\ + \ \n import time\n with open(input_nr_path, 'r') as reader:\n\ + \ line = reader.readline()\n histogram_norm_value = int(line)\n\ + \n accuracy = (batch_size + histogram_norm_value)/ (batch_size +\ + \ epochs+histogram_norm_value)\n val_accuracy = accuracy * 0.9\n\ + \ metrics = {\n \"metrics\": [\n {\n \ + \ \"name\": \"accuracy\", # The name of the metric. Visualized\ + \ as the column name in the runs table.\n \"numberValue\"\ + : accuracy, # The value of the metric. Must be a numeric value.\n \ + \ \"format\": \"PERCENTAGE\", # The optional format of\ + \ the metric. Supported values are \"RAW\" (displayed in raw format)\ + \ and \"PERCENTAGE\" (displayed in percentage format).\n \ + \ },\n {\n \"name\": \"val-accuracy\", #\ + \ The name of the metric. Visualized as the column name in the runs\ + \ table.\n \"numberValue\": val_accuracy, # The value\ + \ of the metric. Must be a numeric value.\n \"format\"\ + : \"PERCENTAGE\", # The optional format of the metric. Supported values\ + \ are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed\ + \ in percentage format).\n },\n ]\n }\n with\ + \ open(mlpipeline_metrics_path, \"w\") as f:\n json.dump(metrics,\ + \ f)\n\n # If this step is to fast, the metrics collector fails as\ + \ the\n # pod is already finished before it can collect the metrics.\n\ + \ time.sleep(10)\n\nimport argparse\n_parser = argparse.ArgumentParser(prog='Train\ + \ e2e', description='This is the simulated train part of our ML pipeline\ + \ where training is performed')\n_parser.add_argument(\"--input-nr\"\ + , dest=\"input_nr_path\", type=str, required=True, default=argparse.SUPPRESS)\n\ + _parser.add_argument(\"--lr\", dest=\"lr\", type=float, required=False,\ + \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--optimizer\",\ + \ dest=\"optimizer\", type=str, required=False, default=argparse.SUPPRESS)\n\ + _parser.add_argument(\"--loss\", dest=\"loss\", type=str, required=False,\ + \ default=argparse.SUPPRESS)\n_parser.add_argument(\"--epochs\", dest=\"\ + epochs\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"\ + --batch-size\", dest=\"batch_size\", type=int, required=False, default=argparse.SUPPRESS)\n\ + _parser.add_argument(\"--mlpipeline-metrics\", dest=\"mlpipeline_metrics_path\"\ + , type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\n\ + _parsed_args = vars(_parser.parse_args())\n\n_outputs = train_e2e(**_parsed_args)\n" + image: python:3.7 + inputs: + artifacts: + - name: prep-e2e-output_nr + path: /tmp/inputs/input_nr/data + parameters: + - name: batch_size + - name: epochs + - name: loss + - name: lr + - name: optimizer + metadata: + annotations: + pipelines.kubeflow.org/arguments.parameters: '{"batch_size": "{{inputs.parameters.batch_size}}", + "epochs": "{{inputs.parameters.epochs}}", "loss": "{{inputs.parameters.loss}}", + "lr": "{{inputs.parameters.lr}}", "optimizer": "{{inputs.parameters.optimizer}}"}' + pipelines.kubeflow.org/component_ref: '{}' + pipelines.kubeflow.org/component_spec: '{"description": "This is the + simulated train part of our ML pipeline where training is performed", + "implementation": {"container": {"args": ["--input-nr", {"inputPath": + "input_nr"}, {"if": {"cond": {"isPresent": "lr"}, "then": ["--lr", + {"inputValue": "lr"}]}}, {"if": {"cond": {"isPresent": "optimizer"}, + "then": ["--optimizer", {"inputValue": "optimizer"}]}}, {"if": {"cond": + {"isPresent": "loss"}, "then": ["--loss", {"inputValue": "loss"}]}}, + {"if": {"cond": {"isPresent": "epochs"}, "then": ["--epochs", {"inputValue": + "epochs"}]}}, {"if": {"cond": {"isPresent": "batch_size"}, "then": + ["--batch-size", {"inputValue": "batch_size"}]}}, "--mlpipeline-metrics", + {"outputPath": "mlpipeline_metrics"}], "command": ["sh", "-ec", "program_path=$(mktemp)\nprintf + \"%s\" \"$0\" > \"$program_path\"\npython3 -u \"$program_path\" \"$@\"\n", + "def _make_parent_dirs_and_return_path(file_path: str):\n import + os\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\n return + file_path\n\ndef train_e2e(\n input_nr_path, # type: ignore # + noqa: F821\n mlpipeline_metrics_path, # type: ignore # noqa: F821\n lr + = 1e-4,\n optimizer = \"Adam\",\n loss = \"categorical_crossentropy\",\n epochs + = 1,\n batch_size = 32,\n):\n \"\"\"\n This is the simulated + train part of our ML pipeline where training is performed\n \"\"\"\n import + json \n import time\n with open(input_nr_path, ''r'') as reader:\n line + = reader.readline()\n histogram_norm_value = int(line)\n\n accuracy + = (batch_size + histogram_norm_value)/ (batch_size + epochs+histogram_norm_value)\n val_accuracy + = accuracy * 0.9\n metrics = {\n \"metrics\": [\n {\n \"name\": + \"accuracy\", # The name of the metric. Visualized as the column + name in the runs table.\n \"numberValue\": accuracy, # + The value of the metric. Must be a numeric value.\n \"format\": + \"PERCENTAGE\", # The optional format of the metric. Supported values + are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed + in percentage format).\n },\n {\n \"name\": + \"val-accuracy\", # The name of the metric. Visualized as the column + name in the runs table.\n \"numberValue\": val_accuracy, # + The value of the metric. Must be a numeric value.\n \"format\": + \"PERCENTAGE\", # The optional format of the metric. Supported values + are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed + in percentage format).\n },\n ]\n }\n with + open(mlpipeline_metrics_path, \"w\") as f:\n json.dump(metrics, + f)\n\n # If this step is to fast, the metrics collector fails as + the\n # pod is already finished before it can collect the metrics.\n time.sleep(10)\n\nimport + argparse\n_parser = argparse.ArgumentParser(prog=''Train e2e'', description=''This + is the simulated train part of our ML pipeline where training is performed'')\n_parser.add_argument(\"--input-nr\", + dest=\"input_nr_path\", type=str, required=True, default=argparse.SUPPRESS)\n_parser.add_argument(\"--lr\", + dest=\"lr\", type=float, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--optimizer\", + dest=\"optimizer\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--loss\", + dest=\"loss\", type=str, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--epochs\", + dest=\"epochs\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--batch-size\", + dest=\"batch_size\", type=int, required=False, default=argparse.SUPPRESS)\n_parser.add_argument(\"--mlpipeline-metrics\", + dest=\"mlpipeline_metrics_path\", type=_make_parent_dirs_and_return_path, + required=True, default=argparse.SUPPRESS)\n_parsed_args = vars(_parser.parse_args())\n\n_outputs + = train_e2e(**_parsed_args)\n"], "image": "python:3.7"}}, "inputs": + [{"name": "input_nr", "type": "Integer"}, {"default": "0.0001", "name": + "lr", "optional": true, "type": "Float"}, {"default": "Adam", "name": + "optimizer", "optional": true, "type": "String"}, {"default": "categorical_crossentropy", + "name": "loss", "optional": true, "type": "String"}, {"default": "1", + "name": "epochs", "optional": true, "type": "Integer"}, {"default": + "32", "name": "batch_size", "optional": true, "type": "Integer"}], + "name": "Train e2e", "outputs": [{"name": "mlpipeline_metrics", "type": + "Metrics"}]}' + pipelines.kubeflow.org/max_cache_staleness: P0D + pipelines.kubeflow.org/task_display_name: Generate dummy metrics + labels: + katib.kubeflow.org/model-training: 'true' + pipelines.kubeflow.org/enable_caching: 'true' + pipelines.kubeflow.org/kfp_sdk_version: 1.8.12 + pipelines.kubeflow.org/pipeline-sdk-type: kfp + name: train-e2e + outputs: + artifacts: + - name: mlpipeline-metrics + path: /tmp/outputs/mlpipeline_metrics/data diff --git a/examples/v1beta1/kubeflow-pipelines/kubeflow-kfpv1-opt-mnist.ipynb b/examples/v1beta1/kubeflow-pipelines/kubeflow-kfpv1-opt-mnist.ipynb new file mode 100644 index 00000000000..6efb26732a0 --- /dev/null +++ b/examples/v1beta1/kubeflow-pipelines/kubeflow-kfpv1-opt-mnist.ipynb @@ -0,0 +1,1972 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Katib parameter tuning over Kubeflow Pipelines (V1)\n", + "\n", + "This example shows how parameter tunning can be done over a multistep Kubeflow pipeline.\n", + "\n", + "The pipeline consists of 4 steps:\n", + "- Download of the training images and labels from the original MNIST publication\n", + "- Prepartion of the training dataset\n", + "- Image pre-processing\n", + "- Model fitting\n", + "\n", + "The pipeline has the model has model fitting parameters as well as image pre-processing parameters exposed as a pipeline parameter for tuning. Katib will be used to explore the question if image preprocessing using a simple histogram normalization might improve a neural network training on MNIST.\n", + "\n", + "## Requirements\n", + "\n", + "This requires a Kubeflow installation with Katib and Pipelines.\n", + "\n", + "Additionally the Katib-Argo integration needs to be setup:\n", + "\n", + "If you are running on a full Kubeflow installation *do not reinstall or update Argo* as this will likely break your installation.\n", + "\n", + "Just run the following commands:\n", + "\n", + "Enable side-car injection:\n", + "\n", + "`kubectl patch namespace argo -p '{\"metadata\":{\"labels\":{\"katib.kubeflow.org/metrics-collector-injection\":\"enabled\"}}}'`\n", + "\n", + "\n", + "Verify that the emissary executor is active (should be default in newer Kubeflow installations):\n", + "\n", + "` kubectl get ConfigMap -n argo workflow-controller-configmap -o yaml | grep containerRuntimeExecutor`\n", + "\n", + "Patch the Katib controller:\n", + "\n", + "`kubectl patch ClusterRole katib-controller -n kubeflow --type=json \\\n", + " -p='[{\"op\": \"add\", \"path\": \"/rules/-\", \"value\": {\"apiGroups\":[\"argoproj.io\"],\"resources\":[\"workflows\"],\"verbs\":[\"get\", \"list\", \"watch\", \"create\", \"delete\"]}}]'\n", + "`\n", + "\n", + "`kubectl patch Deployment katib-controller -n kubeflow --type=json \\\n", + " -p='[{\"op\": \"add\", \"path\": \"/spec/template/spec/containers/0/args/-\", \"value\": \"--trial-resources=Workflow.v1alpha1.argoproj.io\"}]'`\n", + "\n", + "For more details and how to set this up on a partial Kubeflow installation follow:\n", + "https://github.com/kubeflow/katib/tree/master/examples/v1beta1/argo/README.mdd" + ] + }, + { + "attachments": { + "image.png": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAInCAYAAAB+wpi7AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAAEnQAABJ0Ad5mH3gAAI9aSURBVHhe7N0HYBzFoT7wT9d1VfXUu2Rb7r0XsA2mhEAagbwkJCG9kuQ1kvBP8tJeegJ5JLRAQofQTLGNG+69Sy6SbFm9t+td/5nVCWzjIhsbdKfvZ5a729urup39dmZ2NsHhcPSDiIiIKM5YLBblUqX8n4iIiChOMewQERFRXGPYISIiorjGsENERERxjWGHiIiI4hrDDhEREcU1hh0iIiKKaww7REREFNeGxaCC/f39CAaDCAQCiEQi0blE9F6oVCrodDpotVokJCRE544c4XBYKVNk2UJEl4csT2S5olaro3OGt8FBBT/wsCODTigUgt/vVwoneZsolgTCQfSLf8ONLIwMBgP0Gh00as2ICjyyLOEOFMWqcH8YoUg4emt40ev1MOj00Kq1MRF4hk3YkUHH4/EoX1pSUpLyRY7EvVCKTeH+CLq8PcrlsBTpR8QZhDHRCI1GE50Z/2SZIgNPYmIibDabUstFFCucATdcQU/01vDT7wtDHUqA0WiMzhm+hs3pImTY8Xq9DDpEV4CsKXW73cp6NpLIMkWGO6vVyqBDdJnJGlO5jsWSYVEKyAJZ1uww6BBdXrLadiQ2DcvPLMuTWOlXQBRL5PoVa+XKsNnlYdAhIiKiK4H1u0RERBTXGHaIiIgorjHsEBERUVxj2CH6gAUjQbR5OnGw8xg2Nu1SJnldzpP3ERFdjEh/BO6QB8f76rGj9YBSpshLeVvOl/ePNB/4ODs+nw+9vb0oKioaUeOAUHy41HF2ZGHjDftxrOcEah2NIth0odfvhCvoVu43a01I0luQYUxFkTUXo5OLkajWQ5VwcfsnkXAErtZepCSnKAMMjhSdnZ0wm81IT0+PziGKHZc6zk6oP4wmVxtq+upQ72xBj68PjoALAbHTpFNpYdWZkWywId+ShVJbAXLMGdAkXPwRi74+8d78EaSlpUXnDF/DZlBBhh2KZZcSdoKRELp8vajqPYFNzXtwpPuEUiCdjSycylOKsSB7GkYlFSPVkAStaujrCcMOww7FnosNO3IEd1/Ij5POJuxur8DutkNiJ6opeu+7FVlzMD1jAqbbx6PQkgODRo8E8W+oYjHssBmL6H0kCyUZdLa37sffD7+IXW0V5ww6krxPLiOXlY+Rjx2Op6Ygog+O3IGqdTbiXzWr8HrtW+cNOpK8Xy4nl5ePk4+Pdww7RO8jeR6tXe0HsaJuo9JsNZS2c7mMXFY+Rj5WPgcRkSTPo9Xj78UzVa/jSPdxpXl8KORycnn5OPl4+TzxjGGH6H20t6MSe9sPo9PbM6SgI5uxZFXzZ8bcDKPGoDxWPgcRkdTu7caa+u046WiCJ+Qb0sjGhdYcXJe/APOzpymPk4+XzxPPGHaI3gey6UlWFcujrI73NQyp2jhJb8XE1NG4rmABJqWNVjoty4JJPod8PJuziEY2WRvT7unCttZ9cAYufJSVOkGtdE5elDNTCTr5lmz4QgHx+P3K88Rz7Q7DDtH7QO5t9fj70OBqRa/fEZ17brJGZ3xKmVIglSUVipDTDHfQqzRnyeeQzxVr56YhosvLG/Kh1dOpHHl1oaCiU2uRZUrHwuwZSm2xOkGFJlGWRMROU4N4vHwe+XzxKnbDTiSIgNeBrrZ2tIup7ZSpvaMTXS5xf/gD3Bgo78+Nnk4nfJF+8YO6UsRPNeKDs7MHLm9Q7PFHZ18pET+8Thf6ej0IiJsX/IbFChgJutHb4YDHH0J8twqfWygSRp0ILB4RWCS5h2XSJkKv1p12OLk8IkIWSmNTSrE4bzbKU0pQ2V2Np6teQ62jQSnQ5HPI55LPSZeT+DWHvXD2dKHzjDJFTp09Tjj9V3oFO5/B9yfeh9uPK9tzKwi/26W8llf8zK5oSXopZcTFlkNxqtvnQKu7M3oLMKj1SNQY3nXEpixv7ImpmJc1FdcXLFTKkXWN27G6YZvYTgbE99evPI98vngVu2HHeRSVK+/Dt275CD4qpptPmT562+fwrScPo7LlA0ypyvt7Gj/4/N+xoduF3ujsy68Xru5NePjzP8KTKytxxBmdfaV0bcfKB/+B3/3iNewXNy9Y4Ppa4Tj8An58x/14aVctzn+MQPySh6a3ejrgi3YezDHb8fGS6zA7c7Iyns4grVqDSWlj8OGixSiwZGNPeyUePfyiKIT6lOeQ5HPI57rYsX3oAsKivGhdgYfv+TbuOKNMkdMdP34YD2/tii78AXj7/T2Mvz+9A9XR2VdGDbY//SgeuOfveKNV/uais6+ESykjLrYcilPOoAvd/p7oLWBBznRcmz8PpUkF0TkDZHlzVc5MpVzpDTjxQs2b2NqyH8FTDnaQzyOfL17FcM1OAEGvBn7/BNz473fjv37+M/xCmf4D//3tRchY9Wc8v3w7NtR8QH885f2JH2KnC/7wFa7ZEYWgrNlxe4MInO2FXMdRtWE5fvMfz2GH2BN6T9ndOhYzb1qGT356DkrFzQuO+KJLgbFgET539y1YMCYLw39UhiulXwkngy1Pck8rxZCkFD5zM6cgw5imNF1NTR+HjxRfg2SDFTvaDuKNkxtE0Dn9SAn5HANBZ6Tuz14p8ouVNSd2FM/6GL74dpkipzuxLK0Fdc8+jD+u7/iAangG39/7VbMjXke8ludsNTsREdo7NuCZ3z+D55bvR2109iW5lDLiYsuhOBURhcGpOz2yZmdyWjluKFio1ArLGh7ZGfnq3NmYlTkJHb5u5eiriq5qeEIe8Xd95y8rn0c+X7yK8T47emj0eRg3fyEWLFmMJcp0La656hrcMj6Ixj37seNQHdqiS49YwV70Np3Ans3VaPOHlGrfS6ZPR86oUoyflIcUcfOCPyB1InRiL2PKgjHITzPDGJ090simKpvOotTcSLLfzr6Ow8rIpjMyJojCaBYWZA/slclRk/d1HMGmpt3K6Mpn1uDI55DPdbGjKdNQJSOjaAJmvF2myOlG3DizEMWqBqx6aQeqPH5c/Pi2cUSGb28jqvZWoaq2473tQF1KGXGx5VCckqOqmzWm6C3gWE8tGl1tsIudpxtEgJwjdqSuzZunHOjgDnqUnSdZtvQFHO8KNvJ55PPFK/Xdd9/9k+j1D0QoFFJGUU5OToZKdRE/WV8LmmrqsW1nEBNvm4U8sz66kmihUZtRmN2Lbavr4UlMQs60UchRi5VTPKbq0BEcPlyFmtoGNDT1IGxLRaI2BE/LCTTXN6FDkwGb+HurAh1oOlGN6hPtcCVG5zkbcKK+A7XtLpi0vajc1Sr2sOpR33hcPGcNTjY0oT1shTlRC0Oo9d3vL9iH3tZaHNh1SBQQtThR24q+kApqsxWmwV0Th3iNmmPYe+gYasUyytQnfpQaPZJNWhHlQ0DPcRyoOIbKo9WobWlAbZd4T29UwzhnLgrKcpCdGH0uhQMdNYewb+M+bDvSA2NhKmy2JFg0fvi6m3HocJMIQkdQVdMDR78GBpN4rfYqbN93BNU1JwZev6ULzT4DMsSXoAl2iu+lRXxmNzQZZujFd1p9qBWtrY3oFCvawYPiPYnHtPp0UOkSYdUGEHA046D4roImA7RaL9ytzajY1wJ/uA7Hqmpw7NhxNLT3ogdJSBVfhFqVIALaGd9Vnw99bU3wdra/8zca+oCfV4zcM5Kd+i58ZNTAHtihzip0+nqiTVGdEPtSKLblodiaiyJbrjJ8uyyM3qzfopzH5mydDrOMdizKnYlUg+2CgUd2Yg64fEhMTBxRI5R7PB7odDqYTO9sCC6oP6g0P29Y24OEnDyMnl2CzOhdYlOAFEsQQWc73nq1FnkfnoMciwHm8LnWaRMM/Q40H9qNloAR/VojTCqvWBcaxLpwCB1hE1R6Ma/fBX/HCWyvcovfvQPdzR2oq26C238Shw5V4/jxWjT1BeEXZVqqUfzGznx/8vdxznJNLfbsxVsPiNc43zotVyRPBzobqrBt75FomdOC47vr4erVIGXpQoy1ipL17Z9aACF/K2o2bcC6LY3oUWthsicjWZQrpmALampa0VB7HM11daiu80GXYYPe04S6c5VrBrEDdrFlxMWWQwb1QEA79btqaUeDM4Bw3cDfQ/kbiSJ2OJBjackdoQuRZUqnrxsHOo8qt+Wgo/JABqvOpDSHyw7JY5KL4Qy6sal5N9Y17lBGWj5beTU3ewpGJxcpfQkvJOQX7y3cD6Nx+O++6vUDAS4Ow44gT/WR3o/mtfsR1KfBMnY8ygx9cB19BQ/9+XH88+nlWLl2MzZvqYKndBpy0wJofesprH51DbZarsXMHBUMPRvx2j8ewd9f2I+WousxPVsFXdW/8PjyvXhpXzvGpR/Gb7+9BvW9m7B+2yo88/hyrHtrA7b4xqNcBIqcxC60nPb+RAHYthd7Vj6LX//iH3htw3qsfXMXan0m6AvLUJqiEXsnQXj3PIcn/vkEfv335dj01jqsW/kqXqwW4SklC9NLU6AOdMO16UH8/K/P4bHnV+CtfYewudUP87FeJC+cj6J3hZ0T2L18BV55fDUOuMSKvrcB2uKxyLJ1oW3PKvzydytxYs/zePmNVvSYU5Cd64Xvzb/i279/Hm+sWoO31qzEiq2VWNuVjeum5cLi3o7XH1uPV9a0w37jGGS2rsQDP1uLzdvfQkXDWjz6sPgexHve3JUJc3qu+J4ccB17Az/59mYEJxTCntqGhjWr8L8/XI0+wyY896838PK/lovPUYvKhIlYUC4KTF0YgTO+qzfre1G7bSWaD+7HoYybME1shfTDYNs91LCTkJCgFCJ7Ow4rHQHl8vI8Nif6GsRGTo1cc6ZyKogjIsg+Ln6nLe72d9XoSLL5S4ai6wrmKx2ZLzTEO8PO5Qo7gjkCd18XutYdgGr+MpSIdTax+1zrdB5yNPVY/6vv4U33aKjTClGW2IK+Iy/gJ9//PfaoJiAptxAlOI6Orf/Al//Rh+zkEzj45g688M91aItswCMPvopVr7+GTSeC6DWXYf5oo9jhOoqNb7+/ImSEzleumUSQ6Eek/SgazrdOi2WCxzdg84t/x/f+9AI2is+xpr4L7ccdSNGnIP1dYacP7t59ePaeJ7BZ7BAeqWtGew+QMmUSip0r8eBDa7HileXYtXkzXtscROH1o2E+8rL4XOco13ICcFddZBnh3Hpx5VBeIlRnfFcr9lVgQ2M3vC/9Vvw9JkNvL0JJcvQjfsCGGnZkGSBHWd/ZfujtmhoZeGQAks3i41JKlSM3VzdsVYKO7Ix8psGDImQNUIktXzlK60JiMexcRLqIJfJjZSAts0/sOYg/fFev2EPZjofvfgqu8s/ia/e/hFf+8XPc/7kUrBGF1NpDDiQU5CM1S4/New7AHxR/yK5O8Vg1PDYbGppbEQ6Hxax2aJLMKBw3ShSCAZGptqAS8zHvtj+L5/sDHv7SNDhXrMX+RhHEBt7IKWqw/fUVWLPGiYk/fQmPPy/ew29vRLnjGF64X3ayE3uNOIDlz21CG2bhzt+K+59/HK/89EOY4a3EkQMV2Co/h3MnHv7zWzBP+TLuFp/j2Z/fhZ9mN6JCHcI7ffJPVYrZN16P27/+CfHDvx7ff/QXuOO6cSiXfWJlL/6OI8CH/oof/fWn+P7tuVBXb8PjjxzB5Dv/ij//Q7yH+/8bP1qSBu/Gl/FGvRdn7/N9EHXIgmnGL95+z3liL7HygNiTjC5xuk7x7R3ADnweX/vhw8prfG9uGqqfex17vF70nuW7euwGI3LC9VjbGn2KGCMLFNmePiapSOkseCrZXPXKiTV46fjqd3VGPpN8rHwO+VwXcy4buhwsSDQakJFbj7aOELy+863Ta3BYl4MJs4rQ0N6E2gZRIvi8Yn1rB3Jz0e7xobfPKWZ50NnZhtxpE5GbmiReoRZdcKE65W488Ogz4vm+gGU2L46u3XqWjrgXKtdkZ98mnLjgOi0+x8Yt2LJPj5vE53haWd/yMc7WK9bss0mCOWk2vvirT2HuhBtw8+1fwV0/+wJuEMlQVqCgVTwqZxZmfP8hPPnop7E4pQYbz1uuDTzr6S5URpzN+cqhd39Xz/78q/ixfSdWNQbQGKNtkhqxoySPspK1OINN5FKTqx3P1axQRl1/7PBL7+qMfKrBgyLk88jni1dxGnbkRkANtVpuMMIIe7wItbeiqnMUskeJPaJxdmSUFCJr5niMd5+Er92N/qR85OYVIPlkI7pDYRytaofaakHZlCKEmpvF07Sh5mg9PK4IMjPSxbPLry4VRcVFKC7NFwVglnjOPGT7RQgSj3/XkHG97aKc08OpG4Xp8+zIzhHvYdp4lNh1yOhsQEOHBuFICeZ9/ru448sfw0eniftzspExbxYmW4zIDAbg7+2B/0gl1hsWoGzyKMwQnyNv9CiMW7oYU/QG2KIvdTot9GKv3mQxiR91ImxpyUozm1Z+Rf06qPvF6xTakZmdBJspB/ljl+HWn/8CX722GJNKxH3jxqCovAzTfB6EQ7IpZuBZT2dDRpb4XsvFnmZWJjJGFSFLBMXEQOCMwnmQDhptBsrHlyK/IAcZBbnIzExHhvw7ycP0a4+ipluPppzZuF58V7niuyqdNxsTSktQ9O5WnZghw8nMzImYmDZaGQ15kGzSOtxzHNtbD7yrM/Kp5GPkY+VzMOh8EMS3npAgypWQWFfF6tNzvnW6Ec19BmROno5MX1As24f6Xg+qj7Zh/DWTYdGK37mjBb09zag40CR+/xkwGxNFqWKC1ZaN8RNLkZUt1qWSfGRZzEjy+ZS+dqetfv4LlWtO9LrSkHWhdfqEWN96E9Ei9uxvent9m4eJo8pQEn2p06mQoDLAkmyCXmeE0WSBNVkEQbGdVH6VYRsstnRkFaYhXZQ3BlUZFpyvXDtrrr9AGRFd6nTnKYccsuyswHrtXBRPKsNUpewsxdilCzFR7DhYzv6Ew54sBzJNaViWPx/pickirAwEHlmGyDOer2/cgZo+sd06ozPyILm8fJx8vHyeeC5X4jTsyF+uA84+o7hqgkXjRbi9A/XJJUhOtyBV1rzpxMY/PR+j7c3wuTzw6NKQnGRF1smDqO7uxNET4suxZKK4wArbgUo0BhtQVWeGRgSc4gJZLSYTsAhIOclIS9WKmxqoZKgQheFZGwu6OtET0cMlXrMkRf7IxDxrNtLTVUjXt6G5VYVwOAV5JckIdxzA2kfvxZ/u+xv+9PgmHGrsgaxQCbpc6JHt0IYc2FJMSBKfQ51ohDlHhC2xUl/8Oa2NouAWAS1TPFZ5sFEUUiLAlapx7LVH8cRfxXt46EW8sOEY+sS95y4PMsR3kI68bPG9iM8P8T0kqtWiuDoXI7SafJQVm2E2ie9Rq4VGrxfF/MAP0tXcgA5nGD5bJvKi35XeXozs7Czknz3RxYwskx3T7OPFNE4ZD2OwcJFnOZZ9ec5WoyOXkcvKx8jHyuegD4IIHIEg+npSYbWooHWcf51u6dBCV1CKbFcHAs21qGz1or7ehKLZZcjp7YS/uQbHnR7U1WRjTKkZNossU5JgMWWLdSMRGrX4bYj1QqfT4qy9KHwXLtf6nBdep8++vmUofXAurVUnA1ZrEtJlz2FljT5/uXZ25y8jzu7c5dDZy04brPkTMCZDC+tAS0dMMmoSlb42V+fMQq454+0xdmRZIvsFekLesx5lJZeTy8vHycfL54ln5/7dxDLZibf7BOo7TPCrkpGeHEG/SPc+vQ5qlUqJKcpHTxCFkcGPSCSCsDYVZosoiPr3oepEFSpbTAjoilGUakZR9wHU1B9Dhb8Q+pQCjM6IPl6sxBaTDolDSRnBIEJi/QuJgkuufAObOLECa/pFThJ7N/4w+j1NqNq5CZs3bMWmPZWoqDgsppNocXohh6KLRMKisPVfxpFzdWIPLQXJVhXE2xroFNxQie0rV2H9toPYI0JeRUU1auoGjrY4d9ixiu9AhC5z9OYFnfG6ZwiJ7yocOrNuTH5XamjPnaBigiZBjdFJRVicN0cEl7HKIeeyvfxc5H1yGbmsfIx8rHwO+gB4uuDs6EFNTz7sdrXYkF5gnQ6p0W8tRp6+FZGeChyq7cWJvkKk55ahLNKO/laxfnf4UKufjYn5OiTLsCI38lorkkSov2AXRlFuXbBcE3v3F1qnz76+vRenlAeyltJ7/nLt7M5fRpzducuhs5ed8rvSie8q4cLf9TAmD1KwaE3K4eXzs6YqI67L/jrnI++Xy8nl5ePk4+P96M44/HSyAHDBuX0XDruN8KSkwp6ZCLXYUylobYLLIfZ2ZCtBROwVedpQ35gDY6JR7KllICU1G1NmtKKjYjcOunRwaEaLPbZMTJxUj9b1e3BA7DWEC3JQNPBCFyc1Dcmi8LG1taIxIAoYuc4FO9DTrRJ7XxnIsfug7liLJx/ejuqE6bjt9w/goQfvw/1/+AJuHpeLLLG43mgSBWU+jP1+BAMhBMXn6A+HEfJ6lFGaz9/CI16wP4SQfMy5spKzGse2rsWf/9aIUd/6Of7f/z2AB/7yA/z3F67CeHH3kMuc9yjJnokkqxnqoA9e8V3J9xvxt6KrswetHdGFYphZa8TktDG4c+zHsShnOvLMWbCJwsck5ssaHDnJ63KevE8uI5eVj5GPpQ9CEP66k2g+2oBDlmJk2jUwZl9gnc7UQq0uR/nEfli1VaiubsBBfRFSE8Zj0rgATN4jOFbdgcpZk1Gm04lN9UUyDKFcw4XX6bOvbz4ERAiSTWfnJ3bSxHcg/js7ZSDE85dr74ezl50eBJx1qGsIwX3u1BUTZFBJT0zBzcVL8ZGSpZiQOko5t54MMbLGRpYp8lLelvPl/XI5ubx83EgYxiIOP+E7nXg9eaMwblI5ygwGqDNSkKfajrrqLsi+gpD9X/bvx+ZAhggiYsNiESu9CEYTJk5Hm5jvyUyFRQSbnEQgJS2E3bsboVUlIlkueClSU5Fk6YKm9xB271cqeoDqo6htcqHaYEd2eliEnWa0+iwIW2xISxXLuJzY/8i92HiiemDQrqRk6CeUY37Lchzd34hK2eexpQn1b7yMjT7vOTooD/IgHGlEc5vsWBmddSZHH9xdbjSqcmHP0Cg1Vk07NmHrq89hi7j7wgXfZVI2BkXGLqQdXYdXxXfllf3Ft27EwSMV5+gwGXvkUVXJepsocK7Bf0//Er4/9Qv41KgPYVn+PGWS1+U8eZ9cRi4rH0MflIFOvGu3dqLs1hsxLTERSRdapzPF31n8ycrGTIApokZ7azPUs6YgUwSbnDSgp7cXtSf6kJ+dIZa7hKJ4KOWaPHLqQuv0e1rf2kWw60PnuQaWlqc0EZ/7vOXa++GcZecrStn5AY6LfVnJmuApaWPxlfGfxP/M/ha+NP5WEWgWK2WKvJS35Xx5v1zufLXK8SbGw04P3L1r8ff/+C6+96Wv4IvK9H18//89i4Yl/4lPfnIJrhmXDG2CDabk6fi3/1oG3ZHn8djdYrnv/gY/eqoD079xK+aOzUeW+CZUYk9Il5mDcF09Us1GJCXZxN6RAWn2LJys60d+VjLysi8x7KhKMG3pQiyYo8P+X38Fd31NvIefvokjhkIs/cy1mKDXQzt6IWYX98K56QH86itfwTf+8yd4PLAEOVl6ZKkd6O5Lgil1Fu74/lJoDz+HR8Xn+P6f/oknvaNQ3K85RwdlwSz23LIyMcZyECt+/X38Y8VB7DvbSIv2EmSOLsIC8wY898Pv4vtf/gp+taIO1cmzcXVWM3q7wvANnO3gytKWYcaShVg0U4M90e/qe6tOoLI7A2P1achIF19nHGz35d6U3NtKN6RgVFIh5mRNxrKCBcokr8t58j65zEjY8xo+jmLn8r/iF2+XKXL6Hzxfl4S8T30H/3FjPlIT1VBdaJ0Wv1G5KdHaM5EQjkDb04OsvByxgVEjNT0NgZAZXo8WMyZlQqe7hB/0UMq1jCGs05F3r2+/2tmDxkD6OTooCyrxydLHYVSuFz27RBl03//hmcNil+rM1jCNHrhQufbO2Q6uHNW7y86v//pB/OZkCgrCuShKToRlyM3ww5fs3ycDjE1vQY4pUwSacqWZSpYp8lLelvPl/XK5eO6QfKbYHWdHdkATGwGz2LuSRy1k2u2wK1MWsnNHYdq1N2H26FRkKp3+RMGkNiFV7E6pRPFjtliQnFWA3FGTcfUNczHWboKymHx9vRlaQwYmTJ+J8oJUsXOkgsaYhISkMsyeMxHluXJAPvmUBmit2SifVoSsZNNAB8Iz52kssNpzMXZaAdJ0JiTZrLAk26CNJCApw470vHJMnjsLc6aXIUevQkKiDRa9GrbUFFiT7eJz5aJ05lLMLMnEqNFjkJNbgBzr6Z8jvagExRPnYEZeLsbMGIu8DPEaZ/aQVuug0cnwZoEtPQtFoyegIDMZKeLx77w/DbRaA3SJZqQn6WFMzoQ93Y6CsdMwYeJUzCrLEIXmFBSlyLFLkpFRmI+y0SJEJYjvVp+BwvJCFOYmQykvZA3EqfPO+72I11X+luf/rjLHZsPSE0Giz4yM6xZi3GnjfnxwhjrOzvnII3y0YuMhQ41sS5eTvC7nyfveC46zcxHj7EjKb9eKtCzxe3+7TJFTHkZPm4+Zc6djSrZ2YOBL6GE63zqtkxsfQR4SrE0WQacMk6eMQ2FSAvTid99vykFucTnmTCtFliEBavm31p2ybomHilLh9HmnrW9psF6oXDMMZZ0W5Uja6Z8jf+JMjCsZjSljSlA0tgAZIrPI/tJvk+9Va4RBfMe29HTYcwpRIJbPt6lFsXzKui9D+gXLtVxkWS+yjNCKeef8Xs5WDr37u0rJz0RWcTrUWzuQunA+ikefOUbZB2eo4+yciwwxcrwcgwibsulblinyUt6W899ryInFcXYSHA7HpZfSl4EMOr29vSgqKhpRhTGdh78DTXUd6HRpkVpeJjYEoqB1bsKL/7cem46YMP/338cNqaIMHQZhRx7x0OU9+1FUw0EkHIGrtRcpySkwDBxyNyJ0dnbCbBYbebEhJpLnFAs4W1F5qBum4hJk2JNg629Gz/HVuOeOjSj+r29g6bKpmDhManfk0Zmu4PAd/MfXJ96bP4K0tOF/tkOLCLbSMNhcEJ3BcRjbX3wCD/zpcSw/2o6Glna07anA8fYA2tLykCe2X5fSxYGIRqhAN9wn1+KRe36LJ1bsws6Tokw5fhItOytQYSqEwW5BUhw0Y9G5sWaHhp+IH63bnsWKF5/GHzc5lBqchFAyxt74Sdz4mY/hplKj0hfivVXEXh6s2RmeWLNDp+kPI+hoRdXTP8SvV9ahoiUAXb8eBlMRlv7wx/jE9GyUJmmih+9/8Fizc/kM1uww7NCwFOxrQntTPY62DB4TmoiUvHzkFOTAPowGAGPYGZ4YduhM/eEgfC2VONLoQI9b9qTWQKOzInfCOGRbtJAjQA8XDDuXD8MO0WXAsDM8MexQLGPYuXzYZ4eIiIhGhNgNO5EgAl43ejqdyujBw3O/mohiR79y1I6zxwmn23+Ok9ieKYJw0At3TyccvghCl1QQXcrrEtHFiN2w4zyKypVP4wef/zs2dLvOccp/IqIhUk5tsAIP3/Mw/v70DlRHZ5+fEy2Vq/D8D+7EXzZ248SlFESX9LpEdDFiN+wY81E0czG++F83YJIlcWAQKaIYdKy3Fstr1+EvB5/Az3f9Fb/Z8zAernweb9ZvQa1DngOA3h+XWLMTYM0ODS89fgc2t+zBo0dexG/2Poxf7X4Q9x54HM9UvY7d7ZVwBFzRJUeO2A07WhuScooxbX4ZMvUa5azDRLEi0h+BK+jGusbtWFG3EVta9qK6tw5dvh6lY2JNXx0OdVWh3tkcfQQR0fmF+sM41lOLlaJMkTtLBzqPosnVhl4Rftq9XTjYdQyV3dVwirJnpInd00UE+9Db2ozKg51QZdig0wThajqB5toaNLsDqDt0CDXHj+N4p0f8YfuRgnYc3HUIVTXReREDMm3RY5gdDThRcwx7Dx1DbW3twNQn9rY0eiSbtPKYRcDXgqpDR3D4cBVqWtrR4AwgXHcIHWET+rVGmLQBhPztqNm+H0eqj6O6tgNdzggSM22Qr6KMCePpQGdDFbbtPRJ9nVb0hVRQm60w8UC0mHSpp4voC7qwv/MIlp9Yjx6fA6mGJIxOLsTYlDKMSSmGUWuARWeE3ZiKXHNG9FEXj6eLuIjTRfQHlebxDWt7kJCTh9GzS5AZCYnd5OM4UCE2EkerB9bb+kbUunSwJuph1AfR11SDqi1b4cieIcqSRrTXV6OhvRc9SEKqWLEHTi0hypzOBlRt24fD4jlO1PbCk6CBIdkE/ZBetwENTT0I21KRqFUPi1Ol0JVzqaeLOCG2ZW817cCutgrltBAltjyUJxejPKUEOaYMaNQaJOtsyLdkK6ePuFQ8XcQluORDz3v2YOeKTfj9fW7828vfwtwML04+9QDWr9mK2lHXw/nma2jyOtCTPRdTZ8/Gf89sxR9+9RqOdfegK2chZiy7Dfd9aQYM6iC8W/+Ovz27Fo9taYZRI76OgBu95Z/GnZ/6ML59fRl04V64Dv8Lf/jjm9hU2QJnRg50Y6ZiWdUz8H7oD1i09CosyWtBx8l1eODrT2BHnwud4VyUzFiCT//vZ3CV1QCjKoTAkdVYt/xf+K8Xjr0zUN5Nt+FDn/0Ebi7SD5sBrWjoLuXQ81AkjANdR/FI5b+UQu2GwkWYkzkZWaZzHyYdFoHbHw6IAkx9USfw46HnF3HoedgDNL2E/7n7OFQz5+Cm7yzBRF8XXOvuxff/sRe7TvRCnxBCWKVB96Sv476vL8U1E3Ro3r0Sr/7qV6jJXYrK4zXo62lHYtZYFF37Hfz01rHISVKj338UFevewKP3vIT9OsDrH435n/koPv6lazDbGIb6Xa/bLV73//Afj+/CzhM9AwPgmYux9P/9FJ+cmokSq5qH0saxiz30XO5sybLk4cPP42DnMbGDlIVbipdgjAg6WvF7PZuI2BEKyQN9RLA2agwXdbJhHno+DHR2AftrUvCFvz2Cx17+Ne6eJf4mLz6Hbz6bgtvuE/Me/U98bawofHZuxButQfjCB7D8uU1owyzc+duX8Mrzj+OVn34IM7yVOHKgAlu7RNDp3Y6H734KrvLP4mv3v4Rnf/5V/Ni+E6saA2hUfo+yk+IOrPjdE/B97k/40T9ewpP/uwxLMw7iDz9ZgWqHDz7UYPvGw9h7JB9ffUA8x4vitX57I8o1rXjjxe1oFc8Slk9Fca/O2YQ97RVKu/mHixZjbtYUpQbnXGTQafV04oWaN7G1dZ9SqNH7Qaz7zp14+M9vwTzly7hbrPuv/OMPeOyrU5FyYAW2VjXhiHNgyUBIjS2N2bj1ez/Dky//Bb+8oxwpT/4Qj27rw4neLhxZ9Tq2bqxFyk9fwqPPv4TH/jsfltbt+McjZ1v35evuEa97AsVL78Kvn5Cv+3Pcf2ca1jyxGgeOt4oSh+gdskzY1rpfaQovtOTiQ4VXKUFHc46gIzmDLuzpqMRjR15El79XKWfiWdyFHZvdjimLF2N0bg5yMoqRZzejpMSE0vmLUZYj5pVOQXlhIvL1jaht1SAULsG8z38Xd3z5Y/joNDsycrKRMW8WJluMyAwG4O/tgf9IBdZr56J4UhmmjrMjb3Qpxi5diIlqPSxyh97dgq66NuyqHoPR07NQUGJH4czJGDu+EIUHt+Nwsx+9/iD8vk44XL3wROxITROvNe1D+Nitt+DrN41Finga7qmNDI2uNqXjsQw4k9LHKE1Yssr5bGRtTp2zGU8dew2bm/co7fCrG7Yq89/LmdZpKMxItE7Fjff8El+8dR6uFut+RkkRsmdNwyy5NxwMi73igSW1Bj3Gi3Jn/KhRKMwYi8L8Mowt7cAWscPUXrsPJ2tCaOgrxeR5dmTn2FF69TyMS9HCeuQQDnQAwdMqBiPoj3jhdNTB4ddBmyhfdyJKlnwOP//61ZhTlILh33hA7yfZ5LWv47ASWGTTVVlSgVKjc676X9mBeZMoT148vhp72w8r5YtsApN9fuJV3G1fDcZEZOTnwKxWQw09tDotbKmnzNNbYTJpYNF54fGq0N+fgrySZIQ7DmDto/fiT/f9DX96fBMONfbAJ54v6HKhp7YWrYYc2FJMSBKljDrRBmv+BIzJEIWVbA4MuuF3d6G5oxm7XngY//jrvfi/x17Da1uq0ePsQIcnIpJ3BkpmTcbEaSZ0PHUvHvqLeK1HX8K2411IsKUjUTzNcDjXE1153b5e9InCpsCSjRSD7ZzVzN6QH8f76vHGyQ2iIDuCNm8njvfWi9CzG2817VSquWVHZ7pSdNBo0zBqQhI69q3Eyw+Jdfavj+PBl/ai2Rs87agplShbMgpyYRPlj1ZEEbMlDYWlSeju64Pf0QmXKB9qqyuw8fF7cf999+KvT7+FzQdq0S12frq8skkh+kQKI/TmUiy6cwEsjduw8WH5un/Hw69shSMxDTqdfA2id4QjYbED1Qiz1oRMU/o5++PIpqu+gFM5IGJT027UiPKk29+L3W0VWNewHVU9tXEbeEZ2ZYL8o3qbULVzEzZv2IpNeypRUXFYTCfR4vRCnpUpIn5EgYBf6ej5DvG1JeigMyTgnT7VQYRCXWg6egRHK+TzNKHNnYicBZORnyTPuyLCzuyrsfCGWShyVKLmiFhm60qsXrEBK7c3oFs8Azdb8Uvucckjq/Z3HsVxsQflCnqVquf9HUeV+Z7Q4DnA3iEf4wn54Ai4ReGVqFRJy1qgFDF1eLsRkp1Y6coJi79R73HsW7saGzbvxo59cr0+iiNHm9AZCp33EHGVSg2DIRFmEX40IghB7Dp5nC04KcqGSqV86ITXbEfO1DLkij0d9Wl7OiLsmEZj0Rc/hWlZEfQ3ieX3bsfedS/hqVcOorLVhaH35qB4JmtojvacwN6OwyI09yIoyhR59FVFV7XYqeo7y87QQN8eZ8Cl1AybRLmiTtCgyJaLoChP5FFa/XG6AzWyw44ymNdaPPnwdlQnTMdtv38ADz14H+7/wxdw87hcZIlF9EYT0nPzYez3IxgQBZzIR/1hDwLOOtQ1hOCW2yixEVJpUpGSOg+f/uUf8LsHHsDDD/wF99/7C/zPD76A64vNSNP64XUmwpJ/Ne742wP4v4fEMj+4FZPRhR2vrcVh8TTsiRG/ZLOTbIZ6sOJZbGneqxwKKm//38EnsUHsYXV4e6JLvkPunU23j8e3J30GU+3jYNNZcFXuTPzn1C/i06M/jGS97aI6FdJF8rXDWb0Sv/3VbiTM+jS+/ucH8OBff4nf/vijmG81whZdTJI7Qz5RGARDYbGDFITb2YvGxlbkZKTDaDKLwJOL0sm34Bv/9wDuf1CWD/fivj/8P/z7nTdhXjqgO/XPKEJuJBhAb0cmZn3x3/EDWZ78/h788taxqHnuZeytbVD6+RDV9jUoTVH3H3pKqemt6j2JF2pW4cljr6K6r045GOJUsrxIT0zB7aM+hFuKlyq1y0l6C+6adAe+OfHTmJUxCVpVfNYbjvCwI34Irc1o9VkQttiQliqbrZzY/8i92HiiGrVymaRk6CeUY37Lchzd34jKJnkUehPq33gFG31eEVUESw5Sc1MxJWsrDh4KoFuOonrmCM9d27HywT/jZ/f8HW+IksoXnzWFdA6JagNuKVmKq3JmIkfs0SvzNAbcXLxECTDnO7xc7p21ejqUIyZkNTW9T8T6HW5pR304G8ZUEW4sQO+JGux4+F6sdzvRGV1MCnh92PLyClQ1t8Ip1v2mqu1Yvl2N1NRMGHMmoqAgiGzDIezeL8oYuVejlAf/wO9+8RrErNN3dHytcBx+Fj++4248vLLy7U7QRGcanzZKOchhUtoYaNUDzeHjUsuUeVPSyt+edzayg7Ir5EGGMRXqixn2JUbF7jg7vhY01dRj284gJt42C3nmEHoP7UFdkxOBcTdhdg6g17jRfL5548W8UTqRS7aj+sAmrFu7DmvXbsRh61Qku5qQlFkAS8kcEWJsKLC7ULljJ956/RWs2LsXG92JSKqKoGTxNRg3vhilmXoRePpR+fjLWPfmcry8cgcq2vUovuVmLBybjlSDBcn9PfC07sM/nluOjStfw2tra+DInIh5H1mGqwptkJsx9tuJLbIqeCjj7CQkJCh7TKmJyaJgUcOg0WOGfQIW5sxARmLqOfvtyL01OcLy+sYdKE0qxBR7uSichn64J8fZeQ/j7MwrRbrYye3cvRKHKnZi3ao3sengCdQnT0Fhz0HoxyxCbl4Gkp01qNm8EeHUdFTs3oA3X1+BbScC6J/2OXzphvEos5uRmqGFKtyJvQ89Kdb71/DKioNoNxdh7LWLMDvfCKN43Y2DrztnNLK1ZuREjmLd1i1449VX8eaqzXjrUBClt/0brptdihKrjv124phsahrKODtyKAqzzohMY7rS3C2PwJqXNRUTRAgyahPPOUSFbOra0XYQnd4ezBXLj0kuOmcZdDaxOM5O7IYdsfGAxgKrPRdjpxUgTacWK78OFnsB8spGozBJ3K083XnmjRqDQhFkkhLVsKWmwJpsR1Z2LkpnLsXMkkyMGj0GObkFyLGakJqTBpV4BbPFgpT8TGQVp0O9tQOpC+ejeHQeCpJN4r3YoXX6YUhLQVJWEcrGT8bcJVNRbFRBJ/bIreJ5LEkmhKFHZoYd9rxyTJ47C3OmlyFHx6ATi4YadgbJpimrzoQck10ZPFDW8pyrkJGHpsv2eNkZWTZzLcqZiXEppUgUQWmoGHYusiZMbDygz0BheSEKC+ywGm1INWtgSclASqoducVjMHrGIswtTkb+2CnIsycjJVGLRJsdo6ZMRorVLJbLQFH5NMy6ajHmFxth0mlhsFphMttg8AXF09uRll2GCbNmYPrgun/q6+bbYdNbkZVhREhlhMVigz2rALmjJuPqG+ZirF2UIxyUK64NNexIcicqyWCFVWxjRosypdiWpzRNnY0cD6zN04lNLXtwpPs4kg02XFewQFxaL6pJnIMKXoJLHlTw/SI7KTpbUXmoG6biEmTYk2Drb0bP8dW4546NKP6vb2DpsqmYyJNzjUiXMqjgmeQYOn1+p5J2zRqjUsj5wgGx99WqnDKisqtG2VOTTV6Flpzoo4aGgwpexKCCRMPExQ4qeCZ5wEOLu1OUI36YNIkiyCQo5Yoz4FFOJ7Gr/RAMah3mZU/FDQWLoo8aulgcVJBh50K8Teg5tgL33LUGKZ++EwsWTcLEUA26dr6Erz9mxa3/cxs+tKAM+dHFaWS5HGHn9ZNvYU97pdKZsMCapRxhIWty5NEVUllyAT5f/tHoeDwXt0vPsMOwQ7HnvYadk44mvHJiLRpcLcgzZ0KjUqPL16ccxdnt71Oawpflz1f6EOpF6LlYDDuXYNiHnf4wgo5WVD39Q/x6ZR0qWgIDQ7ebirD0hz/GJ6ZnozRJw1M9jFCXI+zIoLOr7ZDSN0ceLiqbxWS1dHlyidKvZ1LaaKX9/VwDD54Pww7DDsWe9xp22kWo2dayH3tF2dLq7VSeT9buyFPSzLBPxHT7OGSb7UrQGeqpZ07FsHMJhn3YEfrDQfhaKnGk0YEetxzbRAONzorcCeIHY5Fj6AwsRyPP5Qg7sm+OPBRdFkiy+UoefSXPfyUPNZeDDlp1l95GyrDDsEOx572GHdnnR5YpcvKG/coYOjLUyCNAZQ1xst6qlDGXimHnEsRC2CE6l8sRdq4khh2GHYo97zXsXGk8ESgRERHRMMOwQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4Nm7DT398fvUZEdHmwXCEi6QMPOyqVClqtFm63G8FgkIUT0WWUkJAAnV6vrGcjiU6nQzgchtfrZZlCdJlpNBplHYslCQ6H4wMtCWSBFAgEEAqFkJiYqHyJsoAmigWR/ggcAZe4HH4bVJVYj9RqDdRBsfHX6sR1dfSe+CfLFLnzJEOewWAYcWGPYps35BOTP3preJHliDqighbqmAg8FotFufzAw44k97zkHpis3ZGhh2JDJBKBS/zN+iP90Bv0MOj10XtoOJA7DiaTSdmJGIk7EDLweDwepWyh2OHz++H3+ZGgSoBZ/H4ZVIcXWZ4YjcaYqdkZVmFHkoFncKLY4HS6cP+Dj4hLJxZftVCZaPiQAWdwGolYpsSmdW9tVCa5kfr6l+8Ul+boPTQcxFq5MuzCDsWevj4HfvvHe5XL65ddgxuuuyZ6DxHRpXlj5WqsWLUaNpsV//HdbyuXRJdqMOywfpCIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7NBl4Xa70dXdDX8ggP7+/uhcIqJzi0Qi8Pv96OnpRXNLC2qOn0BXV1f0XqLLJ8HhcHDLRJfEJQLOE089ixO1dUhNSUZxUSGsVissFjOMxkQkGsSUaIAxceDSYDBArVZHH01E8UwGmYDY+fH7xRTwD1yKKfD2dT+8Ph+8Xq/YWfIot+XOkgw+spyYNHE85s2ZBb1eH31GootnsViUS4YdumQ+UVCt37gZlZVH0NXdg3A4rBRSJqNRCTxyMpvFpZisVguSkmxK8NFptdDqtOJSB624rhPXNRoNEhISos9MRMOVrLmVQSYk1vdQKIRQUEzyMjoFlcuwEl5cbhc8Hq8yyVDjEZNX3h68LiYZfPrF88kyQE6ynJgwfiyWXbMk+opEl45hh94zuRe2Z89+dHR2oqe3D729vcokC6+wKLxkgSgLsUi0Wcsqwk+SzSYKsyQkJ0cvxe3klCRxn0UEIB1UCSqoVAlikpcDE0MQ0ftnMMwol2KS6/DAPHG9fyDkyCDjdLrgcsnJDae4dIrLwduDlx6vRzxGbGjEOqwSU0J0fR64nqDsHMmaG4vZIiYTzHIHyWRCbm42Jk+cEH1HRJeOYYfeM1kABoJBRMIy0EQQECHH4XQq1dC9Ivz0iOAjp4EQ5IDb41EKUVnQKSEmGmZkE5fNahUBSIQfEYDknt3gpZxk4cfAQ3TlyXV6MLycGmQGA4wMNIP3+bw+ZadGBqCBIDQQhgavy3XbYNArtbtyHZY1vGYZaKKX8rYScMSl0WQcCEBiZ0eGII0IQbLWl+i9Ytihy04WcLIKOxgIihAUiF6KKRBQqqt7+xwQvzdlj1DZE4xeekQIktXfGs1Ak5bStCUmucdnNBpF8LHBYpV7fgNNYkoTWfRSLssgRHR+yroZDEX7yHhOa1Z6+3q0ucnpcop1NijW5aDSHPV2E5VsrgqHlOZq8XRi/dQN9MczJkb75Q1cKv31xKVszpahRq7Hg01U2ujlO5NauWRfPrpSGHbofSULyMHOiF6xRzh4KQtbh8OJPhGCZOiRe4sDbfk++Pw+pYCWBaIsPBMNhrcLVFkbJGt9ZOiRgUjeVpaJ3meIFrBEI4HcofDJDr5ndABWLpVOwn6lE7Bct2RfO3lb1sT6xLLycnBZ+Tz94p/cidDr9EqgkWFFXuqit+W6JdcxWSMzeN9py+rk5cB1nWyaVvGgX/rgMOzQsKG0/w9Wk0f7AchL2STW19enHPUlC+KgrCUS02CNkVarGegMLQpd2dY/eCmbxGSHaFl1PlhL9HaHaFlzJEIQa4MoFigdgU+tXYlOA52Ao9fFDoFcZwbDjFJzIy89AzsOyuTxwi9CjuyDoxbhQyPWHaVmRS2m6HWt2KmQ64tcb2SzkjFRTu/U1Jxei2NQwgzXIxruGHZo2FNqg0RB/U7/H9kJeuC6ctnTq+zNyg3CQIfKgf4CskpcFsgDfX5kP6CBvj+yH9Bgx2hZyMs9TnaGpg+KbFoa7PR76vWBTsED170+WSMT7SfjjO4QuAd3CN7pCCz7w8n1ZfA3LH/XSv+X6HU5X4b9tzsBy2ZguXMQ7TMz2Hdm8FKuQ1wXKB4w7FBMGNgIDISZgWngtizYZfjp6e2JBh8RhPoGAlGfmOQGIhwRhb/S4XEgyMg9Wlmzk5KcHA1Ag0HonUu5F8v+A3Slyd+wbE6SgUUJMuL36nI5lUAzEGYGgozDOdARWNbgKCHolA7Ag0c6yuuypsVkEkFFdgQ+T5iRv29ZizPYGXggFJ1+Xa4rRPGCYYdimizg5QZAdqRUmraCsr+BvAwq1fmyH5D4bSudLQc3KHJyi42I7GSpdIZ+u3lroFO0XClsVstAE5i4LjcQcoOhbDzERkQOikh0IbKJSQaZwU6/p11Gr8tDst0ujzJPaZIS4f208WrEb1TeluRvc7BP2jtNSUYluAw2KcnfrjzySWmaUpqkos1UZ0xqNWsvaWRh2KG4JTcWso+P3JAMbGCiGx6xgZF7zLImaHDjIztND/RzGNh7lp2hZagZ3IjIDczAIIkD4Ufugcs967LSYjGVRF+RRqI9e/ejrb1D+Z2kpqQMdPIN+JXflgzcAx2BZQfhUzoMRzsLy47A8nckD7HWnaVj72CHXyXQRDvgD84b7CisLCcvo7dZI0n0bgw7NOLI2iBZ86OMFeJ0Ks0Ebx8GLybZB0gZtl5siIJiGqgxGjj8VjaByY2SDEqyCeL6ZdfghuuuiT4zjUQPPPwYKioPK7V+2dlZSpiWowP7fH6lZkY2B52tdmWw1kWGGNnMNBCsZagZqKl5p/Zm4FKGIAYZokvDsEN0BhmC5NFfSgfoU/oAyRDU1zcwKKKsDZKhiWGHBsOODCLKIday74sqQekILI9msp7SZ+bUvjNytOCB2wNj0BDRlcOwQ3QGGWLO7Ag9ODK0rAVqbm3Fiy+/BjkeEMMODYadzAw75s6eGQ0wFlG4Doz9JPuCyfBzvs7A7D9DdGUNhh12uyeKkhseuZeuDKim1yn9JGR/HZvNiszMDBTk50P26SE6lc1mw+RJEzFm9GgUFuYjw25Xju6T4Uf+fmQTlew8LGt/5G9L/oYGDxEnovcHww7RBcgNk3JEjMHADRS9i/xtyPO6yRod+RuRg13yd0I0vDDsEBERUVxj2CE6C9l/RzmNhdOldE6WkzyCS/blkeR9g/NPneTy8j75eIofcgwnOezAqX/rUCio3CcvT50/OMmxnuSQBrLvFxF9sNhBmegsZGDZum0n9h04iO6eHmWeDDpul0sZtVYeRSP7YZxJjs48ZdJEzJ0zk0faxBE5po6c6hsbo3OgDFMwMDaTBiaTMTr3HfKorEkTx2PO7Bmwp6dH5xLR+4kdlInOQ/bDGDduDHJzc5Q+O3JPXanZidbYnK1mRy4nl5ePk4+n+FFcVIji4kKls/Hg31sGHUlenvo7UO4LhpCRkY6JE8YpJ6Ylog8Www7RWcjgIvfGp02ZhDGjypSB485H3i+Xk8vLx8nHU/yQHZDHjS1Xau1keDnf31eOwJ2fn4t5c2YhNyebNXxEwwBLZKLzkHv0simiqKggepRN9I4oeVvOl/fL5eTyFJ/keDryb1xePloZlkCOlXMmeVh5dlYmJk4Yr4Qj1vARDQ8MO0TnIQ8hLikuxqwZ05GWmio2cKePsyNvy/nyfrkcDzmOX/JvK2vtrlq4ADnZWco5qs4kx9yZMH6sUgPE3wLR8MGwQ3QBcoDBosICXLPkaqXPxuBGTF7K23K+vF8uR/FNdkZOT0vBtUuXICsz47RAI5u2Zk6fhonjxynnuSKi4YNhh+gC5AZNHm1TWlqMRQvmIS01RZkvL+VtOV/ezz35+Cf/xLJpKj8vF7NnzlBCriT75cybOxvjxo5R+vfwt0A0vPDQc6IhCkci6Orswuat29HS2qbs2c8XG7jUtFTlrOg0ssgTxMqhCY4crVL68Fy9cAGysjNhYIdkomGDJwIlukQn6+qVAeOsVgsKC/Kjc2kkamtvR1tbh9JJvbSkRLkkouGDYYdi1uDZyYno8pDNboMTUTxh2KGYJAdw83q9cLvdPCUD0WUi+xwZjUZljCCieMKwQzEnGAwqk6zVkQWzPPqFe6IUKwLhIFxBT/TW8CLXJU2CGupQghJ8uF5RvGDYoZjj8XiUsCPHMjGZTEoBTRQrfOEAenx90VvDT9gfQtgZQFJSEtctihuDYYe/aIoZsglLNl2ZzWYWxkSXmawx5Rn7KV5xi0Exh1XsRER0MRh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0QUI8L9YTiDbrR7u9DkbkOzux3dvj74wwFE+jnQJtG58NBzihnit6ocMZKXlxedQxQ7LvXQ837xLxwJwxv2wxlwod7ZgiZXG/oCTqgT1Mg0pqPAmo20xGSYtUbo1TokiH8XK+gNwNftht1uh1qtjs4lim0cZ4diDsMOxbJLDTvBSAgnnY1YXb8F+zqOwhvyKTU8kegh4uoEFTQqNQqtOZiTOQVX5c6EQa2/6MDDsEPxiGGHYg7DDsWySwk7PX4H9rRXYGvrPpzoa0Cv33nO5iqjxoBMUxrKk0twbf58ZJvsSi3PUDHsUDzioIJERMOUbLqSNToy6Gxo2omKziqlb875+uV4Qj7UOVqwrXU/VjdsQYunXakBIiKGHSKiYUf20ZFNV7JG51hPrVIrdCFKzY4xDfbEVGxt2YeDnceUgEREDDtERMOON+THm3VblKaroQQdrUqDQmsuri2Yj9tHf0ipAZKBRwYlWUtENNIx7BARDSMDh5e7sL/zqNJHZyhKkwqwLH8+FmXPULol69V6HBdBqc7VrJxtnWikY9ghIhpGZN8beXi5POrqQmPnyBqd8pQS3FCwUAk81X0n8a/qVegTIUmGnC5vL9o8XdGliUYuHo1FMeOSj8ZyHceh3Yew50A9eqOz3qGH3piBSTcuw7j0RNi00dn0DuX7a0Gjw4xxH56MHDGLx+pcvKEejSUHDNzUtBv/Or4K7qAXOeYMpCemoNvXiybXO52OZR8d2XQla3Rk0Kl3NuOtpp3Y016JcCSkNF7NsE/ADYWLMM0+TnnM+fBoLIpHPBqLRg53DQ5tWIMXnlyJjRWVOCimisFp707sW/cGXlixHfsaetEXLzX+wT70Np3Ans3VaPWHcOFeH+cRcqC7pQ0NdV1wiZscp/fKkkdhyQEDB8fRyTVlYl7WVCzMnoEsUzp0aq0SdIptebg6ZyYmp49Bs7tNCTp7RdAJRYOO5A37RGDyRG8RjVwMOzRC5KFo0q343kMP4K9ienhw+uN/4Rf/Zsf+B/6MN7YdQ9XQukgMf64aVG1Yjt/8x3PY2euBIzr7kiRNwaLbP4Yvf2sJxoqbrPy6suRggHJk5EGBSBBWrRnX5s9TQk+WMV0EnXylf86i3JnKKSNePbFeqdGRQelUqgSVMhGNdFwLaGRLykLitGW4tawDBq8Hfe8pFRC9d3L0Y3kKCDkyslTZVY3ltetwtKcWNxYuwm2jPoTbR92ImZkTcbyvHo9UvoCq3pNK09WZLCIkpRiSoreIRi722aGYccl9dtpW4am/7cf2xgzc+tDnMEPM0g/cI3jg6T6MF777TdTM/yXmzS5AobcSz69qhR2HUN1RjLKlV2Pph4uR0VuJ5b94Hnt73eiWNUUT5+Aj31qCMvEsWvSiZsM21Bw8DG+eBevWHoLXFwDyZmDS3KX41tLCgZerXYvla7Zh+c6GgdvSmBvx4cWz8OGxKjibd+PhFzuh91Sio88MTYF8jYXiNWqw4b6XseXgSTQgGaakqfjkDz+E8UlGWKNP846T2PfCC3jhkZV4vRbImTEbN935WSwZo0Kk7iyf7apUJFeswM+fO4Red7TBK7kQSdNuwY9uKkVSoAIbVpzAsc4kzP/GfJR1bccL/2yBO6EdquQ2bNnaqTwkfe6/4boFU7Go1KzcptMNtc+OPKlnraMRv9z9t7ePxjJqElFkzcWHixYj15IJvVqrLCNrdGTQkY852yHmNxVdjY+XLkOy3hadc27ss0PxiH12iCRPL/wnD2NXWwEMyWIv2NyL3oaD2LVmNSoDBUgpKESuXYtA01HsfeVFVIqQk1I8DqUpXnhObsVTLx9CizeIIPzobarEvvXr8fKrJ4GCEhSPT0NiZyUOvbECLx3shjdYh31r1mJ/RSdC6eMwvrwM463dqN2xGVv2VqHG4UWwp0Ys8yoOduiQkFGCkgIr9N5WHHr5aRzqUiEhZxzG5Jlh692M51/agyNNffBFP8o7EmFNTYE92w6TJgV5Y4qQkWKEMXK2zxaGu2E/1j+7GvX9BcgQn218TiJMPUex7tU14nvxo8/VjqaqGlQcbEC3PCeTtxFVe9di476TOO7LeftzHNu8BzsO1aEt+i7o0sgjrORJPQssOUrfHMkT8qJahJqV9ZtwrOeE0mS1pmEbDvccF0HHf9agk2FMU04ZYdExfBIx7NAI4YKjsxr71q7DejGtHZxWrcfG1XvRkLkUhcVZyJM1/n4vElwOGOfcgY984SO4floSEqoOY/2LNTDf9Bl8/Bvfxje/tABzipzY8/g6HOvywa0cIOOBOxRBe/9kfOILX8Y3v/MlfGZGMtLbd+OJ7S1w+/vQ5bMgd/wSfPpL38Zd3/oq7rpzCaYldMDR2IgapfeveKLOZhjGLMXC22/HbTeWwd51DBse3wv/qAVY8rVv41tf/QhuX6ZB9QvrcbiqHV3v6lSdgZJJEzF78QzkmCfi+js/jvkTcpAtt5tnfrbZubCF+tGTUITrPvlFfFV8tru+dis+MSsHKcf2oqIrgN6zdtruhN+UA/vkT+Gub34Fd31qLnL6OtBR34zW6BJ0aWQfG7PGiLlZU5TAMth/R/bd2ddxGOsat+PN+s3v6ow8SPb50am0mJg2WunErDml/w/RSMWwQyNECxqPrsCjP7oHPxbTDwen3/wT925IwHXf/QQWlOeJmCAZoVbnIztLjUQZEFyt6G3pxqGWschO7UPQ2w63JQ1JWbkYXb8fx1qD6PMrD0RKUR7mfOFWTDbJ5qVCjB43BhOm29DY1IVAeDSWfON7uP22BZhsbEdbdy/aUnORozcha+DhgkZM+UhLEY+Xta9+B4KtVdhXPxoGfT90aEdvvxq68mkY334EnS29aJEhacjO+GziE+dNvBGf++2P8dGyIAzis7WJV0FyJsaLe8/dGTkTudl5KC8V6VA2eaTbka434MKNJTQUeo1OOXt5eUoxbHozEhLeOYN5RVe1MmDgmZ2RJRl05BnQ5QlB54mwVGoriN5DNLIx7NAIUYSyGV/AT19+Cf8S0ytvT0/gmX/8GP821oqsgRYDQQQCVS6yMzTRQCB1wtn9Gv78+c/gjls+gptv+Sa+84vnsEXcc77Dui02G1KNVkQam9EWDsPXtR0rH/wJvqI8xyfF9FM8sqsatdHlB0awyUNaqhG2gaZmQb7CFjz7i+/ha8rjPo/bP38vXu12ind1sc7y2XytcBx+Ej++499w25A/m128P5t4n9GbdFnJ0CI7Ki/LX4A5mZNh1hqj95yfbAKTh6d/avRNKLHlK8GHiBh2aMTQQCv2kJMz7LCLKePtKR3p6UliY6KC+u2dZ3ElQQON2E68s0Ntg9E6Bx/7rx/g7p//DL/4+S/xm9/+Er/7w3dwc6kF9nd6PJ/G43bD4fNAlSpeu3sTXnxoHQ65inH1v8vn+H9i+hxuGJN7Ss2OpBGBRGzu3l47Zf3KeFz9ma/ju8pr/wK/+tX/4t4H7sEdi8pQdlFdMs74bK7jqNq2Gvf+uRKpH/8OvnqPfP7v4JufueoCNTtq8f5UEP/RFSIDj+xzc23+fHyy7HplpGQZZs5FNnnJQ9G/MPZjGJdSqgQk+RxExLBDdGGJyTClpKLE7oO1aC6mzlmMJfPKMcauQ8cxB0LaCPqjO9DOnj4c23MIXUHZabkNdcdP4ORxD0aNKYa16xj27u9Bj6EAU5YtxlXz56BM24H+fs9ZOhlHiQ2WJjkPJelupOaVY/RU8doLp2FuuQWOEy74AkFEzplIAuiPdKPHEYFY7OycLWirOYG3KjXIWzAP865ZjKmFKUiNdEAeN8QBBD9YerUO+ZYszMmcgpsKr8ZHS6/F0ry5ysjI41PLlH4587KmKUdd3VK8RNw3BxNSR8OqM582Vg/RSMewQ3QhBjvSiwsxb34Ezeu3YOuqdVj75ptYu2YLNu5vRVcgjMHeE0FnF9oPbsSa9euweu3rWHuoHR0JxbhxWgEMBhuMxgT4u47jyI51WLNuA9ZsPoFWbze8CQH4ov1+TqO1QG8fg4XXJiJYW4HdK8Rrr16DtStXY+22k6jv8Z29qUlrgt6khU1Tg4PrN+JQTSc6zjaQrkoPrU4Pk86N1orN2LFRPP/G/dhX1YKAyQ23rx/hgbMT0AdEhhZ5uoh52VPx8ZJlItQsVU4BcV3+AmUaPLx8WcEClCcP1P6wRofodAw7FP9UBiSazbAlGWXX2/NvBlQ6aBPNSEkzi73qhOgKYoZ91FQs/urtSHvrfjz9m3vww1+9gDeOa7Do7lsxxfrOWDcpxjDKU1rx+O9+g//50SNY1ZmFjJs/g9smGWCctBSLJ/dDc/gFPPyje/Cz392PNam3Y/KEfIy1heFzq6ESISU53QKjXqN0VVbO3WUpxNV3fQ2F7duw5T7x2j97EH/8VzsmfOdjmP52p+ozmLOQWlCE6YWd2P/PX+PVTcdwuPssny1jFHInj8OH8g9hze9/hV+L9/XoPi8cY27Cx8tF+vL0IxQ+9fsTm1F1IizJFlhMIigpLya+0XfNo8tNBpjBmh55rquFOTMwP3saxqaUKOPo8KgronPjoIIUMy55UMGIH153EIGwColigy03xucMPBGxnD8AtzuCxBQzdKrBwBNBfyQAV7cT/nA/wmKuRqeH0WqBQfZ/QRt2P/UANm6tg+PmP+Az5QGYtf1Q6xKhN5pg0ctnCcPvdMDjl+9FPEY8tzrRAl3IgwSNXuQsAwwJPjh6w9CJAKUXgWdg8yVX0SA8vS74/CFxTbwnlVZ8FisSNaf2NTpVBOGg+NwOJ7zitTRGmwhQKiSEzvxspy8XES+lFu9Do1FDG/IinJgCoyaIoHfw+0uENuyDyxECdDoYlHAjHiTmOU+bR2ca6qCCHxQOKkjxaHBQQYYdihmXHHbeF9Gws6MJrk88gLtmANZzdFqmkYlhh+j9xxGUiYiIaERg2CG6LPRIyhmF4nETMSoF0HDNIiIaNtiMRTFjeDdjEZ0fm7GI3n9sxiIiIqIRgWGHiIiI4hrDDo1QbTi+fRPeemEjDvYBQQ4VTEQUtxh2aIQSYWfHJqx/cSMOyLBzoZ5rYS8CvXXYt+ko6jtdONtgxFdMfxjwNqFqbxWqazvgiM6mkSXSH0GHtxtVvSexr+MIdrUdEpeHcaT7OJrd7fCEznnSEaIRj2GHaCj8HXDVvIE/fu9xrDrQgNbo7PdFxA90vIVnfv8Mnl2+/5QzpNNI0C/+BSMhdPp6sKllD5469iruO/A4fr/vUfzl4JN47MhLeKtppxJ4iOjsGHaIiIaxUCSMIz01+NP+f+DFmjfR4ulEttmO2ZmTMCtzIsLiX4OzBd2+3ugjiOhMPPScYsZ7OvS8bT8ObF+P+147OnB7TDYMh7qQGrSj9Fc/wsdyAGP9Wixfsw3LdzYMLCONuREfXlyIKf1H8PKv/o6nDvlgKZ+Mhbd+DDdcvxBTTR6g+jXc99JeHDjZM/AYnQmY+Al864ZxmJRnFW+8Ab0VK/Dz5w6h1y1P22lD1phpWPzZT2JuKqCcSQJtaDiwA6//5XUcFLcCGINZN12NpR8uRkbPQSy/5148s60DPUmFmHzDdbjltMdSLLjUQ893tR/C6votOOloxsyMiShPKUaaIVk5T5ZapYYz4IJWpUVaYjKS9YNnabt4PPSc4hEPPacRpA3H923FtnUV6LaOQ1n5OBSFe+F3dWAg1sjTetdj35q12F/RiVD6OIwvL8N4azdqd2zGlr11aEQSsoszYNIkI7MgH9lZKbBqPPB7qrDhmVdxrF2HxBzxOLFMUUI9tr/xJrZXt6HZ14OOhiPY8PxeOGxFyBstlsnRQtNdhRfeOIRWbxBBuNB8aBv2bdmBWvH+SuX7M9SjZv92rNhQB5fGioyiDFhNKUizZyO3MAupOq68I0Gjqw37O46gxd2hnPxzUc4MTBO/z9HJRSi05iDPnImxKaUoSypQgo5s7mr3dmFT8260ejoRlv29iIjlJY0AjiocqGxARV8ebrrz2/jmt76NbywVoSNLD7eygDwUy40unwW545fg01/6Nu761ldx151LMC2hA45GF/qSy3H1J+YgO3EcFtx0A666ajxKzSGxMfGixZuOqYs/ji98TTzuG3fgKx+biuKmCjS09qLZ24uO1lps3dSNsqV34DNfEct87RO4ZVEZdD3dCEQiiPhbUL29AgcPh1Eg3t+X5fv7/DhkB0QAe+MwmhJKMf/js1GaNwlTZl+Faz+2EBNtEHvzypunOCabr2odjUg22LA0bw6KrLkwaM5+0jUZdFo8HdjcvBfLT6zD9tb9IvB0KB2biUY6FpcU9yLHq1DnsaK7ZA6WjAUMWiBp6kyUl47CGGUJeY7uciz9xvdw+20LMNnYjrbuXrSl5iJHb0KWsszZWGFMmo3b/vcX+MRCsZcN8ThvEH32AoxVaSDyiKCG3DYZk/rQfqIdrY1iGWQjZ/p1+OnXF4nApIe+9wSamhPQ5MhFXmo7errb4c8pRZZRi7TGalR08ND4kep4Xz3cQS8KLNkoseVDozp789LgkVo7Ww/imarXlSO2Xq1djy0i+PQGnEonZ6KRjGGH4l5XZzucjiF03uzajpUP/gRfueUjuPmWT4rpp3hkV/X5j34K+4DWFXj4nm/jDuVxn8ftn78Xr3Y70akskIPiGTfgi/d9GubH78LP75DLfARf+Pdf4uH9gCuoLCTUonrX3/Fjcd/Hlef5L/zq6U1K/x0aubp9fdCK4JxlskfnnF2XrxfrGrcrAccfDijhRs5bq8xbB1/Yz8BDI5r67rvv/kn0OtGw5veLAru/HzbbQJ3JUGm6KrGvLoQWVQ6WLiqF7MKpUsbZqUJDUz9SFs/C2MAWvPDIOhzx5WDsLbfhkzcswJLF+VCfcMOaWYyCSfnI8VZj5esOFCydiOLidCT7O+CsfQv3/2Y9nGVXYc6Hb8ZHr5uBudPt8OzsRObceSgpy0WuORFGaxZyR43H1KsWYdG4FCSLDdGK9X0ovaoIyZF6VO3uQUeoFDf895342OLFuGbxUlx73VIsFe9tUr4VSb6j2LS2Bwk5eRg9uwSZAx+NYkioPwxfyB+9dW6yr82qus1YXrsWR3pOoC/gRLO7AxVd1dAkqGHVmZXOyaeSNT5phiSkJybDHfKiz+/Ex0uX4fqCRUqH5mS9DaqE8+/bRkJhhLxBmEwmqFTcD6b4oNcPNPvyF01xT19QJPaMA7C0HMH+eiAQBlw1x9DQ3DTQQTkcFFuYCuza34MeQwGmLFuMq+bPQZm2Q4QrD94Zqk22JXXD6Q7AK2eKvW5/QwXW7PRDVzIBM69djDkTy5AbaYJbbNiUShtfC5oObceL/9yBnuLpmLBwMZbMnoDRKVq0HqlGRzCEgCkTqSkG5GclIqlsMRZcJZaZVQC7GuhpdCOSKJ4nQT6ZQ7yuBy6XvE7xSgaZFIMNvSKwBMIBuIIeEV4cUIugY9aZztqUJR+Tb8lGqa1AqQmSgUh2XJ6SXo4CS47yWKKRjDU7FDMutWYHZg3CrbVw1p7AgS4D0FuL+qrd2LO/CR396Ri1eA7GhmqwY3crnKoItAYvumqqcWDnFlTWdCIhfywKx41GkboZu9duQ6/BCLU5Vewtq6Hrq8WaLa0wZOvR7+tG85FKHNizC0cbHEiecRXKsnXQ1O/F+hfX40BEB1d7I9pF0KppcqM7aSyWXDMGOTYL9L5GePvqcbRWi4h8f0d2YPuuJtS5zCiZU4CUYDsqNu9Bs9ODgDkFyckZsIkdFpUSgigWDLVmJ1GjF+E8XfxtE9AXcIlwo0V5SgmuK1iAUUmFMKjP3kFZPvdxR73SMTnfkqUcvZVqSIree2Gs2aF4NFizw7BDMeOSww6SkJVjgUHVjDf/9iBe37AeVeljYI6kodxghf2quRhVZIe2fjOqdqzFq6+vx+ZdB9E144uYpmlGhj0H+sLpGJetge/IWuzctQO1vmQYymZjQpkFumPLsXXTFry5aj32tvgRnPFpLAnthqp4ATILJ2FaoQXjU4/job8/j/WrV2Pdzmb0Jk3AbXffiUWpWphVBqQUpsKQ0IcDf/gjnhbv782t3TBOno9rP3sDZpoToBYbIM/xnTi2bzP21vYiWHwNJtrlHn30I9KwN9SwI2lUGqVDsjx0XNbYyAEEx6WUQn2Opig58GCdU4Tx9kpU99aJYLRQORw9USPC/RAx7FA8Ggw7HFSQYsZ7GlQwEkTA74XT4VNG1VElGqARV9RIgMYqgpA6jIDTAY9fLCfmJ6hEwEgUQSbkQYLY01bpjUhUh+Bz9METjCBBZ4LBePq8UEQ8r0YDjcEIXcCJiN4KnVjR9AkhRAJudDsDCEfk6qaCRqeHUXndaAuVeFchvw/uXjfk5rBfvDOdeH6jKTE6cGAY/uj7C6t00JuTYNaKx7JmJ2ZcyqCCSsdiEfBl+JHNU+cij9pa07ANe9orkJqYjO9N/pxSq3Ohfjqn4qCCFI8GBxVk2KGY8Z7CDtEH7FJHUB4kg8/Gpl1KB2az1gib3gKnCNHyqKt6Z7Ny5JZs/vpQ0VUoTy5RRlW+GAw7FI84gjIRUQyRNTzuoA91jmbsbqvAuobtWN+4Awc7jykHlU9JH4tr8ueJoFOq1AQR0TsYdoiIYoAMMKOSCzEhbRRyLZmw6MzIMKYqh5Yvzp2NGwoXYYZ9gtLclRBtHCWiAWzGopjBZiyKZe+1GetKYzMWxSM2YxEREdGIwLBDREREcY1hh4iIiOIaww4RERHFNYYdIiIiimsMO0RERBTXGHaIiIgorjHsEBERUVxj2CEiIqK4xrBDREREcY1hh4iIiOIaww4RERHFNYYdIiIiimsMO0RERBTXGHaIiIgorjHsEBERUVxj2CEiIqK4xrBDREREcY1hh4iIiOIaww4RERHFNYYdIiIiimsMO0RERBTXGHaIiIgorjHsUMzp7++PXiMiIrowhh2KGQkJCcplOBxm4CG6zOTaNbiOEcUbhh2KGTqdDmq1Gj09PQw8RJeZWqOByWSCSsXNAsWfBIfDwS0GxYRIJIJgMIhAIKCEHaJYEukXv99IKHpreNFqtTDoDdCptdCI0EMULywWi3LJsEMx5dTAw5qd2NHe3oHde/cr16dPnQy7PV25TsODDDgy8MiJKJ4w7BDR+6ai8jAeePgx5fpXvvg5jB83VrlORHQlDYYdNs4SERFRXGPYISIiorjGsENERERxjWGHiIiI4hrDDhEREcU1hh0iIiKKaww7REREFNcYdoiIiCiuMewQERFRXGPYISIiorjGsENERERxjWGHiIiI4hrDDhEREcU1hh0iIiKKaww7REREFNcYdoiIiCiuMewQERFRXGPYISIiorjGsENERERxjWGHiIiI4hrDDhEREcU1hh0iIiKKaww7REREFNcYdoiIiCiuMewQERFRXGPYISIiorjGsENERERxjWGHiIiI4hrDDhEREcU1hh0iIiKKaww7REREFNcYdoiIiCiuMewQERFRXGPYISIiorjGsENERERxjWGHiIiI4hrDDhEREcU1hh0iIiKKaww7REREFNcYdoiIiCiuMewQERFRXGPYISIiorjGsENERERxjWGHiIiI4hrDDhEREcU1hh0iIiKKaww7REREFNcYdoiIiCiuJTgcjv7odSKi96Szswv1DY1obWuPzhnQ3tGBPXv3K9enTZ0Me3q6cn1QZoYd+Xm5SEtLjc4hInrvLBaLcsmwQ0SXTWNjE/bsO4Cdu/bA5XIh0n/+4kWVkACz2YyZM6Zh2pRJyM3Nid5DRPTeDYYdNmMR0WUjw8qM6VMwZswo6A0GJIgwcy7yPrmMXFY+hkGHiK4Uhh0iuqzsdjuWLV2Mgvw8GAz66Nx3k/fJZeSy8jFERFcKww4RXVYatRrJyUm4etECpR+OTqeL3vMOOU/eJ5eRy8rHEBFdKQw7RHTZaTQaFBcVYsqkicjJzjqtOUtel/PkfXIZuSwR0ZXEsENEl50MNLKZatzYcowfN/a0o6zkdTlP3ieXOV+/HiKiy4Fhh4iumKQkGyZPHI8Z06aIYGNQJnldzpP3ERG9H3joORFdUeFIBG1t7dh/4KBye/KkicjIsEOt4r4WEV1ZHGeHiN43gUAADqdTuW4Vhc/ZOi0TEV1uDDsUkyKRCILBoLLx7L/AgHVENDSyk7hWq1UmonjCsEMx59SgEw6Ho3OJYkOkX/x+I6HoreFFhhyD3gCdWsuj4yiuMOxQzPH7/UrQkQVzcnIy1Go1j+ShmOELB9Dj64veGn76gxHAE1ZO38H1iuIFTxdBMWewRodBh+jyC4dCcLvdSg0qUbxh2KGYMdhHh0GH6PKTaxf7wVG8YtihmMOgQ0REF4Nhh4iIiOIaww4RERHFNYYdIiIiimsMO0REMaBf/POHA2hwtmBveyU2Nu3C5uY9ONJ9HD3+PoT6OfYU0blwnB2KGeK3qhwWm5eXF51DFDsudZydsAgx3SLMVPXU4qSzGV3eXvT5nfCGfVAlqGDVmpFssCLbZEeJLU9MBdCo1EgQ/y5G0BuAr9sNu92uHPFIFA84qCDFHIYdimWXEnZkTU6Lpx0HO49ha8s+VPfWnXMU5gxjGialjcbcrCki8OTDrDVCnTD00MKwQ/GIgwoSEQ1jstmq2d2ON+u34LnqFTjcffy8p5to83TiraadePTIi6jsroEr6FGeg4gYdoiIhh0ZUnxhvwg6m5UaHRlchiIYDqHZ1YGnq17D8b56hCLsx0MkMewQEQ0z/lAAbzXuxJGe4+jzuxAZwsjGshlrYc503Fp2nfKYLSIk1fTVRe8lGtkYdoiIhhF5dnRXyKPU6LS6O5UOyhcig87MjIlYnDsbo5OLoE5QoaKzCif6GniUFpHAsENENIzIfjmd3h7UOZvgCfmic89OHo2VYrBhhn085mROQmpiMmp665WaoDZvl9LnxxlwRZcmGrkYdij+RfzwOnvR2daOtndNHejo6IUrGBF70NHlaej6w4gE3ejtcMDjD4F1CO+dO+hFnaP57f42OpUWRo0BOrX2tMPJZdCRR1xNt0/ANfnzROhJws7Wg0pn5l6/Q6kh6vY50OLujD6CaORi2KH417UdKx/8Cb5yy0dw87umT+O2O36KJw+LjcL5d6LpbHytcBx+AT++4368tKsWTdHZdOlkx+RWT4cI3xHl9rjUMtxUtBiT0sZAq9Yo86QkvQWzMyfj1tLrYNDosa5xO16tXa8crj54FJYz6BKBp1e5TjSSqe++++6fRK8TDWt+vx/9/f2w2WzROUPkOoZ9W9vQ6CrAsv++Ex9bvBjXiGmJnKaXYIqlAf969RCCqdlISU9Fii76OLowlRaqxHTkjCrFmPIcpBl10EbvotPJvjO+kD9669wcIqBU957EcdnfJhJCgSUbU9LHYbIIO46AW0wuEXSsmJ0xCdcVLIBapcaquk3Y1rIfnb6et4OOlJ6Yooy5U2DNic45t0gojJA3CJPJBJWK+8EUH/R6vXLJXzSNEGZY08owZcliXC2mJYPTsqux8JqpyGvdhKaGNrQ4o4vT0KgToUsqwJQFY5CfZoYxOpsunValgU1ngSphoMmq0d2Kqt5apfZmad4cLMiejqtzZymDB+rVOqxt2IYdrQfR4ulQmq5Olag2wKTlX4WIIyhTzLjkEZTbVuGpv+3H9sYM3PrQ5zBDzBrI+pIHnu7DeOG730TN/F9i3qJpmGnrQnWdR2y4u9DjtSIpPxf5RTYY/V04sasG7f4g/Ep4ykLJpDwkiWdRiTl9TS3o7exGyKxDY2MXQmJPGZYMpGXlYVKuSWnyqTrphsfZK/aiQ3BF0lE2owip6EFHXRPqGrrhVfpkGJE1ZjRy7EmwyWoSeTSNfOyRRrT1uBGQr6fVI61sOgpT9bAoNVEBhM77/gSxMexsbcSB2h55S0hESl4+cgpyYB/8QhwNONHQgtpW2an1LO/lTGEvAs52VB7yIrU8F2lpeiT0tqOt5ji8qbnwtHXA4xbv2WCFPq0AM7J9qKtuHvgccl56MWYUJ0GvEe/wXe9PGPz+8qwDt/0daBr8rmSTTlousgId0FrsMGQUIc8qN/a9aDhwAs2dDrhggFYf/Z71Ggx8VS74u05iV3UX/EHZL0YHU3IGcsvLkGUQ+e3izrIwZEMdQdkZdKOyqxp/OfikUosjKYeVZ8/AVbkz4Ql6YdQmKjU4so+ObLrqCzjfFXSkpXlzcUvxUuRbsqJzzo0jKFM8GhxBmc1YFDMuuRnLfRyHdrei0WHGuA9PhqzQf7vnQ8QHf18j9q7cAHfpEhSlu+CsXIVf/m4lTux5Hi+/0YoecxpyJhiQULcWj/3nX/DM66vw6puVONYcRsr8scjVaaBN6ELFa69g/ZNPY2dzNf726DNY/eZKrDzYgYZgOhaWp0DXuQr3P7gWK15Zjl2bN+O1zUEUXj8a5vZNWPvEU7j//ufw+lsbsG7lXnSmFCM5OxPZFhVUoT64jr6Ch/78OP759HKsXL8GGzdvQ71lEYqzLUgX63J/sA2d531/IQRqNmHLS4/h3+97HhvXr8e6N3ehNmCGoXg0RiWrRSAKwrPnBTz5xJP4zaOviGWi78VehtTsLOSa5fmWzuBrQd+xN/CTb29GcEIh7MU6eA+uxWu/vger+wzY+Ny/sPKVF/DyjiPY2GLEvOTDePHhJ/HkU8/jpe1H8FZHJq6dlguLoR/B4xuw+cW/43t/egEbN6zH+jffwPL9XWjoz8KyKdnQqILwnhTPHf2uXtuyFavb/Oha8Qiq2oGujJmYbPfC17cFL/7mETzxzxfw4prd2LK1C5YFY5FpTYRZ3Y9w+zE0vvkQ7vrTc3hj1RqsX70dB06KsFA+A6OTNdCLtHMl8s5Qm7Hkea3ktLllL7whnxJq3EEPTjqalI7KxbY8pdZH1ubIzsjyvlObrgbJGqJpGeMwM2OC8nwXwmYsikdsxiKSelvg3bNWbDSmwZyTijyZhOTRKx1HgA/9FT/660/x/dvLYa7cgRW/ewK+z/0JP/rHS3jyf5dhacZB/OEnK1Dt8GGgb3MPGhvbsGJtCr78f4/gHy//GnfPEhuvVY/jxyta4fCFAbGBQs4szPj+Q3jy0U9jcUoNNt7/Io46xuDG376EV55/DK/8dCKca9bh9dd3oBq9cPVux8N3PwVX+WfxtfvFMs/ej0d+eD08jz2DA9WyU7ATLRd8fzXYvvEw9h7Jx1cfeAnPviie57c3olzTijde3I5WEXTCOIDlz1XBb12Gux895b0cOoBNW4+iS/mMQxMIAFt2Agu//gP85eW/4M9fnYLylffhK392Ifd2Me+JX+JXnxgN+8aX8Wa9Fy0++f62YMs+PW766Ut4+nn5/r6A2/NcaNm5EW+0BuELi/d3ynf10j//igfH16CjrxsHlMog8TkdNXjjJ39Cy/hP4HbxXT37wF344bXVeOw3r2N7Zav4pppwovognnzMjRt/+AAeeFa8zv1fweemR/DM31ag1jX4t/zgyPNZWbRmpY+O7IQ8SAafV06IsHdyPZ6uev1dnZHPVJpUgAJztnIUF9FIx7BDI0QDag88hz986Sv4mpi+ODh999f44ZOtmPyV2zFvXCFy5A5wvw7qfjsyCu3IzE6Crb8bjro27Koeg9HTs1BQYkfhzMkYO74QhQe343CzH73KDnsElrwsTL79NizIz0dJxlTMmDIK48pVqKgVcSIUAcI2WGzpyCpMQ3qKEYauZjR1ZkBvL8bEaeI1c7KQMW8aRusc0Ld3or2+B/4jFVivnYviSWWYOk4sk1uOwukfw3f+56O4emwW0lwt6Lrg+wvC7+uEw9ULT8SO1DTxPNM+hI/degu+ftNYpIgNpgoBeNwtcHhE8FHL95Iv3sun8Y3PXYePzMpHtCFpSLRib2r8okUoLytDfkYBsu12lGT7UTJvAcpKxLzCchSV5mNKcg2a2kJw+3Iw9tpP49/+/Zv4wjw7CnLk+5uE8pxMjPL74PGLjfrhPdgbKIK6bBIWiu8qKzcHY669GhNS05EhW6P8ffA3H8bWg0VIzstFmfiu8spHYeyy+Rh7fC9a67rQ7AojFOyDw9EERyAZJqt4nXELMO+m2/CzO2eh0KwbaOr6gCVq9Li2YB6KrLlKvxxJhhpPyItdbYeUE4PKpquzBZ2EhARYdCbMy5qiDDB4ZeqpiGILww6NEAYYLVkoHD8O48Q0fnCaOhNTFl+Hj10/AeX2wQ62RqjVecjOVMNgEDeDbvjdYkPZ0YxdLzyMf/z1XvzfY6/htS3V6HF2oMMTQUBubAWD1YLMsWVI18r+IVZkZWcjP88KZ1MresNhBJEBqzVJBB2xsLiN1ma06tOhSk9HjkwTKg2QUoz8dDf0kR4017nQU1uLVkMObCkmJMk3eGan4CG9vwyUzJosApUJHU/di4f+ci/+9OhL2Ha8CwkifCVCIzaJeZhy00zkmdpw5J/i/vvux58eX4HjzgRojLZT+jldmEqtRkZBLmzGRGjFc6vFxttsUyOzMDpPa4TeZESq0QW/v198FVakZ6XAZmjHzsfvxV/vk+9vJdYfbITstdIfCqHveDXawmZokpKRIb6rBLUWibkTUJRlQ7pJLBQOIOLpRIezBwfWvoR/PXQv7nvwaTy+4ghaOlvR6fbDHRTfffFYLPy3iQivfQIv/U28zkNP45WtxxBJyUGiSo3h0FtFHmFVaMkVgWWqElgGA4/U5es9Zx8dWSuUrLdiYfZ0EQJHKwMOEhHDDo0YYu+/eAE++p1v45tiuuvtSdb03IK52YmndMDVIUGVgmSrCrq35wURCnWh6egRHK2oREVFE9rcichZMBn5SWKje44tpFqjUfpO9Pv9CPT3IyICUKLBCLNZ3CnPdxQIIKBRo18upzxC7oVrodWFxNUQ/CJFBQIDfZXO70LvT4Sd2Vdj4Q2zUOSoRM0RsczWlVi9YgNWbm9At9jER5CPKR/+MGZPzUZSh7j/4AFUbHwZL7++E1uPdsARfaUrQnY8PrwbW9esx/pdlThwSH6GE6hv74Nb3C0/v/weZAf1dwx+Vypo3v47ye+pDx11NahSvocTqK4NIWX2ROTnJCFJm4z0oulY8oWPYoJKfO4TYpmdb2Hz6lV4dmU1Gr0hpQP4B03Wxih9buzjcVXOTIxPLVOCixxI8Fxkfx7ZEXl25iRcmz8f2Sa7En6IiGGH6MLERkelSUVK6jx8+pd/wO8eeAAPP/AX3H/vL/A/P/gCri82Iy1a7REOheHzeBESG+d+EUB6urrQ0eeAsSAX6SLUnFY7Io94ycxCprMPqu4etAfFPHnkVaAJrW1WsWFPRm6uCem5+TD2+xEMhKAcPPT2qMU9cHqD4lWG8P60chTpRFjyr8Ydf3sA//eQWOYHt2IyurDjtbU4LN9t2AtnTxJKrv43fFM+x/1/xMPfXQJd5V7s3rkftcqbvkK6dmP9Kxvxxh4zlv76AdwrXv+vf/427lw2EcXibpVag/S8Alj0KvQHg/Ap31UIYW89Wlo96JHj5okgkKA2Qa+ZgOu/9gP8VHzOhx+8Hw/e/1v87pffxSdnF6HIGETAmwB/sBw3/fJ3+NWDYpmffQ2fHmfCtkefw36358qGuoska2kWibDzqdEfwgz7BGXcHKvODJM2EYki3MhJjqIs+/aUJRUq4+7cUf4RFFpzTqsNIhrpGHaILsSSg9TcVEzJ2oqDhwLolhtW51FUrnwaP/j837Gh24XBMWo7m1qx8aUVqPcOdAquPFCJo/v8mD99MvS6MzY+StjJht1QDWfTSRypFvOCAWD/Hhzs1aLLYkNaSTL0E8oxv2U5ju5vRKUcolgZtfhZ/PiOu/HwykocwRDenzKK9J/xs3v+jjda5WHQ8g2cIuwDWlfg4Xt+iT8+uApbL6Y38uXQ1Ykehwp9BjuyMwe+mupXn8eWHZtwUN4vv7vJ0zDRtw+eiv3YIr6rsM+L1jdexs6WpoEgZkiCLmcsZhXvQntTFxre/q5OGeFZ+V7+jv+846d4OkZGzZZHUhVZcvHZMbfgp7O+iW9O/Dd8vGQZluXPww0FC3HHmI/gB9O/iu9M/iyuzpkFg1qv1AwR0Tt46DnFjCty6PmZfC1oqqnHtp1BTLxtFvLMehgTtDBY9CJQ9KPy8Zex7s3leHnlDlS061F8y81YODYdqVov2g7tQVNVFQJGA7asegmrXl2FvT3ivlm34LOL85ARPopN63qQkJOH0bNLkCn3NVRW2G0+dNQdxvqXXsYbq1dh+dpGJM65HlddOxPTspLEXrwNBXYXKnfsxFuvv4LXV63Em9uOon/6HVg8rxxjMy2wJV3g/RksSO7vgad1H/7x3HJsXPkaXltbA0fmRMz7yDJcVShfJwlp4XpUHd2J5198FWtXvonlb7XAPPd6XH3NDEzNML27306wD96Oaqx83YGCpRNRXGyGqqkGVdt2wjHhNkzKMyHd6EPfheaNskDXU4XGvWuwdsNGrH31NewLitQTjiDbEEH/mBswNTMJ+cniuzp5GOvEd7ViwxqscKoQru5HQdkUjF4wDePSjcgs06J+3VZsevlFvLhqA9bu6ELadTdj7vQiFCRZYVWFkeKvwNOvvom3VryGFav345gnDdM/ezuuH2dHmlZ1RfrtDPXQ8zPJ4CKbr2RNjRxfJ9lgQ7bZjiJb3sDoyBYRmI2pSg2P9ozzZ10MHnpO8Wjw0HOGHYoZlxx2ZOGvS0ZGYT7KRme+M8je2chRazUWWO25GDutAGlyjBqx6dMaTGKeHVqnH4a0FCRlFaFs/GTMXTIVxUYVdAluNMuw09IH9dSbMSpNjyx7Nkonz8WM2dMwLUcnNiBiE6rPQGF5IQpzk2FWNkqJsKUlwZBoRKJOD2tGFuwFEzD3mrmYUpaJNI0aKrUJqTlp4j1rYbaI0JKZjZyisZh97TJMLbAixTCE96cV91tNsCSZEBaRJTPDDnteOSbPnYU508uQo1chQWtFakoitIkmaHUm2N/1Xga+ondRG6C1ZqN8WhGykk3iE4lwaM1AztgpKE7TiRCl9EA5/7wcEbQsBqQkW2C02ZEhj96augBTxpSJIFSAzMIxKLAlIiX9ne/KliUeOyEHhsMOWHNLkTd7IkZZxXeYmQGtDzCYjDBn5iG3eBzm3TAXY+zi82sNYr4ZGZlm+CIGpKelwp5TitGTp+Oqt/+W0c91mV1q2DnVYOiRoyLL5ix51JVsypK1P/IorPeCYYfi0WDY4QjKFDMueQTl90Ubdj/1ADbuaILrEw/grhmA2O7S5SL7MimjSHchaE5Fuhz1WesFelfj9999C45R83H1Nz6Kq2SSHaaGOoLyB4UjKFM8GhxBmfGdiIa/SADo3oFX7/8rHn18BdbXtKOtqQVtW/bgWMAKvz0N9mEcdIjog8WwQ0TDn9oAZF6Pj9yYCVPjs/jV5z+Cmz/xOdz844OwLF2MG2+chbLookREZ2IzFsWM4d2MJU8EWoeuHh9C2RNRnCSPooneRZeNv70aJ5va0NgtR8ORfVQucKLSYYTNWETvv8FmLIYdihnDO+wQnR/DDtH7j312iIiIaERg2CEiIqK4xrBDREREcY19dihmsM8OxbL32mcnEAmioqsaVb0n0eJuhzfkg16tR5ohCcW2fIxKKkCGMS269MVjnx2KR+yzQ0QUA8L9YTSLcLOybhNW1G3E7rZDOOloQrcITn0BJyq7q1EhpnZvd/QRRHQmhh0iomGszdOF7a37seLkRnR5e5TzYs3KnIgleXNwVc4MjEkuQZreBp1qmB97T/QBYjMWxQw2Y1Esu5RmLNl0JWtzVp7cBK0IM58ouw4T00bBphuomj9Tv/gXjoTFa/mVJi7lnFnKeEQXxmYsikdsxiIiGuZkHx05JWr0StCZkl4Oi9Ycvffd5IlGa/rq8ffDL6CqtxYhEXyIiGGHiGjYkoFF9s3JtWQqNTpGTSJU5zi7uSvowcGuY3jy2KvY034YL59YKy4rEIwEo0sQjVwMO0REw1SLuwOR/ghyTZlK09X5gs6BzqNY07ANld016PX3obKrGm817RSBp1IEnpDSxEU0UjHsEBENI56QV2mK2t1egUZXm9L/psfvwN6OSrR6OhEIv7umRoYZGXj8oQCsOhMSElTIMtmhV+uUx8rARDSSsYMyxQx2UKZYNtQOyvXOZiyvXa8cgSUDjAwq6gQ1Ug1J+HjpdZiZMQEpBlt06dMd72vAc9VvYH/nUfz71C9ghn1C9J4LYwdlikfsoExENAzlmDNwqwg1szMnI0k/UFDnmO1K0Jmb9c68s5G1QO3eLqQlJsOg1kfnEhHDDhHRMKLU4iQm4cNFV2Nu5hQszJ6OZfkLlBods9YIVcLZi+1OXy+O99Wj2+/A1PSxSNafvfaHaCRi2CEiGmZk4Mm3ZGNRzkwsK1iAGdGmq/MFnb3tldjXcUQJRNPt48Xy1ui9RMQ+OxQz2GeHYtl7PTeWPG1Ek6sN7pAPOpUGOrUWwXAI3rBfqdGRQUcepj7VXo6Pl1wHkzYx+sihYZ8dikeDfXYYdihmMOxQLHuvYccd9OK5mhWo6a2DRWtCssGKbn8fOjzdyqVJa1Saum4tu17przPUkZMHMexQPGIHZSKiGCJP/VBoyUGy3opGdxs2N+/F/o6jyiklFmRPx5fH3arU6FxK0CGKd6zZoZjBmh2KZe+1Zkcegt7rd4rJAXfQo4ScCPqVcCM7I8s+OnKE5UvFmh2KR2zGopjDsEOx7L2GnSuNYYfiEZuxiIiIaERg2CEiIqK4xrBDREREcY1hh4iIiOIaww4RERHFNYYdIiIiimsMO0RERBTXGHaIiIgorjHsEBERUVxj2CEiIqK4xrBDREREcY1hh4iIiOIaww4RERHFNYYdIiIiimsMO0RERBTXGHaIiIgorjHsEBERUVxj2CEiIqK4xrBDREREcY1hh4iIiOIaww4RERHFNYYdIiIiimsMO0RERBTXGHYo5vT390evERERXRjDDsUMjUaDhIQEuFwuRCKR6FwiuhxUKhX0er2yjhHFmwSHw8HdZIoJwWBQmWTQMRqNSuHMgpliRSAchCvoid4aXuS6pElQQx1KYOChuGKxWJRLhh2KKaFQCF6vF263m81ZMSQYDMHjGdjQy6Cq1WqU6zQ8yIAj/y4GgyE6hyg+MOxQzJIhh81YseVoVTWeeOpZ5fqnP/VJjBlVplyn4UHW5AxORPGEYYeI3jcVlYfxwMOPKde/8sXPYfy4scp1IqIraTDssIMyERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprDDtEREQU1xh2iIiIKK4x7BAREVFcY9ghIiKiuMawQ0RERHGNYYeILptQKASPx4O+Psdpk9vjjS4B5fqZ98vHyMcSEV0JCQ6Hoz96nYjoPak5fgI7du3BkaPHonMGBIMDIUgyGo3QajXK9UHlY0Zj1oxpKC0pjs4hInrvLBaLcsmaHSK6bDIzMjBmdBmSkpLgdrlPq7kZdGrNj1xGLisfIx9LRHQlMOwQ0WVjNpuU2pm5s2ciPT0NWq02es+7yfvkMnJZ+Rj5WCKiK4Fhh4guK5vViskTJ2DChHFITk6CWv3uYkbOk/fJZeSy8jFERFcKww4RXXYGgwEL589DWUkxTMZ319jIefI+uYxclojoSmLYIaLLTqVKgMVsVpqoRo8uQ0JCQvQeKNflPHmfXEYuS0R0JTHsENEVIUNMRoYdUydPxKQJ46NzoVyX8+R9DDpE9H7goedEdEU5XS5U1xzHtu27lNtzZs9AWWmJUqtDRHQlDR56zrBDRFecw+nEyZP1yvXCwnxYowUQEdGVxLBDMau/vx+RSCR6i4jeK9mPanAiiicMOxST5CkFvF4v3G63EnqI6L3T6/XKyNY8Mo7iDcMOxZxgMKhMslZHFswqlYp7ohQzAuEgXMF3RpIeTuS6pElQQx1KUIIP1yuKFww7FHPkaQZk2LHZbDCZTEoBTRQrfOEAenx90VvDT9gfQtgZUE7fwXWL4sVg2OEvmmKGbMKSTVdmZWwW/nSJLidZY+r3+9k8THGJWwyKOaxiJyKii8GwQ0RERHGNYYeIiIjiGsMOERERxTWGHSKiGBHuD8MZdKPd24Umdxua3e3o9vXBHw4g0s+BNonOhYeeU8wQv1XliJG8vLzoHKLYcamHnveLf+FIGN6wH86AC/XOFjS52tAXcEKdoEamMR0F1mykJSbDrDVCr9YhQfy7WEFvAL5uN+x2O9RqdXQuUWzjODsUcxh2KJZdatgJRkI46WzE6vot2NdxFN6QT6nhiUQPEVcnqKBRqVFozcGczCm4KncmDGr9RQcehh2KRww7FHMYdiiWXUrY6fE7sKe9Altb9+FEXwN6/c5zNlcZNQZkmtJQnlyCa/PnI9tkV2p5hophh+IRBxUkIhqmZNOVrNGRQWdD005UdFYpfXPO1y/HE/KhztGCba37sbphC1o87UoNEBEx7BARDTuyj45supI1Osd6apVaoQtRanaMabAnpmJryz4c7DymBCQiYtghIhp2vCE/3qzbojRdDSXoaFUaFFpzcW3BfNw++kNKDZAMPDIoyVoiopGOYYeIaBgZOLzchf2dR5U+OkNRmlSAZfnzsSh7htItWa/W47gISnWuZuVs60QjHcMOEdEwIvveyMPL5VFXFxo7R9bolKeU4IaChUrgqe47iX9Vr0KfCEky5HR5e9Hm6YouTTRy8WgsihmXfDSW6zgO7T6EPQfq0RuddSqtwYyihZ/EjNRWtB5uQaPDjHEfnowccd8lHZMSEXvSzqPYsKYbCTl5GD27GBnRu2KL7NzahH3LD8NlzUXW9PEoNQ/cc06+FjRV12HbjgAm3D4LeSY9jNG7RrqhHo0lBwzc1LQb/zq+Cu6gFznmDKQnpqDb14sm1zudjmUfHdl0JWt0ZNCpdzbjraad2NNeiXAkpDRezbBPwA2FizDNPk55zPnwaCyKRzwai0YOdw0ObViDF55ciY0VlTgopopTpsojR1HfE4TP50B3Sxsa6rrgkuOYeJtQtbcK1bUdcESf6l3khufM5fpF2Ok7iA0vbsDGHcfRqiwYi0JiasC+V1di04ZDqHEPzD0vbzOaDm3H849swjGXH57obBo6eRSWHDBwcBydXFMm5mVNxcLsGcgypUOn1ipBp9iWh6tzZmJy+hg0u9uUoLNXBJ1QNOhI3rBPBCb+FYgYdmiEyEPRpFvxvYcewF/F9PAp0/1/+g2+OjcJeflTsOj2j+HL31qCsRE/tB1v4ZnfP4Nnl+9HbfRZ3kUsh6EsRzREcjBAOTLyoEAkCKvWjGvz5ymhJ8uYLoJOvtI/Z1HuTOWUEa+eWK/U6MigdCpVgkqZiEY6rgVERMOIHP1YngJCjowsVXZVY3ntOhztqcWNhYtw26gP4fZRN2Jm5kQc76vHI5UvoKr3pNJ0dSaLCEkphqToLaKRS3333Xf/JHqdaFjz+/3o7++HzWaLzhkit+yz03paXxzNwD2n692HDS9txZtvHUP/aAfW/vyfWLe/EpV1DWjp9MNQNh7ZieKxb4/C74Cnbz9eEMutPnW5ktHIDh7FlnU9SNB0w+fagft+/xiWv/oaqvsLkJicgSyl74s8SqYKa+57GI8//ASeeHUL1m7qhm1mEWwGLfRykXdxob1qO9566A/Y1hPA6iefwYtPPY3nN1dgY4cZM62VePmRf+LRR6PzutMwszgJBq1a6btUte1V3PuLv+IZ8V5eeXU7atzidfIKkGWIPn3tWiz/11P47d+exPJVq7G8sQ/dO5pgzitB1vQJKDVf4D3LPjs19di2M4iJt81Cnpl9dgaF+sPwhfzRW+cma2Lk4eJbWvYq/XzC/RGl747saJwqgkueJQs2vVkJOi/UvIlaR6NyItCzdb6UTVyzRChK1Az+gc8tEgoj5A3CZDJBpeJ+MMUHvX6gJGXYoZjx3sLOIezZfwxNvXU4umMHdoppu5wq61DhsmB0phk69yFsXXUYe6sDKLy+DMYTx3DsRAjarCKMnTsZE8oKYBfrjfrtsBMWGwg3+sRyh09drjQb9sBRbF57EFXOEAJJGShOsoh5NTh43ICwORkFpWbova2ofP0f2F6rR8CajaykfiS0H8E+dzIy7UlIsxrOEsrc6Dy+D1ufeQqruvKRnGJFdo5Yyt+F6o2H0eqKIKQ1IskaRMDRiYP7HCieOQGp5h407ViDzav34bB6NEYV25GZ0IHWVg/qnYkonJAOMxpx4F/PY2NFH7qSRmNCfhISRUBqONoHVVE5SqaPQYH6Au9Z2402hp2zGnrYSUCCmGRNjjzxp2yaUvrx+J1wBF0waRPR6GoTYWgfDnYdE/cFzxp0MoxpmJExAWNTy4bUlMWwQ/FoMOzwF00jhA8eZwtOyg7Jp3ZQPnYcR1vcCIZPPcTXCL2xFIs+MRuleZMwZfZVuPZjCzFRZCztaWvMeZZTAlEnOiNWaPKvw13f/Aru+tRcWJpqUSdeszXshk9sqNY+vhf+UQuw5Gvfxre++hHcvkyD6hfW43BVO7rOMzxKpF8NRygHMz50Kz7/nS/gs9eMwqj6dVhZlYrCxWLeN+7Ap68qQVHjJuxp8qOrrw5HdlfhUE0iptz5bXz1W9/GXV9aiFGqLlSt3o6j7gCCXfuxbk8HPMnT8YkvfRvf+dqd+NqsDKTZNPDKFx3Ke353SwpdJBlMzBoj5mZNUQLLYP8d2XdnX8dhrGvcjjfrN7+rM/Ig2edHp9JiYtpopROz5pT+P0QjFcMOjRDn6KD8m7vxu0+WI8mojS53OWVi3OgxWDizEJCH8qbbka43QKmX8jsQbK3CvvrRMOj7oUM7ekWA0ZVPw/j2I+hs6UWLS3mSs9IZ9Ji+7CoUZ8oaGRMMiUnIK9Fj1g3ReeYMJGVlYmJmE7q7w/A1d6HXnYRA8jhMGSseLz9uURkKsowoCbahuSGIwP59OKwbC8OockwvEoWDeK/pV1+LSfYM8e0J7/E909DpNTrl7OXlKcVKk5Ws6RlU0VWtDBh4ZmdkSQYdeQZ0eULQeSIsldoKovcQjWwMO0RXjB02iw1pqdGb7yJPA7AFz/7ie/jaLR/Bzbd8Hrd//l682u1E58ACl09XJ3o0QG9mmohggyt+KmzJIZiS2tDQFBKBpxE+74UOU34f3/MIJkOL7Ki8LH8B5mROhlk7tMZAOcigPDz9U6NvQoktXwk+RMSwQ3QFqZGgUuHc3R9k9cp4XP2Zr+O7P/8ZfvHzX+BXv/pf3PvAPbhjURnKLjSA38Ww2WAJA+buPnSLmwONdg64nWr4XMlIT1Mj1Z4One5CHVnfx/c8wsnAk22y49r8+fhk2fXKSMkyzJyLbPKSh6J/YezHMC6lVAlI8jmIiGGH6AIc8Po8cF2weWaoy0WJDZEmOQ8l6W6k5pVj9NTFWLJwGuaWW+A44YIvEETkcraspaQhSe9CYlcNDtcDQTkIb9tJNLUF0Nifgdw8Lcxjx6GwvwmBpgZUt4lAFAyi7+BenOjtHai1Gcp7Pve2mC6BXq1DviULczKn4KbCq/HR0muxNG+uMjLy+NQypV/OvKxpuKnoatxSvETcNwcTUkfDqjOfNlYP0UjHsEN0NvLolcRUJJl70Fe/D3u2HURVNxA681RFQ13uTFoL9PYxWHhtIoK1Fdi9Yh3Wrl6DtStXY+22k6jv8SkNRpeNORsFpUnIs7Vi7yvrsH61eL1Xd6HGrYd+wjgUp2ihKZyKeSXi47QfwrpX12Hd+g3YtKsG9X2egQ7K7/d7JoUMLfJ0EfOyp+LjJctEqFmqnALiuvwFyiSDzsdLl2FZwQKUJw/U/rBGh+h0DDsU/1QGJJrNsCUZoRM3z7kZOHU5lRYJ6RMwodwEVd1qrHn6CbxeA3jP7BMqlsO7lkuAtz8RlmQLLCa90vCjvKr61HkiZFgKcfVdX0Nh+zZsue8e/PBnD+KP/2rHhO98DNPL885xPi0V1LpEmJLTYDWooFHW4KHMK8SUpdfi6muy0PbUPfjfn4rXe+QgujLH4pbPLsFY8Y60KMfSz96M8ZldOPDIPfjJr/+Mp0LjkJVaiLHmROhVQ3jPKh20iWakpJmhVyewgLmMZIAZrOmR57pamDMD87OnYWxKCZL1Nh51RXQePBEoxYxLPhFoxA+vO4hAWIVEEWRk+Dhr4HnXcmEEnA54/EGExUZcb06CWTz4lANjosLwn7acDWaVDy6nSEY6HQxKuBGrWdgHp+OMeQjC0+uCzx8S10Q4EOEpMcmKRJFY3hnP51QRhIN++FxuhMXevlEng8wQ54lXCPi94j34xDuGeHU1dEYjjCYZZAaefeA78MLtCSCiShD5zAiN+FxqnXi/ynIXeM/98jUCcLsjSEwxi9DIwDNoqCcC/aDwRKAUjwZPBMqwQzHjksMO0TDAsEP0/uNZz4mIiGhEYNghIiKiuMawQ0RERHGNYYeIiIjiGsMOERERxTWGHSIiIoprPPScYgYPPadY9l4PPY/0R9Dl60WP3wF30ItQJAR5ok95wlCb3oIkvRVGzYXObXZuPPSc4hHH2aGYw7BDsexSw06/+BeKhEXI6cPmlr042HEU9c4WeEI+mLSJSDMkY1L6GMzMmIhSW370URePYYfiEcfZISKKATLoHOmpwZ/2/wMv1ryJFk8nss12zM6chFmZExEW/xpE+On29UYfQURnYs0OxQzW7FAsu9SanV3th7C6fgtOOpqV2pvylGKlNkeeJ0utUsMZcEGr0iItMRnJemv0URePNTsUj1izQ0Q0zDW62rC/4wha3B3KyT8X5czAtPRxGJ1chEJrDvLMmRibUoqypAIl6AQjIbR7u7CpeTdaPZ0I98uzoBERww4R0TAlm69qHY1INtiwNG8Oiqy5MGj00XtPJ4NOi6cDm5v3YvmJddjeul8Eng6lYzPRSMewQ0Q0TB3vq1eOvCqwZKPElq8cfXU2MtB0eLuxs/Ugnql6HVW9J/Fq7XpsEcGnN+BUOjkTjWQMO0REw1S3rw9alQZZJnt0ztnJQ9LXNW5XAo4/HFDCjZy3Vpm3Dr6wn4GHRjR2UKaYwQ7KFMuG2kFZ9rWRNTLVfSdxuPu4Mp5OWmKK0j9nYfZ0lKeUwKozR5ceIANOm3jcoa4qbGzejZreOny05FqlP0+KwYpc8Vh1wvk7HbODMsUjdlAmIhqG5FFWKQYbev1OBESIcQU96PM7lLBi1pnO2pQlH5NvyUaprUCpCZJhSAadKenlKLDkXDDoEMU7hh0iomFEHlU1J2syFufOQrEtD9mmDExIG41lBfMxKqkQRk1idMnT+UJ+dPp60OntUToym7XG6D1ExLBDRDTMyFNAXJs/H1flzMQycXl9wUKMSylVam3ORg482OBqxZHuE0qH5hkZE5FqSIreS0QMO0REw9TCnBm4Llqjcz51zialg/LOtgPItWRiun0ckvQDfRWIiB2UKYawgzLFsvd6IlB5RNXGpl1KB2bZRCVP/ukMuJWjruqdzcqRW1mmdHyo6CqUJ5cooypfDHZQpnjEDspERDGkv78f7qAPdY5m7G6rwLqG7VjfuAMHO48pB5VPSR+La/LniaBTCs05mruIRiqGHSKiGCADzKjkQkxIG6U0VVl0ZmQYU5VzZS3OnY0bChdhhn2C0q8nQfwjonewGYtiBpuxKJa912asK43NWBSP2IxFREREIwLDDhEREcU1hh0iIiKKaww7REREFNcYdoiIiCiuMewQERFRXGPYISIiorjGsENERERxjWGHiIjo/7dnN6sNAmEYRsf404B4/5cpuNSqjZJAd924qC/ngMy4d/geRqKJHQAgmtgBAKKJHQAgmtgBAKKJHQAgmtgBAKKJHQAgmtgBAKKJHQAgmtgBAKKJHQAgmtgBAKKJHQAgmtgBAKKJHW5n3/f3DgD+Jna4jaqqznVdV8EDFztO1+eMQRqxw210XVfqui7jOAoeuFjdNKXv+/J4GAvkqaZpMjG4hW3byrIsZZ7nM3bgTrb99f1u3++3/6Vt2/L8epaubkvzih5IMQzDuYodbuV38LjZgWscgXMEz/FAErEDAET7xI6fswBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAEQTOwBANLEDAAQr5QeX0Bq/MwLlxQAAAABJRU5ErkJggg==" + } + }, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Building the base Kubeflow pipeline\n", + "\n", + "The next steps will build up the following Kubeflow pipeline:\n", + "\n", + "![image.png](attachment:image.png)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set default variables\n", + "\n", + "The following default variables should be changed when running the notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [], + "source": [ + "# Namespace to run the workloads under\n", + "USER_NAMESPACE = \"kubeflow\" # On a full installation this would be your user namespace\n", + "# Pipeline service account\n", + "# On a Kubeflow instance on GCP this should be 'default-editor'\n", + "KFP_SERVICE_ACCOUNT = \"pipeline-runner\"\n", + "\n", + "\n", + "# Consmetic variables\n", + "# Pipeline run variables\n", + "KFP_EXPERIMENT = \"katib-kfp-example\"\n", + "KFP_RUN = \"mnist-pipeline-v1\"\n", + "\n", + "# Katib run variables\n", + "KATIB_EXPERIMENT = \"katib-kfp-example-v1\"\n", + "KATIB_E2E_EXPERIMENT = \"katib-kfp-example-e2e-v1\"\n", + "KATIB_WORKLFLOW_COLLECTOR_IMAGE = \"docker.io/kubeflowkatib/kfpv1-metrics-collector:latest\" #\"docker.io/votti/kfpv1-metricscollector:v0.0.10\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install and load required python packages" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: kfp==1.8.12 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (1.8.12)\n", + "Requirement already satisfied: absl-py<2,>=0.9 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (1.4.0)\n", + "Requirement already satisfied: PyYAML<6,>=5.3 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (5.4.1)\n", + "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (2.10.2)\n", + "Requirement already satisfied: google-cloud-storage<2,>=1.20.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (1.44.0)\n", + "Requirement already satisfied: kubernetes<19,>=8.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (18.20.0)\n", + "Requirement already satisfied: google-api-python-client<2,>=1.7.8 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (1.12.11)\n", + "Requirement already satisfied: google-auth<2,>=1.6.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (1.35.0)\n", + "Requirement already satisfied: requests-toolbelt<1,>=0.8.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (0.10.1)\n", + "Requirement already satisfied: cloudpickle<3,>=2.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (2.2.1)\n", + "Requirement already satisfied: kfp-server-api<2.0.0,>=1.1.2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (1.8.5)\n", + "Requirement already satisfied: jsonschema<4,>=3.0.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (3.2.0)\n", + "Requirement already satisfied: tabulate<1,>=0.8.6 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (0.9.0)\n", + "Requirement already satisfied: click<9,>=7.1.2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (8.1.3)\n", + "Requirement already satisfied: Deprecated<2,>=1.2.7 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (1.2.14)\n", + "Requirement already satisfied: strip-hints<1,>=0.1.8 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (0.1.10)\n", + "Requirement already satisfied: docstring-parser<1,>=0.7.3 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (0.15)\n", + "Requirement already satisfied: kfp-pipeline-spec<0.2.0,>=0.1.14 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (0.1.16)\n", + "Requirement already satisfied: fire<1,>=0.3.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (0.5.0)\n", + "Requirement already satisfied: protobuf<4,>=3.13.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (3.20.3)\n", + "Requirement already satisfied: uritemplate<4,>=3.0.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (3.0.1)\n", + "Requirement already satisfied: pydantic<2,>=1.8.2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (1.10.9)\n", + "Requirement already satisfied: typer<1.0,>=0.3.2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp==1.8.12) (0.9.0)\n", + "Requirement already satisfied: wrapt<2,>=1.10 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from Deprecated<2,>=1.2.7->kfp==1.8.12) (1.15.0)\n", + "Requirement already satisfied: six in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from fire<1,>=0.3.1->kfp==1.8.12) (1.16.0)\n", + "Requirement already satisfied: termcolor in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from fire<1,>=0.3.1->kfp==1.8.12) (2.3.0)\n", + "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->kfp==1.8.12) (1.59.1)\n", + "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->kfp==1.8.12) (2.31.0)\n", + "Requirement already satisfied: httplib2<1dev,>=0.15.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.12) (0.22.0)\n", + "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.12) (0.1.0)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.12) (4.2.4)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.12) (0.3.0)\n", + "Requirement already satisfied: setuptools>=40.3.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.12) (67.7.2)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.12) (4.9)\n", + "Requirement already satisfied: google-cloud-core<3.0dev,>=1.6.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.12) (2.3.2)\n", + "Requirement already satisfied: google-resumable-media<3.0dev,>=1.3.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.12) (2.5.0)\n", + "Requirement already satisfied: attrs>=17.4.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from jsonschema<4,>=3.0.1->kfp==1.8.12) (23.1.0)\n", + "Requirement already satisfied: pyrsistent>=0.14.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from jsonschema<4,>=3.0.1->kfp==1.8.12) (0.19.3)\n", + "Requirement already satisfied: urllib3>=1.15 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.12) (1.26.15)\n", + "Requirement already satisfied: certifi in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.12) (2023.5.7)\n", + "Requirement already satisfied: python-dateutil in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.12) (2.8.2)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes<19,>=8.0.0->kfp==1.8.12) (1.6.0)\n", + "Requirement already satisfied: requests-oauthlib in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes<19,>=8.0.0->kfp==1.8.12) (1.3.1)\n", + "Requirement already satisfied: typing-extensions>=4.2.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from pydantic<2,>=1.8.2->kfp==1.8.12) (4.6.3)\n", + "Requirement already satisfied: wheel in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from strip-hints<1,>=0.1.8->kfp==1.8.12) (0.40.0)\n", + "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-resumable-media<3.0dev,>=1.3.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.12) (1.5.0)\n", + "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from httplib2<1dev,>=0.15.0->google-api-python-client<2,>=1.7.8->kfp==1.8.12) (3.1.0)\n", + "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.1->kfp==1.8.12) (0.5.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->kfp==1.8.12) (3.1.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from requests<3.0.0dev,>=2.18.0->google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->kfp==1.8.12) (3.4)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from requests-oauthlib->kubernetes<19,>=8.0.0->kfp==1.8.12) (3.2.2)\n", + "Requirement already satisfied: kubeflow-katib==0.13.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (0.13.0)\n", + "Requirement already satisfied: certifi>=14.05.14 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubeflow-katib==0.13.0) (2023.5.7)\n", + "Requirement already satisfied: six>=1.10 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubeflow-katib==0.13.0) (1.16.0)\n", + "Requirement already satisfied: setuptools>=21.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubeflow-katib==0.13.0) (67.7.2)\n", + "Requirement already satisfied: urllib3>=1.15.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubeflow-katib==0.13.0) (1.26.15)\n", + "Requirement already satisfied: kubernetes>=12.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubeflow-katib==0.13.0) (18.20.0)\n", + "Requirement already satisfied: python-dateutil>=2.5.3 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes>=12.0.0->kubeflow-katib==0.13.0) (2.8.2)\n", + "Requirement already satisfied: pyyaml>=5.4.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes>=12.0.0->kubeflow-katib==0.13.0) (5.4.1)\n", + "Requirement already satisfied: google-auth>=1.0.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes>=12.0.0->kubeflow-katib==0.13.0) (1.35.0)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes>=12.0.0->kubeflow-katib==0.13.0) (1.6.0)\n", + "Requirement already satisfied: requests in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes>=12.0.0->kubeflow-katib==0.13.0) (2.31.0)\n", + "Requirement already satisfied: requests-oauthlib in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from kubernetes>=12.0.0->kubeflow-katib==0.13.0) (1.3.1)\n", + "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=12.0.0->kubeflow-katib==0.13.0) (4.2.4)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=12.0.0->kubeflow-katib==0.13.0) (0.3.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=12.0.0->kubeflow-katib==0.13.0) (4.9)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from requests->kubernetes>=12.0.0->kubeflow-katib==0.13.0) (3.1.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from requests->kubernetes>=12.0.0->kubeflow-katib==0.13.0) (3.4)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from requests-oauthlib->kubernetes>=12.0.0->kubeflow-katib==0.13.0) (3.2.2)\n", + "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/vitoz/mambaforge/envs/katibdev/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=12.0.0->kubeflow-katib==0.13.0) (0.5.0)\n" + ] + } + ], + "source": [ + "# Install required packages (Kubeflow Pipelines and Katib SDK).\n", + "!pip install kfp==1.8.12\n", + "!pip install kubeflow-katib==0.13.0" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from typing import Optional\n", + "from datetime import datetime as dt\n", + "import kfp\n", + "import kfp.components as components\n", + "import kfp.dsl as dsl\n", + "from kfp.components import InputPath, OutputPath, create_component_from_func" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize the Kubeflow pipeline client\n", + "\n", + "Documentation how this is done in various environments: https://www.kubeflow.org/docs/components/pipelines/v1/sdk/connect-api/" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "kfp_client = kfp.Client()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Get the downloader component\n", + "\n", + "This is a publicly available, generic downloader we use to download the raw MNIST data" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "download_data_op = components.load_component_from_url(\n", + " \"https://raw.githubusercontent.com/kubeflow/pipelines/master/components/contrib/web/Download/component.yaml\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Parse the MNIST raw data format\n", + "\n", + "This is a component from text that converts the raw MNIST data format into a tensorflow compatible format." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "parse_mnist_op = components.load_component_from_text(\n", + " \"\"\"\n", + "name: Parse MNIST\n", + "inputs:\n", + "- {name: Images, description: gziped images in the idx format}\n", + "- {name: Labels, description: gziped labels in the idx format}\n", + "outputs:\n", + "- {name: Dataset}\n", + "metadata:\n", + " annotations:\n", + " author: Vito Zanotelli, D-ONE.ai\n", + " description: Based on https://github.com/kubeflow/pipelines/blob/master/components/contrib/sample/Python_script/component.yaml\n", + "implementation:\n", + " container:\n", + " image: tensorflow/tensorflow:2.7.1\n", + " command:\n", + " - sh\n", + " - -ec\n", + " - |\n", + " # This is how additional packages can be installed dynamically\n", + " python3 -m pip install pip idx2numpy\n", + " # Run the rest of the command after installing the packages.\n", + " \"$0\" \"$@\"\n", + " - python3\n", + " - -u # Auto-flush. We want the logs to appear in the console immediately.\n", + " - -c # Inline scripts are easy, but have size limitaions and the error traces do not show source lines.\n", + " - |\n", + " import gzip\n", + " import idx2numpy\n", + " import sys\n", + " from pathlib import Path\n", + " import pickle\n", + " import tensorflow as tf\n", + " img_path = sys.argv[1]\n", + " label_path = sys.argv[2]\n", + " output_path = sys.argv[3]\n", + " with gzip.open(img_path, 'rb') as f:\n", + " x = idx2numpy.convert_from_string(f.read())\n", + " with gzip.open(label_path, 'rb') as f:\n", + " y = idx2numpy.convert_from_string(f.read())\n", + " #one-hot encode the categories\n", + " x_out = tf.convert_to_tensor(x)\n", + " y_out = tf.keras.utils.to_categorical(y)\n", + " Path(output_path).parent.mkdir(parents=True, exist_ok=True)\n", + " with open(output_path, 'wb') as output_file:\n", + " pickle.dump((x_out, y_out), output_file)\n", + " - {inputPath: Images}\n", + " - {inputPath: Labels}\n", + " - {outputPath: Dataset}\n", + "\"\"\"\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Process the images\n", + "\n", + "This does the pre-processing of the images, including a training-validation split.\n", + "\n", + "Here also an optional `histogram_norm` image normalization step can be activated" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "def process(\n", + " data_raw_path: InputPath(str), # type: ignore\n", + " data_processed_path: OutputPath(str), # type: ignore\n", + " val_pct: float = 0.2,\n", + " trainset_flag: bool = True,\n", + " histogram_norm: bool = False,\n", + "):\n", + " \"\"\"\n", + " Here we do all the preprocessing\n", + " if the data path is for training data we:\n", + " (1) Normalize the data\n", + " (2) split the train and val data\n", + " If it is for unseen test data, we:\n", + " (1) Normalize the data\n", + " This function returns in any case the processed data path\n", + " \"\"\"\n", + " # sklearn\n", + " import pickle\n", + " from sklearn.model_selection import train_test_split\n", + " import tensorflow as tf\n", + " import tensorflow_addons as tfa\n", + "\n", + " def img_norm(x):\n", + " x_ = tf.reshape(x, list(x.shape) + [1])\n", + "\n", + " if histogram_norm:\n", + " x_ = tfa.image.equalize(x_)\n", + "\n", + " # Scale between 0-1\n", + " x_ = x_ / 255\n", + " return x_\n", + "\n", + " with open(data_raw_path, \"rb\") as f:\n", + " x, y = pickle.load(f)\n", + " if trainset_flag:\n", + "\n", + " x_ = img_norm(x)\n", + " x_train, x_val, y_train, y_val = train_test_split(\n", + " x_.numpy(), y, test_size=val_pct, stratify=y, random_state=42\n", + " )\n", + "\n", + " with open(data_processed_path, \"wb\") as output_file:\n", + " pickle.dump((x_train, y_train, x_val, y_val), output_file)\n", + "\n", + " else:\n", + " x_ = img_norm(x)\n", + " with open(data_processed_path, \"wb\") as output_file:\n", + " pickle.dump((x_, y), output_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "process_op = create_component_from_func(\n", + " func=process,\n", + " base_image=\"tensorflow/tensorflow:2.7.1\", # Optional\n", + " packages_to_install=[\"scikit-learn\", \"tensorflow-addons[tensorflow]\"], # Optional\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Training component\n", + "\n", + "Component with ML hyperparameters as parameters.\n", + "Note that the `metrics` that should be tracked by Katib need to be\n", + "saved as ML metrics output artifacts.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def train(\n", + " data_train_path: InputPath(str), # type: ignore\n", + " model_out_path: OutputPath(str), # type: ignore\n", + " mlpipeline_metrics_path: OutputPath(\"Metrics\"), # type: ignore # noqa: F821\n", + " lr: float = 1e-4,\n", + " optimizer: str = \"Adam\",\n", + " loss: str = \"categorical_crossentropy\",\n", + " epochs: int = 1,\n", + " batch_size: int = 32,\n", + "):\n", + " \"\"\"\n", + " This is the simulated train part of our ML pipeline where training is performed\n", + " \"\"\"\n", + "\n", + " import tensorflow as tf\n", + " import pickle\n", + " from tensorflow.keras.preprocessing.image import ImageDataGenerator\n", + " import json\n", + "\n", + " with open(data_train_path, \"rb\") as f:\n", + " x_train, y_train, x_val, y_val = pickle.load(f)\n", + "\n", + " model = tf.keras.Sequential(\n", + " [\n", + " tf.keras.layers.Conv2D(\n", + " 64, (3, 3), activation=\"relu\", input_shape=(28, 28, 1)\n", + " ),\n", + " tf.keras.layers.MaxPooling2D(2, 2),\n", + " tf.keras.layers.Conv2D(64, (3, 3), activation=\"relu\"),\n", + " tf.keras.layers.MaxPooling2D(2, 2),\n", + " tf.keras.layers.Flatten(),\n", + " tf.keras.layers.Dense(128, activation=\"relu\"),\n", + " tf.keras.layers.Dense(10, activation=\"softmax\"),\n", + " ]\n", + " )\n", + "\n", + " if optimizer.lower() == \"sgd\":\n", + " optimizer = tf.keras.optimizers.SGD(lr)\n", + " else:\n", + " optimizer = tf.keras.optimizers.Adam(lr)\n", + "\n", + " model.compile(loss=loss, optimizer=optimizer, metrics=[\"accuracy\"])\n", + "\n", + " # fit the model\n", + " model_early_stopping_callback = tf.keras.callbacks.EarlyStopping(\n", + " monitor=\"val_accuracy\", patience=10, verbose=1, restore_best_weights=True\n", + " )\n", + "\n", + " train_datagen = ImageDataGenerator()\n", + "\n", + " validation_datagen = ImageDataGenerator()\n", + " history = model.fit(\n", + " train_datagen.flow(x_train, y_train, batch_size=batch_size),\n", + " epochs=epochs,\n", + " validation_data=validation_datagen.flow(x_val, y_val, batch_size=batch_size),\n", + " shuffle=False,\n", + " callbacks=[model_early_stopping_callback],\n", + " )\n", + "\n", + " model.save(model_out_path, save_format=\"tf\")\n", + "\n", + " metrics = {\n", + " \"metrics\": [\n", + " {\n", + " \"name\": \"accuracy\", # The name of the metric. Visualized as the column name in the runs table.\n", + " \"numberValue\": history.history[\"accuracy\"][\n", + " -1\n", + " ], # The value of the metric. Must be a numeric value.\n", + " \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\n", + " },\n", + " {\n", + " \"name\": \"val-accuracy\", # The name of the metric. Visualized as the column name in the runs table.\n", + " \"numberValue\": history.history[\"val_accuracy\"][\n", + " -1\n", + " ], # The value of the metric. Must be a numeric value.\n", + " \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\n", + " },\n", + " ]\n", + " }\n", + " with open(mlpipeline_metrics_path, \"w\") as f:\n", + " json.dump(metrics, f)\n", + "\n", + "\n", + "train_op = create_component_from_func(\n", + " func=train, base_image=\"tensorflow/tensorflow:2.7.1\", packages_to_install=[\"scipy\"]\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Build the full pipeline\n", + "\n", + "These wires the components to a full pipeline.\n", + "\n", + "The only thing required to make the pipeline Katib compatible is:\n", + "\n", + "1) A pod label to mark the pod from which the metrics tracked by Katib should be collected from: \"katib.kubeflow.org/model-training\", \"true\"\n", + "2) A mark to prevent caching on this pod: `execution_options.caching_strategy.max_cache_staleness = \"P0D\"`\n", + "\n", + "In addition, currently the pod label for caching seems not be added by default and thus the cache is not used. To enable cache usage, the cache label is added to all the steps.\n", + "\n", + "Apart from these two requirements, there is no restriction on how the pipeline is build. The pipeline remains a normal Kubeflow pipeline." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "def _label_cache(step):\n", + " \"\"\"Helper to add pod cache label\n", + "\n", + " Currently there seems to be an issue with pod labeling.\n", + " \"\"\"\n", + " step.add_pod_label(\"pipelines.kubeflow.org/cache_enabled\", \"true\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "@dsl.pipeline(\n", + " name=\"Download MNIST dataset\",\n", + " description=\"A pipeline to train MNIST classification from scratch.\",\n", + ")\n", + "def mnist_training_pipeline(\n", + " lr: float = 1e-4,\n", + " optimizer: str = \"Adam\",\n", + " loss: str = \"categorical_crossentropy\",\n", + " epochs: int = 3,\n", + " batch_size: int = 5,\n", + " histogram_norm: bool = False,\n", + "):\n", + " TRAIN_IMG_URL = \"http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\"\n", + " TRAIN_LAB_URL = \"http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\"\n", + "\n", + " train_imgs = download_data_op(TRAIN_IMG_URL)\n", + " train_imgs.set_display_name(\"Download training images\")\n", + " _label_cache(train_imgs)\n", + "\n", + " train_y = download_data_op(TRAIN_LAB_URL)\n", + " train_y.set_display_name(\"Download training labels\")\n", + " _label_cache(train_y)\n", + "\n", + " mnist_train = parse_mnist_op(train_imgs.output, train_y.output)\n", + " mnist_train.set_display_name(\"Prepare train dataset\")\n", + " _label_cache(mnist_train)\n", + "\n", + " processed_train = (\n", + " process_op(\n", + " mnist_train.output,\n", + " val_pct=0.2,\n", + " trainset_flag=True,\n", + " histogram_norm=histogram_norm,\n", + " )\n", + " .set_cpu_limit(\"1\")\n", + " .set_memory_limit(\"2Gi\")\n", + " .set_display_name(\"Preprocess images\")\n", + " )\n", + " _label_cache(processed_train)\n", + "\n", + " training_output = (\n", + " train_op(\n", + " processed_train.outputs[\"data_processed\"],\n", + " lr=lr,\n", + " optimizer=optimizer,\n", + " epochs=epochs,\n", + " batch_size=batch_size,\n", + " loss=loss,\n", + " )\n", + " .set_cpu_limit(\"1\")\n", + " .set_memory_limit(\"2Gi\")\n", + " )\n", + " training_output.set_display_name(\"Fit the model\")\n", + " # This pod label indicates which pod Katib should collect the metric from.\n", + " # A metrics collecting sidecar container will be added\n", + " training_output.add_pod_label(\"katib.kubeflow.org/model-training\", \"true\")\n", + " # This step needs to run always, as otherwise the metrics for Katib could not\n", + " # be collected.\n", + " training_output.execution_options.caching_strategy.max_cache_staleness = \"P0D\"\n", + "\n", + " return mnist_train.output" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Experiment details." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run details." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "kfp_run = f\"{KFP_RUN}-{dt.today().strftime('%Y-%m-%d-%Hh-%Mm-%Ss')}\"\n", + "run = kfp_client.create_run_from_pipeline_func(\n", + " mnist_training_pipeline,\n", + " mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY,\n", + " # You can optionally override your pipeline_root when submitting the run too:\n", + " # pipeline_root='gs://my-pipeline-root/example-pipeline',\n", + " arguments={\"histogram_norm\": \"0\"},\n", + " experiment_name=KFP_EXPERIMENT,\n", + " run_name=kfp_run,\n", + " # In a multiuser setup, provide the namesapce\n", + " #namespace=USER_NAMESPACE,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Parameter tuning with Katib\n", + "\n", + "We now want to do parameter tuning over the whole pipeline with Katib.\n", + "\n", + "This requires us to build up a specificaiton for the Katib experiment" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First import the Katib python components:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "import yaml\n", + "from typing import List\n", + "\n", + "from kubernetes.client.models import V1ObjectMeta\n", + "from kubeflow.katib import ApiClient\n", + "from kubeflow.katib import KatibClient\n", + "from kubeflow.katib import V1beta1Experiment\n", + "from kubeflow.katib import V1beta1ExperimentSpec\n", + "from kubeflow.katib import V1beta1AlgorithmSpec\n", + "from kubeflow.katib import V1beta1ObjectiveSpec\n", + "from kubeflow.katib import V1beta1ParameterSpec\n", + "from kubeflow.katib import V1beta1FeasibleSpace\n", + "from kubeflow.katib import V1beta1TrialTemplate\n", + "from kubeflow.katib import V1beta1TrialParameterSpec\n", + "from kubeflow.katib import V1beta1MetricsCollectorSpec\n", + "from kubeflow.katib import V1beta1CollectorSpec" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to build a katib experiment, we require a trial spec.\n", + "\n", + "In this case the trial spec is an Argo workflow produced form the Kubeflow pipeline.\n", + "\n", + "This workflow can be run thanks to the Katib-Argo integration that was setup in the requirements section.\n", + "\n", + "\n", + "The Katib Experiment consists of many components, that we next will setup using custom built helper functions:" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Helper functions to build the individual Katib Experiment Components\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def create_trial_spec(\n", + " pipeline,\n", + " params_list: List[dsl.PipelineParam],\n", + " service_account: Optional[str] = None,\n", + "):\n", + " \"\"\"\n", + " Create an Argo workflow specification from a KFP pipeline function\n", + "\n", + " The Argo worklow CRD will be the basis for the trial_template used\n", + " by Katib.\n", + "\n", + " Args:\n", + " pipeline: a kubeflow pipeline function\n", + " params_list (List[dsl.PipelineParam]): a list of mappings of Kubeflow pipeline parameters\n", + " to Katib trialParameters.\n", + " These need to map the pipeline parameter to the Katib parameter.\n", + " Eg: [dsl.PipelineParam(name='lr', value='${trialParameters.learningRate}')]\n", + " here `lr` is the PipelineParam and `trialParameters.learningRate` the Katib trialParameter.\n", + "\n", + " \"\"\"\n", + " compiler = kfp.compiler.Compiler(\n", + " mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY,\n", + " )\n", + " # Here the pipeline parameters are passed.\n", + " # These will be generated in the Katib trials\n", + " trial_spec = compiler._create_workflow(pipeline, params_list=params_list)\n", + " # Somehow the pipeline is configured with the wrong serviceAccountName by default\n", + " if service_account is not None:\n", + " trial_spec[\"spec\"][\"serviceAccountName\"] = service_account\n", + "\n", + " return trial_spec" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "def create_trial_template(\n", + " trial_spec,\n", + " trial_param_specs: List[V1beta1TrialParameterSpec],\n", + " retain_pods: bool = False,\n", + ") -> V1beta1TrialTemplate:\n", + " \"\"\"Generate a trial template from the spec\n", + "\n", + " This takes the Argo workflow CRD and wrapps it as a\n", + " Katib trial template.\n", + " Here the Katib trial parameters are defined.\n", + "\n", + " Args:\n", + " trial_spec (Argo workflow spec): The workflow/pipeline to tune\n", + " trial_params_spec (List[V1beta1TrialParameterSpec]): The trial parameter specifications\n", + " Note that the `name` of the parameters needs to match the names refered to by the\n", + " create_trial_spec `params_list` arguments.\n", + " The `ref` needs to match the names used in the parameter space defined in `V1beta1ParameterSpec`.\n", + "\n", + " Returns:\n", + " V1beta1TrialTemplate: the trial template\n", + " \"\"\"\n", + "\n", + " trial_template = V1beta1TrialTemplate(\n", + " primary_container_name=\"main\", # Name of the primary container returning the metrics in the workflow\n", + " # The label used for the pipeline component returning the pipeline specs\n", + " primary_pod_labels={\"katib.kubeflow.org/model-training\": \"true\"},\n", + " trial_parameters=trial_param_specs,\n", + " trial_spec=trial_spec,\n", + " success_condition='status.[@this].#(phase==\"Succeeded\")#',\n", + " failure_condition='status.[@this].#(phase==\"Failed\")#',\n", + " retain=retain_pods, # Retain completed pods - left hear for easier debugging\n", + " )\n", + " return trial_template" + ] + }, + { + "cell_type": "code", + "execution_count": 96, + "metadata": {}, + "outputs": [], + "source": [ + "def create_metrics_collector_spec(objective: V1beta1ObjectiveSpec):\n", + " \"\"\"This defines the custom metrics collector\n", + "\n", + " This custom metrics connector was built to collect\n", + " Kubeflow pipeline MLmetrics from a step.\n", + "\n", + " Args:\n", + " objective (V1beta1ObjectiveSpec): the objective spec used to get the metrics names\n", + "\n", + " \"\"\"\n", + "\n", + " metric_names = [objective.objective_metric_name] + list(\n", + " objective.additional_metric_names\n", + " )\n", + " collector = V1beta1MetricsCollectorSpec(\n", + " source={\n", + " \"fileSystemPath\": {\n", + " # In KFP v1 this seems to be the hardcoded location\n", + " # for this output file..\n", + " \"path\": \"/tmp/outputs/mlpipeline_metrics/data\",\n", + " \"kind\": \"File\",\n", + " }\n", + " },\n", + " collector=V1beta1CollectorSpec(\n", + " kind=\"Custom\",\n", + " custom_collector={\n", + " \"args\": [\n", + " \"-m\",\n", + " f\"{';'.join(metric_names)}\",\n", + " \"-s\",\n", + " \"katib-db-manager.kubeflow:6789\",\n", + " \"-t\",\n", + " \"$(PodName)\",\n", + " \"-path\",\n", + " \"/tmp/outputs/mlpipeline_metrics\",\n", + " ],\n", + " \"image\": KATIB_WORKLFLOW_COLLECTOR_IMAGE,\n", + " \"imagePullPolicy\": \"Always\",\n", + " \"name\": \"custom-metrics-logger-and-collector\",\n", + " \"env\": [\n", + " {\n", + " # In this setup the PodName can be used to\n", + " # infer the `trial name` required to report back\n", + " # the metrics.\n", + " \"name\": \"PodName\",\n", + " \"valueFrom\": {\"fieldRef\": {\"fieldPath\": \"metadata.name\"}},\n", + " }\n", + " ],\n", + " },\n", + " ),\n", + " )\n", + " return collector" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Final helper function to create experiments from pipelines\n", + "\n", + "\n", + "This helper function is the main entry point to train pipelines." + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "def create_katib_experiment_spec(\n", + " pipeline: dsl.Pipeline,\n", + " pipeline_params: List[dsl.PipelineParam],\n", + " trial_params: List[V1beta1TrialParameterSpec],\n", + " trial_params_space: List[V1beta1ParameterSpec],\n", + " objective: V1beta1ObjectiveSpec,\n", + " algorithm: V1beta1AlgorithmSpec,\n", + " max_trial_count: int = 2,\n", + " max_failed_trial_count: int = 2,\n", + " parallel_trial_count: int = 2,\n", + " pipeline_service_account: Optional[str] = None,\n", + " retain_pods: bool = False,\n", + ") -> V1beta1ExperimentSpec:\n", + " \"\"\"Construct a Katib experiment over a KFP pipeline\n", + "\n", + " Args:\n", + " pipeline (dsl.Pipeline): The Kubeflow Pipeline\n", + " pipeline_params (List[dsl.PipelineParam]): A mapping of trial-parameters to pipeline parameters.\n", + " Example: [\n", + " dsl.PipelineParam(name=\"lr\", value=\"${trialParameters.learningRate}\"),\n", + " ...\n", + " ]\n", + " trial_params (List[V1beta1TrialParameterSpec]): Spec for Trial parameters. Note that name\n", + " and refs need to match the ones used in `pipeline_params` and `trial_params_space`\n", + " Example: [\n", + " V1beta1TrialParameterSpec(\n", + " name=\"learningRate\",\n", + " description=\"Learning rate for the training model\",\n", + " reference=\"learning_rate\",\n", + " ), ...]\n", + " trial_params_space (List[V1beta1ParameterSpec]): The spec for the parameter space explored in the\n", + " Trials\n", + " Example: [\n", + " V1beta1ParameterSpec(\n", + " name=\"learning_rate\",\n", + " parameter_type=\"double\",\n", + " feasible_space=V1beta1FeasibleSpace(min=\"0.00001\", max=\"0.001\"),\n", + " ), ...]\n", + " objective (V1beta1ObjectiveSpec): objective spec. The names used here\n", + " need to match the metrics reported by the pipeline.\n", + " Example: V1beta1ObjectiveSpec(\n", + " type=\"maximize\",\n", + " goal=0.9,\n", + " objective_metric_name=\"val-accuracy\",\n", + " additional_metric_names=[\"accuracy\"],\n", + " )\n", + " algorithm (V1beta1AlgorithmSpec): algorithm spec\n", + " Example: V1beta1AlgorithmSpec(\n", + " algorithm_name=\"random\",\n", + " )\n", + " max_trial_count (int, optional): Max total number of trials. Defaults to 2.\n", + " max_failed_trial_count (int, optional): Number of failed trials tolerated. Defaults to 2.\n", + " parallel_trial_count (int, optional): Number of trials run in parallel. Defaults to 2.\n", + " pipeline_service_account (str | None, optional): Name of the service account to run\n", + " pipelines with. Defaults to None (uses pre-configured default).\n", + " On a Kubeflow GCP deployment this should be set to `default-editor`\n", + " retain_pods (bool): retain pods (good for debugging). Default: false\n", + "\n", + " Returns:\n", + " V1beta1ExperimentSpec: Katib experiment spec\n", + " \"\"\"\n", + "\n", + " trial_spec = create_trial_spec(\n", + " pipeline, pipeline_params, service_account=pipeline_service_account\n", + " )\n", + "\n", + " # Configure parameters for the Trial template.\n", + " trial_template = create_trial_template(\n", + " trial_spec, trial_params, retain_pods=retain_pods\n", + " )\n", + "\n", + " # Metrics collector spec\n", + " metrics_collector = create_metrics_collector_spec(objective=objective)\n", + "\n", + " # Create an Experiment from the above parameters.\n", + " experiment_spec = V1beta1ExperimentSpec(\n", + " # Experimental Budget\n", + " max_trial_count=max_trial_count,\n", + " max_failed_trial_count=max_failed_trial_count,\n", + " parallel_trial_count=parallel_trial_count,\n", + " # Optimization Objective\n", + " objective=objective,\n", + " # Optimization Algorithm\n", + " algorithm=algorithm,\n", + " # Optimization Parameters\n", + " parameters=trial_params_space,\n", + " # Trial Template\n", + " trial_template=trial_template,\n", + " # Metrics collector\n", + " metrics_collector_spec=metrics_collector,\n", + " )\n", + "\n", + " return experiment_spec" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tune the MNIST pipeline using Katib\n", + "\n", + "First prepare all required input" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [], + "source": [ + "pipeline_params = [\n", + " dsl.PipelineParam(name=\"lr\", value=\"${trialParameters.learningRate}\"),\n", + " dsl.PipelineParam(name=\"batch_size\", value=\"${trialParameters.batchSize}\"),\n", + " dsl.PipelineParam(name=\"histogram_norm\", value=\"${trialParameters.histogramNorm}\"),\n", + "]\n", + "trial_params_specs = [\n", + " V1beta1TrialParameterSpec(\n", + " name=\"learningRate\", # the parameter name that is replaced in your template (see Trial Specification).\n", + " description=\"Learning rate for the training model\",\n", + " reference=\"learning_rate\", # the parameter name that experiment’s suggestion returns (parameter name in the Parameters Specification).\n", + " ),\n", + " V1beta1TrialParameterSpec(\n", + " name=\"batchSize\",\n", + " description=\"Batch size for NN training\",\n", + " reference=\"batch_size\",\n", + " ),\n", + " V1beta1TrialParameterSpec(\n", + " name=\"histogramNorm\",\n", + " description=\"Histogram normalization of image on?\",\n", + " reference=\"histogram_norm\",\n", + " ),\n", + "]\n", + "parameter_space = [\n", + " V1beta1ParameterSpec(\n", + " name=\"learning_rate\",\n", + " parameter_type=\"double\",\n", + " feasible_space=V1beta1FeasibleSpace(min=\"0.00001\", max=\"0.001\"),\n", + " ),\n", + " V1beta1ParameterSpec(\n", + " name=\"batch_size\",\n", + " parameter_type=\"int\",\n", + " feasible_space=V1beta1FeasibleSpace(min=\"16\", max=\"64\"),\n", + " ),\n", + " V1beta1ParameterSpec(\n", + " name=\"histogram_norm\",\n", + " parameter_type=\"discrete\",\n", + " feasible_space=V1beta1FeasibleSpace(list=[\"0\", \"1\"]),\n", + " ),\n", + "]\n", + "objective = V1beta1ObjectiveSpec(\n", + " type=\"maximize\",\n", + " goal=0.9,\n", + " objective_metric_name=\"val-accuracy\",\n", + " additional_metric_names=[\"accuracy\"],\n", + ")\n", + "\n", + "algorithm = V1beta1AlgorithmSpec(\n", + " algorithm_name=\"random\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 91, + "metadata": {}, + "outputs": [], + "source": [ + "# Prepare the full spec\n", + "\n", + "katib_spec = create_katib_experiment_spec(\n", + " pipeline=mnist_training_pipeline,\n", + " pipeline_params=pipeline_params,\n", + " trial_params=trial_params_specs,\n", + " trial_params_space=parameter_space,\n", + " objective=objective,\n", + " algorithm=algorithm,\n", + " pipeline_service_account=KFP_SERVICE_ACCOUNT,\n", + " max_trial_count=5,\n", + " parallel_trial_count=5,\n", + " retain_pods=False,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In order to generate a full experiment the api_version, kind and namespace need to be defined:" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "ename": "NameError", + "evalue": "name 'katib_spec' is not defined", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[17], line 11\u001b[0m\n\u001b[1;32m 1\u001b[0m katib_experiment_name \u001b[39m=\u001b[39m (\n\u001b[1;32m 2\u001b[0m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00mKATIB_EXPERIMENT\u001b[39m}\u001b[39;00m\u001b[39m-\u001b[39m\u001b[39m{\u001b[39;00mdt\u001b[39m.\u001b[39mtoday()\u001b[39m.\u001b[39mstrftime(\u001b[39m'\u001b[39m\u001b[39m%\u001b[39m\u001b[39mY-\u001b[39m\u001b[39m%\u001b[39m\u001b[39mm-\u001b[39m\u001b[39m%d\u001b[39;00m\u001b[39m-\u001b[39m\u001b[39m%\u001b[39m\u001b[39mHh-\u001b[39m\u001b[39m%\u001b[39m\u001b[39mMm-\u001b[39m\u001b[39m%\u001b[39m\u001b[39mSs\u001b[39m\u001b[39m'\u001b[39m)\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 3\u001b[0m )\n\u001b[1;32m 4\u001b[0m katib_experiment \u001b[39m=\u001b[39m V1beta1Experiment(\n\u001b[1;32m 5\u001b[0m api_version\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mkubeflow.org/v1beta1\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m 6\u001b[0m kind\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mExperiment\u001b[39m\u001b[39m\"\u001b[39m,\n\u001b[1;32m 7\u001b[0m metadata\u001b[39m=\u001b[39mV1ObjectMeta(\n\u001b[1;32m 8\u001b[0m name\u001b[39m=\u001b[39mkatib_experiment_name,\n\u001b[1;32m 9\u001b[0m namespace\u001b[39m=\u001b[39mUSER_NAMESPACE,\n\u001b[1;32m 10\u001b[0m ),\n\u001b[0;32m---> 11\u001b[0m spec\u001b[39m=\u001b[39mkatib_spec,\n\u001b[1;32m 12\u001b[0m )\n", + "\u001b[0;31mNameError\u001b[0m: name 'katib_spec' is not defined" + ] + } + ], + "source": [ + "katib_experiment_name = (\n", + " f\"{KATIB_EXPERIMENT}-{dt.today().strftime('%Y-%m-%d-%Hh-%Mm-%Ss')}\"\n", + ")\n", + "katib_experiment = V1beta1Experiment(\n", + " api_version=\"kubeflow.org/v1beta1\",\n", + " kind=\"Experiment\",\n", + " metadata=V1ObjectMeta(\n", + " name=katib_experiment_name,\n", + " namespace=USER_NAMESPACE,\n", + " ),\n", + " spec=katib_spec,\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The generated yaml can written out to submit via the web ui:" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "with open(f\"{KATIB_EXPERIMENT}.yaml\", \"w\") as f:\n", + " yaml.dump(ApiClient().sanitize_for_serialization(katib_experiment), f)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Or sumitted via the KatibClient:" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [], + "source": [ + "katib_client = KatibClient()" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'apiVersion': 'kubeflow.org/v1beta1',\n", + " 'kind': 'Experiment',\n", + " 'metadata': {'creationTimestamp': '2023-07-20T19:40:11Z',\n", + " 'generation': 1,\n", + " 'managedFields': [{'apiVersion': 'kubeflow.org/v1beta1',\n", + " 'fieldsType': 'FieldsV1',\n", + " 'fieldsV1': {'f:spec': {'.': {},\n", + " 'f:algorithm': {'.': {}, 'f:algorithmName': {}},\n", + " 'f:maxFailedTrialCount': {},\n", + " 'f:maxTrialCount': {},\n", + " 'f:metricsCollectorSpec': {'.': {},\n", + " 'f:collector': {'.': {},\n", + " 'f:customCollector': {'.': {},\n", + " 'f:args': {},\n", + " 'f:env': {},\n", + " 'f:image': {},\n", + " 'f:imagePullPolicy': {},\n", + " 'f:name': {}},\n", + " 'f:kind': {}},\n", + " 'f:source': {'.': {},\n", + " 'f:fileSystemPath': {'.': {}, 'f:kind': {}, 'f:path': {}}}},\n", + " 'f:objective': {'.': {},\n", + " 'f:additionalMetricNames': {},\n", + " 'f:goal': {},\n", + " 'f:objectiveMetricName': {},\n", + " 'f:type': {}},\n", + " 'f:parallelTrialCount': {},\n", + " 'f:parameters': {},\n", + " 'f:trialTemplate': {'.': {},\n", + " 'f:failureCondition': {},\n", + " 'f:primaryContainerName': {},\n", + " 'f:primaryPodLabels': {'.': {},\n", + " 'f:katib.kubeflow.org/model-training': {}},\n", + " 'f:retain': {},\n", + " 'f:successCondition': {},\n", + " 'f:trialParameters': {},\n", + " 'f:trialSpec': {'.': {},\n", + " 'f:apiVersion': {},\n", + " 'f:kind': {},\n", + " 'f:metadata': {'.': {},\n", + " 'f:annotations': {'.': {},\n", + " 'f:pipelines.kubeflow.org/kfp_sdk_version': {},\n", + " 'f:pipelines.kubeflow.org/pipeline_compilation_time': {},\n", + " 'f:pipelines.kubeflow.org/pipeline_spec': {}},\n", + " 'f:generateName': {},\n", + " 'f:labels': {'.': {},\n", + " 'f:pipelines.kubeflow.org/kfp_sdk_version': {}}},\n", + " 'f:spec': {'.': {},\n", + " 'f:arguments': {'.': {}, 'f:parameters': {}},\n", + " 'f:entrypoint': {},\n", + " 'f:serviceAccountName': {},\n", + " 'f:templates': {}}}}}},\n", + " 'manager': 'OpenAPI-Generator',\n", + " 'operation': 'Update',\n", + " 'time': '2023-07-20T19:40:11Z'}],\n", + " 'name': 'katib-kfp-example-v1-2023-07-20-21h-40m-05s',\n", + " 'namespace': 'kubeflow',\n", + " 'resourceVersion': '6526',\n", + " 'uid': '68d7df06-e02d-4d1e-932c-c3032f7ecaff'},\n", + " 'spec': {'algorithm': {'algorithmName': 'random'},\n", + " 'maxFailedTrialCount': 2,\n", + " 'maxTrialCount': 5,\n", + " 'metricsCollectorSpec': {'collector': {'customCollector': {'args': ['-m',\n", + " 'val-accuracy;accuracy',\n", + " '-s',\n", + " 'katib-db-manager.kubeflow:6789',\n", + " '-t',\n", + " '$(PodName)',\n", + " '-path',\n", + " '/tmp/outputs/mlpipeline_metrics'],\n", + " 'env': [{'name': 'PodName',\n", + " 'valueFrom': {'fieldRef': {'fieldPath': 'metadata.name'}}}],\n", + " 'image': 'docker.io/kubeflowkatib/kfpv1-metrics-collector:latest',\n", + " 'imagePullPolicy': 'Always',\n", + " 'name': 'custom-metrics-logger-and-collector',\n", + " 'resources': {}},\n", + " 'kind': 'Custom'},\n", + " 'source': {'fileSystemPath': {'kind': 'File',\n", + " 'path': '/tmp/outputs/mlpipeline_metrics/data'}}},\n", + " 'objective': {'additionalMetricNames': ['accuracy'],\n", + " 'goal': 0.9,\n", + " 'metricStrategies': [{'name': 'val-accuracy', 'value': 'max'},\n", + " {'name': 'accuracy', 'value': 'max'}],\n", + " 'objectiveMetricName': 'val-accuracy',\n", + " 'type': 'maximize'},\n", + " 'parallelTrialCount': 5,\n", + " 'parameters': [{'feasibleSpace': {'max': '0.001', 'min': '0.00001'},\n", + " 'name': 'learning_rate',\n", + " 'parameterType': 'double'},\n", + " {'feasibleSpace': {'max': '64', 'min': '16'},\n", + " 'name': 'batch_size',\n", + " 'parameterType': 'int'},\n", + " {'feasibleSpace': {'list': ['0', '1']},\n", + " 'name': 'histogram_norm',\n", + " 'parameterType': 'discrete'}],\n", + " 'resumePolicy': 'Never',\n", + " 'trialTemplate': {'failureCondition': 'status.[@this].#(phase==\"Failed\")#',\n", + " 'primaryContainerName': 'main',\n", + " 'primaryPodLabels': {'katib.kubeflow.org/model-training': 'true'},\n", + " 'successCondition': 'status.[@this].#(phase==\"Succeeded\")#',\n", + " 'trialParameters': [{'description': 'Learning rate for the training model',\n", + " 'name': 'learningRate',\n", + " 'reference': 'learning_rate'},\n", + " {'description': 'Batch size for NN training',\n", + " 'name': 'batchSize',\n", + " 'reference': 'batch_size'},\n", + " {'description': 'Histogram normalization of image on?',\n", + " 'name': 'histogramNorm',\n", + " 'reference': 'histogram_norm'}],\n", + " 'trialSpec': {'apiVersion': 'argoproj.io/v1alpha1',\n", + " 'kind': 'Workflow',\n", + " 'metadata': {'annotations': {'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline_compilation_time': '2023-07-20T21:40:03.664402',\n", + " 'pipelines.kubeflow.org/pipeline_spec': '{\"description\": \"A pipeline to download the MNIST dataset files\", \"inputs\": [{\"default\": \"0.0001\", \"name\": \"lr\", \"optional\": true, \"type\": \"Float\"}, {\"default\": \"Adam\", \"name\": \"optimizer\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"categorical_crossentropy\", \"name\": \"loss\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"3\", \"name\": \"epochs\", \"optional\": true, \"type\": \"Integer\"}, {\"default\": \"5\", \"name\": \"batch_size\", \"optional\": true, \"type\": \"Integer\"}, {\"default\": \"False\", \"name\": \"histogram_norm\", \"optional\": true, \"type\": \"Boolean\"}, {\"default\": \"${trialParameters.learningRate}\", \"name\": \"lr\"}, {\"default\": \"${trialParameters.batchSize}\", \"name\": \"batch_size\"}, {\"default\": \"${trialParameters.histogramNorm}\", \"name\": \"histogram_norm\"}], \"name\": \"Download MNIST dataset\"}'},\n", + " 'generateName': 'download-mnist-dataset-',\n", + " 'labels': {'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12'}},\n", + " 'spec': {'arguments': {'parameters': [{'name': 'lr',\n", + " 'value': '${trialParameters.learningRate}'},\n", + " {'name': 'optimizer', 'value': 'Adam'},\n", + " {'name': 'loss', 'value': 'categorical_crossentropy'},\n", + " {'name': 'epochs', 'value': '3'},\n", + " {'name': 'batch_size', 'value': '${trialParameters.batchSize}'},\n", + " {'name': 'histogram_norm',\n", + " 'value': '${trialParameters.histogramNorm}'}]},\n", + " 'entrypoint': 'download-mnist-dataset',\n", + " 'serviceAccountName': 'pipeline-runner',\n", + " 'templates': [{'container': {'args': [],\n", + " 'command': ['sh',\n", + " '-exc',\n", + " 'url=\"$0\"\\noutput_path=\"$1\"\\ncurl_options=\"$2\"\\n\\nmkdir -p \"$(dirname \"$output_path\")\"\\ncurl --get \"$url\" --output \"$output_path\" $curl_options\\n',\n", + " 'http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz',\n", + " '/tmp/outputs/Data/data',\n", + " '--location'],\n", + " 'image': 'byrnedo/alpine-curl@sha256:548379d0a4a0c08b9e55d9d87a592b7d35d9ab3037f4936f5ccd09d0b625a342'},\n", + " 'metadata': {'annotations': {'author': 'Alexey Volkov ',\n", + " 'canonical_location': 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/web/Download/component.yaml',\n", + " 'pipelines.kubeflow.org/arguments.parameters': '{\"Url\": \"http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\", \"curl options\": \"--location\"}',\n", + " 'pipelines.kubeflow.org/component_ref': '{\"digest\": \"2f61f2edf713f214934bd286791877a1a3a37f31a4de4368b90e3b76743f1523\", \"url\": \"https://raw.githubusercontent.com/kubeflow/pipelines/master/components/contrib/web/Download/component.yaml\"}',\n", + " 'pipelines.kubeflow.org/component_spec': '{\"implementation\": {\"container\": {\"command\": [\"sh\", \"-exc\", \"url=\\\\\"$0\\\\\"\\\\noutput_path=\\\\\"$1\\\\\"\\\\ncurl_options=\\\\\"$2\\\\\"\\\\n\\\\nmkdir -p \\\\\"$(dirname \\\\\"$output_path\\\\\")\\\\\"\\\\ncurl --get \\\\\"$url\\\\\" --output \\\\\"$output_path\\\\\" $curl_options\\\\n\", {\"inputValue\": \"Url\"}, {\"outputPath\": \"Data\"}, {\"inputValue\": \"curl options\"}], \"image\": \"byrnedo/alpine-curl@sha256:548379d0a4a0c08b9e55d9d87a592b7d35d9ab3037f4936f5ccd09d0b625a342\"}}, \"inputs\": [{\"name\": \"Url\", \"type\": \"URI\"}, {\"default\": \"--location\", \"description\": \"Additional options given to the curl bprogram. See https://curl.haxx.se/docs/manpage.html\", \"name\": \"curl options\", \"type\": \"string\"}], \"metadata\": {\"annotations\": {\"author\": \"Alexey Volkov \", \"canonical_location\": \"https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/web/Download/component.yaml\"}}, \"name\": \"Download data\", \"outputs\": [{\"name\": \"Data\"}]}',\n", + " 'pipelines.kubeflow.org/task_display_name': 'Download training images'},\n", + " 'labels': {'pipelines.kubeflow.org/cache_enabled': 'true',\n", + " 'pipelines.kubeflow.org/enable_caching': 'true',\n", + " 'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}},\n", + " 'name': 'download-data',\n", + " 'outputs': {'artifacts': [{'name': 'download-data-Data',\n", + " 'path': '/tmp/outputs/Data/data'}]}},\n", + " {'container': {'args': [],\n", + " 'command': ['sh',\n", + " '-exc',\n", + " 'url=\"$0\"\\noutput_path=\"$1\"\\ncurl_options=\"$2\"\\n\\nmkdir -p \"$(dirname \"$output_path\")\"\\ncurl --get \"$url\" --output \"$output_path\" $curl_options\\n',\n", + " 'http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz',\n", + " '/tmp/outputs/Data/data',\n", + " '--location'],\n", + " 'image': 'byrnedo/alpine-curl@sha256:548379d0a4a0c08b9e55d9d87a592b7d35d9ab3037f4936f5ccd09d0b625a342'},\n", + " 'metadata': {'annotations': {'author': 'Alexey Volkov ',\n", + " 'canonical_location': 'https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/web/Download/component.yaml',\n", + " 'pipelines.kubeflow.org/arguments.parameters': '{\"Url\": \"http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\", \"curl options\": \"--location\"}',\n", + " 'pipelines.kubeflow.org/component_ref': '{\"digest\": \"2f61f2edf713f214934bd286791877a1a3a37f31a4de4368b90e3b76743f1523\", \"url\": \"https://raw.githubusercontent.com/kubeflow/pipelines/master/components/contrib/web/Download/component.yaml\"}',\n", + " 'pipelines.kubeflow.org/component_spec': '{\"implementation\": {\"container\": {\"command\": [\"sh\", \"-exc\", \"url=\\\\\"$0\\\\\"\\\\noutput_path=\\\\\"$1\\\\\"\\\\ncurl_options=\\\\\"$2\\\\\"\\\\n\\\\nmkdir -p \\\\\"$(dirname \\\\\"$output_path\\\\\")\\\\\"\\\\ncurl --get \\\\\"$url\\\\\" --output \\\\\"$output_path\\\\\" $curl_options\\\\n\", {\"inputValue\": \"Url\"}, {\"outputPath\": \"Data\"}, {\"inputValue\": \"curl options\"}], \"image\": \"byrnedo/alpine-curl@sha256:548379d0a4a0c08b9e55d9d87a592b7d35d9ab3037f4936f5ccd09d0b625a342\"}}, \"inputs\": [{\"name\": \"Url\", \"type\": \"URI\"}, {\"default\": \"--location\", \"description\": \"Additional options given to the curl bprogram. See https://curl.haxx.se/docs/manpage.html\", \"name\": \"curl options\", \"type\": \"string\"}], \"metadata\": {\"annotations\": {\"author\": \"Alexey Volkov \", \"canonical_location\": \"https://raw.githubusercontent.com/Ark-kun/pipeline_components/master/components/web/Download/component.yaml\"}}, \"name\": \"Download data\", \"outputs\": [{\"name\": \"Data\"}]}',\n", + " 'pipelines.kubeflow.org/task_display_name': 'Download training labels'},\n", + " 'labels': {'pipelines.kubeflow.org/cache_enabled': 'true',\n", + " 'pipelines.kubeflow.org/enable_caching': 'true',\n", + " 'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}},\n", + " 'name': 'download-data-2',\n", + " 'outputs': {'artifacts': [{'name': 'download-data-2-Data',\n", + " 'path': '/tmp/outputs/Data/data'}]}},\n", + " {'dag': {'tasks': [{'name': 'download-data',\n", + " 'template': 'download-data'},\n", + " {'name': 'download-data-2', 'template': 'download-data-2'},\n", + " {'arguments': {'artifacts': [{'from': '{{tasks.download-data-2.outputs.artifacts.download-data-2-Data}}',\n", + " 'name': 'download-data-2-Data'},\n", + " {'from': '{{tasks.download-data.outputs.artifacts.download-data-Data}}',\n", + " 'name': 'download-data-Data'}]},\n", + " 'dependencies': ['download-data', 'download-data-2'],\n", + " 'name': 'parse-mnist',\n", + " 'template': 'parse-mnist'},\n", + " {'arguments': {'artifacts': [{'from': '{{tasks.parse-mnist.outputs.artifacts.parse-mnist-Dataset}}',\n", + " 'name': 'parse-mnist-Dataset'}],\n", + " 'parameters': [{'name': 'histogram_norm',\n", + " 'value': '{{inputs.parameters.histogram_norm}}'}]},\n", + " 'dependencies': ['parse-mnist'],\n", + " 'name': 'process',\n", + " 'template': 'process'},\n", + " {'arguments': {'artifacts': [{'from': '{{tasks.process.outputs.artifacts.process-data_processed}}',\n", + " 'name': 'process-data_processed'}],\n", + " 'parameters': [{'name': 'batch_size',\n", + " 'value': '{{inputs.parameters.batch_size}}'},\n", + " {'name': 'epochs', 'value': '{{inputs.parameters.epochs}}'},\n", + " {'name': 'loss', 'value': '{{inputs.parameters.loss}}'},\n", + " {'name': 'lr', 'value': '{{inputs.parameters.lr}}'},\n", + " {'name': 'optimizer',\n", + " 'value': '{{inputs.parameters.optimizer}}'}]},\n", + " 'dependencies': ['process'],\n", + " 'name': 'train',\n", + " 'template': 'train'}]},\n", + " 'inputs': {'parameters': [{'name': 'batch_size'},\n", + " {'name': 'epochs'},\n", + " {'name': 'histogram_norm'},\n", + " {'name': 'loss'},\n", + " {'name': 'lr'},\n", + " {'name': 'optimizer'}]},\n", + " 'name': 'download-mnist-dataset'},\n", + " {'container': {'args': [],\n", + " 'command': ['sh',\n", + " '-ec',\n", + " '# This is how additional packages can be installed dynamically\\npython3 -m pip install pip idx2numpy\\n# Run the rest of the command after installing the packages.\\n\"$0\" \"$@\"\\n',\n", + " 'python3',\n", + " '-u',\n", + " '-c',\n", + " \"import gzip\\nimport idx2numpy\\nimport sys\\nfrom pathlib import Path\\nimport pickle\\nimport tensorflow as tf\\nimg_path = sys.argv[1]\\nlabel_path = sys.argv[2]\\noutput_path = sys.argv[3]\\nwith gzip.open(img_path, 'rb') as f:\\n x = idx2numpy.convert_from_string(f.read())\\nwith gzip.open(label_path, 'rb') as f:\\n y = idx2numpy.convert_from_string(f.read())\\n#one-hot encode the categories\\nx_out = tf.convert_to_tensor(x)\\ny_out = tf.keras.utils.to_categorical(y)\\nPath(output_path).parent.mkdir(parents=True, exist_ok=True)\\nwith open(output_path, 'wb') as output_file:\\n pickle.dump((x_out, y_out), output_file)\\n\",\n", + " '/tmp/inputs/Images/data',\n", + " '/tmp/inputs/Labels/data',\n", + " '/tmp/outputs/Dataset/data'],\n", + " 'image': 'tensorflow/tensorflow:2.7.1'},\n", + " 'inputs': {'artifacts': [{'name': 'download-data-Data',\n", + " 'path': '/tmp/inputs/Images/data'},\n", + " {'name': 'download-data-2-Data', 'path': '/tmp/inputs/Labels/data'}]},\n", + " 'metadata': {'annotations': {'author': 'Vito Zanotelli, D-ONE.ai',\n", + " 'description': 'Based on https://github.com/kubeflow/pipelines/blob/master/components/contrib/sample/Python_script/component.yaml',\n", + " 'pipelines.kubeflow.org/component_ref': '{\"digest\": \"80825e6ec527562f31b6fdba1bae9a42dae5032c8654f4b9d39cb97a3dc4ed23\"}',\n", + " 'pipelines.kubeflow.org/component_spec': '{\"implementation\": {\"container\": {\"command\": [\"sh\", \"-ec\", \"# This is how additional packages can be installed dynamically\\\\npython3 -m pip install pip idx2numpy\\\\n# Run the rest of the command after installing the packages.\\\\n\\\\\"$0\\\\\" \\\\\"$@\\\\\"\\\\n\", \"python3\", \"-u\", \"-c\", \"import gzip\\\\nimport idx2numpy\\\\nimport sys\\\\nfrom pathlib import Path\\\\nimport pickle\\\\nimport tensorflow as tf\\\\nimg_path = sys.argv[1]\\\\nlabel_path = sys.argv[2]\\\\noutput_path = sys.argv[3]\\\\nwith gzip.open(img_path, \\'rb\\') as f:\\\\n x = idx2numpy.convert_from_string(f.read())\\\\nwith gzip.open(label_path, \\'rb\\') as f:\\\\n y = idx2numpy.convert_from_string(f.read())\\\\n#one-hot encode the categories\\\\nx_out = tf.convert_to_tensor(x)\\\\ny_out = tf.keras.utils.to_categorical(y)\\\\nPath(output_path).parent.mkdir(parents=True, exist_ok=True)\\\\nwith open(output_path, \\'wb\\') as output_file:\\\\n pickle.dump((x_out, y_out), output_file)\\\\n\", {\"inputPath\": \"Images\"}, {\"inputPath\": \"Labels\"}, {\"outputPath\": \"Dataset\"}], \"image\": \"tensorflow/tensorflow:2.7.1\"}}, \"inputs\": [{\"description\": \"gziped images in the idx format\", \"name\": \"Images\"}, {\"description\": \"gziped labels in the idx format\", \"name\": \"Labels\"}], \"metadata\": {\"annotations\": {\"author\": \"Vito Zanotelli, D-ONE.ai\", \"description\": \"Based on https://github.com/kubeflow/pipelines/blob/master/components/contrib/sample/Python_script/component.yaml\"}}, \"name\": \"Parse MNIST\", \"outputs\": [{\"name\": \"Dataset\"}]}',\n", + " 'pipelines.kubeflow.org/task_display_name': 'Prepare train dataset'},\n", + " 'labels': {'pipelines.kubeflow.org/cache_enabled': 'true',\n", + " 'pipelines.kubeflow.org/enable_caching': 'true',\n", + " 'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}},\n", + " 'name': 'parse-mnist',\n", + " 'outputs': {'artifacts': [{'name': 'parse-mnist-Dataset',\n", + " 'path': '/tmp/outputs/Dataset/data'}]}},\n", + " {'container': {'args': ['--data-raw',\n", + " '/tmp/inputs/data_raw/data',\n", + " '--val-pct',\n", + " '0.2',\n", + " '--trainset-flag',\n", + " 'True',\n", + " '--histogram-norm',\n", + " '{{inputs.parameters.histogram_norm}}',\n", + " '--data-processed',\n", + " '/tmp/outputs/data_processed/data'],\n", + " 'command': ['sh',\n", + " '-c',\n", + " '(PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scikit-learn\\' \\'tensorflow-addons[tensorflow]\\' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scikit-learn\\' \\'tensorflow-addons[tensorflow]\\' --user) && \"$0\" \"$@\"',\n", + " 'sh',\n", + " '-ec',\n", + " 'program_path=$(mktemp)\\nprintf \"%s\" \"$0\" > \"$program_path\"\\npython3 -u \"$program_path\" \"$@\"\\n',\n", + " 'def _make_parent_dirs_and_return_path(file_path: str):\\n import os\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\n return file_path\\n\\ndef process(\\n data_raw_path, # type: ignore\\n data_processed_path, # type: ignore\\n val_pct = 0.2,\\n trainset_flag = True,\\n histogram_norm = False,\\n):\\n \"\"\"\\n Here we do all the preprocessing\\n if the data path is for training data we:\\n (1) Normalize the data\\n (2) split the train and val data\\n If it is for unseen test data, we:\\n (1) Normalize the data\\n This function returns in any case the processed data path\\n \"\"\"\\n # sklearn\\n import pickle\\n from sklearn.model_selection import train_test_split\\n import tensorflow as tf\\n import tensorflow_addons as tfa\\n\\n def img_norm(x):\\n x_ = tf.reshape(x, list(x.shape) + [1])\\n\\n if histogram_norm:\\n x_ = tfa.image.equalize(x_)\\n\\n # Scale between 0-1\\n x_ = x_ / 255\\n return x_\\n\\n with open(data_raw_path, \"rb\") as f:\\n x, y = pickle.load(f)\\n if trainset_flag:\\n\\n x_ = img_norm(x)\\n x_train, x_val, y_train, y_val = train_test_split(\\n x_.numpy(), y, test_size=val_pct, stratify=y, random_state=42\\n )\\n\\n with open(data_processed_path, \"wb\") as output_file:\\n pickle.dump((x_train, y_train, x_val, y_val), output_file)\\n\\n else:\\n x_ = img_norm(x)\\n with open(data_processed_path, \"wb\") as output_file:\\n pickle.dump((x_, y), output_file)\\n\\ndef _deserialize_bool(s) -> bool:\\n from distutils.util import strtobool\\n return strtobool(s) == 1\\n\\nimport argparse\\n_parser = argparse.ArgumentParser(prog=\\'Process\\', description=\\'Here we do all the preprocessing\\')\\n_parser.add_argument(\"--data-raw\", dest=\"data_raw_path\", type=str, required=True, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--val-pct\", dest=\"val_pct\", type=float, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--trainset-flag\", dest=\"trainset_flag\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--histogram-norm\", dest=\"histogram_norm\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--data-processed\", dest=\"data_processed_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\n_parsed_args = vars(_parser.parse_args())\\n\\n_outputs = process(**_parsed_args)\\n'],\n", + " 'image': 'tensorflow/tensorflow:2.7.1',\n", + " 'resources': {'limits': {'cpu': '1', 'memory': '2Gi'}}},\n", + " 'inputs': {'artifacts': [{'name': 'parse-mnist-Dataset',\n", + " 'path': '/tmp/inputs/data_raw/data'}],\n", + " 'parameters': [{'name': 'histogram_norm'}]},\n", + " 'metadata': {'annotations': {'pipelines.kubeflow.org/arguments.parameters': '{\"histogram_norm\": \"{{inputs.parameters.histogram_norm}}\", \"trainset_flag\": \"True\", \"val_pct\": \"0.2\"}',\n", + " 'pipelines.kubeflow.org/component_ref': '{}',\n", + " 'pipelines.kubeflow.org/component_spec': '{\"description\": \"Here we do all the preprocessing\", \"implementation\": {\"container\": {\"args\": [\"--data-raw\", {\"inputPath\": \"data_raw\"}, {\"if\": {\"cond\": {\"isPresent\": \"val_pct\"}, \"then\": [\"--val-pct\", {\"inputValue\": \"val_pct\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"trainset_flag\"}, \"then\": [\"--trainset-flag\", {\"inputValue\": \"trainset_flag\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"histogram_norm\"}, \"then\": [\"--histogram-norm\", {\"inputValue\": \"histogram_norm\"}]}}, \"--data-processed\", {\"outputPath\": \"data_processed\"}], \"command\": [\"sh\", \"-c\", \"(PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scikit-learn\\' \\'tensorflow-addons[tensorflow]\\' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scikit-learn\\' \\'tensorflow-addons[tensorflow]\\' --user) && \\\\\"$0\\\\\" \\\\\"$@\\\\\"\", \"sh\", \"-ec\", \"program_path=$(mktemp)\\\\nprintf \\\\\"%s\\\\\" \\\\\"$0\\\\\" > \\\\\"$program_path\\\\\"\\\\npython3 -u \\\\\"$program_path\\\\\" \\\\\"$@\\\\\"\\\\n\", \"def _make_parent_dirs_and_return_path(file_path: str):\\\\n import os\\\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\\\n return file_path\\\\n\\\\ndef process(\\\\n data_raw_path, # type: ignore\\\\n data_processed_path, # type: ignore\\\\n val_pct = 0.2,\\\\n trainset_flag = True,\\\\n histogram_norm = False,\\\\n):\\\\n \\\\\"\\\\\"\\\\\"\\\\n Here we do all the preprocessing\\\\n if the data path is for training data we:\\\\n (1) Normalize the data\\\\n (2) split the train and val data\\\\n If it is for unseen test data, we:\\\\n (1) Normalize the data\\\\n This function returns in any case the processed data path\\\\n \\\\\"\\\\\"\\\\\"\\\\n # sklearn\\\\n import pickle\\\\n from sklearn.model_selection import train_test_split\\\\n import tensorflow as tf\\\\n import tensorflow_addons as tfa\\\\n\\\\n def img_norm(x):\\\\n x_ = tf.reshape(x, list(x.shape) + [1])\\\\n\\\\n if histogram_norm:\\\\n x_ = tfa.image.equalize(x_)\\\\n\\\\n # Scale between 0-1\\\\n x_ = x_ / 255\\\\n return x_\\\\n\\\\n with open(data_raw_path, \\\\\"rb\\\\\") as f:\\\\n x, y = pickle.load(f)\\\\n if trainset_flag:\\\\n\\\\n x_ = img_norm(x)\\\\n x_train, x_val, y_train, y_val = train_test_split(\\\\n x_.numpy(), y, test_size=val_pct, stratify=y, random_state=42\\\\n )\\\\n\\\\n with open(data_processed_path, \\\\\"wb\\\\\") as output_file:\\\\n pickle.dump((x_train, y_train, x_val, y_val), output_file)\\\\n\\\\n else:\\\\n x_ = img_norm(x)\\\\n with open(data_processed_path, \\\\\"wb\\\\\") as output_file:\\\\n pickle.dump((x_, y), output_file)\\\\n\\\\ndef _deserialize_bool(s) -> bool:\\\\n from distutils.util import strtobool\\\\n return strtobool(s) == 1\\\\n\\\\nimport argparse\\\\n_parser = argparse.ArgumentParser(prog=\\'Process\\', description=\\'Here we do all the preprocessing\\')\\\\n_parser.add_argument(\\\\\"--data-raw\\\\\", dest=\\\\\"data_raw_path\\\\\", type=str, required=True, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--val-pct\\\\\", dest=\\\\\"val_pct\\\\\", type=float, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--trainset-flag\\\\\", dest=\\\\\"trainset_flag\\\\\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--histogram-norm\\\\\", dest=\\\\\"histogram_norm\\\\\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--data-processed\\\\\", dest=\\\\\"data_processed_path\\\\\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\\\n_parsed_args = vars(_parser.parse_args())\\\\n\\\\n_outputs = process(**_parsed_args)\\\\n\"], \"image\": \"tensorflow/tensorflow:2.7.1\"}}, \"inputs\": [{\"name\": \"data_raw\", \"type\": \"String\"}, {\"default\": \"0.2\", \"name\": \"val_pct\", \"optional\": true, \"type\": \"Float\"}, {\"default\": \"True\", \"name\": \"trainset_flag\", \"optional\": true, \"type\": \"Boolean\"}, {\"default\": \"False\", \"name\": \"histogram_norm\", \"optional\": true, \"type\": \"Boolean\"}], \"name\": \"Process\", \"outputs\": [{\"name\": \"data_processed\", \"type\": \"String\"}]}',\n", + " 'pipelines.kubeflow.org/task_display_name': 'Preprocess images'},\n", + " 'labels': {'pipelines.kubeflow.org/cache_enabled': 'true',\n", + " 'pipelines.kubeflow.org/enable_caching': 'true',\n", + " 'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}},\n", + " 'name': 'process',\n", + " 'outputs': {'artifacts': [{'name': 'process-data_processed',\n", + " 'path': '/tmp/outputs/data_processed/data'}]}},\n", + " {'container': {'args': ['--data-train',\n", + " '/tmp/inputs/data_train/data',\n", + " '--lr',\n", + " '{{inputs.parameters.lr}}',\n", + " '--optimizer',\n", + " '{{inputs.parameters.optimizer}}',\n", + " '--loss',\n", + " '{{inputs.parameters.loss}}',\n", + " '--epochs',\n", + " '{{inputs.parameters.epochs}}',\n", + " '--batch-size',\n", + " '{{inputs.parameters.batch_size}}',\n", + " '--model-out',\n", + " '/tmp/outputs/model_out/data',\n", + " '--mlpipeline-metrics',\n", + " '/tmp/outputs/mlpipeline_metrics/data'],\n", + " 'command': ['sh',\n", + " '-c',\n", + " '(PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scipy\\' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scipy\\' --user) && \"$0\" \"$@\"',\n", + " 'sh',\n", + " '-ec',\n", + " 'program_path=$(mktemp)\\nprintf \"%s\" \"$0\" > \"$program_path\"\\npython3 -u \"$program_path\" \"$@\"\\n',\n", + " 'def _make_parent_dirs_and_return_path(file_path: str):\\n import os\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\n return file_path\\n\\ndef train(\\n data_train_path, # type: ignore\\n model_out_path, # type: ignore\\n mlpipeline_metrics_path, # type: ignore # noqa: F821\\n lr = 1e-4,\\n optimizer = \"Adam\",\\n loss = \"categorical_crossentropy\",\\n epochs = 1,\\n batch_size = 32,\\n):\\n \"\"\"\\n This is the simulated train part of our ML pipeline where training is performed\\n \"\"\"\\n\\n import tensorflow as tf\\n import pickle\\n from tensorflow.keras.preprocessing.image import ImageDataGenerator\\n import json\\n\\n with open(data_train_path, \"rb\") as f:\\n x_train, y_train, x_val, y_val = pickle.load(f)\\n\\n model = tf.keras.Sequential(\\n [\\n tf.keras.layers.Conv2D(\\n 64, (3, 3), activation=\"relu\", input_shape=(28, 28, 1)\\n ),\\n tf.keras.layers.MaxPooling2D(2, 2),\\n tf.keras.layers.Conv2D(64, (3, 3), activation=\"relu\"),\\n tf.keras.layers.MaxPooling2D(2, 2),\\n tf.keras.layers.Flatten(),\\n tf.keras.layers.Dense(128, activation=\"relu\"),\\n tf.keras.layers.Dense(10, activation=\"softmax\"),\\n ]\\n )\\n\\n if optimizer.lower() == \"sgd\":\\n optimizer = tf.keras.optimizers.SGD(lr)\\n else:\\n optimizer = tf.keras.optimizers.Adam(lr)\\n\\n model.compile(loss=loss, optimizer=optimizer, metrics=[\"accuracy\"])\\n\\n # fit the model\\n model_early_stopping_callback = tf.keras.callbacks.EarlyStopping(\\n monitor=\"val_accuracy\", patience=10, verbose=1, restore_best_weights=True\\n )\\n\\n train_datagen = ImageDataGenerator()\\n\\n validation_datagen = ImageDataGenerator()\\n history = model.fit(\\n train_datagen.flow(x_train, y_train, batch_size=batch_size),\\n epochs=epochs,\\n validation_data=validation_datagen.flow(x_val, y_val, batch_size=batch_size),\\n shuffle=False,\\n callbacks=[model_early_stopping_callback],\\n )\\n\\n model.save(model_out_path, save_format=\"tf\")\\n\\n metrics = {\\n \"metrics\": [\\n {\\n \"name\": \"accuracy\", # The name of the metric. Visualized as the column name in the runs table.\\n \"numberValue\": history.history[\"accuracy\"][\\n -1\\n ], # The value of the metric. Must be a numeric value.\\n \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\\n },\\n {\\n \"name\": \"val-accuracy\", # The name of the metric. Visualized as the column name in the runs table.\\n \"numberValue\": history.history[\"val_accuracy\"][\\n -1\\n ], # The value of the metric. Must be a numeric value.\\n \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\\n },\\n ]\\n }\\n with open(mlpipeline_metrics_path, \"w\") as f:\\n json.dump(metrics, f)\\n\\nimport argparse\\n_parser = argparse.ArgumentParser(prog=\\'Train\\', description=\\'This is the simulated train part of our ML pipeline where training is performed\\')\\n_parser.add_argument(\"--data-train\", dest=\"data_train_path\", type=str, required=True, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--lr\", dest=\"lr\", type=float, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--optimizer\", dest=\"optimizer\", type=str, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--loss\", dest=\"loss\", type=str, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--epochs\", dest=\"epochs\", type=int, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--batch-size\", dest=\"batch_size\", type=int, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--model-out\", dest=\"model_out_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--mlpipeline-metrics\", dest=\"mlpipeline_metrics_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\n_parsed_args = vars(_parser.parse_args())\\n\\n_outputs = train(**_parsed_args)\\n'],\n", + " 'image': 'tensorflow/tensorflow:2.7.1',\n", + " 'resources': {'limits': {'cpu': '1', 'memory': '2Gi'}}},\n", + " 'inputs': {'artifacts': [{'name': 'process-data_processed',\n", + " 'path': '/tmp/inputs/data_train/data'}],\n", + " 'parameters': [{'name': 'batch_size'},\n", + " {'name': 'epochs'},\n", + " {'name': 'loss'},\n", + " {'name': 'lr'},\n", + " {'name': 'optimizer'}]},\n", + " 'metadata': {'annotations': {'pipelines.kubeflow.org/arguments.parameters': '{\"batch_size\": \"{{inputs.parameters.batch_size}}\", \"epochs\": \"{{inputs.parameters.epochs}}\", \"loss\": \"{{inputs.parameters.loss}}\", \"lr\": \"{{inputs.parameters.lr}}\", \"optimizer\": \"{{inputs.parameters.optimizer}}\"}',\n", + " 'pipelines.kubeflow.org/component_ref': '{}',\n", + " 'pipelines.kubeflow.org/component_spec': '{\"description\": \"This is the simulated train part of our ML pipeline where training is performed\", \"implementation\": {\"container\": {\"args\": [\"--data-train\", {\"inputPath\": \"data_train\"}, {\"if\": {\"cond\": {\"isPresent\": \"lr\"}, \"then\": [\"--lr\", {\"inputValue\": \"lr\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"optimizer\"}, \"then\": [\"--optimizer\", {\"inputValue\": \"optimizer\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"loss\"}, \"then\": [\"--loss\", {\"inputValue\": \"loss\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"epochs\"}, \"then\": [\"--epochs\", {\"inputValue\": \"epochs\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"batch_size\"}, \"then\": [\"--batch-size\", {\"inputValue\": \"batch_size\"}]}}, \"--model-out\", {\"outputPath\": \"model_out\"}, \"--mlpipeline-metrics\", {\"outputPath\": \"mlpipeline_metrics\"}], \"command\": [\"sh\", \"-c\", \"(PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scipy\\' || PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location \\'scipy\\' --user) && \\\\\"$0\\\\\" \\\\\"$@\\\\\"\", \"sh\", \"-ec\", \"program_path=$(mktemp)\\\\nprintf \\\\\"%s\\\\\" \\\\\"$0\\\\\" > \\\\\"$program_path\\\\\"\\\\npython3 -u \\\\\"$program_path\\\\\" \\\\\"$@\\\\\"\\\\n\", \"def _make_parent_dirs_and_return_path(file_path: str):\\\\n import os\\\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\\\n return file_path\\\\n\\\\ndef train(\\\\n data_train_path, # type: ignore\\\\n model_out_path, # type: ignore\\\\n mlpipeline_metrics_path, # type: ignore # noqa: F821\\\\n lr = 1e-4,\\\\n optimizer = \\\\\"Adam\\\\\",\\\\n loss = \\\\\"categorical_crossentropy\\\\\",\\\\n epochs = 1,\\\\n batch_size = 32,\\\\n):\\\\n \\\\\"\\\\\"\\\\\"\\\\n This is the simulated train part of our ML pipeline where training is performed\\\\n \\\\\"\\\\\"\\\\\"\\\\n\\\\n import tensorflow as tf\\\\n import pickle\\\\n from tensorflow.keras.preprocessing.image import ImageDataGenerator\\\\n import json\\\\n\\\\n with open(data_train_path, \\\\\"rb\\\\\") as f:\\\\n x_train, y_train, x_val, y_val = pickle.load(f)\\\\n\\\\n model = tf.keras.Sequential(\\\\n [\\\\n tf.keras.layers.Conv2D(\\\\n 64, (3, 3), activation=\\\\\"relu\\\\\", input_shape=(28, 28, 1)\\\\n ),\\\\n tf.keras.layers.MaxPooling2D(2, 2),\\\\n tf.keras.layers.Conv2D(64, (3, 3), activation=\\\\\"relu\\\\\"),\\\\n tf.keras.layers.MaxPooling2D(2, 2),\\\\n tf.keras.layers.Flatten(),\\\\n tf.keras.layers.Dense(128, activation=\\\\\"relu\\\\\"),\\\\n tf.keras.layers.Dense(10, activation=\\\\\"softmax\\\\\"),\\\\n ]\\\\n )\\\\n\\\\n if optimizer.lower() == \\\\\"sgd\\\\\":\\\\n optimizer = tf.keras.optimizers.SGD(lr)\\\\n else:\\\\n optimizer = tf.keras.optimizers.Adam(lr)\\\\n\\\\n model.compile(loss=loss, optimizer=optimizer, metrics=[\\\\\"accuracy\\\\\"])\\\\n\\\\n # fit the model\\\\n model_early_stopping_callback = tf.keras.callbacks.EarlyStopping(\\\\n monitor=\\\\\"val_accuracy\\\\\", patience=10, verbose=1, restore_best_weights=True\\\\n )\\\\n\\\\n train_datagen = ImageDataGenerator()\\\\n\\\\n validation_datagen = ImageDataGenerator()\\\\n history = model.fit(\\\\n train_datagen.flow(x_train, y_train, batch_size=batch_size),\\\\n epochs=epochs,\\\\n validation_data=validation_datagen.flow(x_val, y_val, batch_size=batch_size),\\\\n shuffle=False,\\\\n callbacks=[model_early_stopping_callback],\\\\n )\\\\n\\\\n model.save(model_out_path, save_format=\\\\\"tf\\\\\")\\\\n\\\\n metrics = {\\\\n \\\\\"metrics\\\\\": [\\\\n {\\\\n \\\\\"name\\\\\": \\\\\"accuracy\\\\\", # The name of the metric. Visualized as the column name in the runs table.\\\\n \\\\\"numberValue\\\\\": history.history[\\\\\"accuracy\\\\\"][\\\\n -1\\\\n ], # The value of the metric. Must be a numeric value.\\\\n \\\\\"format\\\\\": \\\\\"PERCENTAGE\\\\\", # The optional format of the metric. Supported values are \\\\\"RAW\\\\\" (displayed in raw format) and \\\\\"PERCENTAGE\\\\\" (displayed in percentage format).\\\\n },\\\\n {\\\\n \\\\\"name\\\\\": \\\\\"val-accuracy\\\\\", # The name of the metric. Visualized as the column name in the runs table.\\\\n \\\\\"numberValue\\\\\": history.history[\\\\\"val_accuracy\\\\\"][\\\\n -1\\\\n ], # The value of the metric. Must be a numeric value.\\\\n \\\\\"format\\\\\": \\\\\"PERCENTAGE\\\\\", # The optional format of the metric. Supported values are \\\\\"RAW\\\\\" (displayed in raw format) and \\\\\"PERCENTAGE\\\\\" (displayed in percentage format).\\\\n },\\\\n ]\\\\n }\\\\n with open(mlpipeline_metrics_path, \\\\\"w\\\\\") as f:\\\\n json.dump(metrics, f)\\\\n\\\\nimport argparse\\\\n_parser = argparse.ArgumentParser(prog=\\'Train\\', description=\\'This is the simulated train part of our ML pipeline where training is performed\\')\\\\n_parser.add_argument(\\\\\"--data-train\\\\\", dest=\\\\\"data_train_path\\\\\", type=str, required=True, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--lr\\\\\", dest=\\\\\"lr\\\\\", type=float, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--optimizer\\\\\", dest=\\\\\"optimizer\\\\\", type=str, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--loss\\\\\", dest=\\\\\"loss\\\\\", type=str, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--epochs\\\\\", dest=\\\\\"epochs\\\\\", type=int, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--batch-size\\\\\", dest=\\\\\"batch_size\\\\\", type=int, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--model-out\\\\\", dest=\\\\\"model_out_path\\\\\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--mlpipeline-metrics\\\\\", dest=\\\\\"mlpipeline_metrics_path\\\\\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\\\n_parsed_args = vars(_parser.parse_args())\\\\n\\\\n_outputs = train(**_parsed_args)\\\\n\"], \"image\": \"tensorflow/tensorflow:2.7.1\"}}, \"inputs\": [{\"name\": \"data_train\", \"type\": \"String\"}, {\"default\": \"0.0001\", \"name\": \"lr\", \"optional\": true, \"type\": \"Float\"}, {\"default\": \"Adam\", \"name\": \"optimizer\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"categorical_crossentropy\", \"name\": \"loss\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"1\", \"name\": \"epochs\", \"optional\": true, \"type\": \"Integer\"}, {\"default\": \"32\", \"name\": \"batch_size\", \"optional\": true, \"type\": \"Integer\"}], \"name\": \"Train\", \"outputs\": [{\"name\": \"model_out\", \"type\": \"String\"}, {\"name\": \"mlpipeline_metrics\", \"type\": \"Metrics\"}]}',\n", + " 'pipelines.kubeflow.org/max_cache_staleness': 'P0D',\n", + " 'pipelines.kubeflow.org/task_display_name': 'Fit the model'},\n", + " 'labels': {'katib.kubeflow.org/model-training': 'true',\n", + " 'pipelines.kubeflow.org/enable_caching': 'true',\n", + " 'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}},\n", + " 'name': 'train',\n", + " 'outputs': {'artifacts': [{'name': 'mlpipeline-metrics',\n", + " 'path': '/tmp/outputs/mlpipeline_metrics/data'},\n", + " {'name': 'train-model_out',\n", + " 'path': '/tmp/outputs/model_out/data'}]}}]}}}}}" + ] + }, + "execution_count": 25, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "katib_client.create_experiment(katib_experiment)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should now be able to observe in the Web UI how the Katib\n", + "Experiment is running.\n", + "\n", + "To see how the `Argo Workflows` are started, you can also check the Kubernetes cluster:\n", + "\n", + "`kubectl get Workflow -n `" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Minimal example pipeline for e2e testing\n", + "\n", + "The following part generates a minimal Katib Experiment for e2e testing" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "metadata": {}, + "outputs": [], + "source": [ + "def prep_e2e(\n", + " output_nr_path: OutputPath(int), # type: ignore # noqa: F821\n", + " histogram_norm: bool = True,\n", + "):\n", + " with open(output_nr_path, 'w') as writer:\n", + " writer.write(str(int(histogram_norm)))\n", + " \n", + "prep_e2e_op = create_component_from_func(\n", + " func=prep_e2e\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [], + "source": [ + "def train_e2e(\n", + " input_nr_path: InputPath(int), # type: ignore # noqa: F821\n", + " mlpipeline_metrics_path: OutputPath(\"Metrics\"), # type: ignore # noqa: F821\n", + " lr: float = 1e-4,\n", + " optimizer: str = \"Adam\",\n", + " loss: str = \"categorical_crossentropy\",\n", + " epochs: int = 1,\n", + " batch_size: int = 32,\n", + "):\n", + " \"\"\"\n", + " This is the simulated train part of our ML pipeline where training is performed\n", + " \"\"\"\n", + " import json \n", + " import time\n", + " with open(input_nr_path, 'r') as reader:\n", + " line = reader.readline()\n", + " histogram_norm_value = int(line)\n", + "\n", + " accuracy = (batch_size + histogram_norm_value)/ (batch_size + epochs+histogram_norm_value)\n", + " val_accuracy = accuracy * 0.9\n", + " metrics = {\n", + " \"metrics\": [\n", + " {\n", + " \"name\": \"accuracy\", # The name of the metric. Visualized as the column name in the runs table.\n", + " \"numberValue\": accuracy, # The value of the metric. Must be a numeric value.\n", + " \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\n", + " },\n", + " {\n", + " \"name\": \"val-accuracy\", # The name of the metric. Visualized as the column name in the runs table.\n", + " \"numberValue\": val_accuracy, # The value of the metric. Must be a numeric value.\n", + " \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\n", + " },\n", + " ]\n", + " }\n", + " with open(mlpipeline_metrics_path, \"w\") as f:\n", + " json.dump(metrics, f)\n", + " \n", + " # If this step is to fast, the metrics collector fails as the\n", + " # pod is already finished before it can collect the metrics.\n", + " time.sleep(10)\n", + "\n", + "\n", + "train_e2e_op = create_component_from_func(\n", + " func=train_e2e\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": {}, + "outputs": [], + "source": [ + "@dsl.pipeline(\n", + " name=\"Minimal KFP1 pipeline for e2e testing\",\n", + " description=\"\",\n", + ")\n", + "def e2e_example_pipeline(\n", + " lr: float = 1e-4,\n", + " optimizer: str = \"Adam\",\n", + " loss: str = \"categorical_crossentropy\",\n", + " epochs: int = 3,\n", + " batch_size: int = 5,\n", + " histogram_norm: bool = False,\n", + "):\n", + " prep_e2e_output = (\n", + " prep_e2e_op(\n", + " histogram_norm=histogram_norm,\n", + " )\n", + " .set_display_name(\"Prepare a dummy output that should be cached\")\n", + " )\n", + " _label_cache(prep_e2e_output)\n", + "\n", + " training_output = (\n", + " train_e2e_op(\n", + " prep_e2e_output.output,\n", + " lr=lr,\n", + " optimizer=optimizer,\n", + " epochs=epochs,\n", + " batch_size=batch_size,\n", + " loss=loss,\n", + " )\n", + " )\n", + " training_output.set_display_name(\"Generate dummy metrics\")\n", + " # This pod label indicates which pod Katib should collect the metric from.\n", + " # A metrics collecting sidecar container will be added\n", + " training_output.add_pod_label(\"katib.kubeflow.org/model-training\", \"true\")\n", + " # This step needs to run always, as otherwise the metrics for Katib could not\n", + " # be collected.\n", + " training_output.execution_options.caching_strategy.max_cache_staleness = \"P0D\"" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "Experiment details." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/html": [ + "Run details." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "kfp_run = f\"e2e-example-{dt.today().strftime('%Y-%m-%d-%Hh-%Mm-%Ss')}\"\n", + "run = kfp_client.create_run_from_pipeline_func(\n", + " e2e_example_pipeline,\n", + " mode=kfp.dsl.PipelineExecutionMode.V1_LEGACY,\n", + " # You can optionally override your pipeline_root when submitting the run too:\n", + " # pipeline_root='gs://my-pipeline-root/example-pipeline',\n", + " arguments={\"histogram_norm\": \"0\"},\n", + " experiment_name=KFP_EXPERIMENT,\n", + " run_name=kfp_run,\n", + " # In a multiuser setup, provide the namesapce\n", + " #namespace=USER_NAMESPACE,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": {}, + "outputs": [], + "source": [ + "# Prepare the full spec\n", + "\n", + "katib_e2e_spec = create_katib_experiment_spec(\n", + " pipeline=e2e_example_pipeline,\n", + " pipeline_params=pipeline_params,\n", + " trial_params=trial_params_specs,\n", + " trial_params_space=parameter_space,\n", + " objective=objective,\n", + " algorithm=algorithm,\n", + " pipeline_service_account=KFP_SERVICE_ACCOUNT,\n", + " max_trial_count=5,\n", + " parallel_trial_count=5,\n", + " retain_pods=False,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "metadata": {}, + "outputs": [], + "source": [ + "# Prepare the experiment\n", + "\n", + "katib_e2e_experiment_name = (\n", + " f\"katib-e2e-{dt.today().strftime('%Y-%m-%d-%Hh-%Mm-%Ss')}\"\n", + ")\n", + "katib_e2e_experiment = V1beta1Experiment(\n", + " api_version=\"kubeflow.org/v1beta1\",\n", + " kind=\"Experiment\",\n", + " metadata=V1ObjectMeta(\n", + " name=katib_e2e_experiment_name,\n", + " namespace=USER_NAMESPACE,\n", + " ),\n", + " spec=katib_e2e_spec,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "metadata": {}, + "outputs": [], + "source": [ + "with open(f\"{KATIB_E2E_EXPERIMENT}.yaml\", \"w\") as f:\n", + " yaml.dump(ApiClient().sanitize_for_serialization(katib_e2e_experiment), f)" + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'apiVersion': 'kubeflow.org/v1beta1',\n", + " 'kind': 'Experiment',\n", + " 'metadata': {'creationTimestamp': '2023-07-20T20:37:59Z',\n", + " 'generation': 1,\n", + " 'managedFields': [{'apiVersion': 'kubeflow.org/v1beta1',\n", + " 'fieldsType': 'FieldsV1',\n", + " 'fieldsV1': {'f:spec': {'.': {},\n", + " 'f:algorithm': {'.': {}, 'f:algorithmName': {}},\n", + " 'f:maxFailedTrialCount': {},\n", + " 'f:maxTrialCount': {},\n", + " 'f:metricsCollectorSpec': {'.': {},\n", + " 'f:collector': {'.': {},\n", + " 'f:customCollector': {'.': {},\n", + " 'f:args': {},\n", + " 'f:env': {},\n", + " 'f:image': {},\n", + " 'f:imagePullPolicy': {},\n", + " 'f:name': {}},\n", + " 'f:kind': {}},\n", + " 'f:source': {'.': {},\n", + " 'f:fileSystemPath': {'.': {}, 'f:kind': {}, 'f:path': {}}}},\n", + " 'f:objective': {'.': {},\n", + " 'f:additionalMetricNames': {},\n", + " 'f:goal': {},\n", + " 'f:objectiveMetricName': {},\n", + " 'f:type': {}},\n", + " 'f:parallelTrialCount': {},\n", + " 'f:parameters': {},\n", + " 'f:trialTemplate': {'.': {},\n", + " 'f:failureCondition': {},\n", + " 'f:primaryContainerName': {},\n", + " 'f:primaryPodLabels': {'.': {},\n", + " 'f:katib.kubeflow.org/model-training': {}},\n", + " 'f:retain': {},\n", + " 'f:successCondition': {},\n", + " 'f:trialParameters': {},\n", + " 'f:trialSpec': {'.': {},\n", + " 'f:apiVersion': {},\n", + " 'f:kind': {},\n", + " 'f:metadata': {'.': {},\n", + " 'f:annotations': {'.': {},\n", + " 'f:pipelines.kubeflow.org/kfp_sdk_version': {},\n", + " 'f:pipelines.kubeflow.org/pipeline_compilation_time': {},\n", + " 'f:pipelines.kubeflow.org/pipeline_spec': {}},\n", + " 'f:generateName': {},\n", + " 'f:labels': {'.': {},\n", + " 'f:pipelines.kubeflow.org/kfp_sdk_version': {}}},\n", + " 'f:spec': {'.': {},\n", + " 'f:arguments': {'.': {}, 'f:parameters': {}},\n", + " 'f:entrypoint': {},\n", + " 'f:serviceAccountName': {},\n", + " 'f:templates': {}}}}}},\n", + " 'manager': 'OpenAPI-Generator',\n", + " 'operation': 'Update',\n", + " 'time': '2023-07-20T20:37:59Z'}],\n", + " 'name': 'katib-e2e-2023-07-20-22h-37m-57s',\n", + " 'namespace': 'kubeflow',\n", + " 'resourceVersion': '11759',\n", + " 'uid': 'c91aa6c9-8a2b-434d-9ab8-c4a317210893'},\n", + " 'spec': {'algorithm': {'algorithmName': 'random'},\n", + " 'maxFailedTrialCount': 2,\n", + " 'maxTrialCount': 5,\n", + " 'metricsCollectorSpec': {'collector': {'customCollector': {'args': ['-m',\n", + " 'val-accuracy;accuracy',\n", + " '-s',\n", + " 'katib-db-manager.kubeflow:6789',\n", + " '-t',\n", + " '$(PodName)',\n", + " '-path',\n", + " '/tmp/outputs/mlpipeline_metrics'],\n", + " 'env': [{'name': 'PodName',\n", + " 'valueFrom': {'fieldRef': {'fieldPath': 'metadata.name'}}}],\n", + " 'image': 'docker.io/votti/kfpv1-metricscollector:v0.0.10',\n", + " 'imagePullPolicy': 'Always',\n", + " 'name': 'custom-metrics-logger-and-collector',\n", + " 'resources': {}},\n", + " 'kind': 'Custom'},\n", + " 'source': {'fileSystemPath': {'kind': 'File',\n", + " 'path': '/tmp/outputs/mlpipeline_metrics/data'}}},\n", + " 'objective': {'additionalMetricNames': ['accuracy'],\n", + " 'goal': 0.9,\n", + " 'metricStrategies': [{'name': 'val-accuracy', 'value': 'max'},\n", + " {'name': 'accuracy', 'value': 'max'}],\n", + " 'objectiveMetricName': 'val-accuracy',\n", + " 'type': 'maximize'},\n", + " 'parallelTrialCount': 5,\n", + " 'parameters': [{'feasibleSpace': {'max': '0.001', 'min': '0.00001'},\n", + " 'name': 'learning_rate',\n", + " 'parameterType': 'double'},\n", + " {'feasibleSpace': {'max': '64', 'min': '16'},\n", + " 'name': 'batch_size',\n", + " 'parameterType': 'int'},\n", + " {'feasibleSpace': {'list': ['0', '1']},\n", + " 'name': 'histogram_norm',\n", + " 'parameterType': 'discrete'}],\n", + " 'resumePolicy': 'Never',\n", + " 'trialTemplate': {'failureCondition': 'status.[@this].#(phase==\"Failed\")#',\n", + " 'primaryContainerName': 'main',\n", + " 'primaryPodLabels': {'katib.kubeflow.org/model-training': 'true'},\n", + " 'successCondition': 'status.[@this].#(phase==\"Succeeded\")#',\n", + " 'trialParameters': [{'description': 'Learning rate for the training model',\n", + " 'name': 'learningRate',\n", + " 'reference': 'learning_rate'},\n", + " {'description': 'Batch size for NN training',\n", + " 'name': 'batchSize',\n", + " 'reference': 'batch_size'},\n", + " {'description': 'Histogram normalization of image on?',\n", + " 'name': 'histogramNorm',\n", + " 'reference': 'histogram_norm'}],\n", + " 'trialSpec': {'apiVersion': 'argoproj.io/v1alpha1',\n", + " 'kind': 'Workflow',\n", + " 'metadata': {'annotations': {'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline_compilation_time': '2023-07-20T22:37:57.355215',\n", + " 'pipelines.kubeflow.org/pipeline_spec': '{\"inputs\": [{\"default\": \"0.0001\", \"name\": \"lr\", \"optional\": true, \"type\": \"Float\"}, {\"default\": \"Adam\", \"name\": \"optimizer\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"categorical_crossentropy\", \"name\": \"loss\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"3\", \"name\": \"epochs\", \"optional\": true, \"type\": \"Integer\"}, {\"default\": \"5\", \"name\": \"batch_size\", \"optional\": true, \"type\": \"Integer\"}, {\"default\": \"False\", \"name\": \"histogram_norm\", \"optional\": true, \"type\": \"Boolean\"}, {\"default\": \"${trialParameters.learningRate}\", \"name\": \"lr\"}, {\"default\": \"${trialParameters.batchSize}\", \"name\": \"batch_size\"}, {\"default\": \"${trialParameters.histogramNorm}\", \"name\": \"histogram_norm\"}], \"name\": \"Minimal KFP1 pipeline for e2e testing\"}'},\n", + " 'generateName': 'minimal-kfp1-pipeline-for-e2e-testing-',\n", + " 'labels': {'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12'}},\n", + " 'spec': {'arguments': {'parameters': [{'name': 'lr',\n", + " 'value': '${trialParameters.learningRate}'},\n", + " {'name': 'optimizer', 'value': 'Adam'},\n", + " {'name': 'loss', 'value': 'categorical_crossentropy'},\n", + " {'name': 'epochs', 'value': '3'},\n", + " {'name': 'batch_size', 'value': '${trialParameters.batchSize}'},\n", + " {'name': 'histogram_norm',\n", + " 'value': '${trialParameters.histogramNorm}'}]},\n", + " 'entrypoint': 'minimal-kfp1-pipeline-for-e2e-testing',\n", + " 'serviceAccountName': 'pipeline-runner',\n", + " 'templates': [{'dag': {'tasks': [{'arguments': {'parameters': [{'name': 'histogram_norm',\n", + " 'value': '{{inputs.parameters.histogram_norm}}'}]},\n", + " 'name': 'prep-e2e',\n", + " 'template': 'prep-e2e'},\n", + " {'arguments': {'artifacts': [{'from': '{{tasks.prep-e2e.outputs.artifacts.prep-e2e-output_nr}}',\n", + " 'name': 'prep-e2e-output_nr'}],\n", + " 'parameters': [{'name': 'batch_size',\n", + " 'value': '{{inputs.parameters.batch_size}}'},\n", + " {'name': 'epochs', 'value': '{{inputs.parameters.epochs}}'},\n", + " {'name': 'loss', 'value': '{{inputs.parameters.loss}}'},\n", + " {'name': 'lr', 'value': '{{inputs.parameters.lr}}'},\n", + " {'name': 'optimizer',\n", + " 'value': '{{inputs.parameters.optimizer}}'}]},\n", + " 'dependencies': ['prep-e2e'],\n", + " 'name': 'train-e2e',\n", + " 'template': 'train-e2e'}]},\n", + " 'inputs': {'parameters': [{'name': 'batch_size'},\n", + " {'name': 'epochs'},\n", + " {'name': 'histogram_norm'},\n", + " {'name': 'loss'},\n", + " {'name': 'lr'},\n", + " {'name': 'optimizer'}]},\n", + " 'name': 'minimal-kfp1-pipeline-for-e2e-testing'},\n", + " {'container': {'args': ['--histogram-norm',\n", + " '{{inputs.parameters.histogram_norm}}',\n", + " '--output-nr',\n", + " '/tmp/outputs/output_nr/data'],\n", + " 'command': ['sh',\n", + " '-ec',\n", + " 'program_path=$(mktemp)\\nprintf \"%s\" \"$0\" > \"$program_path\"\\npython3 -u \"$program_path\" \"$@\"\\n',\n", + " 'def _make_parent_dirs_and_return_path(file_path: str):\\n import os\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\n return file_path\\n\\ndef prep_e2e(\\n output_nr_path, # type: ignore # noqa: F821\\n histogram_norm = True,\\n):\\n with open(output_nr_path, \\'w\\') as writer:\\n writer.write(str(int(histogram_norm)))\\n\\ndef _deserialize_bool(s) -> bool:\\n from distutils.util import strtobool\\n return strtobool(s) == 1\\n\\nimport argparse\\n_parser = argparse.ArgumentParser(prog=\\'Prep e2e\\', description=\\'\\')\\n_parser.add_argument(\"--histogram-norm\", dest=\"histogram_norm\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--output-nr\", dest=\"output_nr_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\n_parsed_args = vars(_parser.parse_args())\\n\\n_outputs = prep_e2e(**_parsed_args)\\n'],\n", + " 'image': 'python:3.7'},\n", + " 'inputs': {'parameters': [{'name': 'histogram_norm'}]},\n", + " 'metadata': {'annotations': {'pipelines.kubeflow.org/arguments.parameters': '{\"histogram_norm\": \"{{inputs.parameters.histogram_norm}}\"}',\n", + " 'pipelines.kubeflow.org/component_ref': '{}',\n", + " 'pipelines.kubeflow.org/component_spec': '{\"implementation\": {\"container\": {\"args\": [{\"if\": {\"cond\": {\"isPresent\": \"histogram_norm\"}, \"then\": [\"--histogram-norm\", {\"inputValue\": \"histogram_norm\"}]}}, \"--output-nr\", {\"outputPath\": \"output_nr\"}], \"command\": [\"sh\", \"-ec\", \"program_path=$(mktemp)\\\\nprintf \\\\\"%s\\\\\" \\\\\"$0\\\\\" > \\\\\"$program_path\\\\\"\\\\npython3 -u \\\\\"$program_path\\\\\" \\\\\"$@\\\\\"\\\\n\", \"def _make_parent_dirs_and_return_path(file_path: str):\\\\n import os\\\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\\\n return file_path\\\\n\\\\ndef prep_e2e(\\\\n output_nr_path, # type: ignore # noqa: F821\\\\n histogram_norm = True,\\\\n):\\\\n with open(output_nr_path, \\'w\\') as writer:\\\\n writer.write(str(int(histogram_norm)))\\\\n\\\\ndef _deserialize_bool(s) -> bool:\\\\n from distutils.util import strtobool\\\\n return strtobool(s) == 1\\\\n\\\\nimport argparse\\\\n_parser = argparse.ArgumentParser(prog=\\'Prep e2e\\', description=\\'\\')\\\\n_parser.add_argument(\\\\\"--histogram-norm\\\\\", dest=\\\\\"histogram_norm\\\\\", type=_deserialize_bool, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--output-nr\\\\\", dest=\\\\\"output_nr_path\\\\\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\\\n_parsed_args = vars(_parser.parse_args())\\\\n\\\\n_outputs = prep_e2e(**_parsed_args)\\\\n\"], \"image\": \"python:3.7\"}}, \"inputs\": [{\"default\": \"True\", \"name\": \"histogram_norm\", \"optional\": true, \"type\": \"Boolean\"}], \"name\": \"Prep e2e\", \"outputs\": [{\"name\": \"output_nr\", \"type\": \"Integer\"}]}',\n", + " 'pipelines.kubeflow.org/task_display_name': 'Prepare a dummy output that should be cached'},\n", + " 'labels': {'pipelines.kubeflow.org/cache_enabled': 'true',\n", + " 'pipelines.kubeflow.org/enable_caching': 'true',\n", + " 'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}},\n", + " 'name': 'prep-e2e',\n", + " 'outputs': {'artifacts': [{'name': 'prep-e2e-output_nr',\n", + " 'path': '/tmp/outputs/output_nr/data'}]}},\n", + " {'container': {'args': ['--input-nr',\n", + " '/tmp/inputs/input_nr/data',\n", + " '--lr',\n", + " '{{inputs.parameters.lr}}',\n", + " '--optimizer',\n", + " '{{inputs.parameters.optimizer}}',\n", + " '--loss',\n", + " '{{inputs.parameters.loss}}',\n", + " '--epochs',\n", + " '{{inputs.parameters.epochs}}',\n", + " '--batch-size',\n", + " '{{inputs.parameters.batch_size}}',\n", + " '--mlpipeline-metrics',\n", + " '/tmp/outputs/mlpipeline_metrics/data'],\n", + " 'command': ['sh',\n", + " '-ec',\n", + " 'program_path=$(mktemp)\\nprintf \"%s\" \"$0\" > \"$program_path\"\\npython3 -u \"$program_path\" \"$@\"\\n',\n", + " 'def _make_parent_dirs_and_return_path(file_path: str):\\n import os\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\n return file_path\\n\\ndef train_e2e(\\n input_nr_path, # type: ignore # noqa: F821\\n mlpipeline_metrics_path, # type: ignore # noqa: F821\\n lr = 1e-4,\\n optimizer = \"Adam\",\\n loss = \"categorical_crossentropy\",\\n epochs = 1,\\n batch_size = 32,\\n):\\n \"\"\"\\n This is the simulated train part of our ML pipeline where training is performed\\n \"\"\"\\n import json \\n import time\\n with open(input_nr_path, \\'r\\') as reader:\\n line = reader.readline()\\n histogram_norm_value = int(line)\\n\\n accuracy = (batch_size + histogram_norm_value)/ (batch_size + epochs+histogram_norm_value)\\n val_accuracy = accuracy * 0.9\\n metrics = {\\n \"metrics\": [\\n {\\n \"name\": \"accuracy\", # The name of the metric. Visualized as the column name in the runs table.\\n \"numberValue\": accuracy, # The value of the metric. Must be a numeric value.\\n \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\\n },\\n {\\n \"name\": \"val-accuracy\", # The name of the metric. Visualized as the column name in the runs table.\\n \"numberValue\": val_accuracy, # The value of the metric. Must be a numeric value.\\n \"format\": \"PERCENTAGE\", # The optional format of the metric. Supported values are \"RAW\" (displayed in raw format) and \"PERCENTAGE\" (displayed in percentage format).\\n },\\n ]\\n }\\n with open(mlpipeline_metrics_path, \"w\") as f:\\n json.dump(metrics, f)\\n\\n # If this step is to fast, the metrics collector fails as the\\n # pod is already finished before it can collect the metrics.\\n time.sleep(10)\\n\\nimport argparse\\n_parser = argparse.ArgumentParser(prog=\\'Train e2e\\', description=\\'This is the simulated train part of our ML pipeline where training is performed\\')\\n_parser.add_argument(\"--input-nr\", dest=\"input_nr_path\", type=str, required=True, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--lr\", dest=\"lr\", type=float, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--optimizer\", dest=\"optimizer\", type=str, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--loss\", dest=\"loss\", type=str, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--epochs\", dest=\"epochs\", type=int, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--batch-size\", dest=\"batch_size\", type=int, required=False, default=argparse.SUPPRESS)\\n_parser.add_argument(\"--mlpipeline-metrics\", dest=\"mlpipeline_metrics_path\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\n_parsed_args = vars(_parser.parse_args())\\n\\n_outputs = train_e2e(**_parsed_args)\\n'],\n", + " 'image': 'python:3.7'},\n", + " 'inputs': {'artifacts': [{'name': 'prep-e2e-output_nr',\n", + " 'path': '/tmp/inputs/input_nr/data'}],\n", + " 'parameters': [{'name': 'batch_size'},\n", + " {'name': 'epochs'},\n", + " {'name': 'loss'},\n", + " {'name': 'lr'},\n", + " {'name': 'optimizer'}]},\n", + " 'metadata': {'annotations': {'pipelines.kubeflow.org/arguments.parameters': '{\"batch_size\": \"{{inputs.parameters.batch_size}}\", \"epochs\": \"{{inputs.parameters.epochs}}\", \"loss\": \"{{inputs.parameters.loss}}\", \"lr\": \"{{inputs.parameters.lr}}\", \"optimizer\": \"{{inputs.parameters.optimizer}}\"}',\n", + " 'pipelines.kubeflow.org/component_ref': '{}',\n", + " 'pipelines.kubeflow.org/component_spec': '{\"description\": \"This is the simulated train part of our ML pipeline where training is performed\", \"implementation\": {\"container\": {\"args\": [\"--input-nr\", {\"inputPath\": \"input_nr\"}, {\"if\": {\"cond\": {\"isPresent\": \"lr\"}, \"then\": [\"--lr\", {\"inputValue\": \"lr\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"optimizer\"}, \"then\": [\"--optimizer\", {\"inputValue\": \"optimizer\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"loss\"}, \"then\": [\"--loss\", {\"inputValue\": \"loss\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"epochs\"}, \"then\": [\"--epochs\", {\"inputValue\": \"epochs\"}]}}, {\"if\": {\"cond\": {\"isPresent\": \"batch_size\"}, \"then\": [\"--batch-size\", {\"inputValue\": \"batch_size\"}]}}, \"--mlpipeline-metrics\", {\"outputPath\": \"mlpipeline_metrics\"}], \"command\": [\"sh\", \"-ec\", \"program_path=$(mktemp)\\\\nprintf \\\\\"%s\\\\\" \\\\\"$0\\\\\" > \\\\\"$program_path\\\\\"\\\\npython3 -u \\\\\"$program_path\\\\\" \\\\\"$@\\\\\"\\\\n\", \"def _make_parent_dirs_and_return_path(file_path: str):\\\\n import os\\\\n os.makedirs(os.path.dirname(file_path), exist_ok=True)\\\\n return file_path\\\\n\\\\ndef train_e2e(\\\\n input_nr_path, # type: ignore # noqa: F821\\\\n mlpipeline_metrics_path, # type: ignore # noqa: F821\\\\n lr = 1e-4,\\\\n optimizer = \\\\\"Adam\\\\\",\\\\n loss = \\\\\"categorical_crossentropy\\\\\",\\\\n epochs = 1,\\\\n batch_size = 32,\\\\n):\\\\n \\\\\"\\\\\"\\\\\"\\\\n This is the simulated train part of our ML pipeline where training is performed\\\\n \\\\\"\\\\\"\\\\\"\\\\n import json \\\\n import time\\\\n with open(input_nr_path, \\'r\\') as reader:\\\\n line = reader.readline()\\\\n histogram_norm_value = int(line)\\\\n\\\\n accuracy = (batch_size + histogram_norm_value)/ (batch_size + epochs+histogram_norm_value)\\\\n val_accuracy = accuracy * 0.9\\\\n metrics = {\\\\n \\\\\"metrics\\\\\": [\\\\n {\\\\n \\\\\"name\\\\\": \\\\\"accuracy\\\\\", # The name of the metric. Visualized as the column name in the runs table.\\\\n \\\\\"numberValue\\\\\": accuracy, # The value of the metric. Must be a numeric value.\\\\n \\\\\"format\\\\\": \\\\\"PERCENTAGE\\\\\", # The optional format of the metric. Supported values are \\\\\"RAW\\\\\" (displayed in raw format) and \\\\\"PERCENTAGE\\\\\" (displayed in percentage format).\\\\n },\\\\n {\\\\n \\\\\"name\\\\\": \\\\\"val-accuracy\\\\\", # The name of the metric. Visualized as the column name in the runs table.\\\\n \\\\\"numberValue\\\\\": val_accuracy, # The value of the metric. Must be a numeric value.\\\\n \\\\\"format\\\\\": \\\\\"PERCENTAGE\\\\\", # The optional format of the metric. Supported values are \\\\\"RAW\\\\\" (displayed in raw format) and \\\\\"PERCENTAGE\\\\\" (displayed in percentage format).\\\\n },\\\\n ]\\\\n }\\\\n with open(mlpipeline_metrics_path, \\\\\"w\\\\\") as f:\\\\n json.dump(metrics, f)\\\\n\\\\n # If this step is to fast, the metrics collector fails as the\\\\n # pod is already finished before it can collect the metrics.\\\\n time.sleep(10)\\\\n\\\\nimport argparse\\\\n_parser = argparse.ArgumentParser(prog=\\'Train e2e\\', description=\\'This is the simulated train part of our ML pipeline where training is performed\\')\\\\n_parser.add_argument(\\\\\"--input-nr\\\\\", dest=\\\\\"input_nr_path\\\\\", type=str, required=True, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--lr\\\\\", dest=\\\\\"lr\\\\\", type=float, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--optimizer\\\\\", dest=\\\\\"optimizer\\\\\", type=str, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--loss\\\\\", dest=\\\\\"loss\\\\\", type=str, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--epochs\\\\\", dest=\\\\\"epochs\\\\\", type=int, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--batch-size\\\\\", dest=\\\\\"batch_size\\\\\", type=int, required=False, default=argparse.SUPPRESS)\\\\n_parser.add_argument(\\\\\"--mlpipeline-metrics\\\\\", dest=\\\\\"mlpipeline_metrics_path\\\\\", type=_make_parent_dirs_and_return_path, required=True, default=argparse.SUPPRESS)\\\\n_parsed_args = vars(_parser.parse_args())\\\\n\\\\n_outputs = train_e2e(**_parsed_args)\\\\n\"], \"image\": \"python:3.7\"}}, \"inputs\": [{\"name\": \"input_nr\", \"type\": \"Integer\"}, {\"default\": \"0.0001\", \"name\": \"lr\", \"optional\": true, \"type\": \"Float\"}, {\"default\": \"Adam\", \"name\": \"optimizer\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"categorical_crossentropy\", \"name\": \"loss\", \"optional\": true, \"type\": \"String\"}, {\"default\": \"1\", \"name\": \"epochs\", \"optional\": true, \"type\": \"Integer\"}, {\"default\": \"32\", \"name\": \"batch_size\", \"optional\": true, \"type\": \"Integer\"}], \"name\": \"Train e2e\", \"outputs\": [{\"name\": \"mlpipeline_metrics\", \"type\": \"Metrics\"}]}',\n", + " 'pipelines.kubeflow.org/max_cache_staleness': 'P0D',\n", + " 'pipelines.kubeflow.org/task_display_name': 'Generate dummy metrics'},\n", + " 'labels': {'katib.kubeflow.org/model-training': 'true',\n", + " 'pipelines.kubeflow.org/enable_caching': 'true',\n", + " 'pipelines.kubeflow.org/kfp_sdk_version': '1.8.12',\n", + " 'pipelines.kubeflow.org/pipeline-sdk-type': 'kfp'}},\n", + " 'name': 'train-e2e',\n", + " 'outputs': {'artifacts': [{'name': 'mlpipeline-metrics',\n", + " 'path': '/tmp/outputs/mlpipeline_metrics/data'}]}}]}}}}}" + ] + }, + "execution_count": 105, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "katib_client.create_experiment(katib_e2e_experiment)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "katibdev", + "language": "python", + "name": "katibdev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + }, + "vscode": { + "interpreter": { + "hash": "346a4e9d8b8e6802b68a0916b92683cfb1882082eeafaaae0a3525ab995e1047" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pkg/metricscollector/v1beta1/common/const.py b/pkg/metricscollector/v1beta1/common/const.py index f3bdf56af46..c155cd04945 100644 --- a/pkg/metricscollector/v1beta1/common/const.py +++ b/pkg/metricscollector/v1beta1/common/const.py @@ -20,6 +20,8 @@ DEFAULT_WAIT_ALL_PROCESSES = "True" # Default value for directory where TF event metrics are reported DEFAULT_METRICS_FILE_DIR = "/log" +# Default value for directory where Kubeflow pipeline metrics are reported +DEFAULT_METRICS_FILE_KFPV1_DIR = "/tmp/outputs/mlpipeline_metrics" # Job finished marker in $$$$.pid file when main process is completed TRAINING_COMPLETED = "completed" diff --git a/pkg/metricscollector/v1beta1/kfp-metricscollector/v1/__init__.py b/pkg/metricscollector/v1beta1/kfp-metricscollector/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/metricscollector/v1beta1/kfp-metricscollector/v1/metrics_loader.py b/pkg/metricscollector/v1beta1/kfp-metricscollector/v1/metrics_loader.py new file mode 100644 index 00000000000..90e1764b7e8 --- /dev/null +++ b/pkg/metricscollector/v1beta1/kfp-metricscollector/v1/metrics_loader.py @@ -0,0 +1,110 @@ +# Copyright 2023 The Kubeflow Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The Kubeflow pipeline metrics collector KFPMetricParser parses the metrics file +# and returns an ObservationLog of the metrics specified. +# Some documentation on the metrics collector file structure can be found here: +# https://v0-6.kubeflow.org/docs/pipelines/sdk/pipelines-metrics/ + +from datetime import datetime +from logging import getLogger, StreamHandler, INFO +import os +from typing import List +import json + +import rfc3339 +import api_pb2 +from pkg.metricscollector.v1beta1.common import const + +class KFPMetricParser: + def __init__(self, metric_names): + self.metric_names = metric_names + + @staticmethod + def find_all_files(directory): + for root, dirs, files in os.walk(directory): + for f in files: + yield os.path.join(root, f) + + def parse_metrics(self, metric_file_path: str) -> List[api_pb2.MetricLog]: + """Parse a kubeflow pipeline metrics file + + Args: + fn (function): path to metrics file + + Returns: + List[api_pb2.MetricLog]: A list of logged metrics + """ + metrics = [] + with open(metric_file_path) as f: + metrics_dict = json.load(f) + for m in metrics_dict["metrics"]: + name = m["name"] + value = m["numberValue"] + if name in self.metric_names: + ml = api_pb2.MetricLog( + time_stamp=rfc3339.rfc3339(datetime.now()), + metric=api_pb2.Metric(name=name, value=str(value)), + ) + metrics.append(ml) + return metrics + +class MetricsCollector: + def __init__(self, metric_names): + self.logger = getLogger(__name__) + handler = StreamHandler() + handler.setLevel(INFO) + self.logger.setLevel(INFO) + self.logger.addHandler(handler) + self.logger.propagate = False + self.metrics = metric_names + self.parser = KFPMetricParser(metric_names) + + def parse_file(self, directory): + """Parses the Kubeflow Pipeline metrics files""" + mls = [] + for f in self.parser.find_all_files(directory): + if os.path.isdir(f): + continue + try: + self.logger.info(f + " will be parsed.") + mls.extend(self.parser.parse_metrics(f)) + except Exception as e: + self.logger.warning("Unexpected error: " + str(e)) + continue + + # Metrics logs must contain at least one objective metric value + # Objective metric is located at first index + is_objective_metric_reported = False + for ml in mls: + if ml.metric.name == self.metrics[0]: + is_objective_metric_reported = True + break + # If objective metrics were not reported, insert unavailable value in the DB + if not is_objective_metric_reported: + mls = [ + api_pb2.MetricLog( + time_stamp=rfc3339.rfc3339(datetime.now()), + metric=api_pb2.Metric( + name=self.metrics[0], value=const.UNAVAILABLE_METRIC_VALUE + ), + ) + ] + self.logger.info( + "Objective metric {} is not found in metrics file, {} value is reported".format( + self.metrics[0], const.UNAVAILABLE_METRIC_VALUE + ) + ) + + return api_pb2.ObservationLog(metric_logs=mls) diff --git a/scripts/v1beta1/build.sh b/scripts/v1beta1/build.sh index 3953f49f54d..b4fa896bc2e 100755 --- a/scripts/v1beta1/build.sh +++ b/scripts/v1beta1/build.sh @@ -71,6 +71,9 @@ docker buildx build --platform "linux/${ARCH}" -t "${REGISTRY}/cert-generator:${ echo -e "\nBuilding file metrics collector image...\n" docker buildx build --platform "linux/${ARCH}" -t "${REGISTRY}/file-metrics-collector:${TAG}" -f ${CMD_PREFIX}/metricscollector/${VERSION}/file-metricscollector/Dockerfile . +echo -e "\nBuilding kfpv1 metrics collector image...\n" +docker buildx build --platform "linux/${ARCH}" -t "${REGISTRY}/kfpv1-metrics-collector:${TAG}" -f ${CMD_PREFIX}/metricscollector/${VERSION}/kfp-metricscollector/v1/Dockerfile . + echo -e "\nBuilding TF Event metrics collector image...\n" if [ "${ARCH}" == "ppc64le" ]; then docker buildx build --platform "linux/${ARCH}" -t "${REGISTRY}/tfevent-metrics-collector:${TAG}" -f ${CMD_PREFIX}/metricscollector/${VERSION}/tfevent-metricscollector/Dockerfile.ppc64le . diff --git a/scripts/v1beta1/push.sh b/scripts/v1beta1/push.sh index 6f0627b4081..d8c7116552f 100755 --- a/scripts/v1beta1/push.sh +++ b/scripts/v1beta1/push.sh @@ -50,6 +50,9 @@ docker push "${REGISTRY}/cert-generator:${TAG}" echo -e "\nPushing file metrics collector image...\n" docker push "${REGISTRY}/file-metrics-collector:${TAG}" +echo -e "\nPushing kfpv1 metrics collector image...\n" +docker push "${REGISTRY}/kfpv1-metrics-collector:${TAG}" + echo -e "\nPushing TF Event metrics collector image...\n" docker push "${REGISTRY}/tfevent-metrics-collector:${TAG}" diff --git a/test/e2e/v1beta1/scripts/gh-actions/run-e2e-experiment.sh b/test/e2e/v1beta1/scripts/gh-actions/run-e2e-experiment.sh index 5a20faa6934..72ae394000f 100755 --- a/test/e2e/v1beta1/scripts/gh-actions/run-e2e-experiment.sh +++ b/test/e2e/v1beta1/scripts/gh-actions/run-e2e-experiment.sh @@ -15,7 +15,12 @@ # limitations under the License. # This shell script is used to run Katib Experiment. -# Input parameter - path to Experiment yaml. +# Input parameters +# - comma separated list of experiment names (exp1,exp2). +# For each experiment name, the script will search the folder +# `examples/v1beta1` for a file "{exp_name}.yaml" that will be +# executed as a katib experiment. Default: "" +# - namespace to execute experiment in. Default: default set -o errexit set -o nounset @@ -24,6 +29,7 @@ set -o pipefail cd "$(dirname "$0")" EXPERIMENT_FILES=${1:-""} IFS="," read -r -a EXPERIMENT_FILE_ARRAY <<< "$EXPERIMENT_FILES" +NAMESPACE=${2:-"default"} echo "Katib deployments" kubectl -n kubeflow get deploy @@ -44,7 +50,7 @@ fi for exp_name in "${EXPERIMENT_FILE_ARRAY[@]}"; do echo "Running Experiment from $exp_name file" exp_path=$(find ../../../../../examples/v1beta1 -name "${exp_name}.yaml") - python run-e2e-experiment.py --experiment-path "${exp_path}" --namespace default \ + python run-e2e-experiment.py --experiment-path "${exp_path}" --namespace "${NAMESPACE}" \ --verbose || (kubectl get pods -n kubeflow && exit 1) done diff --git a/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh b/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh index e2547e2efad..1fa62dc8f2d 100755 --- a/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh +++ b/test/e2e/v1beta1/scripts/gh-actions/setup-katib.sh @@ -23,10 +23,15 @@ cd "$(dirname "$0")" DEPLOY_KATIB_UI=${1:-false} DEPLOY_TRAINING_OPERATOR=${2:-false} WITH_DATABASE_TYPE=${3:-mysql} +# false or a specific KFP version (eg 1.8.1) +DEPLOY_KFP=${4:-false} E2E_TEST_IMAGE_TAG="e2e-test" TRAINING_OPERATOR_VERSION="v1.6.0-rc.0" +KFP_ENV=platform-agnostic-emissary +KFP_BASE_URL="github.com/kubeflow/pipelines/manifests/kustomize" + echo "Start to install Katib" # Update Katib images with `e2e-test`. @@ -61,6 +66,7 @@ if "$DEPLOY_TRAINING_OPERATOR"; then kustomize build "github.com/kubeflow/training-operator/manifests/overlays/standalone?ref=$TRAINING_OPERATOR_VERSION" | kubectl apply -f - fi + echo "Deploying Katib" cd ../../../../../ && WITH_DATABASE_TYPE=$WITH_DATABASE_TYPE make deploy && cd - @@ -80,6 +86,20 @@ kubectl -n kubeflow get svc echo "Katib pods" kubectl -n kubeflow get pod +# If the user wants to deploy kubeflow pipelines, then use the kustomization file for kubeflow pipelines. +# found at: https://github.com/kubeflow/pipelines/tree/master/manifests/kustomize +if [ $DEPLOY_KFP ]; then + KFP_VERSION="$DEPLOY_KFP" + echo "Deploying Kubeflow Pipelines version $KFP_VERSION" + kubectl apply -k "${KFP_BASE_URL}/cluster-scoped-resources/?ref=${KFP_VERSION}" + kubectl wait crd/applications.app.k8s.io --for condition=established --timeout=60s + kubectl apply -k "${KFP_BASE_URL}/env/${KFP_ENV}/?ref=${KFP_VERSION}" + kubectl wait pods -l application-crd-id=kubeflow-pipelines -n kubeflow --for condition=Ready --timeout=1800s + #kubectl port-forward -n kubeflow svc/ml-pipeline-ui 8080:80 + kubectl patch ClusterRole katib-controller -n kubeflow --type=json -p='[{"op": "add", "path": "/rules/-", "value": {"apiGroups":["argoproj.io"],"resources":["workflows"],"verbs":["get", "list", "watch", "create", "delete"]}}]' + kubectl label namespace kubeflow katib.kubeflow.org/metrics-collector-injection=enabled +fi + # Check that Katib is working with 2 Experiments. kubectl apply -f ../../testdata/valid-experiment.yaml kubectl delete -f ../../testdata/valid-experiment.yaml