diff --git a/AutoMLOps_User_Guide.pdf b/AutoMLOps_User_Guide.pdf index deeb949..dfdb265 100644 Binary files a/AutoMLOps_User_Guide.pdf and b/AutoMLOps_User_Guide.pdf differ diff --git a/CHANGELOG.md b/CHANGELOG.md index 9987e65..c80b36c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,25 +7,25 @@ All notable changes to this project will be documented in this file. Major version updates: - Code is now broken down into 5 main operations: generate, provision, deprovision, deploy, and launchAll -- Uses jinja2 templates for store code templates and write to files -- Additional package dependencies using Jinja2 templates and deploy precheck function -- Provides additional warnings and checks for given IAM permissions visibility to the user -- Creates a .gitignore by default now +- Uses jinja2 templates for storing code templates and writing to files +- Additional package dependencies using Jinja2 templates and deploying with precheck function +- Provides additional warnings and checks for giving IAM permissions visibility to the user +- Creates a .gitignore by default - Support for cloud-functions in addition to cloud-run for the submission service - Added 2 new generated folders: provision/ and services/ -- Added naming_prefix parameter to allow for +- Added naming_prefix parameter to allow for differentiating between AutoMLOps pipelines in the same project - Significant generalization in terms of tooling (now allows for specifying provisioning_frameworks, deployment_frameworks, etc.) - Renamed backend src code folder to google_cloud_automlops to avoid naming confusion -- Added enumeration and config files, which are different than previous approach of class inheritance +- Added enum and config files, which is different than previous approach of class inheritance ### Changed - Updated README.md and documentation -- Templatized code has now been pulled how and placed into jinja2 templates. -- Gitops workflow placed into separate folder and file, now will only version AutoMLOps/ directoring instead of the whole cwd. +- Templatized code has now been pulled out and placed into jinja2 templates. +- Gitops workflow placed into separate folder and file, will only version AutoMLOps/ directory instead of the whole cwd. - Reworked deployment workflow and build configuration (submission service and cloud scheduler are now created as part of the provision step). - Update notebook examples -- Changed working and branding descriptions +- Changed wording and branding descriptions - Significant updates to unit tests ### Fixed diff --git a/README.md b/README.md index 46a4542..e63241a 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,12 @@ Based on the above user selection, AutoMLOps will enable up to the following API AutoMLOps will create the following service account and update [IAM permissions](https://cloud.google.com/iam/docs/understanding-roles) during the provision step: 1. Pipeline Runner Service Account (defaults to: vertex-pipelines@PROJECT_ID.iam.gserviceaccount.com). Roles added: -- roles/aiplatform.serviceAgent +- roles/aiplatform.user +- roles/artifactregistry.reader +- roles/bigquery.user +- roles/bigquery.dataEditor +- roles/iam.serviceAccountUser +- roles/storage.admin # Prechecks and Warnings diff --git a/examples/llmops/finetuning-flan-t5/.gitignore b/examples/llmops/finetuning-flan-t5/.gitignore new file mode 100644 index 0000000..383728e --- /dev/null +++ b/examples/llmops/finetuning-flan-t5/.gitignore @@ -0,0 +1,149 @@ +.AutoMLOps-cache/ +.DS_Store +.vscode + +# Terraform +.terraform/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +build/* +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# Local Configs +config.yaml \ No newline at end of file diff --git a/examples/llmops/finetuning-flan-t5/00_llmops_finetuning_flan_t5_example.ipynb b/examples/llmops/finetuning-flan-t5/00_llmops_finetuning_flan_t5_example.ipynb index 33d2b04..cb1a0a1 100644 --- a/examples/llmops/finetuning-flan-t5/00_llmops_finetuning_flan_t5_example.ipynb +++ b/examples/llmops/finetuning-flan-t5/00_llmops_finetuning_flan_t5_example.ipynb @@ -232,7 +232,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 1, "id": "931ff517", "metadata": {}, "outputs": [], @@ -249,7 +249,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 2, "id": "be0be295", "metadata": {}, "outputs": [ @@ -271,7 +271,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 3, "id": "2c36482d", "metadata": {}, "outputs": [ @@ -279,12 +279,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Updated property [core/project].\n", - "\n", - "\n", - "To take a quick anonymous survey, run:\n", - " $ gcloud survey\n", - "\n" + "Updated property [core/project].\r\n" ] } ], @@ -302,7 +297,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "6ee3fdef", "metadata": {}, "outputs": [], @@ -320,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 5, "id": "7e55ef42", "metadata": {}, "outputs": [], @@ -331,7 +326,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 6, "id": "b7796413", "metadata": {}, "outputs": [], @@ -352,7 +347,9 @@ "id": "c7504586", "metadata": {}, "source": [ - "Create a custom serving image for running predictions using FastAPI. **Update [the server](serving/app/main.py) code with your bucket name and model_dir prefix from above.** Then build and push the custom serving image." + "Create a custom serving image for running predictions using FastAPI. **Update [the server](serving/app/main.py) code with your bucket name and model_dir prefix from above.** Then build and push the custom serving image.\n", + "\n", + "The Artifact Registry resource AF_REGISTRY_NAME must exist prior to submitting this build job." ] }, { @@ -377,7 +374,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 7, "id": "45446482", "metadata": {}, "outputs": [], @@ -387,7 +384,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 8, "id": "343ecd63", "metadata": {}, "outputs": [ @@ -396,17 +393,12 @@ "output_type": "stream", "text": [ "Creating Tensorboard\n", - "Creating Tensorboard\n", - "Create Tensorboard backing LRO: projects/45373616427/locations/us-central1/tensorboards/7295189281549582336/operations/1290875785434890240\n", - "Create Tensorboard backing LRO: projects/45373616427/locations/us-central1/tensorboards/7295189281549582336/operations/1290875785434890240\n", - "Tensorboard created. Resource name: projects/45373616427/locations/us-central1/tensorboards/7295189281549582336\n", - "Tensorboard created. Resource name: projects/45373616427/locations/us-central1/tensorboards/7295189281549582336\n", - "To use this Tensorboard in another session:\n", + "Create Tensorboard backing LRO: projects/45373616427/locations/us-central1/tensorboards/7169299598215741440/operations/3844307922502811648\n", + "Tensorboard created. Resource name: projects/45373616427/locations/us-central1/tensorboards/7169299598215741440\n", "To use this Tensorboard in another session:\n", - "tb = aiplatform.Tensorboard('projects/45373616427/locations/us-central1/tensorboards/7295189281549582336')\n", - "tb = aiplatform.Tensorboard('projects/45373616427/locations/us-central1/tensorboards/7295189281549582336')\n", + "tb = aiplatform.Tensorboard('projects/45373616427/locations/us-central1/tensorboards/7169299598215741440')\n", "flan-t5-tensorboard\n", - "projects/45373616427/locations/us-central1/tensorboards/7295189281549582336\n" + "projects/45373616427/locations/us-central1/tensorboards/7169299598215741440\n" ] } ], @@ -455,7 +447,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 9, "id": "781e2b21", "metadata": {}, "outputs": [], @@ -474,7 +466,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 10, "id": "ce8163a5", "metadata": {}, "outputs": [ @@ -501,13 +493,14 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 11, "id": "e75ae116", "metadata": {}, "outputs": [], "source": [ "@AutoMLOps.component(\n", " packages_to_install=[\n", + " 'accelerate==0.20.1',\n", " 'py7zr==0.20.4',\n", " 'nltk==3.7',\n", " 'evaluate==0.4.0',\n", @@ -735,7 +728,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 12, "id": "39c4d7b5", "metadata": {}, "outputs": [], @@ -837,7 +830,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 13, "id": "01996d4c", "metadata": {}, "outputs": [], @@ -880,7 +873,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 14, "id": "bc244ca7", "metadata": {}, "outputs": [], @@ -910,10 +903,26 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "d3162f96", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing directories under AutoMLOps/\n", + "Writing configurations to AutoMLOps/configs/defaults.yaml\n", + "Writing README.md to AutoMLOps/README.md\n", + "Writing kubeflow pipelines code to AutoMLOps/pipelines, AutoMLOps/components\n", + "Writing scripts to AutoMLOps/scripts\n", + "Writing submission service code to AutoMLOps/services\n", + "Writing gcloud provisioning code to AutoMLOps/provision\n", + "Writing cloud build config to AutoMLOps/cloudbuild.yaml\n", + "Code Generation Complete.\n" + ] + } + ], "source": [ "AutoMLOps.generate(project_id=PROJECT_ID, \n", " pipeline_params=pipeline_params, \n", @@ -937,102 +946,104 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "d26d90fe", - "metadata": {}, + "metadata": { + "scrolled": false + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "WARNING: Provisioning requires these permissions:\n", - "-cloudfunctions.functions.get\n", - "-serviceusage.services.use\n", + "-resourcemanager.projects.setIamPolicy\n", + "-pubsub.topics.list\n", "-serviceusage.services.enable\n", - "-cloudfunctions.functions.create\n", "-pubsub.subscriptions.list\n", + "-pubsub.subscriptions.create\n", + "-cloudfunctions.functions.get\n", "-cloudscheduler.jobs.list\n", - "-pubsub.topics.create\n", "-source.repos.list\n", - "-artifactregistry.repositories.create\n", - "-resourcemanager.projects.setIamPolicy\n", "-iam.serviceAccounts.listiam.serviceAccounts.create\n", - "-pubsub.subscriptions.create\n", - "-cloudscheduler.jobs.create\n", + "-artifactregistry.repositories.list\n", + "-artifactregistry.repositories.create\n", + "-storage.buckets.get\n", "-storage.buckets.create\n", "-source.repos.create\n", - "-artifactregistry.repositories.list\n", + "-cloudfunctions.functions.create\n", "-cloudbuild.builds.create\n", + "-serviceusage.services.use\n", + "-pubsub.topics.create\n", "-cloudbuild.builds.list\n", - "-pubsub.topics.list\n", - "-storage.buckets.get\n", + "-cloudscheduler.jobs.create\n", "\n", "You are currently using: srastatter@google.com. Please check your account permissions.\n", "The following are the recommended roles for provisioning:\n", + "-roles/pubsub.editor\n", "-roles/resourcemanager.projectIamAdmin\n", - "-roles/cloudfunctions.admin\n", - "-roles/artifactregistry.admin\n", - "-roles/iam.serviceAccountAdmin\n", "-roles/serviceusage.serviceUsageAdmin\n", - "-roles/aiplatform.serviceAgent\n", - "-roles/cloudscheduler.admin\n", - "-roles/pubsub.editor\n", + "-roles/iam.serviceAccountAdmin\n", "-roles/source.admin\n", + "-roles/cloudfunctions.admin\n", "-roles/cloudbuild.builds.editor\n", + "-roles/artifactregistry.admin\n", + "-roles/cloudscheduler.admin\n", + "-roles/aiplatform.serviceAgent\n", "\n", "\u001b[0;32m Setting up API services in project automlops-sandbox \u001b[0m\n", - "Operation \"operations/acat.p2-45373616427-990bb410-2998-4b37-b37f-09ed724e9519\" finished successfully.\n", + "Operation \"operations/acat.p2-45373616427-52418ef9-25d9-44eb-8afb-80a3ca19640d\" finished successfully.\n", "\u001b[0;32m Setting up Artifact Registry in project automlops-sandbox \u001b[0m\n", "Listing items under project automlops-sandbox, location us-central1.\n", "\n", - "dry-beans-dt-artifact-registry DOCKER STANDARD_REPOSITORY Artifact Registry dry-beans-dt-artifact-registry in us-central1. us-central1 Google-managed key 2023-09-05T11:25:48 2023-09-05T14:47:39 3200.712\n", - "Artifact Registry: dry-beans-dt-artifact-registry already exists in project automlops-sandbox\n", + "flan-t5-samsum-artifact-registry DOCKER STANDARD_REPOSITORY Artifact Registry flan-t5-samsum-artifact-registry in us-central1. us-central1 Google-managed key 2023-09-18T11:05:47 2023-09-18T12:46:01 8226.709\n", + "Artifact Registry: flan-t5-samsum-artifact-registry already exists in project automlops-sandbox\n", "\u001b[0;32m Setting up Storage Bucket in project automlops-sandbox \u001b[0m\n", - "gs://automlops-sandbox-dry-beans-dt-bucket/\n", - "GS Bucket: automlops-sandbox-dry-beans-dt-bucket already exists in project automlops-sandbox\n", + "gs://automlops-sandbox-flan-t5-samsum-bucket/\n", + "GS Bucket: automlops-sandbox-flan-t5-samsum-bucket already exists in project automlops-sandbox\n", "\u001b[0;32m Setting up Pipeline Job Runner Service Account in project automlops-sandbox \u001b[0m\n", "Pipeline Runner Service Account vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com False\n", "Service Account: vertex-pipelines already exists in project automlops-sandbox\n", "\u001b[0;32m Setting up IAM roles for Pipeline Job Runner Service Account in project automlops-sandbox \u001b[0m\n", "\u001b[0;32m Setting up Cloud Source Repository in project automlops-sandbox \u001b[0m\n", - "dry-beans-dt-repository automlops-sandbox https://source.developers.google.com/p/automlops-sandbox/r/dry-beans-dt-repository\n", - "Cloud Source Repository: dry-beans-dt-repository already exists in project automlops-sandbox\n", + "flan-t5-samsum-repository automlops-sandbox https://source.developers.google.com/p/automlops-sandbox/r/flan-t5-samsum-repository\n", + "Cloud Source Repository: flan-t5-samsum-repository already exists in project automlops-sandbox\n", "\u001b[0;32m Setting up Queueing Service in project automlops-sandbox \u001b[0m\n", - "name: projects/automlops-sandbox/topics/dry-beans-dt-queueing-svc\n", - "Pub/Sub Topic: dry-beans-dt-queueing-svc already exists in project automlops-sandbox\n", - "\u001b[0;32m Deploying Cloud Functions: dry-beans-dt-job-submission-svc in project automlops-sandbox \u001b[0m\n", + "name: projects/automlops-sandbox/topics/flan-t5-samsum-queueing-svc\n", + "Pub/Sub Topic: flan-t5-samsum-queueing-svc already exists in project automlops-sandbox\n", + "\u001b[0;32m Deploying Cloud Functions: flan-t5-samsum-job-submission-svc in project automlops-sandbox \u001b[0m\n", "Deploying function (may take a while - up to 2 minutes)...\n", "..\n", - "For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/0dcd7601-a440-4c7e-8ea9-28d1cf927991?project=45373616427\n", - "........................................................................done.\n", + "For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=us-central1/f3249516-f66b-416f-b39b-4a13c74698e8?project=45373616427\n", + ".................................................................done.\n", "availableMemoryMb: 512\n", - "buildId: 0dcd7601-a440-4c7e-8ea9-28d1cf927991\n", - "buildName: projects/45373616427/locations/us-central1/builds/0dcd7601-a440-4c7e-8ea9-28d1cf927991\n", + "buildId: f3249516-f66b-416f-b39b-4a13c74698e8\n", + "buildName: projects/45373616427/locations/us-central1/builds/f3249516-f66b-416f-b39b-4a13c74698e8\n", "dockerRegistry: CONTAINER_REGISTRY\n", "entryPoint: process_request\n", "eventTrigger:\n", " eventType: google.pubsub.topic.publish\n", " failurePolicy: {}\n", - " resource: projects/automlops-sandbox/topics/dry-beans-dt-queueing-svc\n", + " resource: projects/automlops-sandbox/topics/flan-t5-samsum-queueing-svc\n", " service: pubsub.googleapis.com\n", "ingressSettings: ALLOW_ALL\n", "labels:\n", " deployment-tool: cli-gcloud\n", "maxInstances: 3000\n", - "name: projects/automlops-sandbox/locations/us-central1/functions/dry-beans-dt-job-submission-svc\n", + "name: projects/automlops-sandbox/locations/us-central1/functions/flan-t5-samsum-job-submission-svc\n", "runtime: python39\n", "serviceAccountEmail: vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com\n", - "sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/a2ee896a-a9dc-41ee-a129-fc000fc59dc9.zip\n", + "sourceUploadUrl: https://storage.googleapis.com/uploads-961973632599.us-central1.cloudfunctions.appspot.com/7deec24d-6f1f-4ef3-991e-5058eec4b776.zip\n", "status: ACTIVE\n", "timeout: 540s\n", - "updateTime: '2023-09-08T03:06:05.246Z'\n", - "versionId: '2'\n", + "updateTime: '2023-09-18T17:15:39.455Z'\n", + "versionId: '4'\n", "\u001b[0;32m Setting up Cloud Build Trigger in project automlops-sandbox \u001b[0m\n", - "name: dry-beans-dt-build-trigger\n", - "Cloudbuild Trigger already exists in project automlops-sandbox for repo dry-beans-dt-repository\n", + "name: flan-t5-samsum-build-trigger\n", + "Cloudbuild Trigger already exists in project automlops-sandbox for repo flan-t5-samsum-repository\n", "\u001b[0;32m Setting up Cloud Scheduler Job in project automlops-sandbox \u001b[0m\n", - "dry-beans-dt-schedule us-central1 59 11 * * 0 (Etc/UTC) Pub/Sub ENABLED\n", - "Cloud Scheduler Job: dry-beans-dt-schedule already exists in project automlops-sandbox\n" + "flan-t5-samsum-schedule us-central1 59 11 * * 0 (Etc/UTC) Pub/Sub ENABLED\n", + "Cloud Scheduler Job: flan-t5-samsum-schedule already exists in project automlops-sandbox\n" ] } ], @@ -1050,7 +1061,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "17ceb009", "metadata": {}, "outputs": [ @@ -1060,27 +1071,27 @@ "text": [ "WARNING: Running precheck for deploying requires these permissions:\n", "-artifactregistry.repositories.get\n", - "-cloudbuild.builds.get\n", + "-pubsub.topics.get\n", "-resourcemanager.projects.getIamPolicy\n", + "-cloudfunctions.functions.get\n", "-storage.buckets.update\n", + "-source.repos.update\n", + "-cloudbuild.builds.get\n", "-serviceusage.services.get\n", - "-cloudfunctions.functions.get\n", - "-pubsub.topics.get\n", "-iam.serviceAccounts.get\n", - "-source.repos.update\n", "-pubsub.subscriptions.get\n", "\n", "You are currently using: srastatter@google.com. Please check your account permissions.\n", "The following are the recommended roles for deploying with precheck:\n", - "-roles/serviceusage.serviceUsageViewer\n", "-roles/iam.roleViewer\n", + "-roles/iam.serviceAccountUser\n", + "-roles/serviceusage.serviceUsageViewer\n", "-roles/pubsub.viewer\n", "-roles/storage.admin\n", - "-roles/cloudbuild.builds.editor\n", "-roles/source.writer\n", - "-roles/iam.serviceAccountUser\n", - "-roles/cloudfunctions.viewer\n", "-roles/artifactregistry.reader\n", + "-roles/cloudbuild.builds.editor\n", + "-roles/cloudfunctions.viewer\n", "\n", "Checking for required API services in project automlops-sandbox...\n", "Checking for Artifact Registry in project automlops-sandbox...\n", @@ -1094,13 +1105,11 @@ "Checking for Cloud Build Trigger in project automlops-sandbox...\n", "Precheck successfully completed, continuing to deployment.\n", "\n", - "[automlops 1fa5664] Run AutoMLOps\n", - " 7 files changed, 34 insertions(+), 228 deletions(-)\n", - " delete mode 100644 AutoMLOps/components/component_base/src/custom_train_model.py\n", - " delete mode 100644 AutoMLOps/components/custom_train_model/component.yaml\n", - "remote: Waiting for private key checker: 5/5 objects left \n", - "To https://source.developers.google.com/p/automlops-sandbox/r/dry-beans-dt-repository\n", - " 70a23dd..1fa5664 automlops -> automlops\n", + "[automlops d4ac2e3] Run AutoMLOps\n", + " 3 files changed, 15 insertions(+), 14 deletions(-)\n", + "remote: Waiting for private key checker: 3/3 objects left \n", + "To https://source.developers.google.com/p/automlops-sandbox/r/flan-t5-samsum-repository\n", + " 4788c2d..d4ac2e3 automlops -> automlops\n", "Pushing code to automlops branch, triggering build...\n", "Cloud Build job running at: https://console.cloud.google.com/cloud-build/builds;region=us-central1\n", "Please wait for this build job to complete.\n", @@ -1113,16 +1122,16 @@ "# #\n", "#################################################################\n", "\n", - "Google Cloud Storage Bucket: https://console.cloud.google.com/storage/automlops-sandbox-dry-beans-dt-bucket\n", - "Artifact Registry: https://console.cloud.google.com/artifacts/docker/automlops-sandbox/us-central1/dry-beans-dt-artifact-registry\n", + "Google Cloud Storage Bucket: https://console.cloud.google.com/storage/automlops-sandbox-flan-t5-samsum-bucket\n", + "Artifact Registry: https://console.cloud.google.com/artifacts/docker/automlops-sandbox/us-central1/flan-t5-samsum-artifact-registry\n", "Service Accounts: https://console.cloud.google.com/iam-admin/serviceaccounts?project=automlops-sandbox\n", "APIs: https://console.cloud.google.com/apis\n", - "Cloud Source Repository: https://source.cloud.google.com/automlops-sandbox/dry-beans-dt-repository/+/automlops:\n", + "Cloud Source Repository: https://source.cloud.google.com/automlops-sandbox/flan-t5-samsum-repository/+/automlops:\n", "Cloud Build Jobs: https://console.cloud.google.com/cloud-build/builds;region=us-central1\n", "Vertex AI Pipeline Runs: https://console.cloud.google.com/vertex-ai/pipelines/runs\n", "Cloud Build Trigger: https://console.cloud.google.com/cloud-build/triggers;region=us-central1\n", - "Pipeline Job Submission Service (Cloud Functions): https://console.cloud.google.com/functions/details/us-central1/dry-beans-dt-job-submission-svc\n", - "Pub/Sub Queueing Service Topic: https://console.cloud.google.com/cloudpubsub/topic/detail/dry-beans-dt-queueing-svc\n", + "Pipeline Job Submission Service (Cloud Functions): https://console.cloud.google.com/functions/details/us-central1/flan-t5-samsum-job-submission-svc\n", + "Pub/Sub Queueing Service Topic: https://console.cloud.google.com/cloudpubsub/topic/detail/flan-t5-samsum-queueing-svc\n", "Pub/Sub Queueing Service Subscriptions: https://console.cloud.google.com/cloudpubsub/subscription/list\n", "Cloud Scheduler Job: https://console.cloud.google.com/cloudscheduler\n" ] @@ -1150,7 +1159,7 @@ "uri": "gcr.io/deeplearning-platform-release/base-cpu:m102" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -1164,7 +1173,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.12" + "version": "3.9.6" }, "vscode": { "interpreter": { diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/README.md b/examples/llmops/finetuning-flan-t5/AutoMLOps/README.md deleted file mode 100644 index aeb197c..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/README.md +++ /dev/null @@ -1,37 +0,0 @@ -# AutoMLOps - Generated Code Directory - -**Note: This directory contains code generated using AutoMLOps** - -AutoMLOps is a service that generates a production ready MLOps pipeline from Jupyter Notebooks, bridging the gap between Data Science and DevOps and accelerating the adoption and use of Vertex AI. The service generates an MLOps codebase for users to customize, and provides a way to build and manage a CI/CD integrated MLOps pipeline from the notebook. AutoMLOps automatically builds a source repo for versioning, cloudbuild configs and triggers, an artifact registry for storing custom components, gs buckets, service accounts and updated IAM privs for running pipelines, enables APIs (cloud Run, Cloud Build, Artifact Registry, etc.), creates a runner service API in Cloud Run for submitting PipelineJobs to Vertex AI, and a Cloud Scheduler job for submitting PipelineJobs on a recurring basis. These automatic integrations empower data scientists to take their experiments to production more quickly, allowing them to focus on what they do best: providing actionable insights through data. - -# User Guide - -For a user-guide, please view these [slides](https://github.com/GoogleCloudPlatform/automlops/blob/main/AutoMLOps_Implementation_Guide_External.pdf). - -# Layout - -```bash -. -├── cloud_run : Cloud Runner service for submitting PipelineJobs. - ├──run_pipeline : Contains main.py file, Dockerfile and requirements.txt - ├──queueing_svc : Contains files for scheduling and queueing jobs to runner service -├── components : Custom vertex pipeline components. - ├──component_base : Contains all the python files, Dockerfile and requirements.txt - ├──component_a : Components generated using AutoMLOps - ├──... -├── images : Custom container images for training models. -├── pipelines : Vertex ai pipeline definitions. - ├── pipeline.py : Full pipeline definition. - ├── pipeline_runner.py : Sends a PipelineJob to Vertex AI. - ├── runtime_parameters : Variables to be used in a PipelineJob. - ├── pipeline_parameter_values.json : Json containing pipeline parameters. -├── configs : Configurations for defining vertex ai pipeline. - ├── defaults.yaml : PipelineJob configuration variables. -├── scripts : Scripts for manually triggering the cloud run service. - ├── build_components.sh : Submits a Cloud Build job that builds and deploys the components. - ├── build_pipeline_spec.sh : Builds the pipeline specs. - ├── create_resources.sh : Creates an artifact registry and gs bucket if they do not already exist. - ├── run_pipeline.sh : Submit the PipelineJob to Vertex AI. - ├── run_all.sh : Builds components, pipeline specs, and submits the PipelineJob. -└── cloudbuild.yaml : Cloudbuild configuration file for building custom components. -``` diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/main.py b/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/main.py deleted file mode 100644 index a62e485..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/main.py +++ /dev/null @@ -1,190 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -"""Submit pipeline job using Cloud Tasks and create Cloud Scheduler Job.""" -import argparse -import json - -from google.cloud import run_v2 -from google.cloud import scheduler_v1 -from google.cloud import tasks_v2 - -CLOUD_RUN_LOCATION = 'us-central1' -CLOUD_RUN_NAME = 'run-pipeline' -CLOUD_TASKS_QUEUE_LOCATION = 'us-central1' -CLOUD_TASKS_QUEUE_NAME = 'queueing-svc' -PARAMETER_VALUES_PATH = 'queueing_svc/pipeline_parameter_values.json' -PIPELINE_RUNNER_SA = 'vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com' -PROJECT_ID = 'automlops-sandbox' -SCHEDULE_LOCATION = 'us-central1' -SCHEDULE_PATTERN = '59 11 * * 0' -SCHEDULE_NAME = 'AutoMLOps-schedule' - -def get_runner_svc_uri( - cloud_run_location: str, - cloud_run_name: str, - project_id: str): - """Fetches the uri for the given cloud run instance. - - Args: - cloud_run_location: The location of the cloud runner service. - cloud_run_name: The name of the cloud runner service. - project_id: The project ID. - Returns: - str: Uri of the Cloud Run instance. - """ - client = run_v2.ServicesClient() - parent = client.service_path(project_id, cloud_run_location, cloud_run_name) - request = run_v2.GetServiceRequest(name=parent) - response = client.get_service(request=request) - return response.uri - -def get_json_bytes(file_path: str): - """Reads a json file at the specified path and returns as bytes. - - Args: - file_path: Path of the json file. - Returns: - bytes: Encode bytes of the file. - """ - try: - with open(file_path, 'r', encoding='utf-8') as file: - data = json.load(file) - file.close() - except OSError as err: - raise Exception(f'Error reading json file. {err}') from err - return json.dumps(data).encode() - -def create_cloud_task( - cloud_tasks_queue_location: str, - cloud_tasks_queue_name: str, - parameter_values_path: str, - pipeline_runner_sa: str, - project_id: str, - runner_svc_uri: str): - """Create a task to the queue with the runtime parameters. - - Args: - cloud_run_location: The location of the cloud runner service. - cloud_run_name: The name of the cloud runner service. - cloud_tasks_queue_location: The location of the cloud tasks queue. - cloud_tasks_queue_name: The name of the cloud tasks queue. - parameter_values_path: Path to json pipeline params. - pipeline_runner_sa: Service Account to runner PipelineJobs. - project_id: The project ID. - runner_svc_uri: Uri of the Cloud Run instance. - """ - client = tasks_v2.CloudTasksClient() - parent = client.queue_path(project_id, cloud_tasks_queue_location, cloud_tasks_queue_name) - task = { - 'http_request': { - 'http_method': tasks_v2.HttpMethod.POST, - 'url': runner_svc_uri, - 'oidc_token': { - 'service_account_email': pipeline_runner_sa, - 'audience': runner_svc_uri - }, - 'headers': { - 'Content-Type': 'application/json' - } - } - } - task['http_request']['body'] = get_json_bytes(parameter_values_path) - response = client.create_task(request={'parent': parent, 'task': task}) - print(f'Created task {response.name}') - -def create_cloud_scheduler_job( - parameter_values_path: str, - pipeline_runner_sa: str, - project_id: str, - runner_svc_uri: str, - schedule_location: str, - schedule_name: str, - schedule_pattern: str): - """Creates a scheduled pipeline job. - - Args: - parameter_values_path: Path to json pipeline params. - pipeline_runner_sa: Service Account to runner PipelineJobs. - project_id: The project ID. - runner_svc_uri: Uri of the Cloud Run instance. - schedule_location: The location of the scheduler resource. - schedule_name: The name of the scheduler resource. - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - """ - client = scheduler_v1.CloudSchedulerClient() - parent = f'projects/{project_id}/locations/{schedule_location}' - name = f'{parent}/jobs/{schedule_name}' - - request = scheduler_v1.ListJobsRequest(parent=parent) - page_result = client.list_jobs(request=request) - for response in page_result: - if response.name == name: - print(f'Cloud Scheduler {schedule_name} resource already exists in ' - f'project {project_id}.') - return - - oidc_token = scheduler_v1.OidcToken( - service_account_email=pipeline_runner_sa, - audience=runner_svc_uri) - - target = scheduler_v1.HttpTarget( - uri=runner_svc_uri, - http_method=scheduler_v1.HttpMethod(1), # HTTP POST - headers={'Content-Type': 'application/json'}, - body=get_json_bytes(parameter_values_path), - oidc_token=oidc_token) - - job = scheduler_v1.Job( - name=f'{parent}/jobs/{schedule_name}', - description='AutoMLOps cloud scheduled run.', - http_target=target, - schedule=schedule_pattern) - - request = scheduler_v1.CreateJobRequest( - parent=parent, - job=job) - - response = client.create_job(request=request) - print(response) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--setting', type=str, - help='The config file for setting default values.') - args = parser.parse_args() - - uri = get_runner_svc_uri( - cloud_run_location=CLOUD_RUN_LOCATION, - cloud_run_name=CLOUD_RUN_NAME, - project_id=PROJECT_ID) - - if args.setting == 'queue_job': - create_cloud_task( - cloud_tasks_queue_location=CLOUD_TASKS_QUEUE_LOCATION, - cloud_tasks_queue_name=CLOUD_TASKS_QUEUE_NAME, - parameter_values_path=PARAMETER_VALUES_PATH, - pipeline_runner_sa=PIPELINE_RUNNER_SA, - project_id=PROJECT_ID, - runner_svc_uri=uri) - - if args.setting == 'schedule_job': - create_cloud_scheduler_job( - parameter_values_path=PARAMETER_VALUES_PATH, - pipeline_runner_sa=PIPELINE_RUNNER_SA, - project_id=PROJECT_ID, - runner_svc_uri=uri, - schedule_location=SCHEDULE_LOCATION, - schedule_name=SCHEDULE_NAME, - schedule_pattern=SCHEDULE_PATTERN) diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/pipeline_parameter_values.json b/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/pipeline_parameter_values.json deleted file mode 100644 index e515968..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/pipeline_parameter_values.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "endpoint_sa": "vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com", - "project_id": "automlops-sandbox", - "model_dir": "gs://automlops-sandbox-bucket/flan_t5_model/", - "lr": 5e-05, - "epochs": 5, - "logging_steps": 500, - "serving_image_tag": "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/finetuning_flan_t5_base:latest", - "eval_batch": 4, - "region": "us-central1", - "train_batch": 4 -} \ No newline at end of file diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/requirements.txt b/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/requirements.txt deleted file mode 100644 index 06069a4..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/queueing_svc/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -google-cloud -google-cloud-tasks -google-api-python-client -google-cloud-run -google-cloud-scheduler diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/Dockerfile b/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/Dockerfile deleted file mode 100644 index 18ad115..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -FROM python:3.9-slim - -# Allow statements and log messages to immediately appear in the Knative logs -ENV PYTHONUNBUFFERED True - -# Copy local code to the container image. -ENV APP_HOME /app -WORKDIR $APP_HOME -COPY ./ ./ - -# Upgrade pip -RUN python -m pip install --upgrade pip -# Install requirements -RUN pip install --no-cache-dir -r /app/cloud_run/run_pipeline/requirements.txt -# Compile pipeline spec -RUN ./scripts/build_pipeline_spec.sh -# Change Directories -WORKDIR "/app/cloud_run/run_pipeline" -# Run flask api server -CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/main.py b/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/main.py deleted file mode 100644 index f9ca3f0..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/main.py +++ /dev/null @@ -1,104 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -"""Cloud Run to run pipeline spec""" -import logging -import os -from typing import Tuple - -import flask -from google.cloud import aiplatform -import yaml - -app = flask.Flask(__name__) - -logger = logging.getLogger() -log_level = os.environ.get('LOG_LEVEL', 'INFO') -logger.setLevel(log_level) - -CONFIG_FILE = '../../configs/defaults.yaml' -PIPELINE_SPEC_PATH_LOCAL = '../../scripts/pipeline_spec/pipeline_job.json' - -@app.route('/', methods=['POST']) -def process_request() -> flask.Response: - """HTTP web service to trigger pipeline execution. - - Returns: - The response text, or any set of values that can be turned into a - Response object using `make_response` - . - """ - content_type = flask.request.headers['content-type'] - if content_type == 'application/json': - request_json = flask.request.json - - logging.debug('JSON Recieved:') - logging.debug(request_json) - - with open(CONFIG_FILE, 'r', encoding='utf-8') as config_file: - config = yaml.load(config_file, Loader=yaml.FullLoader) - - logging.debug('Calling run_pipeline()') - dashboard_uri, resource_name = run_pipeline( - project_id=config['gcp']['project_id'], - pipeline_root=config['pipelines']['pipeline_storage_path'], - pipeline_runner_sa=config['gcp']['pipeline_runner_service_account'], - pipeline_params=request_json, - pipeline_spec_path=PIPELINE_SPEC_PATH_LOCAL) - return flask.make_response({ - 'dashboard_uri': dashboard_uri, - 'resource_name': resource_name - }, 200) - - else: - raise ValueError(f'Unknown content type: {content_type}') - -def run_pipeline( - project_id: str, - pipeline_root: str, - pipeline_runner_sa: str, - pipeline_params: dict, - pipeline_spec_path: str, - display_name: str = 'mlops-pipeline-run', - enable_caching: bool = False) -> Tuple[str, str]: - """Executes a pipeline run. - - Args: - project_id: The project_id. - pipeline_root: GCS location of the pipeline runs metadata. - pipeline_runner_sa: Service Account to runner PipelineJobs. - pipeline_params: Pipeline parameters values. - pipeline_spec_path: Location of the pipeline spec JSON. - display_name: Name to call the pipeline. - enable_caching: Should caching be enabled (Boolean) - """ - logging.debug('Pipeline Parms Configured:') - logging.debug(pipeline_params) - - aiplatform.init(project=project_id) - job = aiplatform.PipelineJob( - display_name = display_name, - template_path = pipeline_spec_path, - pipeline_root = pipeline_root, - parameter_values = pipeline_params, - enable_caching = enable_caching) - logging.debug('AI Platform job built. Submitting...') - job.submit(service_account=pipeline_runner_sa) - logging.debug('Job sent!') - dashboard_uri = job._dashboard_uri() - resource_name = job.resource_name - return dashboard_uri, resource_name - -if __name__ == '__main__': - app.run(debug=True, host='0.0.0.0', port=int(os.environ.get('PORT', 8080))) diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/requirements.txt b/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/requirements.txt deleted file mode 100644 index 2e0f4c4..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloud_run/run_pipeline/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -kfp<2.0.0 -google-cloud-aiplatform -google-cloud-pipeline-components -Flask -gunicorn -pyyaml diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloudbuild.yaml b/examples/llmops/finetuning-flan-t5/AutoMLOps/cloudbuild.yaml deleted file mode 100644 index d2565e7..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/cloudbuild.yaml +++ /dev/null @@ -1,103 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -steps: -# ============================================================================== -# BUILD CUSTOM IMAGES -# ============================================================================== - - # build the component_base image - - name: "gcr.io/cloud-builders/docker" - args: [ "build", "-t", "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/components/component_base:latest", "." ] - dir: "AutoMLOps/components/component_base" - id: "build_component_base" - waitFor: ["-"] - - # build the run_pipeline image - - name: 'gcr.io/cloud-builders/docker' - args: [ "build", "-t", "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/run_pipeline:latest", "-f", "cloud_run/run_pipeline/Dockerfile", "." ] - dir: "AutoMLOps/" - id: "build_pipeline_runner_svc" - waitFor: ['build_component_base'] - -# ============================================================================== -# PUSH & DEPLOY CUSTOM IMAGES -# ============================================================================== - - # push the component_base image - - name: "gcr.io/cloud-builders/docker" - args: ["push", "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/components/component_base:latest"] - dir: "AutoMLOps/components/component_base" - id: "push_component_base" - waitFor: ["build_pipeline_runner_svc"] - - # push the run_pipeline image - - name: "gcr.io/cloud-builders/docker" - args: ["push", "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/run_pipeline:latest"] - dir: "AutoMLOps/" - id: "push_pipeline_runner_svc" - waitFor: ["push_component_base"] - - # deploy the cloud run service - - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" - entrypoint: gcloud - args: ["run", - "deploy", - "run-pipeline", - "--image", - "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/run_pipeline:latest", - "--region", - "us-central1", - "--service-account", - "vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com",] - id: "deploy_pipeline_runner_svc" - waitFor: ["push_pipeline_runner_svc"] - - # Copy runtime parameters - - name: 'gcr.io/cloud-builders/gcloud' - entrypoint: bash - args: - - '-e' - - '-c' - - | - cp -r AutoMLOps/cloud_run/queueing_svc . - id: "setup_queueing_svc" - waitFor: ["deploy_pipeline_runner_svc"] - - # Install dependencies - - name: python - entrypoint: pip - args: ["install", "-r", "queueing_svc/requirements.txt", "--user"] - id: "install_queueing_svc_deps" - waitFor: ["setup_queueing_svc"] - - # Submit to queue - - name: python - entrypoint: python - args: ["queueing_svc/main.py", "--setting", "queue_job"] - id: "submit_job_to_queue" - waitFor: ["install_queueing_svc_deps"] - - # Create Scheduler Job - - name: python - entrypoint: python - args: ["queueing_svc/main.py", "--setting", "schedule_job"] - id: "schedule_job" - waitFor: ["submit_job_to_queue"] - -images: - # custom component images - - "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/components/component_base:latest" - # Cloud Run image - - "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/run_pipeline:latest" diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/Dockerfile b/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/Dockerfile deleted file mode 100644 index ffbe705..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -FROM us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.1-10:latest -RUN python -m pip install --upgrade pip -COPY requirements.txt . -RUN python -m pip install -r \ - requirements.txt --quiet --no-cache-dir \ - && rm -f requirements.txt -COPY ./src /pipelines/component/src -ENTRYPOINT ["/bin/bash"] diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/requirements.txt b/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/requirements.txt deleted file mode 100644 index e981d58..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -datasets==2.9.0 -evaluate==0.4.0 -google-cloud-aiplatform==1.26.0 -google-cloud-storage==2.7.0 -kfp<2.0.0 -nltk==3.7 -py7zr==0.20.4 -rouge_score==0.1.2 -tensorboard==2.11.2 -transformers==4.25.1 diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/src/deploy_and_test_model.py b/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/src/deploy_and_test_model.py deleted file mode 100644 index bb967c3..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/src/deploy_and_test_model.py +++ /dev/null @@ -1,118 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -import argparse -import json -from kfp.v2.components import executor - -import kfp -from kfp.v2 import dsl -from kfp.v2.dsl import * -from typing import * - -def deploy_and_test_model( - endpoint_sa: str, - project_id: str, - region: str, - serving_image_tag: str -): - """Custom component that uploads a finetuned Flan-T5 from GCS to Vertex Model Registry, - deploys the model to an endpoint for online prediction, and runs a prediction test. - - Args: - endpoint_sa: Service account to run the endpoint prediction service with. - project_id: Project_id. - region: Region. - serving_image_tag: Custom serving image uri. - """ - import pprint as pp - from random import randrange - - from google.cloud import aiplatform - - from datasets import load_dataset - - DATASET_ID = 'samsum' - - aiplatform.init(project=project_id, location=region) - # Check if model exists - models = aiplatform.Model.list() - model_name = 'finetuned-flan-t5' - if 'finetuned-flan-t5' in (m.name for m in models): - parent_model = model_name - model_id = None - is_default_version=False - version_aliases=['experimental', 'finetuned', 'flan-t5'] - version_description='experimental version' - else: - parent_model = None - model_id = model_name - is_default_version=True - version_aliases=['live', 'finetuned', 'flan-t5'] - version_description='live version' - - uploaded_model = aiplatform.Model.upload( - model_id=model_id, - display_name=model_name, - parent_model=parent_model, - is_default_version=is_default_version, - version_aliases=version_aliases, - version_description=version_description, - serving_container_image_uri=serving_image_tag, - serving_container_predict_route='/predict', - serving_container_health_route='/health', - serving_container_ports=[8080], - labels={'created_by': 'automlops-team'}, - ) - - endpoint = uploaded_model.deploy( - machine_type='n1-standard-8', - min_replica_count=1, - max_replica_count=1, - accelerator_type='NVIDIA_TESLA_V100', - accelerator_count=1, - service_account=endpoint_sa, # This SA needs gcs permissions - sync=True - ) - - # Load dataset from the hub - dataset = load_dataset(DATASET_ID) - # select a random test sample - sample = dataset['test'][randrange(len(dataset["test"]))] - - # Test predictions - print('running prediction test...') - try: - resp = endpoint.predict([[sample['dialogue']]]) - print(sample['dialogue']) - pp.pprint(resp) - except Exception as ex: - print('prediction request failed', ex) - -def main(): - """Main executor.""" - parser = argparse.ArgumentParser() - parser.add_argument('--executor_input', type=str) - parser.add_argument('--function_to_execute', type=str) - - args, _ = parser.parse_known_args() - executor_input = json.loads(args.executor_input) - function_to_execute = globals()[args.function_to_execute] - - executor.Executor( - executor_input=executor_input, - function_to_execute=function_to_execute).execute() - -if __name__ == '__main__': - main() diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/src/finetune_t5_model.py b/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/src/finetune_t5_model.py deleted file mode 100644 index cff65e6..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/component_base/src/finetune_t5_model.py +++ /dev/null @@ -1,243 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -import argparse -import json -from kfp.v2.components import executor - -import kfp -from kfp.v2 import dsl -from kfp.v2.dsl import * -from typing import * - -def finetune_t5_model( - model_dir: str, - epochs: int, - eval_batch: int, - logging_steps: int, - lr: float, - train_batch: int -): - """Custom component that finetunes a Flan T5 base model. - - Args: - model_dir: GCS directory to save the model and training artifacts. - epochs: Total number of training epochs to perform. - eval_batch: The batch size per GPU/TPU core/CPU for evaluation. - logging_steps: Number of update steps between two logs. - lr: The initial learning rate for AdamW optimizer. - train_batch: The batch size per GPU/TPU core/CPU for training. - """ - import glob - import logging - import os - - from google.cloud import storage - - from datasets import concatenate_datasets, load_dataset - from huggingface_hub import HfFolder - from transformers import ( - AutoTokenizer, - AutoModelForSeq2SeqLM, - DataCollatorForSeq2Seq, - Seq2SeqTrainer, - Seq2SeqTrainingArguments - ) - from transformers.integrations import TensorBoardCallback - import evaluate - import nltk - import numpy as np - from nltk.tokenize import sent_tokenize - - MODEL_ID='google/flan-t5-base' - DATASET_ID = 'samsum' - - def preprocess_function(sample, padding='max_length'): - # add prefix to the input for t5 - inputs = ['summarize: ' + item for item in sample['dialogue']] - - # tokenize inputs - model_inputs = tokenizer(inputs, max_length=max_source_length, padding=padding, truncation=True) - - # Tokenize targets with the `text_target` keyword argument - labels = tokenizer(text_target=sample['summary'], max_length=max_target_length, padding=padding, truncation=True) - - # If we are padding here, replace all tokenizer.pad_token_id in the labels by -100 when we want to ignore - # padding in the loss. - if padding == 'max_length': - labels['input_ids'] = [ - [(l if l != tokenizer.pad_token_id else -100) for l in label] for label in labels['input_ids'] - ] - - model_inputs['labels'] = labels['input_ids'] - return model_inputs - - # helper function to postprocess text - def postprocess_text(preds, labels): - preds = [pred.strip() for pred in preds] - labels = [label.strip() for label in labels] - - # rougeLSum expects newline after each sentence - preds = ['\n'.join(sent_tokenize(pred)) for pred in preds] - labels = ['\n'.join(sent_tokenize(label)) for label in labels] - - return preds, labels - - def compute_metrics(eval_preds): - preds, labels = eval_preds - if isinstance(preds, tuple): - preds = preds[0] - decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True) - # Replace -100 in the labels as we can't decode them. - labels = np.where(labels != -100, labels, tokenizer.pad_token_id) - decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) - - # Some simple post-processing - decoded_preds, decoded_labels = postprocess_text(decoded_preds, decoded_labels) - - result = metric.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True) - result = {k: round(v * 100, 4) for k, v in result.items()} - prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in preds] - result['gen_len'] = np.mean(prediction_lens) - return result - - def upload_to_gcs(local_directory_path: str, gs_directory_path: str): - client = storage.Client() - - # extract GCS bucket_name - bucket_name = gs_directory_path.split('/')[2] # without gs:// - # extract GCS object_name - object_name = '/'.join(gs_directory_path.split('/')[3:]) - - rel_paths = glob.glob(local_directory_path + '/**', recursive=True) - bucket = client.get_bucket(bucket_name) - for local_file in rel_paths: - remote_path = f'''{object_name}{'/'.join(local_file.split(os.sep)[1:])}''' - logging.info(remote_path) - if os.path.isfile(local_file): - blob = bucket.blob(remote_path) - blob.upload_from_filename(local_file) - - # Load dataset - dataset = load_dataset(DATASET_ID) - # Load tokenizer of FLAN-t5-base - tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) - # load model from the hub - model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_ID) - - nltk.download('punkt') - # Metric - metric = evaluate.load('rouge') - - # Hugging Face repository id - repository_id = f'''{MODEL_ID.split('/')[1]}-{DATASET_ID}''' - - # The maximum total input sequence length after tokenization. - # Sequences longer than this will be truncated, sequences shorter will be padded. - tokenized_inputs = concatenate_datasets([dataset['train'], - dataset['test']]).map(lambda x: tokenizer(x['dialogue'],truncation=True), - batched=True, remove_columns=['dialogue', 'summary']) - max_source_length = max([len(x) for x in tokenized_inputs['input_ids']]) - print(f'Max source length: {max_source_length}') - - # The maximum total sequence length for target text after tokenization. - # Sequences longer than this will be truncated, sequences shorter will be padded." - tokenized_targets = concatenate_datasets([dataset['train'], - dataset['test']]).map(lambda x: tokenizer(x['summary'], truncation=True), - batched=True, remove_columns=['dialogue', 'summary']) - max_target_length = max([len(x) for x in tokenized_targets['input_ids']]) - print(f'Max target length: {max_target_length}') - - tokenized_dataset = dataset.map(preprocess_function, batched=True, remove_columns=['dialogue', 'summary', 'id']) - print(f'''Keys of tokenized dataset: {list(tokenized_dataset['train'].features)}''') - - # we want to ignore tokenizer pad token in the loss - label_pad_token_id = -100 - # Data collator - data_collator = DataCollatorForSeq2Seq( - tokenizer, - model=model, - label_pad_token_id=label_pad_token_id, - pad_to_multiple_of=8 - ) - - # Define training args - training_args = Seq2SeqTrainingArguments( - output_dir=repository_id, - per_device_train_batch_size=train_batch, - per_device_eval_batch_size=eval_batch, - predict_with_generate=True, - fp16=False, # Overflows with fp16 - learning_rate=lr, - num_train_epochs=epochs, - # logging & evaluation strategies - logging_dir=os.environ['AIP_TENSORBOARD_LOG_DIR'], - #logging_dir=f'{repository_id}/logs', - logging_strategy='steps', - logging_steps=logging_steps, - evaluation_strategy='epoch', - save_strategy='epoch', - save_total_limit=2, - load_best_model_at_end=True, - # metric_for_best_model="overall_f1", - # push to hub parameters - report_to='tensorboard', - push_to_hub=False, - hub_strategy='every_save', - hub_model_id=repository_id, - hub_token=HfFolder.get_token(), - ) - - # Create Trainer instance - trainer = Seq2SeqTrainer( - model=model, - args=training_args, - train_dataset=tokenized_dataset['train'], - eval_dataset=tokenized_dataset['test'], - compute_metrics=compute_metrics, - callbacks=[TensorBoardCallback()] - ) - - # Start training - logging.info('Training ....') - trainer.train() - trainer.evaluate() - - # Save tokenizer and model locally - tokenizer.save_pretrained(f'model_tokenizer') - trainer.save_model(f'model_output') - - logging.info('Saving model and tokenizer to GCS ....') - - # Upload model to GCS - upload_to_gcs('model_output', model_dir) - # Upload tokenizer to GCS - upload_to_gcs('model_tokenizer', model_dir) - -def main(): - """Main executor.""" - parser = argparse.ArgumentParser() - parser.add_argument('--executor_input', type=str) - parser.add_argument('--function_to_execute', type=str) - - args, _ = parser.parse_known_args() - executor_input = json.loads(args.executor_input) - function_to_execute = globals()[args.function_to_execute] - - executor.Executor( - executor_input=executor_input, - function_to_execute=function_to_execute).execute() - -if __name__ == '__main__': - main() diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/deploy_and_test_model/component.yaml b/examples/llmops/finetuning-flan-t5/AutoMLOps/components/deploy_and_test_model/component.yaml deleted file mode 100644 index 7c11ed2..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/deploy_and_test_model/component.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -name: deploy_and_test_model -description: Custom component that uploads a finetuned Flan-T5 from GCS to Vertex - Model Registry, -inputs: -- name: endpoint_sa - description: Service account to run the endpoint prediction service with. - type: String -- name: project_id - description: Project_id. - type: String -- name: region - description: Region. - type: String -- name: serving_image_tag - description: Custom serving image uri. - type: String -implementation: - container: - image: us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/components/component_base:latest - command: - - python3 - - /pipelines/component/src/deploy_and_test_model.py - args: - - --executor_input - - executorInput: null - - --function_to_execute - - deploy_and_test_model diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/finetune_t5_model/component.yaml b/examples/llmops/finetuning-flan-t5/AutoMLOps/components/finetune_t5_model/component.yaml deleted file mode 100644 index 6c8e8d4..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/components/finetune_t5_model/component.yaml +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -name: finetune_t5_model -description: Custom component that finetunes a Flan T5 base model. -inputs: -- name: model_dir - description: GCS directory to save the model and training artifacts. - type: String -- name: epochs - description: Total number of training epochs to perform. - type: Integer -- name: eval_batch - description: The batch size per GPU/TPU core/CPU for evaluation. - type: Integer -- name: logging_steps - description: Number of update steps between two logs. - type: Integer -- name: lr - description: The initial learning rate for AdamW optimizer. - type: Float -- name: train_batch - description: The batch size per GPU/TPU core/CPU for training. - type: Integer -implementation: - container: - image: us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/components/component_base:latest - command: - - python3 - - /pipelines/component/src/finetune_t5_model.py - args: - - --executor_input - - executorInput: null - - --function_to_execute - - finetune_t5_model diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/configs/defaults.yaml b/examples/llmops/finetuning-flan-t5/AutoMLOps/configs/defaults.yaml deleted file mode 100644 index f6213dd..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/configs/defaults.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -# These values are descriptive only - do not change. -# Rerun AutoMLOps.generate() to change these values. -gcp: - af_registry_location: us-central1 - af_registry_name: vertex-mlops-af - base_image: us-docker.pkg.dev/vertex-ai/training/pytorch-gpu.1-10:latest - cb_trigger_location: us-central1 - cb_trigger_name: automlops-trigger - cloud_run_location: us-central1 - cloud_run_name: run-pipeline - cloud_tasks_queue_location: us-central1 - cloud_tasks_queue_name: queueing-svc - cloud_schedule_location: us-central1 - cloud_schedule_name: AutoMLOps-schedule - cloud_schedule_pattern: 59 11 * * 0 - cloud_source_repository: AutoMLOps-repo - cloud_source_repository_branch: automlops - gs_bucket_name: automlops-sandbox-bucket - pipeline_runner_service_account: vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com - project_id: automlops-sandbox - vpc_connector: No VPC Specified - -pipelines: - parameter_values_path: pipelines/runtime_parameters/pipeline_parameter_values.json - pipeline_component_directory: components - pipeline_job_spec_path: scripts/pipeline_spec/pipeline_job.json - pipeline_region: us-central1 - pipeline_storage_path: gs://automlops-sandbox-bucket/pipeline_root diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/pipeline.py b/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/pipeline.py deleted file mode 100644 index 362ab40..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/pipeline.py +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -import argparse -import os -from functools import partial -from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component -import kfp -from kfp.v2 import compiler, dsl -from kfp.v2.dsl import * -from typing import * -import yaml - -def load_custom_component(component_name: str): - component_path = os.path.join('components', - component_name, - 'component.yaml') - return kfp.components.load_component_from_file(component_path) - -def create_training_pipeline(pipeline_job_spec_path: str): - deploy_and_test_model = load_custom_component(component_name='deploy_and_test_model') - finetune_t5_model = load_custom_component(component_name='finetune_t5_model') - - finetune_t5_model_custom_training_job_specs = { - 'component_spec': finetune_t5_model, - 'display_name': 'flan-t5-base-finetuning-gpu-tensorboard', - 'machine_type': 'n1-standard-32', - 'accelerator_type': 'NVIDIA_TESLA_V100', - 'accelerator_count': '4', - 'replica_count': '1', - 'service_account': 'vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com', - 'tensorboard': 'projects/45373616427/locations/us-central1/tensorboards/7295189281549582336', - 'base_output_directory': 'gs://automlops-sandbox-bucket/finetune_t5_model/', - } - - finetune_t5_model_job_op = create_custom_training_job_op_from_component(**finetune_t5_model_custom_training_job_specs) - finetune_t5_model = partial(finetune_t5_model_job_op, project='automlops-sandbox') - @dsl.pipeline( - name='finetune-flan-t5-pipeline', - ) - def pipeline( - endpoint_sa: str, - project_id: str, - eval_batch: int, - train_batch: int, - model_dir: str, - lr: float, - epochs: int, - logging_steps: int, - serving_image_tag: str, - region: str): - - finetune_t5_model_task = finetune_t5_model( - model_dir=model_dir, - epochs=epochs, - eval_batch=eval_batch, - lr=lr, - logging_steps=logging_steps, - train_batch=train_batch) - - deploy_and_test_model_task = deploy_and_test_model( - endpoint_sa=endpoint_sa, - project_id=project_id, - region=region, - serving_image_tag=serving_image_tag).after(finetune_t5_model_task) - - compiler.Compiler().compile( - pipeline_func=pipeline, - package_path=pipeline_job_spec_path) - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--config', type=str, - help='The config file for setting default values.') - - args = parser.parse_args() - - with open(args.config, 'r', encoding='utf-8') as config_file: - config = yaml.load(config_file, Loader=yaml.FullLoader) - - pipeline = create_training_pipeline( - pipeline_job_spec_path=config['pipelines']['pipeline_job_spec_path']) diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/pipeline_runner.py b/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/pipeline_runner.py deleted file mode 100644 index 519fd45..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/pipeline_runner.py +++ /dev/null @@ -1,78 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -import argparse -import json -import logging -import os -import yaml - -from google.cloud import aiplatform - -logger = logging.getLogger() -log_level = os.environ.get('LOG_LEVEL', 'INFO') -logger.setLevel(log_level) - -def run_pipeline( - project_id: str, - pipeline_root: str, - pipeline_runner_sa: str, - parameter_values_path: str, - pipeline_spec_path: str, - display_name: str = 'mlops-pipeline-run', - enable_caching: bool = False): - """Executes a pipeline run. - - Args: - project_id: The project_id. - pipeline_root: GCS location of the pipeline runs metadata. - pipeline_runner_sa: Service Account to runner PipelineJobs. - parameter_values_path: Location of parameter values JSON. - pipeline_spec_path: Location of the pipeline spec JSON. - display_name: Name to call the pipeline. - enable_caching: Should caching be enabled (Boolean) - """ - with open(parameter_values_path, 'r') as file: - try: - pipeline_params = json.load(file) - except ValueError as exc: - print(exc) - logging.debug('Pipeline Parms Configured:') - logging.debug(pipeline_params) - - aiplatform.init(project=project_id) - job = aiplatform.PipelineJob( - display_name = display_name, - template_path = pipeline_spec_path, - pipeline_root = pipeline_root, - parameter_values = pipeline_params, - enable_caching = enable_caching) - logging.debug('AI Platform job built. Submitting...') - job.submit(service_account=pipeline_runner_sa) - logging.debug('Job sent!') - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--config', type=str, - help='The config file for setting default values.') - args = parser.parse_args() - - with open(args.config, 'r', encoding='utf-8') as config_file: - config = yaml.load(config_file, Loader=yaml.FullLoader) - - run_pipeline(project_id=config['gcp']['project_id'], - pipeline_root=config['pipelines']['pipeline_storage_path'], - pipeline_runner_sa=config['gcp']['pipeline_runner_service_account'], - parameter_values_path=config['pipelines']['parameter_values_path'], - pipeline_spec_path=config['pipelines']['pipeline_job_spec_path']) diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/runtime_parameters/pipeline_parameter_values.json b/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/runtime_parameters/pipeline_parameter_values.json deleted file mode 100644 index e515968..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/pipelines/runtime_parameters/pipeline_parameter_values.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "endpoint_sa": "vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com", - "project_id": "automlops-sandbox", - "model_dir": "gs://automlops-sandbox-bucket/flan_t5_model/", - "lr": 5e-05, - "epochs": 5, - "logging_steps": 500, - "serving_image_tag": "us-central1-docker.pkg.dev/automlops-sandbox/vertex-mlops-af/finetuning_flan_t5_base:latest", - "eval_batch": 4, - "region": "us-central1", - "train_batch": 4 -} \ No newline at end of file diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/build_components.sh b/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/build_components.sh deleted file mode 100755 index f4bfa55..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/build_components.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -# Submits a Cloud Build job that builds and deploys the components -# This script should run from the AutoMLOps/ directory -# Change directory in case this is not the script root. - -gcloud builds submit .. --config cloudbuild.yaml --timeout=3600 diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/build_pipeline_spec.sh b/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/build_pipeline_spec.sh deleted file mode 100755 index c5155dc..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/build_pipeline_spec.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -# Builds the pipeline specs -# This script should run from the AutoMLOps/ directory -# Change directory in case this is not the script root. - -CONFIG_FILE=configs/defaults.yaml - -python3 -m pipelines.pipeline --config $CONFIG_FILE diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/create_resources.sh b/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/create_resources.sh deleted file mode 100755 index aec536f..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/create_resources.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -# This script will create an artifact registry and gs bucket if they do not already exist. - -GREEN='' -NC='' -AF_REGISTRY_NAME=vertex-mlops-af -AF_REGISTRY_LOCATION=us-central1 -PROJECT_ID=automlops-sandbox -PROJECT_NUMBER=`gcloud projects describe automlops-sandbox --format 'value(projectNumber)'` -BUCKET_NAME=automlops-sandbox-bucket -BUCKET_LOCATION=us-central1 -SERVICE_ACCOUNT_NAME=vertex-pipelines -SERVICE_ACCOUNT_FULL=vertex-pipelines@automlops-sandbox.iam.gserviceaccount.com -CLOUD_SOURCE_REPO=AutoMLOps-repo -CLOUD_SOURCE_REPO_BRANCH=automlops -CB_TRIGGER_LOCATION=us-central1 -CB_TRIGGER_NAME=automlops-trigger -CLOUD_TASKS_QUEUE_LOCATION=us-central1 -CLOUD_TASKS_QUEUE_NAME=queueing-svc - -echo -e "$GREEN Updating required API services in project $PROJECT_ID $NC" -gcloud services enable cloudresourcemanager.googleapis.com \ - aiplatform.googleapis.com \ - artifactregistry.googleapis.com \ - cloudbuild.googleapis.com \ - cloudscheduler.googleapis.com \ - cloudtasks.googleapis.com \ - compute.googleapis.com \ - iam.googleapis.com \ - iamcredentials.googleapis.com \ - ml.googleapis.com \ - run.googleapis.com \ - storage.googleapis.com \ - sourcerepo.googleapis.com - -echo -e "$GREEN Checking for Artifact Registry: $AF_REGISTRY_NAME in project $PROJECT_ID $NC" -if ! (gcloud artifacts repositories list --project="$PROJECT_ID" --location=$AF_REGISTRY_LOCATION | grep -E "(^|[[:blank:]])$AF_REGISTRY_NAME($|[[:blank:]])"); then - - echo "Creating Artifact Registry: ${AF_REGISTRY_NAME} in project $PROJECT_ID" - gcloud artifacts repositories create "$AF_REGISTRY_NAME" \ - --repository-format=docker \ - --location=$AF_REGISTRY_LOCATION \ - --project="$PROJECT_ID" \ - --description="Artifact Registry ${AF_REGISTRY_NAME} in ${AF_REGISTRY_LOCATION}." - -else - - echo "Artifact Registry: ${AF_REGISTRY_NAME} already exists in project $PROJECT_ID" - -fi - - -echo -e "$GREEN Checking for GS Bucket: $BUCKET_NAME in project $PROJECT_ID $NC" -if !(gsutil ls -b gs://$BUCKET_NAME | grep --fixed-strings "$BUCKET_NAME"); then - - echo "Creating GS Bucket: ${BUCKET_NAME} in project $PROJECT_ID" - gsutil mb -l ${BUCKET_LOCATION} gs://$BUCKET_NAME - -else - - echo "GS Bucket: ${BUCKET_NAME} already exists in project $PROJECT_ID" - -fi - -echo -e "$GREEN Checking for Service Account: $SERVICE_ACCOUNT_NAME in project $PROJECT_ID $NC" -if ! (gcloud iam service-accounts list --project="$PROJECT_ID" | grep -E "(^|[[:blank:]])$SERVICE_ACCOUNT_FULL($|[[:blank:]])"); then - - echo "Creating Service Account: ${SERVICE_ACCOUNT_NAME} in project $PROJECT_ID" - gcloud iam service-accounts create $SERVICE_ACCOUNT_NAME \ - --description="For submitting PipelineJobs" \ - --display-name="Pipeline Runner Service Account" -else - - echo "Service Account: ${SERVICE_ACCOUNT_NAME} already exists in project $PROJECT_ID" - -fi - -echo -e "$GREEN Updating required IAM roles in project $PROJECT_ID $NC" -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SERVICE_ACCOUNT_FULL" \ - --role="roles/aiplatform.user" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SERVICE_ACCOUNT_FULL" \ - --role="roles/artifactregistry.reader" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SERVICE_ACCOUNT_FULL" \ - --role="roles/bigquery.user" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SERVICE_ACCOUNT_FULL" \ - --role="roles/bigquery.dataEditor" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SERVICE_ACCOUNT_FULL" \ - --role="roles/iam.serviceAccountUser" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SERVICE_ACCOUNT_FULL" \ - --role="roles/storage.admin" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$SERVICE_ACCOUNT_FULL" \ - --role="roles/run.admin" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ - --role="roles/run.admin" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ - --role="roles/iam.serviceAccountUser" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ - --role="roles/cloudtasks.enqueuer" \ - --no-user-output-enabled - -gcloud projects add-iam-policy-binding $PROJECT_ID \ - --member="serviceAccount:$PROJECT_NUMBER@cloudbuild.gserviceaccount.com" \ - --role="roles/cloudscheduler.admin" \ - --no-user-output-enabled - -echo -e "$GREEN Checking for Cloud Source Repository: $CLOUD_SOURCE_REPO in project $PROJECT_ID $NC" -if ! (gcloud source repos list --project="$PROJECT_ID" | grep -E "(^|[[:blank:]])$CLOUD_SOURCE_REPO($|[[:blank:]])"); then - - echo "Creating Cloud Source Repository: ${CLOUD_SOURCE_REPO} in project $PROJECT_ID" - gcloud source repos create $CLOUD_SOURCE_REPO - -else - - echo "Cloud Source Repository: ${CLOUD_SOURCE_REPO} already exists in project $PROJECT_ID" - -fi - -# Create cloud tasks queue -echo -e "$GREEN Checking for Cloud Tasks Queue: $CLOUD_TASKS_QUEUE_NAME in project $PROJECT_ID $NC" -if ! (gcloud tasks queues list --location $CLOUD_TASKS_QUEUE_LOCATION | grep -E "(^|[[:blank:]])$CLOUD_TASKS_QUEUE_NAME($|[[:blank:]])"); then - - echo "Creating Cloud Tasks Queue: ${CLOUD_TASKS_QUEUE_NAME} in project $PROJECT_ID" - gcloud tasks queues create $CLOUD_TASKS_QUEUE_NAME \ - --location=$CLOUD_TASKS_QUEUE_LOCATION - -else - - echo "Cloud Tasks Queue: ${CLOUD_TASKS_QUEUE_NAME} already exists in project $PROJECT_ID" - -fi - -# Create cloud build trigger -echo -e "$GREEN Checking for Cloudbuild Trigger: $CB_TRIGGER_NAME in project $PROJECT_ID $NC" -if ! (gcloud beta builds triggers list --project="$PROJECT_ID" --region="$CB_TRIGGER_LOCATION" | grep -E "(^|[[:blank:]])name: $CB_TRIGGER_NAME($|[[:blank:]])"); then - - echo "Creating Cloudbuild Trigger on branch $CLOUD_SOURCE_REPO_BRANCH in project $PROJECT_ID for repo ${CLOUD_SOURCE_REPO}" - gcloud beta builds triggers create cloud-source-repositories \ - --region=$CB_TRIGGER_LOCATION \ - --name=$CB_TRIGGER_NAME \ - --repo=$CLOUD_SOURCE_REPO \ - --branch-pattern="$CLOUD_SOURCE_REPO_BRANCH" \ - --build-config=AutoMLOps/cloudbuild.yaml - -else - - echo "Cloudbuild Trigger already exists in project $PROJECT_ID for repo ${CLOUD_SOURCE_REPO}" - -fi diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/pipeline_spec/.gitkeep b/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/pipeline_spec/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/run_all.sh b/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/run_all.sh deleted file mode 100755 index d41bfde..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/run_all.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -# Builds components, pipeline specs, and submits the PipelineJob. -# This script should run from the AutoMLOps/ directory -# Change directory in case this is not the script root. - -GREEN='' -NC='' - -echo -e "${GREEN} BUILDING COMPONENTS ${NC}" -gcloud builds submit .. --config cloudbuild.yaml --timeout=3600 - -echo -e "${GREEN} BUILDING PIPELINE SPEC ${NC}" -./scripts/build_pipeline_spec.sh - -echo -e "${GREEN} RUNNING PIPELINE JOB ${NC}" -./scripts/run_pipeline.sh diff --git a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/run_pipeline.sh b/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/run_pipeline.sh deleted file mode 100755 index 6e50ef5..0000000 --- a/examples/llmops/finetuning-flan-t5/AutoMLOps/scripts/run_pipeline.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# DISCLAIMER: This code is generated as part of the AutoMLOps output. - -# Submits the PipelineJob to Vertex AI -# This script should run from the AutoMLOps/ directory -# Change directory in case this is not the script root. - -CONFIG_FILE=configs/defaults.yaml - -python3 -m pipelines.pipeline_runner --config $CONFIG_FILE diff --git a/google_cloud_automlops/orchestration/kfp/builder.py b/google_cloud_automlops/orchestration/kfp/builder.py index 311a0fb..c121954 100644 --- a/google_cloud_automlops/orchestration/kfp/builder.py +++ b/google_cloud_automlops/orchestration/kfp/builder.py @@ -170,10 +170,12 @@ def build_pipeline(custom_training_job_specs: list, # Add indentation pipeline_scaffold_contents = textwrap.indent(pipeline_scaffold_contents, 4 * ' ') # Construct pipeline.py + project_id = defaults['gcp']['project_id'] write_file(GENERATED_PIPELINE_FILE, pipeline_jinja( components_list, custom_training_job_specs, - pipeline_scaffold_contents), 'w') + pipeline_scaffold_contents, + project_id=project_id), 'w') # Construct pipeline_runner.py write_file(GENERATED_PIPELINE_RUNNER_FILE, pipeline_runner_jinja(), 'w') # Construct requirements.txt @@ -424,7 +426,8 @@ def pipeline_runner_jinja() -> str: def pipeline_jinja( components_list: list, custom_training_job_specs: list, - pipeline_scaffold_contents: str) -> str: + pipeline_scaffold_contents: str, + project_id: str) -> str: """Generates code for the pipeline.py file to be written to the pipelines directory. Args: @@ -432,6 +435,7 @@ def pipeline_jinja( custom_training_job_specs: Specifies the specs to run the training job with. pipeline_scaffold_contents: The contents of the pipeline scaffold file, which can be found at PIPELINE_CACHE_FILE. + project_id: The project ID. Returns: str: pipeline.py file. @@ -443,7 +447,8 @@ def pipeline_jinja( components_list=components_list, custom_training_job_specs=custom_training_job_specs, generated_license=GENERATED_LICENSE, - pipeline_scaffold_contents=pipeline_scaffold_contents) + pipeline_scaffold_contents=pipeline_scaffold_contents, + project_id=project_id) def pipeline_requirements_jinja() -> str: diff --git a/google_cloud_automlops/utils/constants.py b/google_cloud_automlops/utils/constants.py index eb272d2..62dd834 100644 --- a/google_cloud_automlops/utils/constants.py +++ b/google_cloud_automlops/utils/constants.py @@ -123,5 +123,10 @@ # Required IAM Roles for pipeline runner service account IAM_ROLES_RUNNER_SA = set([ - 'roles/aiplatform.serviceAgent' + 'roles/aiplatform.user', + 'roles/artifactregistry.reader', + 'roles/bigquery.user', + 'roles/bigquery.dataEditor', + 'roles/iam.serviceAccountUser', + 'roles/storage.admin' ]) diff --git a/google_cloud_automlops/utils/utils.py b/google_cloud_automlops/utils/utils.py index 02be99d..729a7f1 100644 --- a/google_cloud_automlops/utils/utils.py +++ b/google_cloud_automlops/utils/utils.py @@ -325,8 +325,12 @@ def stringify_job_spec_list(job_spec_list: list) -> list: output = [] for spec in job_spec_list: mapping = {} - mapping['component_spec'] = spec['component_spec'] - mapping['spec_string'] = json.dumps(spec, sort_keys=True, indent=8) + if isinstance(spec['component_spec'], str): + mapping['component_spec'] = spec['component_spec'] + else: + raise ValueError('component_spec must be a string.') + # Remove string quotes from component spec line + mapping['spec_string'] = json.dumps(spec, sort_keys=True, indent=8).replace(f'''"{spec['component_spec']}"''', f'''{spec['component_spec']}''') mapping['spec_string'] = mapping['spec_string'].replace('}', ' }') # align closing bracket output.append(mapping) return output diff --git a/tests/unit/orchestration/kfp/builder_test.py b/tests/unit/orchestration/kfp/builder_test.py index 875a10f..5faddbb 100644 --- a/tests/unit/orchestration/kfp/builder_test.py +++ b/tests/unit/orchestration/kfp/builder_test.py @@ -619,7 +619,7 @@ def test_pipeline_runner_jinja( @pytest.mark.parametrize( - '''components_list, custom_training_job_specs, pipeline_scaffold_contents,''' + '''components_list, custom_training_job_specs, pipeline_scaffold_contents, project_id,''' '''is_included, expected_output_snippets''', [ ( @@ -633,7 +633,7 @@ def test_pipeline_runner_jinja( 'accelerator_count': '1', } ], - 'Pipeline definition goes here', True, + 'Pipeline definition goes here', 'my-project', True, [GENERATED_LICENSE, 'from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', 'def upload_pipeline_spec', @@ -645,7 +645,7 @@ def test_pipeline_runner_jinja( ), ( ['componentA','componentB','componentC'], - None, 'Pipeline definition goes here', True, + None, 'Pipeline definition goes here', 'my-project', True, [GENERATED_LICENSE, 'def upload_pipeline_spec', 'componentA = load_custom_component', @@ -655,7 +655,7 @@ def test_pipeline_runner_jinja( ), ( ['componentA','componentB','componentC'], - None, 'Pipeline definition goes here', False, + None, 'Pipeline definition goes here', 'my-project', False, ['from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', 'componentB_custom_training_job_specs'] ), @@ -665,6 +665,7 @@ def test_pipeline_jinja( components_list: list, custom_training_job_specs: list, pipeline_scaffold_contents: str, + project_id: str, is_included: bool, expected_output_snippets: List[str]): """Tests pipeline_jinja, which generates code for the pipeline.py @@ -679,13 +680,15 @@ def test_pipeline_jinja( custom_training_job_specs: Specifies the specs to run the training job with. pipeline_scaffold_contents: The contents of the pipeline scaffold file, which can be found at PIPELINE_CACHE_FILE. + project_id: The project ID. is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ pipeline_py = pipeline_jinja( components_list, custom_training_job_specs, - pipeline_scaffold_contents) + pipeline_scaffold_contents, + project_id) for snippet in expected_output_snippets: if is_included: diff --git a/tests/unit/utils/utils_test.py b/tests/unit/utils/utils_test.py index d6f1ba4..3365d97 100644 --- a/tests/unit/utils/utils_test.py +++ b/tests/unit/utils/utils_test.py @@ -485,7 +485,7 @@ def test_get_function_source_definition(func: Callable, expected_output: str): '''{\n''' ''' "accelerator_count": 1,\n''' ''' "accelerator_type": "NVIDIA_TESLA_A100",\n''' - ''' "component_spec": "train_model",\n''' + ''' "component_spec": train_model,\n''' ''' "display_name": "train-model-accelerated",\n''' ''' "machine_type": "a2-highgpu-1g"\n }''' }]), @@ -511,7 +511,7 @@ def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: Lis 'spec_string': '''{ "accelerator_count": 1, "accelerator_type": "NVIDIA_TESLA_A100", - "component_spec": "train_model", + "component_spec": train_model, "display_name": "train-model-accelerated", "machine_type": "a2-highgpu-1g" }'''