diff --git a/google_cloud_automlops/tests/__init__.py b/google_cloud_automlops/tests/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/__init__.py b/google_cloud_automlops/tests/unit/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/.gitkeep b/google_cloud_automlops/tests/unit/deployments/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/deployments/__init__.py b/google_cloud_automlops/tests/unit/deployments/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py b/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py b/google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py new file mode 100644 index 0000000..27331f5 --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py @@ -0,0 +1,117 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-module-docstring + +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files + +from typing import List + +import pytest + +from google_cloud_automlops.utils.constants import ( + BASE_DIR, + CLOUDBUILD_TEMPLATES_PATH, + COMPONENT_BASE_RELATIVE_PATH, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH +) +from google_cloud_automlops.utils.utils import render_jinja + +@pytest.mark.parametrize( + '''artifact_repo_location, artifact_repo_name, naming_prefix,''' + '''project_id, pubsub_topic_name, use_ci, is_included,''' + '''expected_output_snippets''', + [ + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-topic', True, True, + ['id: "build_component_base"', + 'id: "push_component_base"', + 'id: "install_pipelines_deps"', + 'id: "build_pipeline_spec"', + 'id: "publish_to_topic"', + '"build", "-t", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', + '"push", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', + 'gcloud pubsub topics publish my-topic'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-topic', False, True, + ['id: "build_component_base"', + '"build", "-t", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-topic', False, False, + ['id: "push_component_base"', + 'id: "install_pipelines_deps"', + 'id: "build_pipeline_spec"', + 'id: "publish_to_topic"', + '"push" "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', + 'gcloud pubsub topics publish my-topic'] + ), + ] +) +def test_create_cloudbuild_jinja( + artifact_repo_location: str, + artifact_repo_name: str, + naming_prefix: str, + project_id: str, + pubsub_topic_name: str, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_cloudbuild_jinja, which generates content for the cloudbuild.yaml. + There are three test cases for this function: + 1. Checks that expected strings are included when use_ci=True. + 2. Checks that expected strings are included when use_ci=False. + 3. Checks that certain strings are not included when use_ci=False. + + Args: + artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). + artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + project_id: The project ID. + pubsub_topic_name: The name of the pubsub topic to publish to. + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + component_base_relative_path = COMPONENT_BASE_RELATIVE_PATH if use_ci else f'{BASE_DIR}{COMPONENT_BASE_RELATIVE_PATH}' + template_file = import_files(CLOUDBUILD_TEMPLATES_PATH) / 'cloudbuild.yaml.j2' + + cloudbuild_config = render_jinja( + template_path=template_file, + artifact_repo_location=artifact_repo_location, + artifact_repo_name=artifact_repo_name, + component_base_relative_path=component_base_relative_path, + generated_licensed=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + naming_prefix=naming_prefix, + project_id=project_id, + pubsub_topic_name=pubsub_topic_name, + use_ci=use_ci + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in cloudbuild_config + elif not is_included: + assert snippet not in cloudbuild_config diff --git a/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py b/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py b/google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py new file mode 100644 index 0000000..a7ed5f6 --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py @@ -0,0 +1,133 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files + +from typing import List + +import pytest + +from google_cloud_automlops.utils.constants import ( + COMPONENT_BASE_RELATIVE_PATH, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH, + GITHUB_ACTIONS_TEMPLATES_PATH +) +from google_cloud_automlops.utils.utils import render_jinja + + +@pytest.mark.parametrize( + '''artifact_repo_location, artifact_repo_name, naming_prefix,''' + '''project_id, project_number, pubsub_topic_name, use_ci, source_repo_branch,''' + '''workload_identity_provider, workload_identity_pool, workload_identity_service_account, is_included,''' + '''expected_output_snippets''', + [ + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-project-number', 'my-topic', True, 'automlops', + 'my-provider', 'my-pool', 'my-sa', True, + ['id: auth', + 'id: build-push-component-base', + 'id: install-pipeline-deps', + 'id: build-pipeline-spec', + 'id: publish-to-topic', + 'us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest', + 'gcloud pubsub topics publish my-topic --message'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-project-number', 'my-topic', False, 'automlops', + 'my-provider', 'my-pool', 'my-sa', True, + ['id: build-push-component-base', + 'us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-project-number', 'my-topic', False, 'automlops', + 'my-provider', 'my-pool', 'my-sa', False, + ['id: install-pipeline-deps', + 'id: build-pipeline-spec', + 'id: publish-to-topic', + 'gcloud pubsub topics publish my-topic --message'] + ), + ] +) +def test_create_github_actions_jinja( + artifact_repo_location: str, + artifact_repo_name: str, + naming_prefix: str, + project_id: str, + project_number: str, + pubsub_topic_name: str, + use_ci: bool, + source_repo_branch: str, + workload_identity_pool: str, + workload_identity_provider: str, + workload_identity_service_account: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_github_actions_jinja, which generates content for the github actions file. + There are three test cases for this function: + 1. Checks that expected strings are included when use_ci=True. + 2. Checks that expected strings are included when use_ci=False. + 3. Checks that certain strings are not included when use_ci=False. + + Args: + artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). + artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + project_id: The project ID. + project_number: The project number. + pubsub_topic_name: The name of the pubsub topic to publish to. + source_repo_branch: The branch to use in the source repository. + use_ci: Flag that determines whether to use Cloud CI/CD. + workload_identity_pool: Pool for workload identity federation. + workload_identity_provider: Provider for workload identity federation. + workload_identity_service_account: Service account for workload identity federation. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + + template_file = import_files(GITHUB_ACTIONS_TEMPLATES_PATH) / 'github_actions.yaml.j2' + github_actions_config = render_jinja( + template_path=template_file, + artifact_repo_location=artifact_repo_location, + artifact_repo_name=artifact_repo_name, + component_base_relative_path=COMPONENT_BASE_RELATIVE_PATH, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + naming_prefix=naming_prefix, + project_id=project_id, + project_number=project_number, + pubsub_topic_name=pubsub_topic_name, + source_repo_branch=source_repo_branch, + use_ci=use_ci, + workload_identity_pool=workload_identity_pool, + workload_identity_provider=workload_identity_provider, + workload_identity_service_account=workload_identity_service_account + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in github_actions_config + elif not is_included: + assert snippet not in github_actions_config diff --git a/google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep b/google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep b/google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/orchestration/__init__.py b/google_cloud_automlops/tests/unit/orchestration/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep b/google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep b/google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py b/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py b/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py new file mode 100644 index 0000000..08d0c6e --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py @@ -0,0 +1,993 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +import json +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files +import os +from typing import List + +import pytest +import pytest_mock + +from google_cloud_automlops.utils.constants import ( + BASE_DIR, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH, + KFP_TEMPLATES_PATH, + PINNED_KFP_VERSION, +) +import google_cloud_automlops.orchestration.kfp.builder +from google_cloud_automlops.orchestration.kfp.builder import ( + build_component, + build_pipeline, + build_services, +) +import google_cloud_automlops.utils.utils +from google_cloud_automlops.utils.utils import ( + make_dirs, + read_yaml_file, + render_jinja, + write_yaml_file +) + + +DEFAULTS = { + 'gcp': { + 'artifact_repo_location': 'us-central1', + 'project_id': 'my_project', + 'artifact_repo_name': 'my_af_registry', + 'naming_prefix': 'my-prefix', + 'pipeline_job_runner_service_account': 'my-service-account@service.com', + 'pipeline_job_submission_service_type': 'cloud-functions', + 'setup_model_monitoring': True + }, + 'pipelines': { + 'gs_pipeline_job_spec_path': 'gs://my-bucket/pipeline_root/my-prefix/pipeline_job.json', + 'pipeline_storage_path': 'gs://my-bucket/pipeline_root/' + } +} + +TEMP_YAML = { + 'name': 'create_dataset', + 'description': 'Custom component that takes in a BQ table and writes it to GCS.', + 'inputs': [ + { + 'name': 'bq_table', + 'description': 'The source biquery table.', + 'type': 'String', + }, + { + 'name': 'data_path', + 'description': 'The gcs location to write the csv.', + 'type': 'String', + }, + { + 'name': 'project_id', + 'description': 'The project ID.', + 'type': 'String'}, + ], + 'implementation': { + 'container': { + 'image': 'TBD', + 'command': [ + 'sh', + '-c', + 'if ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \\\n --no-warn-script-location && "$0" "$@"\n\n', + 'def create_dataset(\n bq_table: str,\n data_path: str,\n project_id: str\n):\n """Custom component that takes in a BQ table and writes it to GCS.\n\n Args:\n bq_table: The source biquery table.\n data_path: The gcs location to write the csv.\n project_id: The project ID.\n """\n from google.cloud import bigquery\n import pandas as pd\n from sklearn import preprocessing\n\n bq_client = bigquery.Client(project=project_id)\n\n def get_query(bq_input_table: str) -> str:\n """Generates BQ Query to read data.\n\n Args:\n bq_input_table: The full name of the bq input table to be read into\n the dataframe (e.g. ..)\n Returns: A BQ query string.\n """\n return f\'\'\'\n SELECT *\n FROM `{bq_input_table}`\n \'\'\'\n\n def load_bq_data(query: str, client: bigquery.Client) -> pd.DataFrame:\n """Loads data from bq into a Pandas Dataframe for EDA.\n Args:\n query: BQ Query to generate data.\n client: BQ Client used to execute query.\n Returns:\n pd.DataFrame: A dataframe with the requested data.\n """\n df = client.query(query).to_dataframe()\n return df\n\n dataframe = load_bq_data(get_query(bq_table), bq_client)\n le = preprocessing.LabelEncoder()\n dataframe[\'Class\'] = le.fit_transform(dataframe[\'Class\'])\n dataframe.to_csv(data_path, index=False)\n', + ], + 'args': [ + '--executor_input', + {'executorInput': None}, + '--function_to_execute', + 'create_dataset', + ], + } + }, +} + + +@pytest.fixture(name='temp_yaml_dict', params=[TEMP_YAML]) +def fixture_temp_yaml_dict(request: pytest.FixtureRequest, tmpdir: pytest.FixtureRequest): + """Writes temporary yaml file fixture using defaults parameterized + dictionaries during pytest session scope. + + Args: + request: Pytest fixture special object that provides information + about the fixture. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + + Returns: + dict: Path of yaml file and dictionary it contains. + """ + yaml_path = tmpdir.join('test.yaml') + write_yaml_file(yaml_path, request.param, 'w') + return {'path': yaml_path, 'vals': request.param} + + +@pytest.fixture(name='defaults_dict', params=[DEFAULTS]) +def fixture_defaults_dict(request: pytest.FixtureRequest, tmpdir: pytest.FixtureRequest): + """Writes temporary yaml file fixture using defaults parameterized + dictionaries during pytest session scope. + + Args: + request: Pytest fixture special object that provides information + about the fixture. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + + Returns: + dict: Path of yaml file and dictionary it contains. + """ + yaml_path = tmpdir.join('defaults.yaml') + write_yaml_file(yaml_path, request.param, 'w') + return {'path': yaml_path, 'vals': request.param} + + +@pytest.fixture(name='expected_component_dict') +def fixture_expected_component_dict(): + """Creates the expected component dictionary, which is the temporary yaml + file with a change to the implementation key. + + Returns: + dict: Expected component dictionary generated from the component + builder. + """ + expected = TEMP_YAML + expected['implementation'] = { + 'container': { + 'image': 'us-central1-docker.pkg.dev/my_project/my_af_registry/my-prefix/components/component_base:latest', + 'command': ['python3', '/pipelines/component/src/create_dataset.py'], + 'args': [ + '--executor_input', + {'executorInput': None}, + '--function_to_execute', + 'create_dataset', + ], + } + } + return expected + + +def test_build_component(mocker: pytest_mock.MockerFixture, + tmpdir: pytest.FixtureRequest, + temp_yaml_dict: pytest.FixtureRequest, + defaults_dict: pytest.FixtureRequest, + expected_component_dict: pytest.FixtureRequest): + """Tests build_component, which Constructs and writes component.yaml and + {component_name}.py files. + + Args: + mocker: Mocker to patch directories. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + temp_yaml_dict: Locally defined temp_yaml_file Pytest fixture. + defaults_dict: Locally defined defaults_dict Pytest fixture. + expected_component_dict: Locally defined expected_component_dict + Pytest fixture. + """ + # Patch filepath constants to point to test path. + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'BASE_DIR', + f'{tmpdir}/') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_DEFAULTS_FILE', + defaults_dict['path']) + + # Extract component name, create required directories, run build_component + component_name = TEMP_YAML['name'] + make_dirs([f'{tmpdir}/components/component_base/src']) + build_component(temp_yaml_dict['path']) + + # Ensure correct files are created with build_component call + assert os.path.exists(f'{tmpdir}/components/{component_name}/component.yaml') + assert os.path.exists(f'{tmpdir}/components/component_base/src/{component_name}.py') + + # Load component.yaml file and compare to the expected output in test_data + created_component_dict = read_yaml_file(f'{tmpdir}/components/{component_name}/component.yaml') + assert created_component_dict == expected_component_dict + + +@pytest.mark.parametrize( + 'custom_training_job_specs, pipeline_parameter_values', + [ + ( + [{'component_spec': 'mycomp1', 'other': 'myother'}], + { + 'bq_table': 'automlops-sandbox.test_dataset.dry-beans', + 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 13:00:41.379753', + 'data_path': 'gs://automlops-sandbox-bucket/data.csv', + 'project_id': 'automlops-sandbox', + 'region': 'us-central1' + }, + ), + ( + [ + { + 'component_spec': 'train_model', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': '1', + } + ], + { + 'bq_table': 'automlops-sandbox.test_dataset.dry-beans', + 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 13:00:41.379753', + 'data_path': 'gs://automlops-sandbox-bucket/data.csv', + 'project_id': 'automlops-sandbox', + 'region': 'us-central1' + }, + ), + ( + [ + { + 'component_spec': 'test_model', + 'display_name': 'test-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': '1', + } + ], + { + 'bq_table': 'automlops-sandbox.test_dataset.dry-beans2', + 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 14:00:41.379753', + 'data_path': 'gs://automlops-sandbox-bucket/data2.csv', + 'project_id': 'automlops-sandbox', + 'region': 'us-central1' + }, + ) + ] +) +def test_build_pipeline(mocker: pytest_mock.MockerFixture, + tmpdir: pytest.FixtureRequest, + defaults_dict: pytest.FixtureRequest, + custom_training_job_specs: List[dict], + pipeline_parameter_values: dict): + """Tests build_pipeline, which constructs and writes pipeline.py, + pipeline_runner.py, and pipeline_parameter_values.json files. + + Args: + mocker: Mocker to patch directories. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + defaults_dict: Locally defined defaults_dict Pytest fixture. + custom_training_job_specs (List[dict]): Specifies the specs to run the training job with. + pipeline_parameter_values (dict): Dictionary of runtime parameters for the PipelineJob. + """ + # Patch constants and other functions + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'BASE_DIR', + f'{tmpdir}/') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_DEFAULTS_FILE', + defaults_dict['path']) + mocker.patch.object(google_cloud_automlops.utils.utils, + 'CACHE_DIR', + f'{tmpdir}/.AutoMLOps-cache') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'PIPELINE_CACHE_FILE', + f'{tmpdir}/.AutoMLOps-cache/pipeline_scaffold.py') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_PIPELINE_FILE', + f'{tmpdir}/pipelines/pipeline.py') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_PIPELINE_RUNNER_FILE', + f'{tmpdir}/pipelines/pipeline_runner.py') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_PIPELINE_REQUIREMENTS_FILE', + f'{tmpdir}/pipelines/requirements.txt') + + # Create required directory and file for build_pipeline + make_dirs([f'{tmpdir}/pipelines/runtime_parameters', f'{tmpdir}/.AutoMLOps-cache']) + os.system(f'touch {tmpdir}/.AutoMLOps-cache/pipeline_scaffold.py') + build_pipeline(custom_training_job_specs, pipeline_parameter_values) + + # Ensure correct files were created + assert os.path.exists(f'{tmpdir}/pipelines/pipeline.py') + assert os.path.exists(f'{tmpdir}/pipelines/pipeline_runner.py') + assert os.path.exists(f'{tmpdir}/pipelines/requirements.txt') + assert os.path.exists(f'{tmpdir}/pipelines/runtime_parameters/pipeline_parameter_values.json') + + # Ensure pipeline_parameter_values.json was created as expected + with open(f'{tmpdir}/pipelines/runtime_parameters/pipeline_parameter_values.json', mode='r', encoding='utf-8') as f: + pipeline_params_dict = json.load(f) + assert pipeline_params_dict == pipeline_parameter_values + + +def test_build_services(mocker: pytest_mock.MockerFixture, + tmpdir: pytest.FixtureRequest, + defaults_dict: pytest.FixtureRequest): + """Tests build_services, which Constructs and writes a Dockerfile, requirements.txt, and + main.py to the services/submission_service directory. + + Args: + mocker: Mocker to patch directories. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + defaults_dict: Locally defined defaults_dict Pytest fixture. + """ + # Patch filepath constants to point to test path. + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'BASE_DIR', + f'{tmpdir}/') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_DEFAULTS_FILE', + defaults_dict['path']) + + # create required directories, run build_services + make_dirs([f'{tmpdir}/services/submission_service']) + build_services() + + # Ensure correct files are created with build_services call + assert os.path.exists(f'{tmpdir}/services/submission_service/Dockerfile') + assert os.path.exists(f'{tmpdir}/services/submission_service/requirements.txt') + assert os.path.exists(f'{tmpdir}/services/submission_service/main.py') + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline --config $CONFIG_FILE'])] +) +def test_build_pipeline_spec_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests build_pipeline_spec_jinja, which generates code for build_pipeline_spec.sh + which builds the pipeline specs. There is one test case for this function: + 1. Checks for the apache license and the pipeline compile command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_pipeline_spec.sh.j2' + build_pipeline_spec_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in build_pipeline_spec_script + elif not is_included: + assert snippet not in build_pipeline_spec_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600'])] +) +def test_build_components_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests build_components_jinja, which generates code for build_components.sh + which builds the components. There is one test case for this function: + 1. Checks for the apache license and the builds submit command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_components.sh.j2' + build_components_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in build_components_script + elif not is_included: + assert snippet not in build_components_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline_runner --config $CONFIG_FILE'])] +) +def test_run_pipeline_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests run_pipeline_jinja, which generates code for run_pipeline.sh + which runs the pipeline locally. There is one test case for this function: + 1. Checks for the apache license and the pipeline runner command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_pipeline.sh.j2' + run_pipeline_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in run_pipeline_script + elif not is_included: + assert snippet not in run_pipeline_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600', + './scripts/build_pipeline_spec.sh', './scripts/run_pipeline.sh'])] +) +def test_run_all_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests run_all_jinja, which generates code for run_all.sh + which builds runs all other shell scripts. There is one test case for this function: + 1. Checks for the apache license and the builds submit, the pipeline compile, and the pipeline runner commands. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_all.sh.j2' + run_all_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in run_all_script + elif not is_included: + assert snippet not in run_all_script + + +@pytest.mark.parametrize( + 'pubsub_topic_name, is_included, expected_output_snippets', + [('my-topic', True, [GENERATED_LICENSE, 'gcloud pubsub topics publish my-topic'])] +) +def test_publish_to_topic_jinja( + pubsub_topic_name: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests publish_to_topic_jinja, which generates code for publish_to_topic.sh + which submits a message to the pipeline job submission service. + There is one test case for this function: + 1. Checks for the apache license and the pubsub publish command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'publish_to_topic.sh.j2' + publish_to_topic_script = render_jinja( + template_path=template_file, + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + pubsub_topic_name=pubsub_topic_name + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in publish_to_topic_script + elif not is_included: + assert snippet not in publish_to_topic_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python3 -m model_monitoring.monitor --config $CONFIG_FILE'])] +) +def test_create_model_monitoring_job_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_model_monitoring_job_jinja, which generates code for create_model_monitoring_job.sh + which creates a Model Monitoring Job in Vertex AI for a deployed model endpoint. + There is one test case for this function: + 1. Checks for the apache license and the monitor command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'create_model_monitoring_job.sh.j2' + create_model_monitoring_job_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in create_model_monitoring_job_script + elif not is_included: + assert snippet not in create_model_monitoring_job_script + + +@pytest.mark.parametrize( + 'setup_model_monitoring, use_ci, is_included, expected_output_snippets', + [ + ( + False, True, True, + ['AutoMLOps - Generated Code Directory', + '├── components', + '├── configs', + '├── images', + '├── provision', + '├── scripts', + '├── services', + '├── README.md', + '└── cloudbuild.yaml'] + ), + ( + True, True, True, + ['AutoMLOps - Generated Code Directory', + '├── components', + '├── configs', + '├── images', + '├── model_monitoring', + '├── provision', + '├── scripts', + '├── services', + '├── README.md', + '└── cloudbuild.yaml'] + ), + ( + False, False, False, + ['├── publish_to_topic.sh' + '├── services', + '├── create_model_monitoring_job.sh', + '├── model_monitoring'] + ), + ] +) +def test_readme_jinja( + setup_model_monitoring: bool, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests readme_jinja, which generates code for readme.md which + is a readme markdown file to describe the contents of the + generated AutoMLOps code repo. There are three test cases for this function: + 1. Checks that certain directories and files exist when use_ci=True and setup_model_monitoring=False. + 2. Checks that certain directories and files exist when use_ci=True and setup_model_monitoring=True. + 3. Checks that certain directories and files do not exist when use_ci=False. + + Args: + setup_model_monitoring: Boolean parameter which specifies whether to set up a Vertex AI Model Monitoring Job. + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH) / 'README.md.j2' + readme_str = render_jinja( + template_path=template_file, + setup_model_monitoring=setup_model_monitoring, + use_ci=use_ci + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in readme_str + elif not is_included: + assert snippet not in readme_str + + +@pytest.mark.parametrize( + 'base_image, is_included, expected_output_snippets', + [('my-base-image', True, [GENERATED_LICENSE, 'FROM my-base-image'])] +) +def test_component_base_dockerfile_jinja( + base_image: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests readme_jinja, which generates code for a Dockerfile + to be written to the component_base directory. There is one + test case for this function: + 1. Checks for the apache license and the FROM image line. + + Args: + base_image: The image to use in the component base dockerfile. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2' + component_base_dockerfile = render_jinja( + template_path=template_file, + base_image=base_image, + generated_license=GENERATED_LICENSE + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in component_base_dockerfile + elif not is_included: + assert snippet not in component_base_dockerfile + + +@pytest.mark.parametrize( + 'custom_code_contents, kfp_spec_bool, is_included, expected_output_snippets', + [ + ( + 'this is some custom code', True, True, + [GENERATED_LICENSE, + 'this is some custom code', + 'def main():'] + ), + ( + 'this is some custom code', False, True, + [GENERATED_LICENSE, + 'this is some custom code', + 'def main():', + 'import kfp', + 'from kfp.v2.dsl import *'] + ), + ( + 'this is some custom code', True, False, + ['import kfp', + 'from kfp.v2.dsl import *'] + ) + ] +) +def test_component_base_task_file_jinja( + custom_code_contents: str, + kfp_spec_bool: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests component_base_task_file_jinja, which generates code + for the task.py file to be written to the component_base/src directory. + There are three test cases for this function: + 1. Checks for the apache license, the custom_code_contents, and a main function when using kfp spec (kfp spec comes with kfp imports by default). + 2. Checks for the apache license, the custom_code_contents, a main function, and kfp imports when not using kfp spec. + 3. Checks that the kfp imports are not included in the string when using kfp spec (kfp spec comes with kfp imports by default). + + Args: + custom_code_contents: Code inside of the component, specified by the user. + kfp_spec_bool: Boolean that specifies whether components are defined using kfp. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2' + component_base_task_file = render_jinja( + template_path=template_file, + custom_code_contents=custom_code_contents, + generated_license=GENERATED_LICENSE, + kfp_spec_bool=kfp_spec_bool) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in component_base_task_file + elif not is_included: + assert snippet not in component_base_task_file + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_pipeline_runner_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_runner_jinja, which generates code for the pipeline_runner.py + file to be written to the pipelines directory. There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2' + pipeline_runner_py = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in pipeline_runner_py + elif not is_included: + assert snippet not in pipeline_runner_py + + +@pytest.mark.parametrize( + '''components_list, custom_training_job_specs, pipeline_scaffold_contents, project_id,''' + '''is_included, expected_output_snippets''', + [ + ( + ['componentA','componentB','componentC'], + [ + { + 'component_spec': 'componentB', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': '1', + } + ], + 'Pipeline definition goes here', 'my-project', True, + [GENERATED_LICENSE, + 'from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', + 'def upload_pipeline_spec', + 'componentA = load_custom_component', + 'componentB = load_custom_component', + 'componentC = load_custom_component', + 'componentB_custom_training_job_specs', + 'Pipeline definition goes here'] + ), + ( + ['componentA','componentB','componentC'], + None, 'Pipeline definition goes here', 'my-project', True, + [GENERATED_LICENSE, + 'def upload_pipeline_spec', + 'componentA = load_custom_component', + 'componentB = load_custom_component', + 'componentC = load_custom_component', + 'Pipeline definition goes here'] + ), + ( + ['componentA','componentB','componentC'], + None, 'Pipeline definition goes here', 'my-project', False, + ['from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', + 'componentB_custom_training_job_specs'] + ), + ] +) +def test_pipeline_jinja( + components_list: list, + custom_training_job_specs: list, + pipeline_scaffold_contents: str, + project_id: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_jinja, which generates code for the pipeline.py + file to be written to the pipelines directory. + There are three test cases for this function: + 1. Checks for the apache license and relevant code elements when custom_training_job_specs is not None. + 2. Checks for the apache license and relevant code elements when custom_training_job_specs is None. + 3. Checks that the output does not contain custom_training_job_specs code elements when custom_training_job_specs is None. + + Args: + components_list: Contains the names or paths of all component yamls in the dir. + custom_training_job_specs: Specifies the specs to run the training job with. + pipeline_scaffold_contents: The contents of the pipeline scaffold file, + which can be found at PIPELINE_CACHE_FILE. + project_id: The project ID. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2' + pipeline_py = render_jinja( + template_path=template_file, + components_list=components_list, + custom_training_job_specs=custom_training_job_specs, + generated_license=GENERATED_LICENSE, + pipeline_scaffold_contents=pipeline_scaffold_contents, + project_id=project_id) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in pipeline_py + elif not is_included: + assert snippet not in pipeline_py + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform'])] +) +def test_pipeline_requirements_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_requirements_jinja, which generates code for a requirements.txt + to be written to the pipelines directory. There is one test case for this function: + 1. Checks for the pinned kfp version, and the google-cloud-aiplatform dep. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2' + pipeline_requirements_py = render_jinja( + template_path=template_file, + pinned_kfp_version=PINNED_KFP_VERSION + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in pipeline_requirements_py + elif not is_included: + assert snippet not in pipeline_requirements_py + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python:3.9-slim', + 'CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app'])] +) +def test_submission_service_dockerfile_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_requirements_jinja, which generates code for a Dockerfile to be + written to the serivces/submission_service directory. There is one test case for this function: + 1. Checks for the apache license and relevant dockerfile elements. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2' + submission_service_dockerfile = render_jinja( + template_path=template_file, + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in submission_service_dockerfile + elif not is_included: + assert snippet not in submission_service_dockerfile + + +@pytest.mark.parametrize( + 'pipeline_job_submission_service_type, is_included, expected_output_snippets', + [('cloud-functions', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'google-cloud-storage', 'functions-framework==3.*']), + ('cloud-functions', False, ['gunicorn']), + ('cloud-run', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'google-cloud-storage', 'gunicorn']), + ('cloud-run', False, ['functions-framework==3.*']),] +) +def test_submission_service_requirements_jinja( + pipeline_job_submission_service_type: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests submission_service_requirements_jinja, which generates code + for a requirements.txt to be written to the serivces/submission_service directory. + There are four test cases for this function: + 1. Checks for the pinned kfp version, the google-cloud-aiplatform, google-cloud-storage and function-framework deps when set to cloud-functions. + 2. Checks that gunicorn dep is not included when set to cloud-functions. + 3. Checks for the pinned kfp version, the google-cloud-aiplatform, google-cloud-storage and gunicorn deps when set to cloud-run. + 4. Checks that functions-framework dep is not included when set to cloud-run. + + Args: + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2' + submission_service_requirements = render_jinja( + template_path=template_file, + pinned_kfp_version=PINNED_KFP_VERSION, + pipeline_job_submission_service_type=pipeline_job_submission_service_type + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in submission_service_requirements + elif not is_included: + assert snippet not in submission_service_requirements + + +@pytest.mark.parametrize( + '''naming_prefix, pipeline_root, pipeline_job_runner_service_account, pipeline_job_submission_service_type,''' + '''project_id, setup_model_monitoring, is_included, expected_output_snippets''', + [ + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', + 'my-project', False, True, + [GENERATED_LICENSE, + 'from google.cloud import aiplatform', + 'import functions_framework', + '@functions_framework.http', + 'def process_request(request: flask.Request)', + '''base64_message = request_json['data']['data']'''] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', + 'my-project', False, False, + ['app = flask.Flask', + '''@app.route('/', methods=['POST'])''', + 'request = flask.request', + '''base64_message = request_json['message']['data']''', + '''if __name__ == '__main__':''', + '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))''', + 'from google.cloud import storage', + 'NAMING_PREFIX', + 'def read_gs_auto_retraining_params_file()'] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', + 'my-project', False, True, + [GENERATED_LICENSE, + 'from google.cloud import aiplatform', + 'app = flask.Flask', + '''@app.route('/', methods=['POST'])''', + 'request = flask.request', + '''base64_message = request_json['message']['data']''', + '''if __name__ == '__main__':''', + '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))'''] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', + 'my-project', False, False, + ['import functions_framework', + '@functions_framework.http', + 'def process_request(request: flask.Request)', + '''base64_message = request_json['data']['data']''', + 'from google.cloud import storage', + 'NAMING_PREFIX', + 'def read_gs_auto_retraining_params_file()'] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', + 'my-project', True, True, + ['from google.cloud import storage', + 'NAMING_PREFIX', + 'def read_gs_auto_retraining_params_file()', + '''if data_payload['logName'] == f'projects/{PROJECT_ID}/logs/aiplatform.googleapis.com%2Fmodel_monitoring_anomaly':'''] + ), + ] +) +def test_submission_service_main_jinja( + naming_prefix: str, + pipeline_root: str, + pipeline_job_runner_service_account: str, + pipeline_job_submission_service_type: str, + project_id: str, + setup_model_monitoring: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests submission_service_main_jinja, which generates content + for main.py to be written to the serivces/submission_service directory. + There are five test cases for this function: + 1. Checks for functions_framework code elements when set to cloud-functions. + 2. Checks that Flask app code elements are not included when set to cloud-functions. + 3. Checks for Flask app code elements when set to cloud-run. + 4. Checks that functions_framework code elements are not included when set to cloud-run. + 5. Checks that model_monitoring auto retraining code elements exists when setup_model_monitoring is True. + + Args: + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + pipeline_root: GS location where to store metadata from pipeline runs. + pipeline_job_runner_service_account: Service Account to runner PipelineJobs. + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + project_id: The project ID. + setup_model_monitoring: Boolean parameter which specifies whether to set up a Vertex AI Model Monitoring Job. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2' + submission_service_main_py = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + naming_prefix=naming_prefix, + pipeline_root=pipeline_root, + pipeline_job_runner_service_account=pipeline_job_runner_service_account, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + project_id=project_id, + setup_model_monitoring=setup_model_monitoring) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in submission_service_main_py + elif not is_included: + assert snippet not in submission_service_main_py diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py b/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py new file mode 100644 index 0000000..a66c74c --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py @@ -0,0 +1,318 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for kfp scaffold module.""" + +# pylint: disable=anomalous-backslash-in-string +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring + +from contextlib import nullcontext as does_not_raise +import os +from typing import Callable, List, NamedTuple +from typing import Optional + +import pytest + +from google_cloud_automlops.orchestration.kfp.scaffold import ( + create_component_scaffold, + create_pipeline_scaffold, + get_packages_to_install_command, + get_compile_step, + get_function_parameters, + get_pipeline_decorator, + get_function_return_types, +) +from google_cloud_automlops.utils.constants import DEFAULT_PIPELINE_NAME +import google_cloud_automlops.utils.utils +from google_cloud_automlops.utils.utils import get_function_source_definition, read_yaml_file + + +def add(a: int, b: int) -> NamedTuple('output', [('sum', int)]): + """Testing + + Args: + a (int): Integer a + b (int): Integer b + + Returns: + int: Sum of a and b + """ + return a + b + + +def sub(a, b): + return a - b + + +def div(a: float, b: float): + """Testing + + Args: + a (float): Float a + b (float): Float b + """ + return a/b + + +@pytest.mark.parametrize( + 'func, packages_to_install, expectation, has_return_type', + [ + (add, None, does_not_raise(), True), + (add, ['pandas', 'pytest'], does_not_raise(), True), + (sub, None, pytest.raises(TypeError), False) + ] +) +def test_create_component_scaffold(func: Callable, packages_to_install: list, expectation, has_return_type: bool): + """Tests create_component_scaffold, which creates a tmp component scaffold + which will be used by the formalize function. Code is temporarily stored in + component_spec['implementation']['container']['command']. + + Args: + func (Callable): The python function to create a component from. The function + should have type annotations for all its arguments, indicating how + it is intended to be used (e.g. as an input/output Artifact object, + a plain parameter, or a path to a file). + packages_to_install (list): A list of optional packages to install before + executing func. These will always be installed at component runtime. + expectation: Any corresponding expected errors for each + set of parameters. + has_return_type: boolean indicating if the function has a return type hint. + This is used to determine if an 'outputs' key should exist in the component scaffold. + """ + with expectation: + create_component_scaffold(func=func, + packages_to_install=packages_to_install) + + # Assert the yaml exists + func_path = f'.AutoMLOps-cache/{func.__name__}.yaml' + assert os.path.exists(func_path) + + # Assert yaml contains correct keys + component_spec = read_yaml_file(func_path) + outputs_key = ['outputs'] if has_return_type else [] + assert set(component_spec.keys()) == set(['name', 'description', 'inputs', 'implementation', *outputs_key]) + assert list(component_spec['implementation'].keys()) == ['container'] + assert list(component_spec['implementation']['container'].keys()) == ['image', 'command', 'args'] + + # Remove temporary files + os.remove(func_path) + os.rmdir('.AutoMLOps-cache') + + +@pytest.mark.parametrize( + 'func, packages_to_install', + [ + (add, None), + (add, ['pandas']), + (sub, ['pandas', 'kfp', 'pytest']) + ] +) +def test_get_packages_to_install_command(func: Callable, packages_to_install: list): + """Tests get_packages_to_install_command, which returns a list of + formatted list of commands, including code for tmp storage. + + Args: + func (Callable): The python function to create a component from. The function + should have type annotations for all its arguments, indicating how + it is intended to be used (e.g. as an input/output Artifact object, + a plain parameter, or a path to a file). + packages_to_install (list): A list of optional packages to install before + executing func. These will always be installed at component runtime. + """ + newline = '\n' + if not packages_to_install: + packages_to_install = [] + install_python_packages_script = ( + f'''if ! [ -x "$(command -v pip)" ]; then\n''' + f''' python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\n''' + f'''fi\n''' + f'''PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \{newline}''' + f''' --no-warn-script-location {' '.join([repr(str(package)) for package in packages_to_install])} && "$0" "$@"\n''' + f'''\n''') + assert get_packages_to_install_command(func, packages_to_install) == ['sh', '-c', install_python_packages_script, get_function_source_definition(func=func)] + + +@pytest.mark.parametrize( + 'func, params, expectation', + [ + ( + add, + [ + {'description': 'Integer a', 'name': 'a', 'type': 'Integer'}, + {'description': 'Integer b', 'name': 'b', 'type': 'Integer'} + ], + does_not_raise() + ), + ( + sub, + None, + pytest.raises(TypeError) + ), + ( + div, + [ + {'description': 'Float a', 'name': 'a', 'type': 'Float'}, + {'description': 'Float b', 'name': 'b', 'type': 'Float'} + ], + does_not_raise() + ) + ] +) +def test_get_function_parameters(func: Callable, params: List[dict], expectation): + """Tests get_function_parameters, which returns a formatted list of + parameters. + + Args: + func (Callable): The python function to create a component from. The function + should have type annotations for all its arguments, indicating how + it is intended to be used (e.g. as an input/output Artifact object, + a plain parameter, or a path to a file). + params (List[dict]): Params list with types converted to kubeflow spec. + expectation: Any corresponding expected errors for each + set of parameters. + """ + with expectation: + assert params == get_function_parameters(func=func) + + +@pytest.mark.parametrize( + 'func, name, description', + [ + (add, 'Add', 'This is a test'), + (sub, 'Sub', 'Test 2'), + (div, None, None) + ] +) +def test_create_pipeline_scaffold(mocker, func: Callable, name: str, description: str): + """Tests create_pipeline_scaffold, which creates a temporary pipeline + scaffold which will be used by the formalize function. + + Args: + mocker: Mocker used to patch constants to test in tempoarary + environment. + func (Callable): The python function to create a pipeline from. The + function should have type annotations for all its arguments, + indicating how it is intended to be used (e.g. as an input/output + Artifact object, a plain parameter, or a path to a file). + name (str): The name of the pipeline. + description (str): Short description of what the pipeline does. + """ + mocker.patch.object(google_cloud_automlops.utils.utils, 'CACHE_DIR', '.') + create_pipeline_scaffold(func=func, name=name, description=description) + fold = '.AutoMLOps-cache' + file_path = 'pipeline_scaffold.py' + assert os.path.exists(os.path.join(fold, file_path)) + os.remove(os.path.join(fold, file_path)) + os.rmdir(fold) + + +@pytest.mark.parametrize( + 'name, description', + [ + ('Name1', 'Description1'), + ('Name2', 'Description2'), + (None, None), + ] +) +def test_get_pipeline_decorator(name: str, description: str): + """Tests get_pipeline_decorator, which creates the kfp pipeline decorator. + + Args: + name (str): The name of the pipeline. + description (str): Short description of what the pipeline does. + """ + desc_str = f''' description='{description}',\n''' if description else '' + decorator = ( + f'''@dsl.pipeline''' + f'''(\n name='{DEFAULT_PIPELINE_NAME if not name else name}',\n''' + f'''{desc_str}''' + f''')\n''' + ) + assert decorator == get_pipeline_decorator(name=name, description=description) + + +@pytest.mark.parametrize( + 'func_name', + ['func1', 'func2'] +) +def test_get_compile_step(func_name: str): + """Tests get_compile_step, which creates the compile function call. + + Args: + func_name (str): The name of the pipeline function. + """ + assert get_compile_step(func_name=func_name) == ( + f'\n' + f'compiler.Compiler().compile(\n' + f' pipeline_func={func_name},\n' + f' package_path=pipeline_job_spec_path)\n' + f'\n' + ) + + +@pytest.mark.parametrize( + 'return_annotation, return_types, expectation', + [ + ( + NamedTuple('output', [('sum', int)]), + [{'description': None, 'name': 'sum', 'type': 'Integer'},], + does_not_raise() + ), + ( + NamedTuple('output', [('first', str), ('last', str)]), + [{'description': None, 'name': 'first', 'type': 'String'}, + {'description': None, 'name': 'last', 'type': 'String'},], + does_not_raise() + ), + ( + Optional[NamedTuple('output', [('count', int)])], + None, + pytest.raises(TypeError) + ), + ( + int, + None, + pytest.raises(TypeError) + ),( + None, + None, + pytest.raises(TypeError) + ), + ( + 'NO_ANNOTATION', + None, + does_not_raise() + ) + ] +) +def test_get_function_return_types(return_annotation, return_types: List[dict], expectation): + """Tests get_function_outputs, which returns a formatted list of + return types. + + Args: + annotation (Any): The return type to test. + return_types (List[dict]): The return type converted into the kubeflow output spec. + expectation: Any corresponding expected errors for each + set of parameters. + """ + + def func(): + ... + + if return_annotation != 'NO_ANNOTATION': + func.__annotations__ = {'return' : return_annotation} + + with expectation: + assert return_types == get_function_return_types(func=func) diff --git a/google_cloud_automlops/tests/unit/provisioning/__init__.py b/google_cloud_automlops/tests/unit/provisioning/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py b/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py b/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py new file mode 100644 index 0000000..37e837d --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py @@ -0,0 +1,220 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +from typing import List + +import pytest + +from google_cloud_automlops.provisioning.gcloud.builder import provision_resources_script_jinja + +@pytest.mark.parametrize( + '''artifact_repo_location, artifact_repo_name, artifact_repo_type, build_trigger_location,''' + '''build_trigger_name, deployment_framework, naming_prefix, pipeline_job_runner_service_account,''' + '''pipeline_job_submission_service_location, pipeline_job_submission_service_name, pipeline_job_submission_service_type,''' + '''project_id, pubsub_topic_name,''' + '''required_apis,''' + '''schedule_location, schedule_name, schedule_pattern,''' + '''source_repo_branch, source_repo_name, source_repo_type, storage_bucket_location, storage_bucket_name,''' + '''use_ci, vpc_connector, is_included,''' + '''expected_output_snippets''', + [ + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'my-vpc-connector', True, + ['gcloud artifacts repositories create', 'gcloud iam service-accounts create', + 'gsutil mb -l ${STORAGE_BUCKET_LOCATION} gs://$STORAGE_BUCKET_NAME', 'gcloud iam service-accounts create', + 'gcloud projects add-iam-policy-binding', 'gcloud source repos create', 'gcloud pubsub topics create', + 'gcloud functions deploy', 'gcloud beta builds triggers create', 'gcloud scheduler jobs create pubsub', + '--vpc-connector=my-vpc-connector'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-run', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', True, + ['gcloud artifacts repositories create', 'gcloud iam service-accounts create', + 'gsutil mb -l ${STORAGE_BUCKET_LOCATION} gs://$STORAGE_BUCKET_NAME', 'gcloud iam service-accounts create', + 'gcloud projects add-iam-policy-binding', 'gcloud source repos create', 'gcloud pubsub topics create', + 'gcloud builds submit ${BASE_DIR}services/submission_service', 'gcloud run deploy', 'gcloud pubsub subscriptions create', + 'gcloud beta builds triggers create', 'gcloud scheduler jobs create pubsub'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-run', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'some-other-source-repository', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud source repos create', 'cloud beta builds triggers create'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-run', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', 'No Schedule Specified', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud scheduler jobs create pubsub', '--vpc_connector='] + ), + ( + 'us-central1', 'my-registry', 'some-other-repo-type', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud artifacts repositories create'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'some-other-deployment-framework', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud beta builds triggers create'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + False, 'No VPC Specified', False, + ['gcloud pubsub topics create', 'gcloud beta builds triggers create', + 'gcloud functions deploy', 'gcloud run deploy', 'gcloud scheduler jobs create pubsub'] + ) + ] +) +def test_provision_resources_script_jinja( + artifact_repo_location: str, + artifact_repo_name: str, + artifact_repo_type: str, + build_trigger_location: str, + build_trigger_name: str, + deployment_framework: str, + naming_prefix: str, + pipeline_job_runner_service_account: str, + pipeline_job_submission_service_location: str, + pipeline_job_submission_service_name: str, + pipeline_job_submission_service_type: str, + project_id: str, + pubsub_topic_name: str, + required_apis: list, + schedule_location: str, + schedule_name: str, + schedule_pattern: str, + source_repo_branch: str, + source_repo_name: str, + source_repo_type: str, + storage_bucket_location: str, + storage_bucket_name: str, + use_ci: bool, + vpc_connector: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests provision_resources_script_jinja, which generates code for + provision_resources.sh which sets up the project's environment. + There are seven test cases for this function: + 1. Checks for relevant gcloud commands when using the following tooling: + artifact-registry, cloud-build, cloud-functions, cloud scheduler, cloud-source-repositories, and a vpc connector + 2. Checks for relevant gcloud commands when using the following tooling: + artifact-registry, cloud-build, cloud-run, cloud scheduler, cloud-source-repositories, and no vpc connector + 3. Checks that gcloud source repo commands are not included when not using cloud-source-repositories. + 4. Checks that gcloud scheduler command and vpc_connector flag are not included when not specifying a vpc connector or schedule. + 5. Checks that gcloud artifacts command is not included when not using artifact-registry. + 6. Checks that gcloud beta builds triggers command is not included when not using cloud-build. + 7. Checks for that CI/CD elements are not included when use_ci=False. + + Args: + artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). + artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). + artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) + build_trigger_location: The location of the build trigger (for cloud build). + build_trigger_name: The name of the build trigger (for cloud build). + deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + pipeline_job_runner_service_account: Service Account to run PipelineJobs. + pipeline_job_submission_service_location: The location of the cloud submission service. + pipeline_job_submission_service_name: The name of the cloud submission service. + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + project_id: The project ID. + pubsub_topic_name: The name of the pubsub topic to publish to. + required_apis: List of APIs that are required to run the service. + schedule_location: The location of the scheduler resource. + schedule_name: The name of the scheduler resource. + schedule_pattern: Cron formatted value used to create a Scheduled retrain job. + source_repo_branch: The branch to use in the source repository. + source_repo_name: The name of the source repository to use. + source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) + storage_bucket_location: Region of the GS bucket. + storage_bucket_name: GS bucket name where pipeline run metadata is stored. + use_ci: Flag that determines whether to use Cloud CI/CD. + vpc_connector: The name of the vpc connector to use. + """ + provision_resources_script = provision_resources_script_jinja( + artifact_repo_location=artifact_repo_location, + artifact_repo_name=artifact_repo_name, + artifact_repo_type=artifact_repo_type, + build_trigger_location=build_trigger_location, + build_trigger_name=build_trigger_name, + deployment_framework=deployment_framework, + naming_prefix=naming_prefix, + pipeline_job_runner_service_account=pipeline_job_runner_service_account, + pipeline_job_submission_service_location=pipeline_job_submission_service_location, + pipeline_job_submission_service_name=pipeline_job_submission_service_name, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + project_id=project_id, + pubsub_topic_name=pubsub_topic_name, + required_apis=required_apis, + schedule_location=schedule_location, + schedule_name=schedule_name, + schedule_pattern=schedule_pattern, + source_repo_branch=source_repo_branch, + source_repo_name=source_repo_name, + source_repo_type=source_repo_type, + storage_bucket_location=storage_bucket_location, + storage_bucket_name=storage_bucket_name, + use_ci=use_ci, + vpc_connector=vpc_connector) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in provision_resources_script + elif not is_included: + assert snippet not in provision_resources_script diff --git a/google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep b/google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep b/google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py b/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py b/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py new file mode 100644 index 0000000..03e5c72 --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py @@ -0,0 +1,522 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +from typing import List + +import pytest + +from google_cloud_automlops.utils.constants import GENERATED_LICENSE +from google_cloud_automlops.provisioning.terraform.builder import ( + create_environment_data_tf_jinja, + create_environment_iam_tf_jinja, + create_environment_main_tf_jinja, + create_environment_outputs_tf_jinja, + create_environment_provider_tf_jinja, + create_environment_variables_tf_jinja, + create_environment_versions_tf_jinja, + create_provision_resources_script_jinja, + create_state_bucket_variables_tf_jinja, + create_state_bucket_main_tf_jinja +) + + +@pytest.mark.parametrize( + 'required_apis, use_ci, is_included, expected_output_snippets', + [ + ( + ['apiA', 'apiB'], True, True, + [GENERATED_LICENSE, 'archive_cloud_functions_submission_service', + 'enable_apis = [\n' + ' "apiA",\n' + ' "apiB",\n' + ' ]' + ] + ), + ( + ['apiA', 'apiB'], False, False, + ['archive_cloud_functions_submission_service'] + ) + ] +) +def test_create_environment_data_tf_jinja( + required_apis: List, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_data_tf_jinja, which generates code for environment/data.tf, + the terraform hcl script that contains terraform remote backend and org project details. + There are two test cases for this function: + 1. Checks for the apache license and relevant terraform blocks. + 2. Checks for that the archive statement is not included when use_ci=False. + + Args: + required_apis: List of APIs that are required to run the service. + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + data_tf_str = create_environment_data_tf_jinja(required_apis, use_ci) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in data_tf_str + elif not is_included: + assert snippet not in data_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_environment_iam_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_iam_tf_jinja, which generates code for environment/iam.tf, the terraform hcl + script that contains service accounts iam bindings for project's environment. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + iam_tf_str = create_environment_iam_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in iam_tf_str + elif not is_included: + assert snippet not in iam_tf_str + + +@pytest.mark.parametrize( + '''artifact_repo_type, deployment_framework, naming_prefix,''' + '''pipeline_job_submission_service_type, schedule_pattern,''' + '''source_repo_type, use_ci, vpc_connector, is_included,''' + '''expected_output_snippets''', + [ + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, 'my-vpc-connector', True, + ['resource "google_artifact_registry_repository"', + 'resource "google_storage_bucket"', + 'resource "google_sourcerepo_repository"', + 'resource "google_pubsub_topic"', + 'resource "google_storage_bucket_object"', + 'resource "google_cloudfunctions_function"', + 'vpc_connector =', + 'resource "google_cloudbuild_trigger"', + 'resource "google_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, 'my-vpc-connector', True, + ['resource "google_artifact_registry_repository"', + 'resource "google_storage_bucket"', + 'resource "google_sourcerepo_repository"', + 'resource "null_resource" "build_and_push_submission_service"', + 'module "cloud_run"', + 'run.googleapis.com/vpc-access-connector', + 'module "pubsub"', + 'resource "google_cloudbuild_trigger"', + 'resource "google_cloud_scheduler_job"'] + ), + ( + 'some-other-repo', 'cloud-build', 'my-prefix', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['resource "google_artifact_registry_repository"', 'vpc_connector ='] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['run.googleapis.com/vpc-access-connector'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['resource "google_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'some-deployment-framework', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['resource "google_cloudbuild_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', True, 'No VPC Specified', False, + ['resource "google_sourcerepo_repository"', 'resource "google_cloudbuild_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', False, 'No VPC Specified', False, + ['resource "null_resource" "build_and_push_submission_service"', + 'module "cloud_run"', + 'module "pubsub"', + 'resource "google_pubsub_topic"', + 'resource "google_storage_bucket_object"', + 'resource "google_cloudfunctions_function"', + 'resource "google_cloudbuild_trigger"', + 'resource "google_cloud_scheduler_job"'] + ), + ] +) +def test_create_environment_main_tf_jinja( + artifact_repo_type: str, + deployment_framework: str, + naming_prefix: str, + pipeline_job_submission_service_type: str, + schedule_pattern: str, + source_repo_type: str, + use_ci: bool, + vpc_connector: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_main_environment_tf_jinja, which generates code for environment/main.tf, the terraform hcl + script that contains terraform resources configs to deploy resources in the project. + There are eight test cases for this function: + 1. Checks for relevant terraform blocks when using the following tooling: + artifact-registry, cloud-build, cloud-functions, cloud scheduler, cloud-source-repositories, and a vpc connector + 2. Checks for relevant terraform blocks when using the following tooling: + artifact-registry, cloud-build, cloud-run, cloud scheduler, cloud-source-repositories, and a vpc connector + 3. Checks that the artifact-registry terraform block is not included when not using artifact-registry. + 4. Checks that the vpc-connector element is not included when not using a vpc connector. + 5. Checks that the cloud scheduler terraform block is not included when not using a cloud schedule. + 6. Checks that the cloud build trigger terraform block is not included when not using cloud-build. + 7. Checks that the cloud source repositories and cloud build trigger terraform blocks are not included when not using cloud-source-repositories. + 8. Checks for that CI/CD infra terraform blocks are not included when use_ci=False. + + Args: + artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) + deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + schedule_pattern: Cron formatted value used to create a Scheduled retrain job. + source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) + use_ci: Flag that determines whether to use Cloud CI/CD. + vpc_connector: The name of the vpc connector to use. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + main_tf_str = create_environment_main_tf_jinja( + artifact_repo_type=artifact_repo_type, + deployment_framework=deployment_framework, + naming_prefix=naming_prefix, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + schedule_pattern=schedule_pattern, + source_repo_type=source_repo_type, + use_ci=use_ci, + vpc_connector=vpc_connector) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in main_tf_str + elif not is_included: + assert snippet not in main_tf_str + + +@pytest.mark.parametrize( + '''artifact_repo_type, deployment_framework,''' + '''pipeline_job_submission_service_type, schedule_pattern,''' + '''source_repo_type, use_ci, is_included,''' + '''expected_output_snippets''', + [ + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, True, + ['output "enabled_apis"', + 'output "create_pipeline_job_runner_service_account_email"', + 'output "create_artifact_registry"', + 'output "create_storage_bucket"', + 'output "create_storage_bucket_names"', + 'output "create_cloud_source_repository"', + 'output "create_pubsub_topic"', + 'output "create_cloud_function"', + 'output "create_cloud_build_trigger"', + 'output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, True, + ['output "enabled_apis"', + 'output "create_pipeline_job_runner_service_account_email"', + 'output "create_artifact_registry"', + 'output "create_storage_bucket"', + 'output "create_storage_bucket_names"', + 'output "create_cloud_source_repository"', + 'output "cloud_run_id"', + 'output "create_pubsub_subscription"', + 'output "create_cloud_build_trigger"', + 'output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job"'] + ), + ( + 'some-other-repo', 'cloud-build', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, False, + ['output "create_artifact_registry"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, False, + ['output "create_cloud_function"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, False, + ['output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'some-deployment-framework', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, False, + ['output "create_cloud_build_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', True, False, + ['output "create_cloud_source_repository"', + 'output "create_cloud_build_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', False, False, + ['resource "null_resource" "build_and_push_submission_service"', + 'output "cloud_run_id"' + 'output "create_pubsub_subscription"', + 'output "create_pubsub_topic"', + 'output "create_cloud_function"', + 'output "create_cloud_build_trigger"', + 'output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job" '] + ), + ] +) +def test_create_environment_outputs_tf_jinja( + artifact_repo_type: str, + deployment_framework: str, + pipeline_job_submission_service_type: str, + schedule_pattern: str, + source_repo_type: str, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_outputs_tf_jinja, which gnerates code for environment/outputs.tf, the terraform hcl + script that contains outputs from project's environment. + There are eight test cases for this function: + 1. Checks for relevant terraform output blocks when using the following tooling: + artifact-registry, cloud-build, cloud-functions, cloud scheduler, and cloud-source-repositories + 2. Checks for relevant terraform output blocks when using the following tooling: + artifact-registry, cloud-build, cloud-run, cloud scheduler, and cloud-source-repositories + 3. Checks that the artifact-registry terraform output block is not included when not using artifact-registry. + 4. Checks that the cloud functions terraform output block is not included when using cloud-run. + 5. Checks that the cloud scheduler terraform output blocks are not included when not using a cloud schedule. + 6. Checks that the cloud build trigger terraform output block is not included when not using cloud-build. + 7. Checks that the cloud source repositories and cloud build trigger output blocks are not included when not using cloud-source-repositories. + 8. Checks for that CI/CD infra terraform output blocks are not included when use_ci=False. + + Args: + artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) + deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + schedule_pattern: Cron formatted value used to create a Scheduled retrain job. + source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + main_tf_str = create_environment_outputs_tf_jinja( + artifact_repo_type=artifact_repo_type, + deployment_framework=deployment_framework, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + schedule_pattern=schedule_pattern, + source_repo_type=source_repo_type, + use_ci=use_ci) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in main_tf_str + elif not is_included: + assert snippet not in main_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_environment_provider_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_provider_tf_jinja, which generates code for environment/provider.tf, the terraform hcl + script that contains teraform providers used to deploy project's environment. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + provider_tf_str = create_environment_provider_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in provider_tf_str + elif not is_included: + assert snippet not in provider_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_environment_variables_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_variables_tf_jinja, which generates code for environment/variables.tf, + the terraform hcl script that contains variables used to deploy project's environment. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + variables_tf_str = create_environment_variables_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in variables_tf_str + elif not is_included: + assert snippet not in variables_tf_str + + +@pytest.mark.parametrize( + 'storage_bucket_name, is_included, expected_output_snippets', + [('my-storage-bucket', True, [GENERATED_LICENSE, 'bucket = "my-storage-bucket-tfstate"'])] +) +def test_create_environment_versions_tf_jinja( + storage_bucket_name: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_versions_tf_jinja, which generates code for environment/versions.tf, + the terraform hcl script that contains teraform version information. + There is one test case for this function: + 1. Checks for the apache license and state file storage_bucket backend. + + Args: + storage_bucket_name: GS bucket name where pipeline run metadata is stored. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + versions_tf_str = create_environment_versions_tf_jinja(storage_bucket_name=storage_bucket_name) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in versions_tf_str + elif not is_included: + assert snippet not in versions_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, '#!/bin/bash'])] +) +def test_create_provision_resources_script_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_provision_resources_script_jinja, which generates code for provision_resources.sh + which sets up the project's environment using terraform. + There is one test case for this function: + 1. Checks for the apache license and the Bash shebang. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + provision_resources_script = create_provision_resources_script_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in provision_resources_script + elif not is_included: + assert snippet not in provision_resources_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_state_bucket_variables_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_state_bucket_variables_tf_jinja, which generates code for state_bucket/variables.tf, + the terraform hcl script that contains variables used for the state_bucket. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + variables_tf_str = create_state_bucket_variables_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in variables_tf_str + elif not is_included: + assert snippet not in variables_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_state_bucket_main_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_main_state_bucket_tf_jinja, which generates code for state_bucket/main.tf, the terraform hcl + script that contains terraform resources configs to create the state_bucket. + There are eight test cases for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + main_tf_str = create_state_bucket_main_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in main_tf_str + elif not is_included: + assert snippet not in main_tf_str diff --git a/google_cloud_automlops/tests/unit/utils/__init__.py b/google_cloud_automlops/tests/unit/utils/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/utils/utils_test.py b/google_cloud_automlops/tests/unit/utils/utils_test.py new file mode 100644 index 0000000..110503f --- /dev/null +++ b/google_cloud_automlops/tests/unit/utils/utils_test.py @@ -0,0 +1,531 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for utils module.""" + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring + +from contextlib import nullcontext as does_not_raise +import os +from typing import Callable, List + +import pandas as pd +import pytest +import pytest_mock +import yaml + +import google_cloud_automlops.utils.utils +from google_cloud_automlops.utils.utils import ( + delete_file, + execute_process, + get_components_list, + get_function_source_definition, + is_component_config, + make_dirs, + read_file, + read_yaml_file, + stringify_job_spec_list, + update_params, + validate_use_ci, + write_and_chmod, + write_file, + write_yaml_file +) + + +# Define simple functions to be used in tests +def func1(x): + return x + 1 + + +def func2(x, y): + return x + y + + +def func3(x, y, z): + return x + y + z + + +def func4(): + + def inner_func(): + res = 1 + 1 + return res + + return inner_func() + + +@pytest.mark.parametrize( + 'directories, existance, expectation', + [ + (['dir1', 'dir2'], [True, True], does_not_raise()), + (['dir1', 'dir1'], [True, False], does_not_raise()), + (['\0', 'dir1'], [True, True], pytest.raises(ValueError)) + ] +) +def test_make_dirs(directories: List[str], existance: List[bool], expectation): + """Tests make_dirs, which creates a list of directories if they do not + already exist. There are three test cases for this function: + 1. Expected outcome, folders created as expected. + 2. Duplicate folder names given, expecting only one folder created. + 3. Invalid folder name given, expects an error. + + Args: + directories (List[str]): List of directories to be created. + existance (List[bool]): List of booleans indicating whether the listed directories to + be created are expected to exist after invoking make_dirs. + expectation: Any corresponding expected errors for each set of parameters. + """ + with expectation: + make_dirs(directories=directories) + for directory, exist in zip(directories, existance): + assert os.path.exists(directory) == exist + if exist: + os.rmdir(directory) + + +@pytest.mark.parametrize( + 'filepath, content1, content2, expectation', + [ + ( + 'test.yaml', + {'key1': 'value1', 'key2': 'value2'}, + None, + does_not_raise() + ), + ( + 'test.yaml', + {'key1': 'value1', False: 'random stuff'}, + r'-A fails', + pytest.raises(yaml.YAMLError) + ) + ] +) +def test_read_yaml_file(filepath: str, content1: dict, content2: str, expectation): + """Tests read_yaml_file, which reads a yaml file and returns the file + contents as a dict. There are two sets of test cases for this function: + 1. Expected outcome, file read in with correct content. + 2. File to be read is not in standard yaml format, expects a yaml error. + + Args: + filepath (str): Path to yaml file to be read. + content1 (dict): First set of content to be included in the yaml at the given + file path. + content2 (str): Second set of content to be included in the yaml at the given + file path. + expectation: Any corresponding expected errors for each set of + parameters. + """ + with open(file=filepath, mode='w', encoding='utf-8') as file: + if content1: + yaml.dump(content1, file) + if content2: + yaml.dump(content2, file) + with expectation: + assert read_yaml_file(filepath=filepath) == content1 + os.remove(path=filepath) + + +@pytest.mark.parametrize( + 'filepath, mode, expectation', + [ + ('test.yaml', 'w', does_not_raise()), + ('/nonexistent/directory', 'w', pytest.raises(FileNotFoundError)), + ('test.yaml', 'r', pytest.raises(IOError)) + ] +) +def test_write_yaml(filepath: str, mode: str, expectation): + """Tests write_yaml_file, which writes a yaml file. There are three sets of + test cases for this function: + 1. Expected outcome, yaml is written correctly. + 2. Invalid file path given, expecting a FileNotFoundError. + 3. Invalid mode given, expecting an IOError. + + Args: + filepath (str): Path for yaml file to be written. + mode (str): Read/write mode to be used. + expectation: Any corresponding expected errors for each set of + parameters. + """ + contents = {'key1': 'value1', 'key2': 'value2'} + with expectation: + write_yaml_file( + filepath=filepath, + contents=contents, + mode=mode + ) + with open(file=filepath, mode='r', encoding='utf-8') as file: + assert yaml.safe_load(file) == contents + os.remove(path=filepath) + + +@pytest.mark.parametrize( + 'filepath, text, write_file_bool, expectation', + [ + ('test.txt', 'This is a text file.', True, does_not_raise()), + ('fail', '', False, pytest.raises(FileNotFoundError)) + ] +) +def test_read_file(filepath: str, text: str, write_file_bool: bool, expectation): + """Tests read_file, which reads a text file in as a string. There are two + sets of test cases for this function: + 1. Expected outcome, file is read correctly. + 2. Invalid file path given (file was not written), expecting a + FileNotFoundError. + + Args: + filepath (str): Path for file to be read from. + text (str): Text expected to be read from the given file. + write_file_bool (bool): Whether or not the file should be written for this + test case. + expectation: Any corresponding expected errors for each set of + parameters. + """ + if write_file_bool: + with open(file=filepath, mode='w', encoding='utf-8') as file: + file.write(text) + with expectation: + assert read_file(filepath=filepath) == text + if os.path.exists(filepath): + os.remove(filepath) + + +@pytest.mark.parametrize( + 'filepath, text, mode, expectation', + [ + ('test.txt', 'This is a test file.', 'w', does_not_raise()), + (15, 'This is a test file.', 'w', pytest.raises(OSError)) + ] +) +def test_write_file(filepath: str, text: str, mode: str, expectation): + """Tests write_file, which writes a string to a text file. There are two + test cases for this function: + 1. Expected outcome, file is written as expected. + 2. Invalid file path given (file was not written), expecting + an OSError. + + Args: + filepath (str): Path for file to be written. + text (str): Content to be written to the file at the given filepath. + mode (str): Read/write mode to be used. + expectation: Any corresponding expected errors for each set of + parameters. + """ + with expectation: + write_file( + filepath=filepath, + text=text, + mode=mode + ) + assert os.path.exists(filepath) + with open(file=filepath, mode='r', encoding='utf-8') as file: + assert text == file.read() + os.remove(filepath) + + +def test_write_and_chmod(): + """Tests write_and_chmod, which writes a file at the specified path + and chmods the file to allow for execution. + """ + # Create a file. + with open(file='test.txt', mode='w', encoding='utf-8') as file: + file.write('This is a test file.') + + # Call the `write_and_chmod` function. + write_and_chmod('test.txt', 'This is a test file.') + + # Assert that the file exists and is executable. + assert os.path.exists('test.txt') + assert os.access('test.txt', os.X_OK) + + # Assert that the contents of the file are correct. + with open(file='test.txt', mode='r', encoding='utf-8') as file: + contents = file.read() + assert contents == 'This is a test file.' + os.remove('test.txt') + + +@pytest.mark.parametrize( + 'file_to_delete, valid_file', + [ + ('test.txt', True), + ('fake.txt', False) + ] +) +def test_delete_file(file_to_delete: str, valid_file: bool): + """Tests delete_file, which deletes a file at the specified path. + There are two test cases for this function: + 1. Create a valid file and call delete_file, which is expected to successfully delete the file. + 2. Pass in a nonexistent file and call delete_file, which is expected to pass. + + Args: + file_to_delete (str): Name of file to delete. + valid_file (bool): Whether or not the file to delete actually exists.""" + if not valid_file: + with does_not_raise(): + delete_file(file_to_delete) + else: + with open(file=file_to_delete, mode='w', encoding='utf-8') as file: + file.write('This is a test file.') + delete_file(file_to_delete) + assert not os.path.exists(file_to_delete) + + +@pytest.mark.parametrize( + 'comp_path, comp_name, patch_cwd, expectation', + [ + (['component.yaml'], ['component'], True, does_not_raise()), + ([], [], True, does_not_raise()), + (['component.yaml'], ['component'], False, pytest.raises(FileNotFoundError)) + ] +) +def test_get_components_list(mocker: pytest_mock.MockerFixture, + comp_path: List[str], + comp_name: List[str], + patch_cwd: bool, + expectation): + """Tests get_components_list, which reads yamls in .AutoMLOps-cache directory, + verifies they are component yamls, and returns the name of the files. There + are three test cases for this function: + 1. Expected outcome, component list is pulled as expected. + 2. Verifies an empty list comes back if no YAMLs are present. + 3. Call function with a nonexistent dir, expecting OSError. + + Args: + mocker: Mocker to patch the cache directory for component files. + comp_path (List[str]): Path(s) to component yamls. + comp_name (List[str]): Name(s) of components. + patch_cwd (bool): Boolean flag indicating whether to patch the current working + directory from CACHE_DIR to root + expectation: Any corresponding expected errors for each set of + parameters. + """ + if patch_cwd: + mocker.patch.object(google_cloud_automlops.utils.utils, 'CACHE_DIR', '.') + if comp_path: + for file in comp_path: + with open(file=file, mode='w', encoding='utf-8') as f: + yaml.dump( + { + 'name': 'value1', + 'inputs': 'value2', + 'implementation': 'value3' + }, + f) + with expectation: + assert get_components_list(full_path=False) == comp_name + assert get_components_list(full_path=True) == [os.path.join('.', file) for file in comp_path] + for file in comp_path: + if os.path.exists(file): + os.remove(file) + + +@pytest.mark.parametrize( + 'yaml_contents, expectation', + [ + ( + { + 'name': 'value1', + 'inputs': 'value2', + 'implementation': 'value3' + }, + True + ), + ( + { + 'name': 'value1', + 'inputs': 'value2' + }, + False + ) + ] +) +def test_is_component_config(yaml_contents: dict, expectation: bool): + """Tests is_component_config, which which checks to see if the given file is + a component yaml. There are two test cases for this function: + 1. A valid component is given, expecting return value True. + 2. An invalid component is given, expecting return value False. + + Args: + yaml_contents (dict): Component configurations to be written to yaml file. + expected (bool): Expectation of whether or not the configuration is valid. + """ + with open(file='component.yaml', mode='w', encoding='utf-8') as f: + yaml.dump(yaml_contents, f) + assert expectation == is_component_config('component.yaml') + os.remove('component.yaml') + + +@pytest.mark.parametrize( + 'command, to_null, expectation', + [ + ('touch test.txt', False, False), + ('not a real command', False, True), + ('echo "howdy"', True, False) + ] +) +def test_execute_process(command: str, to_null: bool, expectation: bool): + """Tests execute_process, which executes an external shell process. There + are two test cases for this function: + 1. A valid command to create a file, which is expected to run successfully. + 2. An invalid command, which is expected to raise a RunTime Error. + 3. A valid command to output a string, which is expected to send output to null + + Args: + command (str): Command that is to be executed. + expectation (bool): Whether or not an error is expected to be raised. + """ + if expectation: + with pytest.raises(RuntimeError): + execute_process(command=command, to_null=to_null) + elif to_null: + assert execute_process(command=command, to_null=to_null) is None + else: + execute_process(command=command, to_null=to_null) + assert os.path.exists('test.txt') + os.remove('test.txt') + + +@pytest.mark.parametrize( + 'sch_pattern, setup_model_monitoring, use_ci, expectation', + [ + ('No Schedule Specified', False, True, does_not_raise()), + ('No Schedule Specified', False, False, does_not_raise()), + ('Schedule', False, False, pytest.raises(ValueError)), + ('Schedule', True, True, does_not_raise()), + ('Schedule', True, False, pytest.raises(ValueError)) + ] +) +def test_validate_use_ci(sch_pattern: str, + setup_model_monitoring: bool, + use_ci: bool, + expectation): + """Tests validate_use_ci, which validates the inputted schedule + parameter and the setup_model_monitoring parameter. There are + five test cases for this function, which tests each + combination of sch_pattern and setup_model_monitoring for the expected results. + + Args: + sch_pattern (str): Cron formatted value used to create a Scheduled retrain job. + setup_model_monitoring (bool): Boolean parameter which specifies whether to set + up a Vertex AI Model Monitoring Job. + use_ci (bool): Flag that determines whether to use Cloud Run CI/CD. + expectation: Any corresponding expected errors for each set of parameters. + """ + with expectation: + validate_use_ci(schedule_pattern=sch_pattern, + setup_model_monitoring=setup_model_monitoring, + use_ci=use_ci) + + +@pytest.mark.parametrize( + 'params, expected_output', + [ + ([{'name': 'param1', 'type': int}], [{'name': 'param1', 'type': 'Integer'}]), + ([{'name': 'param2', 'type': str}], [{'name': 'param2', 'type': 'String'}]), + ([{'name': 'param3', 'type': float}], [{'name': 'param3', 'type': 'Float'}]), + ([{'name': 'param4', 'type': bool}], [{'name': 'param4', 'type': 'Boolean'}]), + ([{'name': 'param5', 'type': list}], [{'name': 'param5', 'type': 'JsonArray'}]), + ([{'name': 'param6', 'type': dict}], [{'name': 'param6', 'type': 'JsonObject'}]), + ([{'name': 'param6', 'type': pd.DataFrame}], None) + ] +) +def test_update_params(params: List[dict], expected_output: List[dict]): + """Tests the update_params function, which reformats the source code type + labels as strings. There are seven test cases for this function, which test + for updating different parameter types. + + Args: + params (List[dict]): Pipeline parameters. A list of dictionaries, each param is a dict containing keys: + 'name': required, str param name. + 'type': required, python primitive type. + 'description': optional, str param desc. + expected_output (List[dict]): Expectation of whether or not the configuration is valid. + """ + if expected_output is not None: + assert expected_output == update_params(params=params) + else: + with pytest.raises(ValueError): + assert update_params(params=params) + + +@pytest.mark.parametrize( + 'func, expected_output', + [ + (func1, 'def func1(x):\n return x + 1\n'), + (func2, 'def func2(x, y):\n return x + y\n'), + (func3, 'def func3(x, y, z):\n return x + y + z\n'), + (func4, 'def func4():\n\n def inner_func():\n res = 1 + 1\n return res\n\n return inner_func()\n') + ] +) +def test_get_function_source_definition(func: Callable, expected_output: str): + """Tests get_function_source_definition, which returns a formatted string of + the source code. + + Args: + func (Callable): Function to pull source definition from. + expected_output (str): Expected source definition of the given function. + """ + assert expected_output == get_function_source_definition(func=func) + + +@pytest.mark.parametrize( + 'job_spec_list, expected_output', + [ + ([{'component_spec': 'train_model', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': 1}], + [{'component_spec': 'train_model', + 'spec_string': + '''{\n''' + ''' "accelerator_count": 1,\n''' + ''' "accelerator_type": "NVIDIA_TESLA_A100",\n''' + ''' "component_spec": train_model,\n''' + ''' "display_name": "train-model-accelerated",\n''' + ''' "machine_type": "a2-highgpu-1g"\n }''' + }]), + ] +) +def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: List[dict]): + """Tests the stringify_job_spec_list function, takes in a list of custom training job spec + dictionaries and turns them into strings. + + Args: + job_spec: Dictionary with job spec info. e.g. + input = [{ + 'component_spec': 'train_model', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': 1 + }] + expected_output (List[dict]): Dictionary with key value pair for component_spec, + and a string representation of the full dictionary e.g. + output = [{ + 'component_spec': 'train_model', + 'spec_string': '''{ + "accelerator_count": 1, + "accelerator_type": "NVIDIA_TESLA_A100", + "component_spec": train_model, + "display_name": "train-model-accelerated", + "machine_type": "a2-highgpu-1g" + }''' + }] + """ + + formatted_spec = stringify_job_spec_list(job_spec_list=job_spec_list) + assert formatted_spec == expected_output