From ca80b499f0129219c3d5a8dc8afb1e75c3ae3fbc Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 7 Feb 2024 15:21:04 -0500 Subject: [PATCH 1/9] Created render_jinja function to eliminate redundant jinja functions --- .../deployments/cloudbuild/builder.py | 57 +- .../deployments/github_actions/builder.py | 82 +-- .../deployments/gitops/git_utils.py | 19 +- .../orchestration/kfp/builder.py | 382 +++++--------- .../provisioning/gcloud/builder.py | 136 +---- .../provisioning/pulumi/builder.py | 123 +---- .../provisioning/terraform/builder.py | 497 ++++++------------ google_cloud_automlops/utils/utils.py | 16 + 8 files changed, 391 insertions(+), 921 deletions(-) diff --git a/google_cloud_automlops/deployments/cloudbuild/builder.py b/google_cloud_automlops/deployments/cloudbuild/builder.py index c96671c..ab33fa0 100644 --- a/google_cloud_automlops/deployments/cloudbuild/builder.py +++ b/google_cloud_automlops/deployments/cloudbuild/builder.py @@ -24,7 +24,10 @@ from jinja2 import Template -from google_cloud_automlops.utils.utils import write_file +from google_cloud_automlops.utils.utils import ( + render_jinja, + write_file +) from google_cloud_automlops.utils.constants import ( BASE_DIR, CLOUDBUILD_TEMPLATES_PATH, @@ -48,46 +51,18 @@ def build(config: CloudBuildConfig): config.use_ci: Flag that determines whether to use Cloud CI/CD. """ # Write cloud build config - write_file(GENERATED_CLOUDBUILD_FILE, create_cloudbuild_jinja( - config.artifact_repo_location, - config.artifact_repo_name, - config.naming_prefix, - config.project_id, - config.pubsub_topic_name, - config.use_ci), 'w') - -def create_cloudbuild_jinja( - artifact_repo_location: str, - artifact_repo_name: str, - naming_prefix: str, - project_id: str, - pubsub_topic_name: str, - use_ci: bool) -> str: - """Generates content for the cloudbuild.yaml, to be written to the base_dir. - This file contains the ci/cd manifest for AutoMLOps. - - Args: - artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). - artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - project_id: The project ID. - pubsub_topic_name: The name of the pubsub topic to publish to. - use_ci: Flag that determines whether to use Cloud CI/CD. - - Returns: - str: Contents of cloudbuild.yaml. - """ - component_base_relative_path = COMPONENT_BASE_RELATIVE_PATH if use_ci else f'{BASE_DIR}{COMPONENT_BASE_RELATIVE_PATH}' - template_file = import_files(CLOUDBUILD_TEMPLATES_PATH) / 'cloudbuild.yaml.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - artifact_repo_location=artifact_repo_location, - artifact_repo_name=artifact_repo_name, + component_base_relative_path = COMPONENT_BASE_RELATIVE_PATH if config.use_ci else f'{BASE_DIR}{COMPONENT_BASE_RELATIVE_PATH}' + write_file( + filepath=GENERATED_CLOUDBUILD_FILE, + text=render_jinja( + template_path=import_files(CLOUDBUILD_TEMPLATES_PATH) / 'cloudbuild.yaml.j2', + artifact_repo_location=config.artifact_repo_location, + artifact_repo_name=config.artifact_repo_name, component_base_relative_path=component_base_relative_path, generated_license=GENERATED_LICENSE, generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, - naming_prefix=naming_prefix, - project_id=project_id, - pubsub_topic_name=pubsub_topic_name, - use_ci=use_ci) + naming_prefix=config.naming_prefix, + project_id=config.project_id, + pubsub_topic_name=config.pubsub_topic_name, + use_ci=config.use_ci), + mode='w') diff --git a/google_cloud_automlops/deployments/github_actions/builder.py b/google_cloud_automlops/deployments/github_actions/builder.py index b6a5ee2..2e8c360 100644 --- a/google_cloud_automlops/deployments/github_actions/builder.py +++ b/google_cloud_automlops/deployments/github_actions/builder.py @@ -24,7 +24,11 @@ from jinja2 import Template -from google_cloud_automlops.utils.utils import write_file +from google_cloud_automlops.utils.utils import ( + render_jinja, + write_file +) + from google_cloud_automlops.utils.constants import ( GENERATED_GITHUB_ACTIONS_FILE, COMPONENT_BASE_RELATIVE_PATH, @@ -52,65 +56,23 @@ def build(config: GitHubActionsConfig): config.workload_identity_service_account: Service account for workload identity federation. """ # Write github actions config - write_file(GENERATED_GITHUB_ACTIONS_FILE, create_github_actions_jinja( - config.artifact_repo_location, - config.artifact_repo_name, - config.naming_prefix, - config.project_id, - config.project_number, - config.pubsub_topic_name, - config.source_repo_branch, - config.use_ci, - config.workload_identity_pool, - config.workload_identity_provider, - config.workload_identity_service_account), 'w') - -def create_github_actions_jinja( - artifact_repo_location: str, - artifact_repo_name: str, - naming_prefix: str, - project_id: str, - project_number: str, - pubsub_topic_name: str, - source_repo_branch: str, - use_ci: bool, - workload_identity_pool: str, - workload_identity_provider: str, - workload_identity_service_account: str) -> str: - """Generates content for the github_actions.yaml, to be written to the .github/workflows directory. - This file contains the ci/cd manifest for AutoMLOps. - - Args: - artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). - artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - project_id: The project ID. - project_number: The project number. - pubsub_topic_name: The name of the pubsub topic to publish to. - source_repo_branch: The branch to use in the source repository. - use_ci: Flag that determines whether to use Cloud CI/CD. - workload_identity_pool: Pool for workload identity federation. - workload_identity_provider: Provider for workload identity federation. - workload_identity_service_account: Service account for workload identity federation. - - Returns: - str: Contents of github_actions.yaml. - """ - template_file = import_files(GITHUB_ACTIONS_TEMPLATES_PATH) / 'github_actions.yaml.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - artifact_repo_location=artifact_repo_location, - artifact_repo_name=artifact_repo_name, + write_file( + filepath=GENERATED_GITHUB_ACTIONS_FILE, + text=render_jinja( + template_path=import_files(GITHUB_ACTIONS_TEMPLATES_PATH) / 'github_actions.yaml.j2', + artifact_repo_location=config.artifact_repo_location, + artifact_repo_name=config.artifact_repo_name, component_base_relative_path=COMPONENT_BASE_RELATIVE_PATH, generated_license=GENERATED_LICENSE, generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, - naming_prefix=naming_prefix, - project_id=project_id, - project_number=project_number, - pubsub_topic_name=pubsub_topic_name, - source_repo_branch=source_repo_branch, - use_ci=use_ci, - workload_identity_pool=workload_identity_pool, - workload_identity_provider=workload_identity_provider, - workload_identity_service_account=workload_identity_service_account) + naming_prefix=config.naming_prefix, + project_id=config.project_id, + project_number=config.project_number, + pubsub_topic_name=config.pubsub_topic_name, + source_repo_branch=config.source_repo_branch, + use_ci=config.use_ci, + workload_identity_pool=config.workload_identity_pool, + workload_identity_provider=config.workload_identity_provider, + workload_identity_service_account=config.workload_identity_service_account + ), + mode='w') diff --git a/google_cloud_automlops/deployments/gitops/git_utils.py b/google_cloud_automlops/deployments/gitops/git_utils.py index f9ff2db..08baedd 100644 --- a/google_cloud_automlops/deployments/gitops/git_utils.py +++ b/google_cloud_automlops/deployments/gitops/git_utils.py @@ -39,6 +39,7 @@ from google_cloud_automlops.utils.utils import ( execute_process, read_yaml_file, + render_jinja, write_file ) from google_cloud_automlops.deployments.enums import ( @@ -76,7 +77,11 @@ def git_workflow(): has_remote_branch = subprocess.check_output( [f'''git -C {BASE_DIR} ls-remote origin {defaults['gcp']['source_repository_branch']}'''], shell=True, stderr=subprocess.STDOUT) - write_file(f'{BASE_DIR}.gitignore', _create_gitignore_jinja(), 'w') + write_file( + f'{BASE_DIR}.gitignore', + render_jinja(template_path=import_files(GITOPS_TEMPLATES_PATH) / 'gitignore.j2'), + 'w') + # This will initialize the branch, a second push will be required to trigger the cloudbuild job after initializing if not has_remote_branch: execute_process(f'git -C {BASE_DIR} add .gitignore', to_null=False) @@ -102,15 +107,3 @@ def git_workflow(): if deployment_framework == Deployer.CLOUDBUILD.value: logging.info( f'''Cloud Build job running at: https://console.cloud.google.com/cloud-build/builds;region={defaults['gcp']['build_trigger_location']}''') - - -def _create_gitignore_jinja() -> str: - """Generates code for .gitignore file. - - Returns: - str: .gitignore file. - """ - template_file = import_files(GITOPS_TEMPLATES_PATH) / 'gitignore.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render() diff --git a/google_cloud_automlops/orchestration/kfp/builder.py b/google_cloud_automlops/orchestration/kfp/builder.py index c121954..1e47a8c 100644 --- a/google_cloud_automlops/orchestration/kfp/builder.py +++ b/google_cloud_automlops/orchestration/kfp/builder.py @@ -33,6 +33,7 @@ make_dirs, read_file, read_yaml_file, + render_jinja, is_using_kfp_spec, write_and_chmod, write_file, @@ -70,12 +71,50 @@ def build(config: KfpConfig): """ # Write scripts for building pipeline, building components, running pipeline, and running all files - write_and_chmod(GENERATED_PIPELINE_SPEC_SH_FILE, build_pipeline_spec_jinja()) - write_and_chmod(GENERATED_BUILD_COMPONENTS_SH_FILE, build_components_jinja()) - write_and_chmod(GENERATED_RUN_PIPELINE_SH_FILE, run_pipeline_jinja()) - write_and_chmod(GENERATED_RUN_ALL_SH_FILE, run_all_jinja()) + scripts_path = import_files(KFP_TEMPLATES_PATH + '.scripts') + + # Write script for building pipeline + write_and_chmod( + GENERATED_PIPELINE_SPEC_SH_FILE, + render_jinja( + template_path=scripts_path / 'build_pipeline_spec.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR)) + + # Write script for building components + write_and_chmod( + GENERATED_BUILD_COMPONENTS_SH_FILE, + render_jinja( + template_path=scripts_path / 'build_components.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR)) + + # Write script for running pipeline + write_and_chmod( + GENERATED_RUN_PIPELINE_SH_FILE, + render_jinja( + template_path=scripts_path / 'run_pipeline.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR)) + + # Write script for running all files + write_and_chmod( + GENERATED_RUN_ALL_SH_FILE, + render_jinja( + template_path=scripts_path / 'run_all.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR)) + + # If using CI, write script for publishing to pubsub topic if config.use_ci: - write_and_chmod(GENERATED_PUBLISH_TO_TOPIC_FILE, publish_to_topic_jinja(pubsub_topic_name=config.pubsub_topic_name)) + write_and_chmod( + GENERATED_PUBLISH_TO_TOPIC_FILE, + render_jinja( + template_path=scripts_path / 'publish_to_topic.sh.j2', + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + pubsub_topic_name=config.pubsub_topic_name)) # Create components and pipelines components_path_list = get_components_list(full_path=True) @@ -87,10 +126,21 @@ def build(config: KfpConfig): write_file(f'{BASE_DIR}scripts/pipeline_spec/.gitkeep', '', 'w') # Write readme.md to description the contents of the directory - write_file(f'{BASE_DIR}README.md', readme_jinja(config.use_ci), 'w') + write_file( + f'{BASE_DIR}README.md', + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH) / 'README.md.j2', + use_ci=config.use_ci), + 'w') # Write dockerfile to the component base directory - write_file(f'{GENERATED_COMPONENT_BASE}/Dockerfile', component_base_dockerfile_jinja(config.base_image), 'w') + write_file( + f'{GENERATED_COMPONENT_BASE}/Dockerfile', + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2', + base_image=config.base_image, + generated_license=GENERATED_LICENSE), + 'w') # Write requirements.txt to the component base directory write_file(f'{GENERATED_COMPONENT_BASE}/requirements.txt', create_component_base_requirements(), 'w') @@ -137,7 +187,15 @@ def build_component(component_path: str): + '.py') # Write task script to component base - write_file(task_filepath, component_base_task_file_jinja(custom_code_contents, kfp_spec_bool), 'w') + write_file( + task_filepath, + + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2', + custom_code_contents=custom_code_contents, + generated_license=GENERATED_LICENSE, + kfp_spec_bool=kfp_spec_bool), + 'w') # Update component_spec to include correct image and startup command component_spec['implementation']['container']['image'] = compspec_image @@ -163,25 +221,48 @@ def build_pipeline(custom_training_job_specs: list, pipeline_parameter_values: Dictionary of runtime parameters for the PipelineJob. """ defaults = read_yaml_file(GENERATED_DEFAULTS_FILE) + # Get the names of the components components_list = get_components_list(full_path=False) + # Read pipeline definition pipeline_scaffold_contents = read_file(PIPELINE_CACHE_FILE) + # Add indentation pipeline_scaffold_contents = textwrap.indent(pipeline_scaffold_contents, 4 * ' ') + # Construct pipeline.py project_id = defaults['gcp']['project_id'] - write_file(GENERATED_PIPELINE_FILE, pipeline_jinja( - components_list, - custom_training_job_specs, - pipeline_scaffold_contents, - project_id=project_id), 'w') + write_file( + GENERATED_PIPELINE_FILE, + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2', + components_list=components_list, + custom_training_job_specs=custom_training_job_specs, + generated_license=GENERATED_LICENSE, + pipeline_scaffold_contents=pipeline_scaffold_contents, + project_id=project_id), + 'w') + # Construct pipeline_runner.py - write_file(GENERATED_PIPELINE_RUNNER_FILE, pipeline_runner_jinja(), 'w') + write_file( + GENERATED_PIPELINE_RUNNER_FILE, + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2', + generated_license=GENERATED_LICENSE), + 'w') + # Construct requirements.txt - write_file(GENERATED_PIPELINE_REQUIREMENTS_FILE, pipeline_requirements_jinja(), 'w') + write_file( + GENERATED_PIPELINE_REQUIREMENTS_FILE, + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2', + pinned_kfp_version=PINNED_KFP_VERSION), + 'w') + # Add pipeline_spec_path to dict pipeline_parameter_values['gs_pipeline_spec_path'] = defaults['pipelines']['gs_pipeline_job_spec_path'] + # Construct pipeline_parameter_values.json serialized_params = json.dumps(pipeline_parameter_values, indent=4) write_file(BASE_DIR + GENERATED_PARAMETER_VALUES_PATH, serialized_params, 'w') @@ -198,18 +279,34 @@ def build_services(): submission_service_base = BASE_DIR + 'services/submission_service' # Write cloud run dockerfile - write_file(f'{submission_service_base}/Dockerfile', submission_service_dockerfile_jinja(), 'w') + write_file( + f'{submission_service_base}/Dockerfile', + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2', + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE), + 'w') # Write requirements files for cloud run base and queueing svc - write_file(f'{submission_service_base}/requirements.txt', submission_service_requirements_jinja( - pipeline_job_submission_service_type=defaults['gcp']['pipeline_job_submission_service_type']), 'w') + write_file( + f'{submission_service_base}/requirements.txt', + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2', + pinned_kfp_version=PINNED_KFP_VERSION, + pipeline_job_submission_service_type=defaults['gcp']['pipeline_job_submission_service_type']), + 'w') # Write main code files for cloud run base and queueing svc - write_file(f'{submission_service_base}/main.py', submission_service_main_jinja( - pipeline_root=defaults['pipelines']['pipeline_storage_path'], - pipeline_job_runner_service_account=defaults['gcp']['pipeline_job_runner_service_account'], - pipeline_job_submission_service_type=defaults['gcp']['pipeline_job_submission_service_type'], - project_id=defaults['gcp']['project_id']), 'w') + write_file( + f'{submission_service_base}/main.py', + render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2', + generated_license=GENERATED_LICENSE, + pipeline_root=defaults['pipelines']['pipeline_storage_path'], + pipeline_job_runner_service_account=defaults['gcp']['pipeline_job_runner_service_account'], + pipeline_job_submission_service_type=defaults['gcp']['pipeline_job_submission_service_type'], + project_id=defaults['gcp']['project_id']), + 'w') def create_component_base_requirements(): @@ -281,242 +378,3 @@ def create_component_base_requirements(): # Stringify and sort reqs_str = ''.join(r+'\n' for r in sorted(set_of_requirements)) return reqs_str - - -def build_pipeline_spec_jinja() -> str: - """Generates code for build_pipeline_spec.sh which builds the pipeline specs. - - Returns: - str: build_pipeline_spec.sh script. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_pipeline_spec.sh.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR) - - -def build_components_jinja() -> str: - """Generates code for build_components.sh which builds the components. - - Returns: - str: build_components.sh script. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_components.sh.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR) - - -def run_pipeline_jinja() -> str: - """Generates code for run_pipeline.sh which runs the pipeline locally. - - Returns: - str: run_pipeline.sh script. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_pipeline.sh.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR) - - -def run_all_jinja() -> str: - """Generates code for run_all.sh which builds runs all other shell scripts. - - Returns: - str: run_all.sh script. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_all.sh.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR) - - -def publish_to_topic_jinja(pubsub_topic_name: str) -> str: - """Generates code for publish_to_topic.sh which submits a message to the - pipeline job submission service. - - Args: - pubsub_topic_name: The name of the pubsub topic to publish to. - - Returns: - str: publish_to_topic.sh script. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'publish_to_topic.sh.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - base_dir=BASE_DIR, - generated_license=GENERATED_LICENSE, - generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, - pubsub_topic_name=pubsub_topic_name) - - -def readme_jinja(use_ci: str) -> str: - """Generates code for readme.md which is a readme markdown file to describe the contents of the - generated AutoMLOps code repo. - - Args: - use_ci: Flag that determines whether to use Cloud CI/CD. - - Returns: - str: README.md file. - """ - template_file = import_files(KFP_TEMPLATES_PATH) / 'README.md.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render(use_ci=use_ci) - - -def component_base_dockerfile_jinja(base_image: str) -> str: - """Generates code for a Dockerfile to be written to the component_base directory. - - Args: - base_image: The image to use in the component base dockerfile. - - Returns: - str: Dockerfile file. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - base_image=base_image, - generated_license=GENERATED_LICENSE) - - -def component_base_task_file_jinja(custom_code_contents: str, kfp_spec_bool: str) -> str: - """Generates code for the task.py file to be written to the component_base/src directory. - - Args: - custom_code_contents: Code inside of the component, specified by the user. - kfp_spec_bool: Boolean that specifies whether components are defined using kfp. - - Returns: - str: Contents of the task.py file. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - custom_code_contents=custom_code_contents, - generated_license=GENERATED_LICENSE, - kfp_spec_bool=kfp_spec_bool) - - -def pipeline_runner_jinja() -> str: - """Generates code for the pipeline_runner.py file to be written to the pipelines directory. - - Returns: - str: pipeline_runner.py file. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render(generated_license=GENERATED_LICENSE) - - -def pipeline_jinja( - components_list: list, - custom_training_job_specs: list, - pipeline_scaffold_contents: str, - project_id: str) -> str: - """Generates code for the pipeline.py file to be written to the pipelines directory. - - Args: - components_list: Contains the names or paths of all component yamls in the dir. - custom_training_job_specs: Specifies the specs to run the training job with. - pipeline_scaffold_contents: The contents of the pipeline scaffold file, - which can be found at PIPELINE_CACHE_FILE. - project_id: The project ID. - - Returns: - str: pipeline.py file. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - components_list=components_list, - custom_training_job_specs=custom_training_job_specs, - generated_license=GENERATED_LICENSE, - pipeline_scaffold_contents=pipeline_scaffold_contents, - project_id=project_id) - - -def pipeline_requirements_jinja() -> str: - """Generates code for a requirements.txt to be written to the pipelines directory. - - Returns: - str: requirements.txt file for pipelines. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render(pinned_kfp_version=PINNED_KFP_VERSION) - - -def submission_service_dockerfile_jinja() -> str: - """Generates code for a Dockerfile to be written to the serivces/submission_service directory. - - Returns: - str: Dockerfile file. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - base_dir=BASE_DIR, - generated_license=GENERATED_LICENSE) - - -def submission_service_requirements_jinja(pipeline_job_submission_service_type: str) -> str: - """Generates code for a requirements.txt to be written to the serivces/submission_service directory. - - Args: - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - - Returns: - str: requirements.txt file for submission_service. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - pinned_kfp_version=PINNED_KFP_VERSION, - pipeline_job_submission_service_type=pipeline_job_submission_service_type) - - -def submission_service_main_jinja( - pipeline_root: str, - pipeline_job_runner_service_account: str, - pipeline_job_submission_service_type: str, - project_id: str) -> str: - """Generates content for main.py to be written to the serivces/submission_service directory. - This file contains code for running a flask service that will act as a pipeline job submission service. - - Args: - pipeline_root: GS location where to store metadata from pipeline runs. - pipeline_job_runner_service_account: Service Account to runner PipelineJobs. - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - project_id: The project ID. - - Returns: - str: Content of serivces/submission_service main.py. - """ - template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE, - pipeline_root=pipeline_root, - pipeline_job_runner_service_account=pipeline_job_runner_service_account, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - project_id=project_id) diff --git a/google_cloud_automlops/provisioning/gcloud/builder.py b/google_cloud_automlops/provisioning/gcloud/builder.py index be36ced..53aa256 100644 --- a/google_cloud_automlops/provisioning/gcloud/builder.py +++ b/google_cloud_automlops/provisioning/gcloud/builder.py @@ -29,6 +29,7 @@ from google_cloud_automlops.utils.utils import ( get_required_apis, read_yaml_file, + render_jinja, write_and_chmod ) from google_cloud_automlops.utils.constants import ( @@ -75,118 +76,35 @@ def build( defaults = read_yaml_file(GENERATED_DEFAULTS_FILE) required_apis = get_required_apis(defaults) # create provision_resources.sh - write_and_chmod(GENERATED_RESOURCES_SH_FILE, provision_resources_script_jinja( - artifact_repo_location=config.artifact_repo_location, - artifact_repo_name=config.artifact_repo_name, - artifact_repo_type=config.artifact_repo_type, - build_trigger_location=config.build_trigger_location, - build_trigger_name=config.build_trigger_name, - deployment_framework=config.deployment_framework, - naming_prefix=config.naming_prefix, - pipeline_job_runner_service_account=config.pipeline_job_runner_service_account, - pipeline_job_submission_service_location=config.pipeline_job_submission_service_location, - pipeline_job_submission_service_name=config.pipeline_job_submission_service_name, - pipeline_job_submission_service_type=config.pipeline_job_submission_service_type, - project_id=project_id, - pubsub_topic_name=config.pubsub_topic_name, - required_apis=required_apis, - schedule_location=config.schedule_location, - schedule_name=config.schedule_name, - schedule_pattern=config.schedule_pattern, - source_repo_branch=config.source_repo_branch, - source_repo_name=config.source_repo_name, - source_repo_type=config.source_repo_type, - storage_bucket_location=config.storage_bucket_location, - storage_bucket_name=config.storage_bucket_name, - use_ci=config.use_ci, - vpc_connector=config.vpc_connector)) - - -def provision_resources_script_jinja( - artifact_repo_location: str, - artifact_repo_name: str, - artifact_repo_type: str, - build_trigger_location: str, - build_trigger_name: str, - deployment_framework: str, - naming_prefix: str, - pipeline_job_runner_service_account: str, - pipeline_job_submission_service_location: str, - pipeline_job_submission_service_name: str, - pipeline_job_submission_service_type: str, - project_id: str, - pubsub_topic_name: str, - required_apis: list, - schedule_location: str, - schedule_name: str, - schedule_pattern: str, - source_repo_branch: str, - source_repo_name: str, - source_repo_type: str, - storage_bucket_location: str, - storage_bucket_name: str, - use_ci: bool, - vpc_connector: str) -> str: - """Generates code for provision_resources.sh which sets up the project's environment. - - Args: - artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). - artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). - artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) - build_trigger_location: The location of the build trigger (for cloud build). - build_trigger_name: The name of the build trigger (for cloud build). - deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - pipeline_job_runner_service_account: Service Account to run PipelineJobs. - pipeline_job_submission_service_location: The location of the cloud submission service. - pipeline_job_submission_service_name: The name of the cloud submission service. - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - project_id: The project ID. - pubsub_topic_name: The name of the pubsub topic to publish to. - required_apis: List of APIs that are required to run the service. - schedule_location: The location of the scheduler resource. - schedule_name: The name of the scheduler resource. - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - source_repo_branch: The branch to use in the source repository. - source_repo_name: The name of the source repository to use. - source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) - storage_bucket_location: Region of the GS bucket. - storage_bucket_name: GS bucket name where pipeline run metadata is stored. - use_ci: Flag that determines whether to use Cloud CI/CD. - vpc_connector: The name of the vpc connector to use. - - Returns: - str: provision_resources.sh shell script. - """ - template_file = import_files(GCLOUD_TEMPLATES_PATH) / 'provision_resources.sh.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - artifact_repo_location=artifact_repo_location, - artifact_repo_name=artifact_repo_name, - artifact_repo_type=artifact_repo_type, + write_and_chmod( + GENERATED_RESOURCES_SH_FILE, + render_jinja( + template_path=import_files(GCLOUD_TEMPLATES_PATH) / 'provision_resources.sh.j2', + artifact_repo_location=config.artifact_repo_location, + artifact_repo_name=config.artifact_repo_name, + artifact_repo_type=config.artifact_repo_type, base_dir=BASE_DIR, - build_trigger_location=build_trigger_location, - build_trigger_name=build_trigger_name, - deployment_framework=deployment_framework, + build_trigger_location=config.build_trigger_location, + build_trigger_name=config.build_trigger_name, + deployment_framework=config.deployment_framework, generated_license=GENERATED_LICENSE, generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, - naming_prefix=naming_prefix, - pipeline_job_runner_service_account=pipeline_job_runner_service_account, - pipeline_job_submission_service_location=pipeline_job_submission_service_location, - pipeline_job_submission_service_name=pipeline_job_submission_service_name, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, + naming_prefix=config.naming_prefix, + pipeline_job_runner_service_account=config.pipeline_job_runner_service_account, + pipeline_job_submission_service_location=config.pipeline_job_submission_service_location, + pipeline_job_submission_service_name=config.pipeline_job_submission_service_name, + pipeline_job_submission_service_type=config.pipeline_job_submission_service_type, project_id=project_id, - pubsub_topic_name=pubsub_topic_name, + pubsub_topic_name=config.pubsub_topic_name, required_apis=required_apis, required_iam_roles=IAM_ROLES_RUNNER_SA, - schedule_location=schedule_location, - schedule_name=schedule_name, - schedule_pattern=schedule_pattern, - source_repo_branch=source_repo_branch, - source_repo_name=source_repo_name, - source_repo_type=source_repo_type, - storage_bucket_location=storage_bucket_location, - storage_bucket_name=storage_bucket_name, - use_ci=use_ci, - vpc_connector=vpc_connector) + schedule_location=config.schedule_location, + schedule_name=config.schedule_name, + schedule_pattern=config.schedule_pattern, + source_repo_branch=config.source_repo_branch, + source_repo_name=config.source_repo_name, + source_repo_type=config.source_repo_type, + storage_bucket_location=config.storage_bucket_location, + storage_bucket_name=config.storage_bucket_name, + use_ci=config.use_ci, + vpc_connector=config.vpc_connector)) diff --git a/google_cloud_automlops/provisioning/pulumi/builder.py b/google_cloud_automlops/provisioning/pulumi/builder.py index eb49c4c..69815b5 100644 --- a/google_cloud_automlops/provisioning/pulumi/builder.py +++ b/google_cloud_automlops/provisioning/pulumi/builder.py @@ -22,6 +22,7 @@ from google_cloud_automlops.utils.utils import ( write_file, + render_jinja, make_dirs, ) @@ -77,105 +78,39 @@ def build( pulumi_folder = pipeline_model_name + '/' # create Pulumi.yaml - write_file(pulumi_folder + 'Pulumi.yaml', _create_pulumi_yaml_jinja( - pipeline_model_name=pipeline_model_name, - pulumi_runtime=config.pulumi_runtime), 'w' - ) - - # create Pulumi.dev.yaml - write_file(pulumi_folder + 'Pulumi.dev.yaml', _create_pulumi_dev_yaml_jinja( - project_id=project_id, - pipeline_model_name=pipeline_model_name, - region=config.region, - gcs_bucket_name=gcs_bucket_name), 'w' - ) - - # create python __main__.py - if config.pulumi_runtime == PulumiRuntime.PYTHON: - write_file(pulumi_folder + '__main__.py', _create_main_python_jinja( - artifact_repo_name=artifact_repo_name, - source_repo_name=source_repo_name, - cloudtasks_queue_name=cloudtasks_queue_name, - cloud_build_trigger_name=cloud_build_trigger_name), 'w' - ) - - -def _create_pulumi_yaml_jinja( - pipeline_model_name: str, - pulumi_runtime: str, -) -> str: - """Generates code for Pulumi.yaml, the pulumi script that contains details to deploy project's GCP environment. - - Args: - config.pipeline_model_name: Name of the model being deployed. - config.pulumi_runtime: The pulumi runtime option (default: PulumiRuntime.PYTHON). - - Returns: - str: Pulumi.yaml config script. - """ - - with open(PULUMI_TEMPLATES_PATH / 'Pulumi.yaml.jinja', 'r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( + write_file( + pulumi_folder + 'Pulumi.yaml', + render_jinja( + template_path=PULUMI_TEMPLATES_PATH / 'Pulumi.yaml.jinja', generated_license=GENERATED_LICENSE, pipeline_model_name=pipeline_model_name, - pulumi_runtime=pulumi_runtime.value, - ) - - -def _create_pulumi_dev_yaml_jinja( - project_id: str, - pipeline_model_name: str, - region: str, - gcs_bucket_name: str, -) -> str: - """Generates code for Pulumi.dev.yaml, the pulumi script that contains details to deploy dev environment config. - - Args: - project_id: The project ID. - config.pipeline_model_name: Name of the model being deployed. - config.region: region used in gcs infrastructure config. - config.gcs_bucket_name: gcs bucket name to use as part of the model infrastructure. - - Returns: - str: Pulumi.dev.yaml config script. - """ + pulumi_runtime=config.pulumi_runtime.value), + 'w' + ) - with open(PULUMI_TEMPLATES_PATH / 'Pulumi.dev.yaml.jinja', 'r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( + # create Pulumi.dev.yaml + write_file( + pulumi_folder + 'Pulumi.dev.yaml', + render_jinja( + template_path=PULUMI_TEMPLATES_PATH / 'Pulumi.dev.yaml.jinja', generated_license=GENERATED_LICENSE, project_id=project_id, pipeline_model_name=pipeline_model_name, - region=region, - gcs_bucket_name=gcs_bucket_name, - ) - - -def _create_main_python_jinja( - artifact_repo_name, - source_repo_name, - cloudtasks_queue_name, - cloud_build_trigger_name, -) -> str: - """Generates code for __main__.py, the pulumi script that creates the primary resources. - - Args: - artifact_repo_name: name of the artifact registry for the model infrastructure. - source_repo_name: source repository used as part of the the model infra. - cloudtasks_queue_name: name of the task queue used for model scheduling. - cloud_build_trigger_name: name of the cloud build trigger for the model infra. - - Returns: - str: Main pulumi script. - """ + region=config.region, + gcs_bucket_name=gcs_bucket_name), + 'w' + ) - with open(PULUMI_TEMPLATES_PATH / 'python/__main__.py.jinja', 'r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE, - artifact_repo_name=artifact_repo_name, - source_repo_name=source_repo_name, - cloudtasks_queue_name=cloudtasks_queue_name, - cloud_build_trigger_name=cloud_build_trigger_name, + # create python __main__.py + if config.pulumi_runtime == PulumiRuntime.PYTHON: + write_file( + pulumi_folder + '__main__.py', + render_jinja( + template_path=PULUMI_TEMPLATES_PATH / 'python/__main__.py.jinja', + generated_license=GENERATED_LICENSE, + artifact_repo_name=artifact_repo_name, + source_repo_name=source_repo_name, + cloudtasks_queue_name=cloudtasks_queue_name, + cloud_build_trigger_name=cloud_build_trigger_name), + 'w' ) diff --git a/google_cloud_automlops/provisioning/terraform/builder.py b/google_cloud_automlops/provisioning/terraform/builder.py index 66a7f0f..7c5797d 100644 --- a/google_cloud_automlops/provisioning/terraform/builder.py +++ b/google_cloud_automlops/provisioning/terraform/builder.py @@ -28,6 +28,7 @@ from google_cloud_automlops.utils.utils import ( get_required_apis, read_yaml_file, + render_jinja, write_and_chmod, write_file ) @@ -81,368 +82,180 @@ def build( """ defaults = read_yaml_file(GENERATED_DEFAULTS_FILE) required_apis = get_required_apis(defaults) - # create environment/data.tf - write_file(f'{BASE_DIR}provision/environment/data.tf', create_environment_data_tf_jinja( - required_apis=required_apis, - use_ci=config.use_ci,), 'w') - # create environment/iam.tf - write_file(f'{BASE_DIR}provision/environment/iam.tf', create_environment_iam_tf_jinja(), 'w') - # create environment/main.tf - write_file(f'{BASE_DIR}provision/environment/main.tf', create_environment_main_tf_jinja( - artifact_repo_type=config.artifact_repo_type, - deployment_framework=config.deployment_framework, - naming_prefix=config.naming_prefix, - pipeline_job_submission_service_type=config.pipeline_job_submission_service_type, - schedule_pattern=config.schedule_pattern, - source_repo_type=config.source_repo_type, - use_ci=config.use_ci, - vpc_connector=config.vpc_connector), 'w') - # create environment/outputs.tf - write_file(f'{BASE_DIR}provision/environment/outputs.tf', create_environment_outputs_tf_jinja( - artifact_repo_type=config.artifact_repo_type, - deployment_framework=config.deployment_framework, - pipeline_job_submission_service_type=config.pipeline_job_submission_service_type, - schedule_pattern=config.schedule_pattern, - source_repo_type=config.source_repo_type, - use_ci=config.use_ci), 'w') - # create environment/provider.tf - write_file(f'{BASE_DIR}provision/environment/provider.tf', create_environment_provider_tf_jinja(), 'w') - # create environment/variables.tf - write_file(f'{BASE_DIR}provision/environment/variables.tf', create_environment_variables_tf_jinja(), 'w') - # create environment/variables.auto.tfvars - if config.deployment_framework == Deployer.CLOUDBUILD.value: - write_file(f'{BASE_DIR}provision/environment/variables.auto.tfvars', create_environment_variables_auto_tfvars_jinja( - artifact_repo_location=config.artifact_repo_location, - artifact_repo_name=config.artifact_repo_name, - build_trigger_location=config.build_trigger_location, - build_trigger_name=config.build_trigger_name, - pipeline_job_runner_service_account=config.pipeline_job_runner_service_account, - pipeline_job_submission_service_location=config.pipeline_job_submission_service_location, - pipeline_job_submission_service_name=config.pipeline_job_submission_service_name, - project_id=project_id, - provision_credentials_key=config.provision_credentials_key, - pubsub_topic_name=config.pubsub_topic_name, - schedule_location=config.schedule_location, - schedule_name=config.schedule_name, - schedule_pattern=config.schedule_pattern, - source_repo_branch=config.source_repo_branch, - source_repo_name=config.source_repo_name, - storage_bucket_location=config.storage_bucket_location, - storage_bucket_name=config.storage_bucket_name, - vpc_connector=config.vpc_connector), 'w') - #TODO: implement workload identity as optional - if config.deployment_framework == Deployer.GITHUB_ACTIONS.value: - write_file(f'{BASE_DIR}provision/environment/variables.auto.tfvars', create_environment_variables_auto_tfvars_jinja( - artifact_repo_location=config.artifact_repo_location, - artifact_repo_name=config.artifact_repo_name, - build_trigger_location=config.build_trigger_location, - build_trigger_name=config.build_trigger_name, - pipeline_job_runner_service_account=config.pipeline_job_runner_service_account, - pipeline_job_submission_service_location=config.pipeline_job_submission_service_location, - pipeline_job_submission_service_name=config.pipeline_job_submission_service_name, - project_id=project_id, - provision_credentials_key=config.provision_credentials_key, - pubsub_topic_name=config.pubsub_topic_name, - schedule_location=config.schedule_location, - schedule_name=config.schedule_name, - schedule_pattern=config.schedule_pattern, - source_repo_branch=config.source_repo_branch, - source_repo_name=config.source_repo_name, - storage_bucket_location=config.storage_bucket_location, - storage_bucket_name=config.storage_bucket_name, - vpc_connector=config.vpc_connector), 'w') - # create environment/versions.tf - write_file(f'{BASE_DIR}provision/environment/versions.tf', create_environment_versions_tf_jinja(storage_bucket_name=config.storage_bucket_name), 'w') - # create provision_resources.sh - write_and_chmod(GENERATED_RESOURCES_SH_FILE, create_provision_resources_script_jinja()) - # create state_bucket/main.tf - write_file(f'{BASE_DIR}provision/state_bucket/main.tf', create_state_bucket_main_tf_jinja(), 'w') - # create state_bucket/variables.tf - write_file(f'{BASE_DIR}provision/state_bucket/variables.tf', create_state_bucket_variables_tf_jinja(), 'w') - # create state_bucket/variables.auto.tfvars - write_file(f'{BASE_DIR}provision/state_bucket/variables.auto.tfvars', create_state_bucket_variables_auto_tfvars_jinja( - project_id=project_id, - storage_bucket_location=config.storage_bucket_location, - storage_bucket_name=config.storage_bucket_name), 'w') - - -def create_environment_data_tf_jinja( - required_apis: list, - use_ci: bool) -> str: - """Generates code for environment/data.tf, the terraform hcl script that contains terraform remote backend and org project details. - Args: - required_apis: List of APIs that are required to run the service. - use_ci: Flag that determines whether to use Cloud CI/CD. - - Returns: - str: environment/data.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'data.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( + # create environment/data.tf + write_file( + filepath=f'{BASE_DIR}provision/environment/data.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'data.tf.j2', generated_license=GENERATED_LICENSE, required_apis=required_apis, required_iam_roles=IAM_ROLES_RUNNER_SA, - use_ci=use_ci) - - -def create_environment_iam_tf_jinja() -> str: - """Generates code for environment/iam.tf, the terraform hcl script that contains service accounts iam bindings for project's environment. + use_ci=config.use_ci + ), + mode='w') - Returns: - str: environment/iam.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'iam.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render(generated_license=GENERATED_LICENSE) - - -def create_environment_main_tf_jinja( - artifact_repo_type: str, - deployment_framework: str, - naming_prefix: str, - pipeline_job_submission_service_type: str, - schedule_pattern: str, - source_repo_type: str, - use_ci: bool, - vpc_connector: str) -> str: - """Generates code for environment/main.tf, the terraform hcl script that contains terraform resources configs to deploy resources in the project. - - Args: - artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) - deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) - use_ci: Flag that determines whether to use Cloud CI/CD. - vpc_connector: The name of the vpc connector to use. + # create environment/iam.tf + write_file( + filepath=f'{BASE_DIR}provision/environment/iam.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'iam.tf.j2', + generated_license=GENERATED_LICENSE + ), + mode='w') - Returns: - str: environment/main.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'main.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - artifact_repo_type=artifact_repo_type, + # create environment/main.tf + write_file( + filepath=f'{BASE_DIR}provision/environment/main.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'main.tf.j2', + artifact_repo_type=config.artifact_repo_type, base_dir=BASE_DIR, - deployment_framework=deployment_framework, + deployment_framework=config.deployment_framework, generated_license=GENERATED_LICENSE, generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, - naming_prefix=naming_prefix, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - schedule_pattern=schedule_pattern, - source_repo_type=source_repo_type, - use_ci=use_ci, - vpc_connector=vpc_connector) - - -def create_environment_outputs_tf_jinja( - artifact_repo_type: str, - deployment_framework: str, - pipeline_job_submission_service_type: str, - schedule_pattern: str, - source_repo_type: str, - use_ci: bool) -> str: - """Generates code for environment/outputs.tf, the terraform hcl script that contains outputs from project's environment. - - Args: - artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) - deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) - use_ci: Flag that determines whether to use Cloud CI/CD. + naming_prefix=config.naming_prefix, + pipeline_job_submission_service_type=config.pipeline_job_submission_service_type, + schedule_pattern=config.schedule_pattern, + source_repo_type=config.source_repo_type, + use_ci=config.use_ci, + vpc_connector=config.vpc_connector + ), + mode='w') - Returns: - str: environment/outputs.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'outputs.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - artifact_repo_type=artifact_repo_type, - deployment_framework=deployment_framework, + # create environment/outputs.tf + write_file( + filepath=f'{BASE_DIR}provision/environment/outputs.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'outputs.tf.j2', + artifact_repo_type=config.artifact_repo_type, + deployment_framework=config.deployment_framework, generated_license=GENERATED_LICENSE, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - schedule_pattern=schedule_pattern, - source_repo_type=source_repo_type, - use_ci=use_ci) - - -def create_environment_provider_tf_jinja() -> str: - """Generates code for environment/provider.tf, the terraform hcl script that contains teraform providers used to deploy project's environment. - - Returns: - str: environment/provider.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'provider.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE) - - -def create_environment_variables_tf_jinja() -> str: - """Generates code for environment/variables.tf, the terraform hcl script that contains variables used to deploy project's environment. - - Returns: - str: environment/variables.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE) - - -def create_environment_variables_auto_tfvars_jinja( - artifact_repo_location: str, - artifact_repo_name: str, - build_trigger_location: str, - build_trigger_name: str, - pipeline_job_runner_service_account: str, - pipeline_job_submission_service_location: str, - pipeline_job_submission_service_name: str, - project_id: str, - provision_credentials_key: str, - pubsub_topic_name: str, - schedule_location: str, - schedule_name: str, - schedule_pattern: str, - source_repo_branch: str, - source_repo_name: str, - storage_bucket_location: str, - storage_bucket_name: str, - vpc_connector: str) -> str: - """Generates code for environment/variables.auto.tfvars, the terraform hcl script that contains teraform arguments for variables used to deploy project's environment. + pipeline_job_submission_service_type=config.pipeline_job_submission_service_type, + schedule_pattern=config.schedule_pattern, + source_repo_type=config.source_repo_type, + use_ci=config.use_ci + ), + mode='w') - Args: - artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). - artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). - build_trigger_location: The location of the build trigger (for cloud build). - build_trigger_name: The name of the build trigger (for cloud build). - pipeline_job_runner_service_account: Service Account to run PipelineJobs. - pipeline_job_submission_service_location: The location of the cloud submission service. - pipeline_job_submission_service_name: The name of the cloud submission service. - project_id: The project ID. - pubsub_topic_name: The name of the pubsub topic to publish to. - schedule_location: The location of the scheduler resource. - schedule_name: The name of the scheduler resource. - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - source_repo_branch: The branch to use in the source repository. - source_repo_name: The name of the source repository to use. - storage_bucket_location: Region of the GS bucket. - storage_bucket_name: GS bucket name where pipeline run metadata is stored. - vpc_connector: The name of the vpc connector to use. + # create environment/provider.tf + write_file( + filepath=f'{BASE_DIR}provision/environment/provider.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'provider.tf.j2', + generated_license=GENERATED_LICENSE + ), + mode='w') - Returns: - str: environment/variables.auto.tfvars file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.auto.tfvars.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - artifact_repo_location=artifact_repo_location, - artifact_repo_name=artifact_repo_name, - build_trigger_location=build_trigger_location, - build_trigger_name=build_trigger_name, - generated_license=GENERATED_LICENSE, - pipeline_job_runner_service_account=pipeline_job_runner_service_account, - pipeline_job_submission_service_location=pipeline_job_submission_service_location, - pipeline_job_submission_service_name=pipeline_job_submission_service_name, - project_id=project_id, - provision_credentials_key=provision_credentials_key, - pubsub_topic_name=pubsub_topic_name, - schedule_location=schedule_location, - schedule_name=schedule_name, - schedule_pattern=schedule_pattern, - source_repo_branch=source_repo_branch, - source_repo_name=source_repo_name, - storage_bucket_location=storage_bucket_location, - storage_bucket_name=storage_bucket_name, - vpc_connector=vpc_connector) + # create environment/variables.tf + write_file( + filepath=f'{BASE_DIR}provision/environment/variables.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.tf.j2', + generated_license=GENERATED_LICENSE + ), + mode='w') + # create environment/variables.auto.tfvars + if config.deployment_framework == Deployer.CLOUDBUILD.value: + write_file( + filepath=f'{BASE_DIR}provision/environment/variables.auto.tfvars', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.auto.tfvars.j2', + artifact_repo_location=config.artifact_repo_location, + artifact_repo_name=config.artifact_repo_name, + build_trigger_location=config.build_trigger_location, + build_trigger_name=config.build_trigger_name, + generated_license=GENERATED_LICENSE, + pipeline_job_runner_service_account=config.pipeline_job_runner_service_account, + pipeline_job_submission_service_location=config.pipeline_job_submission_service_location, + pipeline_job_submission_service_name=config.pipeline_job_submission_service_name, + project_id=project_id, + provision_credentials_key=config.provision_credentials_key, + pubsub_topic_name=config.pubsub_topic_name, + schedule_location=config.schedule_location, + schedule_name=config.schedule_name, + schedule_pattern=config.schedule_pattern, + source_repo_branch=config.source_repo_branch, + source_repo_name=config.source_repo_name, + storage_bucket_location=config.storage_bucket_location, + storage_bucket_name=config.storage_bucket_name, + vpc_connector=config.vpc_connector + ), + mode='w') -def create_environment_versions_tf_jinja(storage_bucket_name: str) -> str: - """Generates code for environment/versions.tf, the terraform hcl script that contains teraform version information. - Args: - storage_bucket_name: GS bucket name where pipeline run metadata is stored. + #TODO: implement workload identity as optional + if config.deployment_framework == Deployer.GITHUB_ACTIONS.value: + write_file( + filepath=f'{BASE_DIR}provision/environment/variables.auto.tfvars', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.auto.tfvars.j2', + artifact_repo_location=config.artifact_repo_location, + artifact_repo_name=config.artifact_repo_name, + build_trigger_location=config.build_trigger_location, + build_trigger_name=config.build_trigger_name, + generated_license=GENERATED_LICENSE, + pipeline_job_runner_service_account=config.pipeline_job_runner_service_account, + pipeline_job_submission_service_location=config.pipeline_job_submission_service_location, + pipeline_job_submission_service_name=config.pipeline_job_submission_service_name, + project_id=project_id, + provision_credentials_key=config.provision_credentials_key, + pubsub_topic_name=config.pubsub_topic_name, + schedule_location=config.schedule_location, + schedule_name=config.schedule_name, + schedule_pattern=config.schedule_pattern, + source_repo_branch=config.source_repo_branch, + source_repo_name=config.source_repo_name, + storage_bucket_location=config.storage_bucket_location, + storage_bucket_name=config.storage_bucket_name, + vpc_connector=config.vpc_connector + ), + mode='w') - Returns: - str: environment/versions.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'versions.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( + # create environment/versions.tf + write_file( + filepath=f'{BASE_DIR}provision/environment/versions.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'versions.tf.j2', generated_license=GENERATED_LICENSE, - storage_bucket_name=storage_bucket_name) - + storage_bucket_name=config.storage_bucket_name + ), + mode='w') -def create_provision_resources_script_jinja() -> str: - """Generates code for provision_resources.sh which sets up the project's environment using terraform. - - Returns: - str: provision_resources.sh shell script. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH) / 'provision_resources.sh.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( + # create provision_resources.sh + write_and_chmod( + filepath=GENERATED_RESOURCES_SH_FILE, + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH) / 'provision_resources.sh.j2', base_dir=BASE_DIR, - generated_license=GENERATED_LICENSE) - - -def create_state_bucket_variables_tf_jinja() -> str: - """Generates code for state_bucket/variables.tf, the terraform hcl script that contains variables used for the state_bucket. - - Returns: - str: state_bucket/variables.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'variables.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE) - + generated_license=GENERATED_LICENSE + )) -def create_state_bucket_variables_auto_tfvars_jinja( - project_id: str, - storage_bucket_location: str, - storage_bucket_name: str) -> str: - """Generates code for state_bucket/variables.auto.tfvars, the terraform hcl script that contains teraform arguments for variables used for the state_bucket. - Uses the string f'{storage_bucket_name}-bucket-tfstate' for the name of the storage state bucket. + # create state_bucket/main.tf + write_file( + filepath=f'{BASE_DIR}provision/state_bucket/main.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'main.tf.j2', + generated_license=GENERATED_LICENSE + ), + mode='w') - Args: - project_id: The project ID. - storage_bucket_location: Region of the GS bucket. - storage_bucket_name: GS bucket name where pipeline run metadata is stored. + # create state_bucket/variables.tf + write_file( + filepath=f'{BASE_DIR}provision/state_bucket/variables.tf', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'variables.tf.j2', + generated_license=GENERATED_LICENSE + ), + mode='w') - Returns: - str: environment/variables.auto.tfvars file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'variables.auto.tfvars.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( + # create state_bucket/variables.auto.tfvars + write_file( + filepath=f'{BASE_DIR}provision/state_bucket/variables.auto.tfvars', + text=render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'variables.auto.tfvars.j2', project_id=project_id, - storage_bucket_location=storage_bucket_location, - storage_bucket_name=storage_bucket_name) - - -def create_state_bucket_main_tf_jinja() -> str: - """Generates code for state_bucket/main.tf, the terraform hcl script that contains terraform resources configs to create the state_bucket. - - Returns: - str: state_bucket/main.tf file. - """ - template_file = import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'main.tf.j2' - with template_file.open('r', encoding='utf-8') as f: - template = Template(f.read()) - return template.render( - generated_license=GENERATED_LICENSE) + storage_bucket_location=config.storage_bucket_location, + storage_bucket_name=config.storage_bucket_name + ), + mode='w') diff --git a/google_cloud_automlops/utils/utils.py b/google_cloud_automlops/utils/utils.py index 8ce6631..0c0a080 100644 --- a/google_cloud_automlops/utils/utils.py +++ b/google_cloud_automlops/utils/utils.py @@ -30,6 +30,7 @@ from packaging import version import yaml +from jinja2 import Template from googleapiclient import discovery import google.auth @@ -940,3 +941,18 @@ def resources_generation_manifest(defaults: dict): if defaults['gcp']['schedule_pattern'] != DEFAULT_SCHEDULE_PATTERN: logging.info( 'Cloud Scheduler Job: https://console.cloud.google.com/cloudscheduler') + +def render_jinja(template_path, **template_vars): + """Renders a Jinja2 template with provided variables. + + Args: + template_path (str): The path to the Jinja2 template file. + **template_vars: Keyword arguments representing variables to substitute + in the template. + + Returns: + str: The rendered template as a string. + """ + with template_path.open('r', encoding='utf-8') as f: + template = Template(f.read()) + return template.render(**template_vars) \ No newline at end of file From 2b5ccf0ae201c3a69b0c4b0d0430dafa78829fd9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 7 Feb 2024 15:29:19 -0500 Subject: [PATCH 2/9] Pylint fixes --- .../deployments/cloudbuild/builder.py | 4 +-- .../deployments/github_actions/builder.py | 4 +-- .../deployments/gitops/git_utils.py | 4 +-- .../orchestration/kfp/builder.py | 27 +++++++++---------- .../provisioning/gcloud/builder.py | 2 +- .../provisioning/pulumi/builder.py | 2 -- .../provisioning/terraform/builder.py | 24 ++++++++--------- google_cloud_automlops/utils/utils.py | 2 +- 8 files changed, 28 insertions(+), 41 deletions(-) diff --git a/google_cloud_automlops/deployments/cloudbuild/builder.py b/google_cloud_automlops/deployments/cloudbuild/builder.py index ab33fa0..534d1d3 100644 --- a/google_cloud_automlops/deployments/cloudbuild/builder.py +++ b/google_cloud_automlops/deployments/cloudbuild/builder.py @@ -22,8 +22,6 @@ # Try backported to PY<37 `importlib_resources` from importlib_resources import files as import_files -from jinja2 import Template - from google_cloud_automlops.utils.utils import ( render_jinja, write_file @@ -53,7 +51,7 @@ def build(config: CloudBuildConfig): # Write cloud build config component_base_relative_path = COMPONENT_BASE_RELATIVE_PATH if config.use_ci else f'{BASE_DIR}{COMPONENT_BASE_RELATIVE_PATH}' write_file( - filepath=GENERATED_CLOUDBUILD_FILE, + filepath=GENERATED_CLOUDBUILD_FILE, text=render_jinja( template_path=import_files(CLOUDBUILD_TEMPLATES_PATH) / 'cloudbuild.yaml.j2', artifact_repo_location=config.artifact_repo_location, diff --git a/google_cloud_automlops/deployments/github_actions/builder.py b/google_cloud_automlops/deployments/github_actions/builder.py index 2e8c360..90f639c 100644 --- a/google_cloud_automlops/deployments/github_actions/builder.py +++ b/google_cloud_automlops/deployments/github_actions/builder.py @@ -22,8 +22,6 @@ # Try backported to PY<37 `importlib_resources` from importlib_resources import files as import_files -from jinja2 import Template - from google_cloud_automlops.utils.utils import ( render_jinja, write_file @@ -57,7 +55,7 @@ def build(config: GitHubActionsConfig): """ # Write github actions config write_file( - filepath=GENERATED_GITHUB_ACTIONS_FILE, + filepath=GENERATED_GITHUB_ACTIONS_FILE, text=render_jinja( template_path=import_files(GITHUB_ACTIONS_TEMPLATES_PATH) / 'github_actions.yaml.j2', artifact_repo_location=config.artifact_repo_location, diff --git a/google_cloud_automlops/deployments/gitops/git_utils.py b/google_cloud_automlops/deployments/gitops/git_utils.py index 08baedd..8db2e52 100644 --- a/google_cloud_automlops/deployments/gitops/git_utils.py +++ b/google_cloud_automlops/deployments/gitops/git_utils.py @@ -28,8 +28,6 @@ import os import subprocess -from jinja2 import Template - from google_cloud_automlops.utils.constants import ( BASE_DIR, GENERATED_DEFAULTS_FILE, @@ -79,7 +77,7 @@ def git_workflow(): write_file( f'{BASE_DIR}.gitignore', - render_jinja(template_path=import_files(GITOPS_TEMPLATES_PATH) / 'gitignore.j2'), + render_jinja(template_path=import_files(GITOPS_TEMPLATES_PATH) / 'gitignore.j2'), 'w') # This will initialize the branch, a second push will be required to trigger the cloudbuild job after initializing diff --git a/google_cloud_automlops/orchestration/kfp/builder.py b/google_cloud_automlops/orchestration/kfp/builder.py index 1e47a8c..019eb59 100644 --- a/google_cloud_automlops/orchestration/kfp/builder.py +++ b/google_cloud_automlops/orchestration/kfp/builder.py @@ -25,8 +25,6 @@ import re import textwrap -from jinja2 import Template - from google_cloud_automlops.utils.utils import ( execute_process, get_components_list, @@ -72,10 +70,10 @@ def build(config: KfpConfig): # Write scripts for building pipeline, building components, running pipeline, and running all files scripts_path = import_files(KFP_TEMPLATES_PATH + '.scripts') - + # Write script for building pipeline write_and_chmod( - GENERATED_PIPELINE_SPEC_SH_FILE, + GENERATED_PIPELINE_SPEC_SH_FILE, render_jinja( template_path=scripts_path / 'build_pipeline_spec.sh.j2', generated_license=GENERATED_LICENSE, @@ -83,7 +81,7 @@ def build(config: KfpConfig): # Write script for building components write_and_chmod( - GENERATED_BUILD_COMPONENTS_SH_FILE, + GENERATED_BUILD_COMPONENTS_SH_FILE, render_jinja( template_path=scripts_path / 'build_components.sh.j2', generated_license=GENERATED_LICENSE, @@ -91,7 +89,7 @@ def build(config: KfpConfig): # Write script for running pipeline write_and_chmod( - GENERATED_RUN_PIPELINE_SH_FILE, + GENERATED_RUN_PIPELINE_SH_FILE, render_jinja( template_path=scripts_path / 'run_pipeline.sh.j2', generated_license=GENERATED_LICENSE, @@ -99,7 +97,7 @@ def build(config: KfpConfig): # Write script for running all files write_and_chmod( - GENERATED_RUN_ALL_SH_FILE, + GENERATED_RUN_ALL_SH_FILE, render_jinja( template_path=scripts_path / 'run_all.sh.j2', generated_license=GENERATED_LICENSE, @@ -108,7 +106,7 @@ def build(config: KfpConfig): # If using CI, write script for publishing to pubsub topic if config.use_ci: write_and_chmod( - GENERATED_PUBLISH_TO_TOPIC_FILE, + GENERATED_PUBLISH_TO_TOPIC_FILE, render_jinja( template_path=scripts_path / 'publish_to_topic.sh.j2', base_dir=BASE_DIR, @@ -130,7 +128,7 @@ def build(config: KfpConfig): f'{BASE_DIR}README.md', render_jinja( template_path=import_files(KFP_TEMPLATES_PATH) / 'README.md.j2', - use_ci=config.use_ci), + use_ci=config.use_ci), 'w') # Write dockerfile to the component base directory @@ -188,8 +186,7 @@ def build_component(component_path: str): # Write task script to component base write_file( - task_filepath, - + task_filepath, render_jinja( template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2', custom_code_contents=custom_code_contents, @@ -234,7 +231,7 @@ def build_pipeline(custom_training_job_specs: list, # Construct pipeline.py project_id = defaults['gcp']['project_id'] write_file( - GENERATED_PIPELINE_FILE, + GENERATED_PIPELINE_FILE, render_jinja( template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2', components_list=components_list, @@ -246,7 +243,7 @@ def build_pipeline(custom_training_job_specs: list, # Construct pipeline_runner.py write_file( - GENERATED_PIPELINE_RUNNER_FILE, + GENERATED_PIPELINE_RUNNER_FILE, render_jinja( template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2', generated_license=GENERATED_LICENSE), @@ -254,7 +251,7 @@ def build_pipeline(custom_training_job_specs: list, # Construct requirements.txt write_file( - GENERATED_PIPELINE_REQUIREMENTS_FILE, + GENERATED_PIPELINE_REQUIREMENTS_FILE, render_jinja( template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2', pinned_kfp_version=PINNED_KFP_VERSION), @@ -305,7 +302,7 @@ def build_services(): pipeline_root=defaults['pipelines']['pipeline_storage_path'], pipeline_job_runner_service_account=defaults['gcp']['pipeline_job_runner_service_account'], pipeline_job_submission_service_type=defaults['gcp']['pipeline_job_submission_service_type'], - project_id=defaults['gcp']['project_id']), + project_id=defaults['gcp']['project_id']), 'w') diff --git a/google_cloud_automlops/provisioning/gcloud/builder.py b/google_cloud_automlops/provisioning/gcloud/builder.py index 53aa256..5a12114 100644 --- a/google_cloud_automlops/provisioning/gcloud/builder.py +++ b/google_cloud_automlops/provisioning/gcloud/builder.py @@ -77,7 +77,7 @@ def build( required_apis = get_required_apis(defaults) # create provision_resources.sh write_and_chmod( - GENERATED_RESOURCES_SH_FILE, + GENERATED_RESOURCES_SH_FILE, render_jinja( template_path=import_files(GCLOUD_TEMPLATES_PATH) / 'provision_resources.sh.j2', artifact_repo_location=config.artifact_repo_location, diff --git a/google_cloud_automlops/provisioning/pulumi/builder.py b/google_cloud_automlops/provisioning/pulumi/builder.py index 69815b5..225b4f6 100644 --- a/google_cloud_automlops/provisioning/pulumi/builder.py +++ b/google_cloud_automlops/provisioning/pulumi/builder.py @@ -18,8 +18,6 @@ # pylint: disable=line-too-long # pylint: disable=unused-import -from jinja2 import Template - from google_cloud_automlops.utils.utils import ( write_file, render_jinja, diff --git a/google_cloud_automlops/provisioning/terraform/builder.py b/google_cloud_automlops/provisioning/terraform/builder.py index 7c5797d..5f4f19a 100644 --- a/google_cloud_automlops/provisioning/terraform/builder.py +++ b/google_cloud_automlops/provisioning/terraform/builder.py @@ -23,8 +23,6 @@ # Try backported to PY<37 `importlib_resources` from importlib_resources import files as import_files -from jinja2 import Template - from google_cloud_automlops.utils.utils import ( get_required_apis, read_yaml_file, @@ -85,7 +83,7 @@ def build( # create environment/data.tf write_file( - filepath=f'{BASE_DIR}provision/environment/data.tf', + filepath=f'{BASE_DIR}provision/environment/data.tf', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'data.tf.j2', generated_license=GENERATED_LICENSE, @@ -97,7 +95,7 @@ def build( # create environment/iam.tf write_file( - filepath=f'{BASE_DIR}provision/environment/iam.tf', + filepath=f'{BASE_DIR}provision/environment/iam.tf', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'iam.tf.j2', generated_license=GENERATED_LICENSE @@ -106,7 +104,7 @@ def build( # create environment/main.tf write_file( - filepath=f'{BASE_DIR}provision/environment/main.tf', + filepath=f'{BASE_DIR}provision/environment/main.tf', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'main.tf.j2', artifact_repo_type=config.artifact_repo_type, @@ -140,7 +138,7 @@ def build( # create environment/provider.tf write_file( - filepath=f'{BASE_DIR}provision/environment/provider.tf', + filepath=f'{BASE_DIR}provision/environment/provider.tf', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'provider.tf.j2', generated_license=GENERATED_LICENSE @@ -159,7 +157,7 @@ def build( # create environment/variables.auto.tfvars if config.deployment_framework == Deployer.CLOUDBUILD.value: write_file( - filepath=f'{BASE_DIR}provision/environment/variables.auto.tfvars', + filepath=f'{BASE_DIR}provision/environment/variables.auto.tfvars', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.auto.tfvars.j2', artifact_repo_location=config.artifact_repo_location, @@ -187,7 +185,7 @@ def build( #TODO: implement workload identity as optional if config.deployment_framework == Deployer.GITHUB_ACTIONS.value: write_file( - filepath=f'{BASE_DIR}provision/environment/variables.auto.tfvars', + filepath=f'{BASE_DIR}provision/environment/variables.auto.tfvars', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.auto.tfvars.j2', artifact_repo_location=config.artifact_repo_location, @@ -214,7 +212,7 @@ def build( # create environment/versions.tf write_file( - filepath=f'{BASE_DIR}provision/environment/versions.tf', + filepath=f'{BASE_DIR}provision/environment/versions.tf', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'versions.tf.j2', generated_license=GENERATED_LICENSE, @@ -224,7 +222,7 @@ def build( # create provision_resources.sh write_and_chmod( - filepath=GENERATED_RESOURCES_SH_FILE, + filepath=GENERATED_RESOURCES_SH_FILE, text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH) / 'provision_resources.sh.j2', base_dir=BASE_DIR, @@ -233,7 +231,7 @@ def build( # create state_bucket/main.tf write_file( - filepath=f'{BASE_DIR}provision/state_bucket/main.tf', + filepath=f'{BASE_DIR}provision/state_bucket/main.tf', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'main.tf.j2', generated_license=GENERATED_LICENSE @@ -242,7 +240,7 @@ def build( # create state_bucket/variables.tf write_file( - filepath=f'{BASE_DIR}provision/state_bucket/variables.tf', + filepath=f'{BASE_DIR}provision/state_bucket/variables.tf', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'variables.tf.j2', generated_license=GENERATED_LICENSE @@ -251,7 +249,7 @@ def build( # create state_bucket/variables.auto.tfvars write_file( - filepath=f'{BASE_DIR}provision/state_bucket/variables.auto.tfvars', + filepath=f'{BASE_DIR}provision/state_bucket/variables.auto.tfvars', text=render_jinja( template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'variables.auto.tfvars.j2', project_id=project_id, diff --git a/google_cloud_automlops/utils/utils.py b/google_cloud_automlops/utils/utils.py index 0c0a080..439766e 100644 --- a/google_cloud_automlops/utils/utils.py +++ b/google_cloud_automlops/utils/utils.py @@ -955,4 +955,4 @@ def render_jinja(template_path, **template_vars): """ with template_path.open('r', encoding='utf-8') as f: template = Template(f.read()) - return template.render(**template_vars) \ No newline at end of file + return template.render(**template_vars) From a31d735861256e7ea53f1e894963e319d87b7bd0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 7 Feb 2024 15:31:09 -0500 Subject: [PATCH 3/9] More Pylint fixes --- google_cloud_automlops/provisioning/pulumi/builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google_cloud_automlops/provisioning/pulumi/builder.py b/google_cloud_automlops/provisioning/pulumi/builder.py index 225b4f6..b2cafce 100644 --- a/google_cloud_automlops/provisioning/pulumi/builder.py +++ b/google_cloud_automlops/provisioning/pulumi/builder.py @@ -77,7 +77,7 @@ def build( # create Pulumi.yaml write_file( - pulumi_folder + 'Pulumi.yaml', + pulumi_folder + 'Pulumi.yaml', render_jinja( template_path=PULUMI_TEMPLATES_PATH / 'Pulumi.yaml.jinja', generated_license=GENERATED_LICENSE, @@ -88,7 +88,7 @@ def build( # create Pulumi.dev.yaml write_file( - pulumi_folder + 'Pulumi.dev.yaml', + pulumi_folder + 'Pulumi.dev.yaml', render_jinja( template_path=PULUMI_TEMPLATES_PATH / 'Pulumi.dev.yaml.jinja', generated_license=GENERATED_LICENSE, @@ -102,7 +102,7 @@ def build( # create python __main__.py if config.pulumi_runtime == PulumiRuntime.PYTHON: write_file( - pulumi_folder + '__main__.py', + pulumi_folder + '__main__.py', render_jinja( template_path=PULUMI_TEMPLATES_PATH / 'python/__main__.py.jinja', generated_license=GENERATED_LICENSE, From 7df0e557401557fba7252ebae78fa434e043cc24 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 9 Feb 2024 11:43:18 -0500 Subject: [PATCH 4/9] Adjusted test scripts to account for lost functions --- google_cloud_automlops/utils/utils.py | 2 +- .../deployments/cloudbuild/builder_test.py | 96 ---- .../github_actions/builder_test.py | 115 ---- tests/unit/orchestration/kfp/builder_test.py | 532 ------------------ .../unit/provisioning/gcloud/builder_test.py | 220 -------- .../provisioning/terraform/builder_test.py | 522 ----------------- tests/unit/utils/utils_test.py | 31 + 7 files changed, 32 insertions(+), 1486 deletions(-) delete mode 100644 tests/unit/deployments/cloudbuild/builder_test.py delete mode 100644 tests/unit/deployments/github_actions/builder_test.py delete mode 100644 tests/unit/provisioning/gcloud/builder_test.py delete mode 100644 tests/unit/provisioning/terraform/builder_test.py diff --git a/google_cloud_automlops/utils/utils.py b/google_cloud_automlops/utils/utils.py index 439766e..a4ac109 100644 --- a/google_cloud_automlops/utils/utils.py +++ b/google_cloud_automlops/utils/utils.py @@ -953,6 +953,6 @@ def render_jinja(template_path, **template_vars): Returns: str: The rendered template as a string. """ - with template_path.open('r', encoding='utf-8') as f: + with open(template_path, 'r', encoding='utf-8') as f: template = Template(f.read()) return template.render(**template_vars) diff --git a/tests/unit/deployments/cloudbuild/builder_test.py b/tests/unit/deployments/cloudbuild/builder_test.py deleted file mode 100644 index a1a91c5..0000000 --- a/tests/unit/deployments/cloudbuild/builder_test.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright 2023 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=line-too-long -# pylint: disable=missing-module-docstring - -from typing import List - -import pytest - -from google_cloud_automlops.deployments.cloudbuild.builder import create_cloudbuild_jinja - -@pytest.mark.parametrize( - '''artifact_repo_location, artifact_repo_name, naming_prefix,''' - '''project_id, pubsub_topic_name, use_ci, is_included,''' - '''expected_output_snippets''', - [ - ( - 'us-central1', 'my-artifact-repo', 'my-prefix', - 'my-project', 'my-topic', True, True, - ['id: "build_component_base"', - 'id: "push_component_base"', - 'id: "install_pipelines_deps"', - 'id: "build_pipeline_spec"', - 'id: "publish_to_topic"', - '"build", "-t", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', - '"push", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', - 'gcloud pubsub topics publish my-topic'] - ), - ( - 'us-central1', 'my-artifact-repo', 'my-prefix', - 'my-project', 'my-topic', False, True, - ['id: "build_component_base"', - '"build", "-t", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"'] - ), - ( - 'us-central1', 'my-artifact-repo', 'my-prefix', - 'my-project', 'my-topic', False, False, - ['id: "push_component_base"', - 'id: "install_pipelines_deps"', - 'id: "build_pipeline_spec"', - 'id: "publish_to_topic"', - '"push" "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', - 'gcloud pubsub topics publish my-topic'] - ), - ] -) -def test_create_cloudbuild_jinja( - artifact_repo_location: str, - artifact_repo_name: str, - naming_prefix: str, - project_id: str, - pubsub_topic_name: str, - use_ci: bool, - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_cloudbuild_jinja, which generates content for the cloudbuild.yaml. - There are three test cases for this function: - 1. Checks that expected strings are included when use_ci=True. - 2. Checks that expected strings are included when use_ci=False. - 3. Checks that certain strings are not included when use_ci=False. - - Args: - artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). - artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - project_id: The project ID. - pubsub_topic_name: The name of the pubsub topic to publish to. - use_ci: Flag that determines whether to use Cloud CI/CD. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - cloudbuild_config = create_cloudbuild_jinja( - artifact_repo_location, - artifact_repo_name, - naming_prefix, - project_id, - pubsub_topic_name, - use_ci) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in cloudbuild_config - elif not is_included: - assert snippet not in cloudbuild_config diff --git a/tests/unit/deployments/github_actions/builder_test.py b/tests/unit/deployments/github_actions/builder_test.py deleted file mode 100644 index 692afc6..0000000 --- a/tests/unit/deployments/github_actions/builder_test.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2023 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=line-too-long -# pylint: disable=missing-function-docstring -# pylint: disable=missing-module-docstring - -from typing import List - -import pytest - -from google_cloud_automlops.deployments.github_actions.builder import create_github_actions_jinja - - -@pytest.mark.parametrize( - '''artifact_repo_location, artifact_repo_name, naming_prefix,''' - '''project_id, project_number, pubsub_topic_name, use_ci, source_repo_branch,''' - '''workload_identity_provider, workload_identity_pool, workload_identity_service_account, is_included,''' - '''expected_output_snippets''', - [ - ( - 'us-central1', 'my-artifact-repo', 'my-prefix', - 'my-project', 'my-project-number', 'my-topic', True, 'automlops', - 'my-provider', 'my-pool', 'my-sa', True, - ['id: auth', - 'id: build-push-component-base', - 'id: install-pipeline-deps', - 'id: build-pipeline-spec', - 'id: publish-to-topic', - 'us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest', - 'gcloud pubsub topics publish my-topic --message'] - ), - ( - 'us-central1', 'my-artifact-repo', 'my-prefix', - 'my-project', 'my-project-number', 'my-topic', False, 'automlops', - 'my-provider', 'my-pool', 'my-sa', True, - ['id: build-push-component-base', - 'us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest'] - ), - ( - 'us-central1', 'my-artifact-repo', 'my-prefix', - 'my-project', 'my-project-number', 'my-topic', False, 'automlops', - 'my-provider', 'my-pool', 'my-sa', False, - ['id: install-pipeline-deps', - 'id: build-pipeline-spec', - 'id: publish-to-topic', - 'gcloud pubsub topics publish my-topic --message'] - ), - ] -) -def test_create_github_actions_jinja( - artifact_repo_location: str, - artifact_repo_name: str, - naming_prefix: str, - project_id: str, - project_number: str, - pubsub_topic_name: str, - use_ci: bool, - source_repo_branch: str, - workload_identity_pool: str, - workload_identity_provider: str, - workload_identity_service_account: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_github_actions_jinja, which generates content for the github actions file. - There are three test cases for this function: - 1. Checks that expected strings are included when use_ci=True. - 2. Checks that expected strings are included when use_ci=False. - 3. Checks that certain strings are not included when use_ci=False. - - Args: - artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). - artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - project_id: The project ID. - project_number: The project number. - pubsub_topic_name: The name of the pubsub topic to publish to. - source_repo_branch: The branch to use in the source repository. - use_ci: Flag that determines whether to use Cloud CI/CD. - workload_identity_pool: Pool for workload identity federation. - workload_identity_provider: Provider for workload identity federation. - workload_identity_service_account: Service account for workload identity federation. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - - github_actions_config = create_github_actions_jinja( - artifact_repo_location, - artifact_repo_name, - naming_prefix, - project_id, - project_number, - pubsub_topic_name, - source_repo_branch, - use_ci, - workload_identity_pool, - workload_identity_provider, - workload_identity_service_account) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in github_actions_config - elif not is_included: - assert snippet not in github_actions_config diff --git a/tests/unit/orchestration/kfp/builder_test.py b/tests/unit/orchestration/kfp/builder_test.py index 5faddbb..7a93d60 100644 --- a/tests/unit/orchestration/kfp/builder_test.py +++ b/tests/unit/orchestration/kfp/builder_test.py @@ -23,29 +23,11 @@ import pytest import pytest_mock -from google_cloud_automlops.utils.constants import ( - GENERATED_LICENSE, - PINNED_KFP_VERSION -) import google_cloud_automlops.orchestration.kfp.builder from google_cloud_automlops.orchestration.kfp.builder import ( build_component, build_pipeline, build_services, - build_pipeline_spec_jinja, - build_components_jinja, - run_pipeline_jinja, - run_all_jinja, - publish_to_topic_jinja, - readme_jinja, - component_base_dockerfile_jinja, - component_base_task_file_jinja, - pipeline_runner_jinja, - pipeline_jinja, - pipeline_requirements_jinja, - submission_service_dockerfile_jinja, - submission_service_requirements_jinja, - submission_service_main_jinja ) import google_cloud_automlops.utils.utils from google_cloud_automlops.utils.utils import ( @@ -345,517 +327,3 @@ def test_build_services(mocker: pytest_mock.MockerFixture, assert os.path.exists(f'{tmpdir}/services/submission_service/Dockerfile') assert os.path.exists(f'{tmpdir}/services/submission_service/requirements.txt') assert os.path.exists(f'{tmpdir}/services/submission_service/main.py') - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline --config $CONFIG_FILE'])] -) -def test_build_pipeline_spec_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests build_pipeline_spec_jinja, which generates code for build_pipeline_spec.sh - which builds the pipeline specs. There is one test case for this function: - 1. Checks for the apache license and the pipeline compile command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - build_pipeline_spec_script = build_pipeline_spec_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in build_pipeline_spec_script - elif not is_included: - assert snippet not in build_pipeline_spec_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600'])] -) -def test_build_components_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests build_components_jinja, which generates code for build_components.sh - which builds the components. There is one test case for this function: - 1. Checks for the apache license and the builds submit command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - build_components_script = build_components_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in build_components_script - elif not is_included: - assert snippet not in build_components_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline_runner --config $CONFIG_FILE'])] -) -def test_run_pipeline_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests run_pipeline_jinja, which generates code for run_pipeline.sh - which runs the pipeline locally. There is one test case for this function: - 1. Checks for the apache license and the pipeline runner command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - run_pipeline_script = run_pipeline_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in run_pipeline_script - elif not is_included: - assert snippet not in run_pipeline_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600', - './scripts/build_pipeline_spec.sh', './scripts/run_pipeline.sh'])] -) -def test_run_all_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests run_all_jinja, which generates code for run_all.sh - which builds runs all other shell scripts. There is one test case for this function: - 1. Checks for the apache license and the builds submit, the pipeline compile, and the pipeline runner commands. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - run_all_script = run_all_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in run_all_script - elif not is_included: - assert snippet not in run_all_script - - -@pytest.mark.parametrize( - 'pubsub_topic_name, is_included, expected_output_snippets', - [('my-topic', True, [GENERATED_LICENSE, 'gcloud pubsub topics publish my-topic'])] -) -def test_publish_to_topic_jinja( - pubsub_topic_name: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests publish_to_topic_jinja, which generates code for publish_to_topic.sh - which submits a message to the pipeline job submission service. - There is one test case for this function: - 1. Checks for the apache license and the pubsub publish command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - publish_to_topic_script = publish_to_topic_jinja(pubsub_topic_name=pubsub_topic_name) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in publish_to_topic_script - elif not is_included: - assert snippet not in publish_to_topic_script - - -@pytest.mark.parametrize( - 'use_ci, is_included, expected_output_snippets', - [ - ( - True, True, - ['AutoMLOps - Generated Code Directory', - '├── components', - '├── configs', - '├── images', - '├── provision', - '├── scripts', - '├── services', - '├── README.md', - '└── cloudbuild.yaml'] - ), - ( - False, False, - ['├── publish_to_topic.sh' - '├── services'] - ), - ] -) -def test_readme_jinja( - use_ci: bool, - is_included: bool, - expected_output_snippets: List[str]): - """Tests readme_jinja, which generates code for readme.md which - is a readme markdown file to describe the contents of the - generated AutoMLOps code repo. There are two test cases for this function: - 1. Checks that certain directories and files exist when use_ci=True. - 2. Checks that certain directories and files do not exist when use_ci=False. - - Args: - use_ci: Flag that determines whether to use Cloud CI/CD. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - readme_str = readme_jinja(use_ci=use_ci) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in readme_str - elif not is_included: - assert snippet not in readme_str - - -@pytest.mark.parametrize( - 'base_image, is_included, expected_output_snippets', - [('my-base-image', True, [GENERATED_LICENSE, 'FROM my-base-image'])] -) -def test_component_base_dockerfile_jinja( - base_image: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests readme_jinja, which generates code for a Dockerfile - to be written to the component_base directory. There is one - test case for this function: - 1. Checks for the apache license and the FROM image line. - - Args: - base_image: The image to use in the component base dockerfile. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - component_base_dockerfile = component_base_dockerfile_jinja(base_image) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in component_base_dockerfile - elif not is_included: - assert snippet not in component_base_dockerfile - - -@pytest.mark.parametrize( - 'custom_code_contents, kfp_spec_bool, is_included, expected_output_snippets', - [ - ( - 'this is some custom code', True, True, - [GENERATED_LICENSE, - 'this is some custom code', - 'def main():'] - ), - ( - 'this is some custom code', False, True, - [GENERATED_LICENSE, - 'this is some custom code', - 'def main():', - 'import kfp', - 'from kfp.v2.dsl import *'] - ), - ( - 'this is some custom code', True, False, - ['import kfp', - 'from kfp.v2.dsl import *'] - ) - ] -) -def test_component_base_task_file_jinja( - custom_code_contents: str, - kfp_spec_bool: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests component_base_task_file_jinja, which generates code - for the task.py file to be written to the component_base/src directory. - There are three test cases for this function: - 1. Checks for the apache license, the custom_code_contents, and a main function when using kfp spec (kfp spec comes with kfp imports by default). - 2. Checks for the apache license, the custom_code_contents, a main function, and kfp imports when not using kfp spec. - 3. Checks that the kfp imports are not included in the string when using kfp spec (kfp spec comes with kfp imports by default). - - Args: - custom_code_contents: Code inside of the component, specified by the user. - kfp_spec_bool: Boolean that specifies whether components are defined using kfp. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - component_base_task_file = component_base_task_file_jinja(custom_code_contents, kfp_spec_bool) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in component_base_task_file - elif not is_included: - assert snippet not in component_base_task_file - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE])] -) -def test_pipeline_runner_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_runner_jinja, which generates code for the pipeline_runner.py - file to be written to the pipelines directory. There is one test case for this function: - 1. Checks for the apache license. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - pipeline_runner_py = pipeline_runner_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in pipeline_runner_py - elif not is_included: - assert snippet not in pipeline_runner_py - - -@pytest.mark.parametrize( - '''components_list, custom_training_job_specs, pipeline_scaffold_contents, project_id,''' - '''is_included, expected_output_snippets''', - [ - ( - ['componentA','componentB','componentC'], - [ - { - 'component_spec': 'componentB', - 'display_name': 'train-model-accelerated', - 'machine_type': 'a2-highgpu-1g', - 'accelerator_type': 'NVIDIA_TESLA_A100', - 'accelerator_count': '1', - } - ], - 'Pipeline definition goes here', 'my-project', True, - [GENERATED_LICENSE, - 'from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', - 'def upload_pipeline_spec', - 'componentA = load_custom_component', - 'componentB = load_custom_component', - 'componentC = load_custom_component', - 'componentB_custom_training_job_specs', - 'Pipeline definition goes here'] - ), - ( - ['componentA','componentB','componentC'], - None, 'Pipeline definition goes here', 'my-project', True, - [GENERATED_LICENSE, - 'def upload_pipeline_spec', - 'componentA = load_custom_component', - 'componentB = load_custom_component', - 'componentC = load_custom_component', - 'Pipeline definition goes here'] - ), - ( - ['componentA','componentB','componentC'], - None, 'Pipeline definition goes here', 'my-project', False, - ['from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', - 'componentB_custom_training_job_specs'] - ), - ] -) -def test_pipeline_jinja( - components_list: list, - custom_training_job_specs: list, - pipeline_scaffold_contents: str, - project_id: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_jinja, which generates code for the pipeline.py - file to be written to the pipelines directory. - There are three test cases for this function: - 1. Checks for the apache license and relevant code elements when custom_training_job_specs is not None. - 2. Checks for the apache license and relevant code elements when custom_training_job_specs is None. - 3. Checks that the output does not contain custom_training_job_specs code elements when custom_training_job_specs is None. - - Args: - components_list: Contains the names or paths of all component yamls in the dir. - custom_training_job_specs: Specifies the specs to run the training job with. - pipeline_scaffold_contents: The contents of the pipeline scaffold file, - which can be found at PIPELINE_CACHE_FILE. - project_id: The project ID. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - pipeline_py = pipeline_jinja( - components_list, - custom_training_job_specs, - pipeline_scaffold_contents, - project_id) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in pipeline_py - elif not is_included: - assert snippet not in pipeline_py - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform'])] -) -def test_pipeline_requirements_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_requirements_jinja, which generates code for a requirements.txt - to be written to the pipelines directory. There is one test case for this function: - 1. Checks for the pinned kfp version, and the google-cloud-aiplatform dep. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - pipeline_requirements_py = pipeline_requirements_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in pipeline_requirements_py - elif not is_included: - assert snippet not in pipeline_requirements_py - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'python:3.9-slim', - 'CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app'])] -) -def test_submission_service_dockerfile_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_requirements_jinja, which generates code for a Dockerfile to be - written to the serivces/submission_service directory. There is one test case for this function: - 1. Checks for the apache license and relevant dockerfile elements. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - submission_service_dockerfile = submission_service_dockerfile_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in submission_service_dockerfile - elif not is_included: - assert snippet not in submission_service_dockerfile - - -@pytest.mark.parametrize( - 'pipeline_job_submission_service_type, is_included, expected_output_snippets', - [('cloud-functions', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'functions-framework==3.*']), - ('cloud-functions', False, ['gunicorn']), - ('cloud-run', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'gunicorn']), - ('cloud-run', False, ['functions-framework==3.*']),] -) -def test_submission_service_requirements_jinja( - pipeline_job_submission_service_type: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests submission_service_requirements_jinja, which generates code - for a requirements.txt to be written to the serivces/submission_service directory. - There are four test cases for this function: - 1. Checks for the pinned kfp version, the google-cloud-aiplatform and function-framework deps when set to cloud-functions. - 2. Checks that gunicorn dep is not included when set to cloud-functions. - 3. Checks for the pinned kfp version, the google-cloud-aiplatform and gunicorn deps when set to cloud-run. - 4. Checks that functions-framework dep is not included when set to cloud-run. - - Args: - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - submission_service_requirements = submission_service_requirements_jinja(pipeline_job_submission_service_type=pipeline_job_submission_service_type) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in submission_service_requirements - elif not is_included: - assert snippet not in submission_service_requirements - - -@pytest.mark.parametrize( - '''pipeline_root, pipeline_job_runner_service_account, pipeline_job_submission_service_type,''' - '''project_id, is_included, expected_output_snippets''', - [ - ( - 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', - 'my-project', True, - [GENERATED_LICENSE, - 'from google.cloud import aiplatform', - 'import functions_framework', - '@functions_framework.http', - 'def process_request(request: flask.Request)', - '''base64_message = request_json['data']['data']'''] - ), - ( - 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', - 'my-project', False, - ['app = flask.Flask', - '''@app.route('/', methods=['POST'])''', - 'request = flask.request', - '''base64_message = request_json['message']['data']''', - '''if __name__ == '__main__':''', - '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))'''] - ), - ( - 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', - 'my-project', True, - [GENERATED_LICENSE, - 'from google.cloud import aiplatform', - 'app = flask.Flask', - '''@app.route('/', methods=['POST'])''', - 'request = flask.request', - '''base64_message = request_json['message']['data']''', - '''if __name__ == '__main__':''', - '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))'''] - ), - ( - 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', - 'my-project', False, - ['import functions_framework', - '@functions_framework.http', - 'def process_request(request: flask.Request)', - '''base64_message = request_json['data']['data']'''] - ), - ] -) -def test_submission_service_main_jinja( - pipeline_root: str, - pipeline_job_runner_service_account: str, - pipeline_job_submission_service_type: str, - project_id: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests submission_service_main_jinja, which generates content - for main.py to be written to the serivces/submission_service directory. - There are four test cases for this function: - 1. Checks for functions_framework code elements when set to cloud-functions. - 2. Checks that Flask app code elements are not included when set to cloud-functions. - 3. Checks for Flask app code elements when set to cloud-run. - 4. Checks that functions_framework code elements are not included when set to cloud-run. - - Args: - pipeline_root: GS location where to store metadata from pipeline runs. - pipeline_job_runner_service_account: Service Account to runner PipelineJobs. - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - project_id: The project ID. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - submission_service_main_py = submission_service_main_jinja( - pipeline_root=pipeline_root, - pipeline_job_runner_service_account=pipeline_job_runner_service_account, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - project_id=project_id) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in submission_service_main_py - elif not is_included: - assert snippet not in submission_service_main_py diff --git a/tests/unit/provisioning/gcloud/builder_test.py b/tests/unit/provisioning/gcloud/builder_test.py deleted file mode 100644 index 7c7a561..0000000 --- a/tests/unit/provisioning/gcloud/builder_test.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright 2023 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=line-too-long -# pylint: disable=missing-function-docstring -# pylint: disable=missing-module-docstring - -from typing import List - -import pytest - -from google_cloud_automlops.provisioning.gcloud.builder import provision_resources_script_jinja - -@pytest.mark.parametrize( - '''artifact_repo_location, artifact_repo_name, artifact_repo_type, build_trigger_location,''' - '''build_trigger_name, deployment_framework, naming_prefix, pipeline_job_runner_service_account,''' - '''pipeline_job_submission_service_location, pipeline_job_submission_service_name, pipeline_job_submission_service_type,''' - '''project_id, pubsub_topic_name,''' - '''required_apis,''' - '''schedule_location, schedule_name, schedule_pattern,''' - '''source_repo_branch, source_repo_name, source_repo_type, storage_bucket_location, storage_bucket_name,''' - '''use_ci, vpc_connector, is_included,''' - '''expected_output_snippets''', - [ - ( - 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', - 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', - 'us-central1', 'my-submission-svc', 'cloud-functions', - 'my-project', 'my-topic-name', - ['apiA','apiB','apiC'], - 'us-central1', 'my-schedule', '0 12 * * *', - 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', - True, 'my-vpc-connector', True, - ['gcloud artifacts repositories create', 'gcloud iam service-accounts create', - 'gsutil mb -l ${STORAGE_BUCKET_LOCATION} gs://$STORAGE_BUCKET_NAME', 'gcloud iam service-accounts create', - 'gcloud projects add-iam-policy-binding', 'gcloud source repos create', 'gcloud pubsub topics create', - 'gcloud functions deploy', 'gcloud beta builds triggers create', 'gcloud scheduler jobs create pubsub', - '--vpc-connector=my-vpc-connector'] - ), - ( - 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', - 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', - 'us-central1', 'my-submission-svc', 'cloud-run', - 'my-project', 'my-topic-name', - ['apiA','apiB','apiC'], - 'us-central1', 'my-schedule', '0 12 * * *', - 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', - True, 'No VPC Specified', True, - ['gcloud artifacts repositories create', 'gcloud iam service-accounts create', - 'gsutil mb -l ${STORAGE_BUCKET_LOCATION} gs://$STORAGE_BUCKET_NAME', 'gcloud iam service-accounts create', - 'gcloud projects add-iam-policy-binding', 'gcloud source repos create', 'gcloud pubsub topics create', - 'gcloud builds submit ${BASE_DIR}services/submission_service', 'gcloud run deploy', 'gcloud pubsub subscriptions create', - 'gcloud beta builds triggers create', 'gcloud scheduler jobs create pubsub'] - ), - ( - 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', - 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', - 'us-central1', 'my-submission-svc', 'cloud-run', - 'my-project', 'my-topic-name', - ['apiA','apiB','apiC'], - 'us-central1', 'my-schedule', '0 12 * * *', - 'my-branch', 'my-repo', 'some-other-source-repository', 'us-central1', 'my-bucket', - True, 'No VPC Specified', False, - ['gcloud source repos create', 'cloud beta builds triggers create'] - ), - ( - 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', - 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', - 'us-central1', 'my-submission-svc', 'cloud-run', - 'my-project', 'my-topic-name', - ['apiA','apiB','apiC'], - 'us-central1', 'my-schedule', 'No Schedule Specified', - 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', - True, 'No VPC Specified', False, - ['gcloud scheduler jobs create pubsub', '--vpc_connector='] - ), - ( - 'us-central1', 'my-registry', 'some-other-repo-type', 'us-central1', - 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', - 'us-central1', 'my-submission-svc', 'cloud-functions', - 'my-project', 'my-topic-name', - ['apiA','apiB','apiC'], - 'us-central1', 'my-schedule', '0 12 * * *', - 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', - True, 'No VPC Specified', False, - ['gcloud artifacts repositories create'] - ), - ( - 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', - 'my-trigger', 'some-other-deployment-framework', 'my-prefix', 'my-service-account@serviceaccount.com', - 'us-central1', 'my-submission-svc', 'cloud-functions', - 'my-project', 'my-topic-name', - ['apiA','apiB','apiC'], - 'us-central1', 'my-schedule', '0 12 * * *', - 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', - True, 'No VPC Specified', False, - ['gcloud beta builds triggers create'] - ), - ( - 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', - 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', - 'us-central1', 'my-submission-svc', 'cloud-functions', - 'my-project', 'my-topic-name', - ['apiA','apiB','apiC'], - 'us-central1', 'my-schedule', '0 12 * * *', - 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', - False, 'No VPC Specified', False, - ['gcloud pubsub topics create', 'gcloud beta builds triggers create', - 'gcloud functions deploy', 'gcloud run deploy', 'gcloud scheduler jobs create pubsub'] - ) - ] -) -def test_provision_resources_script_jinja( - artifact_repo_location: str, - artifact_repo_name: str, - artifact_repo_type: str, - build_trigger_location: str, - build_trigger_name: str, - deployment_framework: str, - naming_prefix: str, - pipeline_job_runner_service_account: str, - pipeline_job_submission_service_location: str, - pipeline_job_submission_service_name: str, - pipeline_job_submission_service_type: str, - project_id: str, - pubsub_topic_name: str, - required_apis: list, - schedule_location: str, - schedule_name: str, - schedule_pattern: str, - source_repo_branch: str, - source_repo_name: str, - source_repo_type: str, - storage_bucket_location: str, - storage_bucket_name: str, - use_ci: bool, - vpc_connector: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests provision_resources_script_jinja, which generates code for - provision_resources.sh which sets up the project's environment. - There are seven test cases for this function: - 1. Checks for relevant gcloud commands when using the following tooling: - artifact-registry, cloud-build, cloud-functions, cloud scheduler, cloud-source-repositories, and a vpc connector - 2. Checks for relevant gcloud commands when using the following tooling: - artifact-registry, cloud-build, cloud-run, cloud scheduler, cloud-source-repositories, and no vpc connector - 3. Checks that gcloud source repo commands are not included when not using cloud-source-repositories. - 4. Checks that gcloud scheduler command and vpc_connector flag are not included when not specifying a vpc connector or schedule. - 5. Checks that gcloud artifacts command is not included when not using artifact-registry. - 6. Checks that gcloud beta builds triggers command is not included when not using cloud-build. - 7. Checks for that CI/CD elements are not included when use_ci=False. - - Args: - artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). - artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). - artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) - build_trigger_location: The location of the build trigger (for cloud build). - build_trigger_name: The name of the build trigger (for cloud build). - deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - pipeline_job_runner_service_account: Service Account to run PipelineJobs. - pipeline_job_submission_service_location: The location of the cloud submission service. - pipeline_job_submission_service_name: The name of the cloud submission service. - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - project_id: The project ID. - pubsub_topic_name: The name of the pubsub topic to publish to. - required_apis: List of APIs that are required to run the service. - schedule_location: The location of the scheduler resource. - schedule_name: The name of the scheduler resource. - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - source_repo_branch: The branch to use in the source repository. - source_repo_name: The name of the source repository to use. - source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) - storage_bucket_location: Region of the GS bucket. - storage_bucket_name: GS bucket name where pipeline run metadata is stored. - use_ci: Flag that determines whether to use Cloud CI/CD. - vpc_connector: The name of the vpc connector to use. - """ - provision_resources_script = provision_resources_script_jinja( - artifact_repo_location=artifact_repo_location, - artifact_repo_name=artifact_repo_name, - artifact_repo_type=artifact_repo_type, - build_trigger_location=build_trigger_location, - build_trigger_name=build_trigger_name, - deployment_framework=deployment_framework, - naming_prefix=naming_prefix, - pipeline_job_runner_service_account=pipeline_job_runner_service_account, - pipeline_job_submission_service_location=pipeline_job_submission_service_location, - pipeline_job_submission_service_name=pipeline_job_submission_service_name, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - project_id=project_id, - pubsub_topic_name=pubsub_topic_name, - required_apis=required_apis, - schedule_location=schedule_location, - schedule_name=schedule_name, - schedule_pattern=schedule_pattern, - source_repo_branch=source_repo_branch, - source_repo_name=source_repo_name, - source_repo_type=source_repo_type, - storage_bucket_location=storage_bucket_location, - storage_bucket_name=storage_bucket_name, - use_ci=use_ci, - vpc_connector=vpc_connector) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in provision_resources_script - elif not is_included: - assert snippet not in provision_resources_script diff --git a/tests/unit/provisioning/terraform/builder_test.py b/tests/unit/provisioning/terraform/builder_test.py deleted file mode 100644 index a531ecb..0000000 --- a/tests/unit/provisioning/terraform/builder_test.py +++ /dev/null @@ -1,522 +0,0 @@ -# Copyright 2023 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=line-too-long -# pylint: disable=missing-function-docstring -# pylint: disable=missing-module-docstring - -from typing import List - -import pytest - -from google_cloud_automlops.utils.constants import GENERATED_LICENSE -from google_cloud_automlops.provisioning.terraform.builder import ( - create_environment_data_tf_jinja, - create_environment_iam_tf_jinja, - create_environment_main_tf_jinja, - create_environment_outputs_tf_jinja, - create_environment_provider_tf_jinja, - create_environment_variables_tf_jinja, - create_environment_versions_tf_jinja, - create_provision_resources_script_jinja, - create_state_bucket_variables_tf_jinja, - create_state_bucket_main_tf_jinja -) - - -@pytest.mark.parametrize( - 'required_apis, use_ci, is_included, expected_output_snippets', - [ - ( - ['apiA', 'apiB'], True, True, - [GENERATED_LICENSE, 'archive_cloud_functions_submission_service', - 'enable_apis = [\n' - ' "apiA",\n' - ' "apiB",\n' - ' ]' - ] - ), - ( - ['apiA', 'apiB'], False, False, - ['archive_cloud_functions_submission_service'] - ) - ] -) -def test_create_environment_data_tf_jinja( - required_apis: List, - use_ci: bool, - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_environment_data_tf_jinja, which generates code for environment/data.tf, - the terraform hcl script that contains terraform remote backend and org project details. - There are two test cases for this function: - 1. Checks for the apache license and relevant terraform blocks. - 2. Checks for that the archive statement is not included when use_ci=False. - - Args: - required_apis: List of APIs that are required to run the service. - use_ci: Flag that determines whether to use Cloud CI/CD. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - data_tf_str = create_environment_data_tf_jinja(required_apis, use_ci) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in data_tf_str - elif not is_included: - assert snippet not in data_tf_str - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE])] -) -def test_create_environment_iam_tf_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_environment_iam_tf_jinja, which generates code for environment/iam.tf, the terraform hcl - script that contains service accounts iam bindings for project's environment. - There is one test case for this function: - 1. Checks for the apache license. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - iam_tf_str = create_environment_iam_tf_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in iam_tf_str - elif not is_included: - assert snippet not in iam_tf_str - - -@pytest.mark.parametrize( - '''artifact_repo_type, deployment_framework, naming_prefix,''' - '''pipeline_job_submission_service_type, schedule_pattern,''' - '''source_repo_type, use_ci, vpc_connector, is_included,''' - '''expected_output_snippets''', - [ - ( - 'artifact-registry', 'cloud-build', 'my-prefix', - 'cloud-functions', '0 12 * * *', - 'cloud-source-repositories', True, 'my-vpc-connector', True, - ['resource "google_artifact_registry_repository"', - 'resource "google_storage_bucket"', - 'resource "google_sourcerepo_repository"', - 'resource "google_pubsub_topic"', - 'resource "google_storage_bucket_object"', - 'resource "google_cloudfunctions_function"', - 'vpc_connector =', - 'resource "google_cloudbuild_trigger"', - 'resource "google_cloud_scheduler_job"'] - ), - ( - 'artifact-registry', 'cloud-build', 'my-prefix', - 'cloud-run', '0 12 * * *', - 'cloud-source-repositories', True, 'my-vpc-connector', True, - ['resource "google_artifact_registry_repository"', - 'resource "google_storage_bucket"', - 'resource "google_sourcerepo_repository"', - 'resource "null_resource" "build_and_push_submission_service"', - 'module "cloud_run"', - 'run.googleapis.com/vpc-access-connector', - 'module "pubsub"', - 'resource "google_cloudbuild_trigger"', - 'resource "google_cloud_scheduler_job"'] - ), - ( - 'some-other-repo', 'cloud-build', 'my-prefix', - 'cloud-functions', '0 12 * * *', - 'cloud-source-repositories', True, 'No VPC Specified', False, - ['resource "google_artifact_registry_repository"', 'vpc_connector ='] - ), - ( - 'artifact-registry', 'cloud-build', 'my-prefix', - 'cloud-run', '0 12 * * *', - 'cloud-source-repositories', True, 'No VPC Specified', False, - ['run.googleapis.com/vpc-access-connector'] - ), - ( - 'artifact-registry', 'cloud-build', 'my-prefix', - 'cloud-functions', 'No Schedule Specified', - 'cloud-source-repositories', True, 'No VPC Specified', False, - ['resource "google_cloud_scheduler_job"'] - ), - ( - 'artifact-registry', 'some-deployment-framework', 'my-prefix', - 'cloud-functions', 'No Schedule Specified', - 'cloud-source-repositories', True, 'No VPC Specified', False, - ['resource "google_cloudbuild_trigger"'] - ), - ( - 'artifact-registry', 'cloud-build', 'my-prefix', - 'cloud-functions', 'No Schedule Specified', - 'some-other-code-repo', True, 'No VPC Specified', False, - ['resource "google_sourcerepo_repository"', 'resource "google_cloudbuild_trigger"'] - ), - ( - 'artifact-registry', 'cloud-build', 'my-prefix', - 'cloud-functions', 'No Schedule Specified', - 'some-other-code-repo', False, 'No VPC Specified', False, - ['resource "null_resource" "build_and_push_submission_service"', - 'module "cloud_run"', - 'module "pubsub"', - 'resource "google_pubsub_topic"', - 'resource "google_storage_bucket_object"', - 'resource "google_cloudfunctions_function"', - 'resource "google_cloudbuild_trigger"', - 'resource "google_cloud_scheduler_job"'] - ), - ] -) -def test_create_environment_main_tf_jinja( - artifact_repo_type: str, - deployment_framework: str, - naming_prefix: str, - pipeline_job_submission_service_type: str, - schedule_pattern: str, - source_repo_type: str, - use_ci: bool, - vpc_connector: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_main_environment_tf_jinja, which generates code for environment/main.tf, the terraform hcl - script that contains terraform resources configs to deploy resources in the project. - There are eight test cases for this function: - 1. Checks for relevant terraform blocks when using the following tooling: - artifact-registry, cloud-build, cloud-functions, cloud scheduler, cloud-source-repositories, and a vpc connector - 2. Checks for relevant terraform blocks when using the following tooling: - artifact-registry, cloud-build, cloud-run, cloud scheduler, cloud-source-repositories, and a vpc connector - 3. Checks that the artifact-registry terraform block is not included when not using artifact-registry. - 4. Checks that the vpc-connector element is not included when not using a vpc connector. - 5. Checks that the cloud scheduler terraform block is not included when not using a cloud schedule. - 6. Checks that the cloud build trigger terraform block is not included when not using cloud-build. - 7. Checks that the cloud source repositories and cloud build trigger terraform blocks are not included when not using cloud-source-repositories. - 8. Checks for that CI/CD infra terraform blocks are not included when use_ci=False. - - Args: - artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) - deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) - use_ci: Flag that determines whether to use Cloud CI/CD. - vpc_connector: The name of the vpc connector to use. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - main_tf_str = create_environment_main_tf_jinja( - artifact_repo_type=artifact_repo_type, - deployment_framework=deployment_framework, - naming_prefix=naming_prefix, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - schedule_pattern=schedule_pattern, - source_repo_type=source_repo_type, - use_ci=use_ci, - vpc_connector=vpc_connector) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in main_tf_str - elif not is_included: - assert snippet not in main_tf_str - - -@pytest.mark.parametrize( - '''artifact_repo_type, deployment_framework,''' - '''pipeline_job_submission_service_type, schedule_pattern,''' - '''source_repo_type, use_ci, is_included,''' - '''expected_output_snippets''', - [ - ( - 'artifact-registry', 'cloud-build', - 'cloud-functions', '0 12 * * *', - 'cloud-source-repositories', True, True, - ['output "enabled_apis"', - 'output "create_pipeline_job_runner_service_account_email"', - 'output "create_artifact_registry"', - 'output "create_storage_bucket"', - 'output "create_storage_bucket_names"', - 'output "create_cloud_source_repository"', - 'output "create_pubsub_topic"', - 'output "create_cloud_function"', - 'output "create_cloud_build_trigger"', - 'output "create_cloud_scheduler_name"', - 'output "create_cloud_scheduler_job"'] - ), - ( - 'artifact-registry', 'cloud-build', - 'cloud-run', '0 12 * * *', - 'cloud-source-repositories', True, True, - ['output "enabled_apis"', - 'output "create_pipeline_job_runner_service_account_email"', - 'output "create_artifact_registry"', - 'output "create_storage_bucket"', - 'output "create_storage_bucket_names"', - 'output "create_cloud_source_repository"', - 'output "cloud_run_id"', - 'output "create_pubsub_subscription"', - 'output "create_cloud_build_trigger"', - 'output "create_cloud_scheduler_name"', - 'output "create_cloud_scheduler_job"'] - ), - ( - 'some-other-repo', 'cloud-build', - 'cloud-functions', '0 12 * * *', - 'cloud-source-repositories', True, False, - ['output "create_artifact_registry"'] - ), - ( - 'artifact-registry', 'cloud-build', - 'cloud-run', '0 12 * * *', - 'cloud-source-repositories', True, False, - ['output "create_cloud_function"'] - ), - ( - 'artifact-registry', 'cloud-build', - 'cloud-functions', 'No Schedule Specified', - 'cloud-source-repositories', True, False, - ['output "create_cloud_scheduler_name"', - 'output "create_cloud_scheduler_job"'] - ), - ( - 'artifact-registry', 'some-deployment-framework', - 'cloud-functions', 'No Schedule Specified', - 'cloud-source-repositories', True, False, - ['output "create_cloud_build_trigger"'] - ), - ( - 'artifact-registry', 'cloud-build', - 'cloud-functions', 'No Schedule Specified', - 'some-other-code-repo', True, False, - ['output "create_cloud_source_repository"', - 'output "create_cloud_build_trigger"'] - ), - ( - 'artifact-registry', 'cloud-build', - 'cloud-functions', 'No Schedule Specified', - 'some-other-code-repo', False, False, - ['resource "null_resource" "build_and_push_submission_service"', - 'output "cloud_run_id"' - 'output "create_pubsub_subscription"', - 'output "create_pubsub_topic"', - 'output "create_cloud_function"', - 'output "create_cloud_build_trigger"', - 'output "create_cloud_scheduler_name"', - 'output "create_cloud_scheduler_job" '] - ), - ] -) -def test_create_environment_outputs_tf_jinja( - artifact_repo_type: str, - deployment_framework: str, - pipeline_job_submission_service_type: str, - schedule_pattern: str, - source_repo_type: str, - use_ci: bool, - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_environment_outputs_tf_jinja, which gnerates code for environment/outputs.tf, the terraform hcl - script that contains outputs from project's environment. - There are eight test cases for this function: - 1. Checks for relevant terraform output blocks when using the following tooling: - artifact-registry, cloud-build, cloud-functions, cloud scheduler, and cloud-source-repositories - 2. Checks for relevant terraform output blocks when using the following tooling: - artifact-registry, cloud-build, cloud-run, cloud scheduler, and cloud-source-repositories - 3. Checks that the artifact-registry terraform output block is not included when not using artifact-registry. - 4. Checks that the cloud functions terraform output block is not included when using cloud-run. - 5. Checks that the cloud scheduler terraform output blocks are not included when not using a cloud schedule. - 6. Checks that the cloud build trigger terraform output block is not included when not using cloud-build. - 7. Checks that the cloud source repositories and cloud build trigger output blocks are not included when not using cloud-source-repositories. - 8. Checks for that CI/CD infra terraform output blocks are not included when use_ci=False. - - Args: - artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) - deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - schedule_pattern: Cron formatted value used to create a Scheduled retrain job. - source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) - use_ci: Flag that determines whether to use Cloud CI/CD. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - main_tf_str = create_environment_outputs_tf_jinja( - artifact_repo_type=artifact_repo_type, - deployment_framework=deployment_framework, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - schedule_pattern=schedule_pattern, - source_repo_type=source_repo_type, - use_ci=use_ci) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in main_tf_str - elif not is_included: - assert snippet not in main_tf_str - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE])] -) -def test_create_environment_provider_tf_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_environment_provider_tf_jinja, which generates code for environment/provider.tf, the terraform hcl - script that contains teraform providers used to deploy project's environment. - There is one test case for this function: - 1. Checks for the apache license. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - provider_tf_str = create_environment_provider_tf_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in provider_tf_str - elif not is_included: - assert snippet not in provider_tf_str - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE])] -) -def test_create_environment_variables_tf_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_environment_variables_tf_jinja, which generates code for environment/variables.tf, - the terraform hcl script that contains variables used to deploy project's environment. - There is one test case for this function: - 1. Checks for the apache license. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - variables_tf_str = create_environment_variables_tf_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in variables_tf_str - elif not is_included: - assert snippet not in variables_tf_str - - -@pytest.mark.parametrize( - 'storage_bucket_name, is_included, expected_output_snippets', - [('my-storage-bucket', True, [GENERATED_LICENSE, 'bucket = "my-storage-bucket-tfstate"'])] -) -def test_create_environment_versions_tf_jinja( - storage_bucket_name: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_environment_versions_tf_jinja, which generates code for environment/versions.tf, - the terraform hcl script that contains teraform version information. - There is one test case for this function: - 1. Checks for the apache license and state file storage_bucket backend. - - Args: - storage_bucket_name: GS bucket name where pipeline run metadata is stored. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - versions_tf_str = create_environment_versions_tf_jinja(storage_bucket_name=storage_bucket_name) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in versions_tf_str - elif not is_included: - assert snippet not in versions_tf_str - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, '#!/bin/bash'])] -) -def test_create_provision_resources_script_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_provision_resources_script_jinja, which generates code for provision_resources.sh - which sets up the project's environment using terraform. - There is one test case for this function: - 1. Checks for the apache license and the Bash shebang. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - provision_resources_script = create_provision_resources_script_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in provision_resources_script - elif not is_included: - assert snippet not in provision_resources_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE])] -) -def test_create_state_bucket_variables_tf_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_state_bucket_variables_tf_jinja, which generates code for state_bucket/variables.tf, - the terraform hcl script that contains variables used for the state_bucket. - There is one test case for this function: - 1. Checks for the apache license. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - variables_tf_str = create_state_bucket_variables_tf_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in variables_tf_str - elif not is_included: - assert snippet not in variables_tf_str - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE])] -) -def test_create_state_bucket_main_tf_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_main_state_bucket_tf_jinja, which generates code for state_bucket/main.tf, the terraform hcl - script that contains terraform resources configs to create the state_bucket. - There are eight test cases for this function: - 1. Checks for the apache license. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - main_tf_str = create_state_bucket_main_tf_jinja() - - for snippet in expected_output_snippets: - if is_included: - assert snippet in main_tf_str - elif not is_included: - assert snippet not in main_tf_str diff --git a/tests/unit/utils/utils_test.py b/tests/unit/utils/utils_test.py index 3365d97..1fddba4 100644 --- a/tests/unit/utils/utils_test.py +++ b/tests/unit/utils/utils_test.py @@ -19,6 +19,7 @@ from contextlib import nullcontext as does_not_raise import os +import tempfile from typing import Callable, List import pandas as pd @@ -26,6 +27,12 @@ import pytest_mock import yaml +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files + import google_cloud_automlops.utils.utils from google_cloud_automlops.utils.utils import ( delete_file, @@ -36,6 +43,7 @@ make_dirs, read_file, read_yaml_file, + render_jinja, stringify_job_spec_list, update_params, validate_schedule, @@ -520,3 +528,26 @@ def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: Lis formatted_spec = stringify_job_spec_list(job_spec_list=job_spec_list) assert formatted_spec == expected_output + +@pytest.mark.parametrize( + "template_string, template_vars, expected_output", + [ + ('Hello {{ name1 }} my name is {{ name2 }}', {'name1': 'Alice', 'name2': 'John'}, 'Hello Alice my name is John'), + ('The answer is: {{ result }}', {'result': 42}, 'The answer is: 42'), + ] +) +def test_render_jinja(template_string, template_vars, expected_output): + """Tests the render_jinja function using temporary files.""" + + with tempfile.TemporaryDirectory() as tmpdirname: # Creates temp directory + template_path = os.path.join(tmpdirname, "template.txt.j2") + + # Write the template to the temporary file + with open(template_path, 'w') as f: + f.write(template_string) + + # Call the render_jinja function + result = render_jinja(template_path, **template_vars) + + # Assertion + assert result == expected_output \ No newline at end of file From 39c427ba39e274547b7c79dab0ff2525bc4255e5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 9 Feb 2024 11:47:44 -0500 Subject: [PATCH 5/9] Pylint fix --- tests/unit/utils/utils_test.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/tests/unit/utils/utils_test.py b/tests/unit/utils/utils_test.py index 1fddba4..4a0f6a0 100644 --- a/tests/unit/utils/utils_test.py +++ b/tests/unit/utils/utils_test.py @@ -27,12 +27,6 @@ import pytest_mock import yaml -try: - from importlib.resources import files as import_files -except ImportError: - # Try backported to PY<37 `importlib_resources` - from importlib_resources import files as import_files - import google_cloud_automlops.utils.utils from google_cloud_automlops.utils.utils import ( delete_file, @@ -530,24 +524,24 @@ def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: Lis assert formatted_spec == expected_output @pytest.mark.parametrize( - "template_string, template_vars, expected_output", + 'template_string, template_vars, expected_output', [ ('Hello {{ name1 }} my name is {{ name2 }}', {'name1': 'Alice', 'name2': 'John'}, 'Hello Alice my name is John'), - ('The answer is: {{ result }}', {'result': 42}, 'The answer is: 42'), + ('The answer is: {{ result }}', {'result': 42}, 'The answer is: 42'), ] ) def test_render_jinja(template_string, template_vars, expected_output): """Tests the render_jinja function using temporary files.""" with tempfile.TemporaryDirectory() as tmpdirname: # Creates temp directory - template_path = os.path.join(tmpdirname, "template.txt.j2") + template_path = os.path.join(tmpdirname, 'template.txt.j2') # Write the template to the temporary file - with open(template_path, 'w') as f: + with open(template_path, 'w', encoding='utf-8') as f: f.write(template_string) - # Call the render_jinja function + # Call the render_jinja function result = render_jinja(template_path, **template_vars) - # Assertion - assert result == expected_output \ No newline at end of file + # Assertion + assert result == expected_output \ No newline at end of file From 557baeaf67d950fc9d6cb33ddbbe66bc1b26bd21 Mon Sep 17 00:00:00 2001 From: Allegra Noto Date: Fri, 9 Feb 2024 11:49:43 -0500 Subject: [PATCH 6/9] Pylint --- tests/unit/utils/utils_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils/utils_test.py b/tests/unit/utils/utils_test.py index 4a0f6a0..668279a 100644 --- a/tests/unit/utils/utils_test.py +++ b/tests/unit/utils/utils_test.py @@ -544,4 +544,4 @@ def test_render_jinja(template_string, template_vars, expected_output): result = render_jinja(template_path, **template_vars) # Assertion - assert result == expected_output \ No newline at end of file + assert result == expected_output From cb2125af8d2d6125b55bf60035c7f4bc6175a9a8 Mon Sep 17 00:00:00 2001 From: Allegra Noto Date: Fri, 23 Feb 2024 12:20:46 -0500 Subject: [PATCH 7/9] Restored testing files to be adjusted as render jinjas --- google_cloud_automlops/tests/__init__.py | 13 + google_cloud_automlops/tests/unit/__init__.py | 13 + .../tests/unit/deployments/.gitkeep | 0 .../tests/unit/deployments/__init__.py | 13 + .../unit/deployments/cloudbuild/__init__.py | 13 + .../deployments/cloudbuild/builder_test.py | 117 +++ .../deployments/github_actions/__init__.py | 13 + .../github_actions/builder_test.py | 133 +++ .../tests/unit/deployments/gitlab_ci/.gitkeep | 0 .../tests/unit/deployments/jenkins/.gitkeep | 0 .../tests/unit/orchestration/__init__.py | 13 + .../tests/unit/orchestration/airflow/.gitkeep | 0 .../tests/unit/orchestration/argo/.gitkeep | 0 .../tests/unit/orchestration/kfp/__init__.py | 13 + .../unit/orchestration/kfp/builder_test.py | 993 ++++++++++++++++++ .../unit/orchestration/kfp/scaffold_test.py | 318 ++++++ .../tests/unit/provisioning/__init__.py | 13 + .../unit/provisioning/gcloud/__init__.py | 13 + .../unit/provisioning/gcloud/builder_test.py | 220 ++++ .../tests/unit/provisioning/pulumi/.gitkeep | 0 .../unit/provisioning/terraform/.gitkeep | 0 .../unit/provisioning/terraform/__init__.py | 13 + .../provisioning/terraform/builder_test.py | 522 +++++++++ .../tests/unit/utils/__init__.py | 13 + .../tests/unit/utils/utils_test.py | 531 ++++++++++ 25 files changed, 2977 insertions(+) create mode 100644 google_cloud_automlops/tests/__init__.py create mode 100644 google_cloud_automlops/tests/unit/__init__.py create mode 100644 google_cloud_automlops/tests/unit/deployments/.gitkeep create mode 100644 google_cloud_automlops/tests/unit/deployments/__init__.py create mode 100644 google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py create mode 100644 google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py create mode 100644 google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py create mode 100644 google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py create mode 100644 google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep create mode 100644 google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep create mode 100644 google_cloud_automlops/tests/unit/orchestration/__init__.py create mode 100644 google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep create mode 100644 google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep create mode 100644 google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py create mode 100644 google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py create mode 100644 google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py create mode 100644 google_cloud_automlops/tests/unit/provisioning/__init__.py create mode 100644 google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py create mode 100644 google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py create mode 100644 google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep create mode 100644 google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep create mode 100644 google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py create mode 100644 google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py create mode 100644 google_cloud_automlops/tests/unit/utils/__init__.py create mode 100644 google_cloud_automlops/tests/unit/utils/utils_test.py diff --git a/google_cloud_automlops/tests/__init__.py b/google_cloud_automlops/tests/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/__init__.py b/google_cloud_automlops/tests/unit/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/.gitkeep b/google_cloud_automlops/tests/unit/deployments/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/deployments/__init__.py b/google_cloud_automlops/tests/unit/deployments/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py b/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py b/google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py new file mode 100644 index 0000000..27331f5 --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py @@ -0,0 +1,117 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-module-docstring + +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files + +from typing import List + +import pytest + +from google_cloud_automlops.utils.constants import ( + BASE_DIR, + CLOUDBUILD_TEMPLATES_PATH, + COMPONENT_BASE_RELATIVE_PATH, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH +) +from google_cloud_automlops.utils.utils import render_jinja + +@pytest.mark.parametrize( + '''artifact_repo_location, artifact_repo_name, naming_prefix,''' + '''project_id, pubsub_topic_name, use_ci, is_included,''' + '''expected_output_snippets''', + [ + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-topic', True, True, + ['id: "build_component_base"', + 'id: "push_component_base"', + 'id: "install_pipelines_deps"', + 'id: "build_pipeline_spec"', + 'id: "publish_to_topic"', + '"build", "-t", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', + '"push", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', + 'gcloud pubsub topics publish my-topic'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-topic', False, True, + ['id: "build_component_base"', + '"build", "-t", "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-topic', False, False, + ['id: "push_component_base"', + 'id: "install_pipelines_deps"', + 'id: "build_pipeline_spec"', + 'id: "publish_to_topic"', + '"push" "us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest"', + 'gcloud pubsub topics publish my-topic'] + ), + ] +) +def test_create_cloudbuild_jinja( + artifact_repo_location: str, + artifact_repo_name: str, + naming_prefix: str, + project_id: str, + pubsub_topic_name: str, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_cloudbuild_jinja, which generates content for the cloudbuild.yaml. + There are three test cases for this function: + 1. Checks that expected strings are included when use_ci=True. + 2. Checks that expected strings are included when use_ci=False. + 3. Checks that certain strings are not included when use_ci=False. + + Args: + artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). + artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + project_id: The project ID. + pubsub_topic_name: The name of the pubsub topic to publish to. + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + component_base_relative_path = COMPONENT_BASE_RELATIVE_PATH if use_ci else f'{BASE_DIR}{COMPONENT_BASE_RELATIVE_PATH}' + template_file = import_files(CLOUDBUILD_TEMPLATES_PATH) / 'cloudbuild.yaml.j2' + + cloudbuild_config = render_jinja( + template_path=template_file, + artifact_repo_location=artifact_repo_location, + artifact_repo_name=artifact_repo_name, + component_base_relative_path=component_base_relative_path, + generated_licensed=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + naming_prefix=naming_prefix, + project_id=project_id, + pubsub_topic_name=pubsub_topic_name, + use_ci=use_ci + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in cloudbuild_config + elif not is_included: + assert snippet not in cloudbuild_config diff --git a/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py b/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py b/google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py new file mode 100644 index 0000000..a7ed5f6 --- /dev/null +++ b/google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py @@ -0,0 +1,133 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files + +from typing import List + +import pytest + +from google_cloud_automlops.utils.constants import ( + COMPONENT_BASE_RELATIVE_PATH, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH, + GITHUB_ACTIONS_TEMPLATES_PATH +) +from google_cloud_automlops.utils.utils import render_jinja + + +@pytest.mark.parametrize( + '''artifact_repo_location, artifact_repo_name, naming_prefix,''' + '''project_id, project_number, pubsub_topic_name, use_ci, source_repo_branch,''' + '''workload_identity_provider, workload_identity_pool, workload_identity_service_account, is_included,''' + '''expected_output_snippets''', + [ + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-project-number', 'my-topic', True, 'automlops', + 'my-provider', 'my-pool', 'my-sa', True, + ['id: auth', + 'id: build-push-component-base', + 'id: install-pipeline-deps', + 'id: build-pipeline-spec', + 'id: publish-to-topic', + 'us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest', + 'gcloud pubsub topics publish my-topic --message'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-project-number', 'my-topic', False, 'automlops', + 'my-provider', 'my-pool', 'my-sa', True, + ['id: build-push-component-base', + 'us-central1-docker.pkg.dev/my-project/my-artifact-repo/my-prefix/components/component_base:latest'] + ), + ( + 'us-central1', 'my-artifact-repo', 'my-prefix', + 'my-project', 'my-project-number', 'my-topic', False, 'automlops', + 'my-provider', 'my-pool', 'my-sa', False, + ['id: install-pipeline-deps', + 'id: build-pipeline-spec', + 'id: publish-to-topic', + 'gcloud pubsub topics publish my-topic --message'] + ), + ] +) +def test_create_github_actions_jinja( + artifact_repo_location: str, + artifact_repo_name: str, + naming_prefix: str, + project_id: str, + project_number: str, + pubsub_topic_name: str, + use_ci: bool, + source_repo_branch: str, + workload_identity_pool: str, + workload_identity_provider: str, + workload_identity_service_account: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_github_actions_jinja, which generates content for the github actions file. + There are three test cases for this function: + 1. Checks that expected strings are included when use_ci=True. + 2. Checks that expected strings are included when use_ci=False. + 3. Checks that certain strings are not included when use_ci=False. + + Args: + artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). + artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + project_id: The project ID. + project_number: The project number. + pubsub_topic_name: The name of the pubsub topic to publish to. + source_repo_branch: The branch to use in the source repository. + use_ci: Flag that determines whether to use Cloud CI/CD. + workload_identity_pool: Pool for workload identity federation. + workload_identity_provider: Provider for workload identity federation. + workload_identity_service_account: Service account for workload identity federation. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + + template_file = import_files(GITHUB_ACTIONS_TEMPLATES_PATH) / 'github_actions.yaml.j2' + github_actions_config = render_jinja( + template_path=template_file, + artifact_repo_location=artifact_repo_location, + artifact_repo_name=artifact_repo_name, + component_base_relative_path=COMPONENT_BASE_RELATIVE_PATH, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + naming_prefix=naming_prefix, + project_id=project_id, + project_number=project_number, + pubsub_topic_name=pubsub_topic_name, + source_repo_branch=source_repo_branch, + use_ci=use_ci, + workload_identity_pool=workload_identity_pool, + workload_identity_provider=workload_identity_provider, + workload_identity_service_account=workload_identity_service_account + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in github_actions_config + elif not is_included: + assert snippet not in github_actions_config diff --git a/google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep b/google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep b/google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/orchestration/__init__.py b/google_cloud_automlops/tests/unit/orchestration/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep b/google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep b/google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py b/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py b/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py new file mode 100644 index 0000000..08d0c6e --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py @@ -0,0 +1,993 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +import json +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files +import os +from typing import List + +import pytest +import pytest_mock + +from google_cloud_automlops.utils.constants import ( + BASE_DIR, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH, + KFP_TEMPLATES_PATH, + PINNED_KFP_VERSION, +) +import google_cloud_automlops.orchestration.kfp.builder +from google_cloud_automlops.orchestration.kfp.builder import ( + build_component, + build_pipeline, + build_services, +) +import google_cloud_automlops.utils.utils +from google_cloud_automlops.utils.utils import ( + make_dirs, + read_yaml_file, + render_jinja, + write_yaml_file +) + + +DEFAULTS = { + 'gcp': { + 'artifact_repo_location': 'us-central1', + 'project_id': 'my_project', + 'artifact_repo_name': 'my_af_registry', + 'naming_prefix': 'my-prefix', + 'pipeline_job_runner_service_account': 'my-service-account@service.com', + 'pipeline_job_submission_service_type': 'cloud-functions', + 'setup_model_monitoring': True + }, + 'pipelines': { + 'gs_pipeline_job_spec_path': 'gs://my-bucket/pipeline_root/my-prefix/pipeline_job.json', + 'pipeline_storage_path': 'gs://my-bucket/pipeline_root/' + } +} + +TEMP_YAML = { + 'name': 'create_dataset', + 'description': 'Custom component that takes in a BQ table and writes it to GCS.', + 'inputs': [ + { + 'name': 'bq_table', + 'description': 'The source biquery table.', + 'type': 'String', + }, + { + 'name': 'data_path', + 'description': 'The gcs location to write the csv.', + 'type': 'String', + }, + { + 'name': 'project_id', + 'description': 'The project ID.', + 'type': 'String'}, + ], + 'implementation': { + 'container': { + 'image': 'TBD', + 'command': [ + 'sh', + '-c', + 'if ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \\\n --no-warn-script-location && "$0" "$@"\n\n', + 'def create_dataset(\n bq_table: str,\n data_path: str,\n project_id: str\n):\n """Custom component that takes in a BQ table and writes it to GCS.\n\n Args:\n bq_table: The source biquery table.\n data_path: The gcs location to write the csv.\n project_id: The project ID.\n """\n from google.cloud import bigquery\n import pandas as pd\n from sklearn import preprocessing\n\n bq_client = bigquery.Client(project=project_id)\n\n def get_query(bq_input_table: str) -> str:\n """Generates BQ Query to read data.\n\n Args:\n bq_input_table: The full name of the bq input table to be read into\n the dataframe (e.g. ..)\n Returns: A BQ query string.\n """\n return f\'\'\'\n SELECT *\n FROM `{bq_input_table}`\n \'\'\'\n\n def load_bq_data(query: str, client: bigquery.Client) -> pd.DataFrame:\n """Loads data from bq into a Pandas Dataframe for EDA.\n Args:\n query: BQ Query to generate data.\n client: BQ Client used to execute query.\n Returns:\n pd.DataFrame: A dataframe with the requested data.\n """\n df = client.query(query).to_dataframe()\n return df\n\n dataframe = load_bq_data(get_query(bq_table), bq_client)\n le = preprocessing.LabelEncoder()\n dataframe[\'Class\'] = le.fit_transform(dataframe[\'Class\'])\n dataframe.to_csv(data_path, index=False)\n', + ], + 'args': [ + '--executor_input', + {'executorInput': None}, + '--function_to_execute', + 'create_dataset', + ], + } + }, +} + + +@pytest.fixture(name='temp_yaml_dict', params=[TEMP_YAML]) +def fixture_temp_yaml_dict(request: pytest.FixtureRequest, tmpdir: pytest.FixtureRequest): + """Writes temporary yaml file fixture using defaults parameterized + dictionaries during pytest session scope. + + Args: + request: Pytest fixture special object that provides information + about the fixture. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + + Returns: + dict: Path of yaml file and dictionary it contains. + """ + yaml_path = tmpdir.join('test.yaml') + write_yaml_file(yaml_path, request.param, 'w') + return {'path': yaml_path, 'vals': request.param} + + +@pytest.fixture(name='defaults_dict', params=[DEFAULTS]) +def fixture_defaults_dict(request: pytest.FixtureRequest, tmpdir: pytest.FixtureRequest): + """Writes temporary yaml file fixture using defaults parameterized + dictionaries during pytest session scope. + + Args: + request: Pytest fixture special object that provides information + about the fixture. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + + Returns: + dict: Path of yaml file and dictionary it contains. + """ + yaml_path = tmpdir.join('defaults.yaml') + write_yaml_file(yaml_path, request.param, 'w') + return {'path': yaml_path, 'vals': request.param} + + +@pytest.fixture(name='expected_component_dict') +def fixture_expected_component_dict(): + """Creates the expected component dictionary, which is the temporary yaml + file with a change to the implementation key. + + Returns: + dict: Expected component dictionary generated from the component + builder. + """ + expected = TEMP_YAML + expected['implementation'] = { + 'container': { + 'image': 'us-central1-docker.pkg.dev/my_project/my_af_registry/my-prefix/components/component_base:latest', + 'command': ['python3', '/pipelines/component/src/create_dataset.py'], + 'args': [ + '--executor_input', + {'executorInput': None}, + '--function_to_execute', + 'create_dataset', + ], + } + } + return expected + + +def test_build_component(mocker: pytest_mock.MockerFixture, + tmpdir: pytest.FixtureRequest, + temp_yaml_dict: pytest.FixtureRequest, + defaults_dict: pytest.FixtureRequest, + expected_component_dict: pytest.FixtureRequest): + """Tests build_component, which Constructs and writes component.yaml and + {component_name}.py files. + + Args: + mocker: Mocker to patch directories. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + temp_yaml_dict: Locally defined temp_yaml_file Pytest fixture. + defaults_dict: Locally defined defaults_dict Pytest fixture. + expected_component_dict: Locally defined expected_component_dict + Pytest fixture. + """ + # Patch filepath constants to point to test path. + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'BASE_DIR', + f'{tmpdir}/') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_DEFAULTS_FILE', + defaults_dict['path']) + + # Extract component name, create required directories, run build_component + component_name = TEMP_YAML['name'] + make_dirs([f'{tmpdir}/components/component_base/src']) + build_component(temp_yaml_dict['path']) + + # Ensure correct files are created with build_component call + assert os.path.exists(f'{tmpdir}/components/{component_name}/component.yaml') + assert os.path.exists(f'{tmpdir}/components/component_base/src/{component_name}.py') + + # Load component.yaml file and compare to the expected output in test_data + created_component_dict = read_yaml_file(f'{tmpdir}/components/{component_name}/component.yaml') + assert created_component_dict == expected_component_dict + + +@pytest.mark.parametrize( + 'custom_training_job_specs, pipeline_parameter_values', + [ + ( + [{'component_spec': 'mycomp1', 'other': 'myother'}], + { + 'bq_table': 'automlops-sandbox.test_dataset.dry-beans', + 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 13:00:41.379753', + 'data_path': 'gs://automlops-sandbox-bucket/data.csv', + 'project_id': 'automlops-sandbox', + 'region': 'us-central1' + }, + ), + ( + [ + { + 'component_spec': 'train_model', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': '1', + } + ], + { + 'bq_table': 'automlops-sandbox.test_dataset.dry-beans', + 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 13:00:41.379753', + 'data_path': 'gs://automlops-sandbox-bucket/data.csv', + 'project_id': 'automlops-sandbox', + 'region': 'us-central1' + }, + ), + ( + [ + { + 'component_spec': 'test_model', + 'display_name': 'test-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': '1', + } + ], + { + 'bq_table': 'automlops-sandbox.test_dataset.dry-beans2', + 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 14:00:41.379753', + 'data_path': 'gs://automlops-sandbox-bucket/data2.csv', + 'project_id': 'automlops-sandbox', + 'region': 'us-central1' + }, + ) + ] +) +def test_build_pipeline(mocker: pytest_mock.MockerFixture, + tmpdir: pytest.FixtureRequest, + defaults_dict: pytest.FixtureRequest, + custom_training_job_specs: List[dict], + pipeline_parameter_values: dict): + """Tests build_pipeline, which constructs and writes pipeline.py, + pipeline_runner.py, and pipeline_parameter_values.json files. + + Args: + mocker: Mocker to patch directories. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + defaults_dict: Locally defined defaults_dict Pytest fixture. + custom_training_job_specs (List[dict]): Specifies the specs to run the training job with. + pipeline_parameter_values (dict): Dictionary of runtime parameters for the PipelineJob. + """ + # Patch constants and other functions + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'BASE_DIR', + f'{tmpdir}/') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_DEFAULTS_FILE', + defaults_dict['path']) + mocker.patch.object(google_cloud_automlops.utils.utils, + 'CACHE_DIR', + f'{tmpdir}/.AutoMLOps-cache') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'PIPELINE_CACHE_FILE', + f'{tmpdir}/.AutoMLOps-cache/pipeline_scaffold.py') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_PIPELINE_FILE', + f'{tmpdir}/pipelines/pipeline.py') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_PIPELINE_RUNNER_FILE', + f'{tmpdir}/pipelines/pipeline_runner.py') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_PIPELINE_REQUIREMENTS_FILE', + f'{tmpdir}/pipelines/requirements.txt') + + # Create required directory and file for build_pipeline + make_dirs([f'{tmpdir}/pipelines/runtime_parameters', f'{tmpdir}/.AutoMLOps-cache']) + os.system(f'touch {tmpdir}/.AutoMLOps-cache/pipeline_scaffold.py') + build_pipeline(custom_training_job_specs, pipeline_parameter_values) + + # Ensure correct files were created + assert os.path.exists(f'{tmpdir}/pipelines/pipeline.py') + assert os.path.exists(f'{tmpdir}/pipelines/pipeline_runner.py') + assert os.path.exists(f'{tmpdir}/pipelines/requirements.txt') + assert os.path.exists(f'{tmpdir}/pipelines/runtime_parameters/pipeline_parameter_values.json') + + # Ensure pipeline_parameter_values.json was created as expected + with open(f'{tmpdir}/pipelines/runtime_parameters/pipeline_parameter_values.json', mode='r', encoding='utf-8') as f: + pipeline_params_dict = json.load(f) + assert pipeline_params_dict == pipeline_parameter_values + + +def test_build_services(mocker: pytest_mock.MockerFixture, + tmpdir: pytest.FixtureRequest, + defaults_dict: pytest.FixtureRequest): + """Tests build_services, which Constructs and writes a Dockerfile, requirements.txt, and + main.py to the services/submission_service directory. + + Args: + mocker: Mocker to patch directories. + tmpdir: Pytest fixture that provides a temporary directory unique + to the test invocation. + defaults_dict: Locally defined defaults_dict Pytest fixture. + """ + # Patch filepath constants to point to test path. + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'BASE_DIR', + f'{tmpdir}/') + mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, + 'GENERATED_DEFAULTS_FILE', + defaults_dict['path']) + + # create required directories, run build_services + make_dirs([f'{tmpdir}/services/submission_service']) + build_services() + + # Ensure correct files are created with build_services call + assert os.path.exists(f'{tmpdir}/services/submission_service/Dockerfile') + assert os.path.exists(f'{tmpdir}/services/submission_service/requirements.txt') + assert os.path.exists(f'{tmpdir}/services/submission_service/main.py') + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline --config $CONFIG_FILE'])] +) +def test_build_pipeline_spec_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests build_pipeline_spec_jinja, which generates code for build_pipeline_spec.sh + which builds the pipeline specs. There is one test case for this function: + 1. Checks for the apache license and the pipeline compile command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_pipeline_spec.sh.j2' + build_pipeline_spec_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in build_pipeline_spec_script + elif not is_included: + assert snippet not in build_pipeline_spec_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600'])] +) +def test_build_components_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests build_components_jinja, which generates code for build_components.sh + which builds the components. There is one test case for this function: + 1. Checks for the apache license and the builds submit command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_components.sh.j2' + build_components_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in build_components_script + elif not is_included: + assert snippet not in build_components_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline_runner --config $CONFIG_FILE'])] +) +def test_run_pipeline_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests run_pipeline_jinja, which generates code for run_pipeline.sh + which runs the pipeline locally. There is one test case for this function: + 1. Checks for the apache license and the pipeline runner command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_pipeline.sh.j2' + run_pipeline_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in run_pipeline_script + elif not is_included: + assert snippet not in run_pipeline_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600', + './scripts/build_pipeline_spec.sh', './scripts/run_pipeline.sh'])] +) +def test_run_all_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests run_all_jinja, which generates code for run_all.sh + which builds runs all other shell scripts. There is one test case for this function: + 1. Checks for the apache license and the builds submit, the pipeline compile, and the pipeline runner commands. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_all.sh.j2' + run_all_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in run_all_script + elif not is_included: + assert snippet not in run_all_script + + +@pytest.mark.parametrize( + 'pubsub_topic_name, is_included, expected_output_snippets', + [('my-topic', True, [GENERATED_LICENSE, 'gcloud pubsub topics publish my-topic'])] +) +def test_publish_to_topic_jinja( + pubsub_topic_name: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests publish_to_topic_jinja, which generates code for publish_to_topic.sh + which submits a message to the pipeline job submission service. + There is one test case for this function: + 1. Checks for the apache license and the pubsub publish command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'publish_to_topic.sh.j2' + publish_to_topic_script = render_jinja( + template_path=template_file, + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + pubsub_topic_name=pubsub_topic_name + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in publish_to_topic_script + elif not is_included: + assert snippet not in publish_to_topic_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python3 -m model_monitoring.monitor --config $CONFIG_FILE'])] +) +def test_create_model_monitoring_job_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_model_monitoring_job_jinja, which generates code for create_model_monitoring_job.sh + which creates a Model Monitoring Job in Vertex AI for a deployed model endpoint. + There is one test case for this function: + 1. Checks for the apache license and the monitor command. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'create_model_monitoring_job.sh.j2' + create_model_monitoring_job_script = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in create_model_monitoring_job_script + elif not is_included: + assert snippet not in create_model_monitoring_job_script + + +@pytest.mark.parametrize( + 'setup_model_monitoring, use_ci, is_included, expected_output_snippets', + [ + ( + False, True, True, + ['AutoMLOps - Generated Code Directory', + '├── components', + '├── configs', + '├── images', + '├── provision', + '├── scripts', + '├── services', + '├── README.md', + '└── cloudbuild.yaml'] + ), + ( + True, True, True, + ['AutoMLOps - Generated Code Directory', + '├── components', + '├── configs', + '├── images', + '├── model_monitoring', + '├── provision', + '├── scripts', + '├── services', + '├── README.md', + '└── cloudbuild.yaml'] + ), + ( + False, False, False, + ['├── publish_to_topic.sh' + '├── services', + '├── create_model_monitoring_job.sh', + '├── model_monitoring'] + ), + ] +) +def test_readme_jinja( + setup_model_monitoring: bool, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests readme_jinja, which generates code for readme.md which + is a readme markdown file to describe the contents of the + generated AutoMLOps code repo. There are three test cases for this function: + 1. Checks that certain directories and files exist when use_ci=True and setup_model_monitoring=False. + 2. Checks that certain directories and files exist when use_ci=True and setup_model_monitoring=True. + 3. Checks that certain directories and files do not exist when use_ci=False. + + Args: + setup_model_monitoring: Boolean parameter which specifies whether to set up a Vertex AI Model Monitoring Job. + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH) / 'README.md.j2' + readme_str = render_jinja( + template_path=template_file, + setup_model_monitoring=setup_model_monitoring, + use_ci=use_ci + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in readme_str + elif not is_included: + assert snippet not in readme_str + + +@pytest.mark.parametrize( + 'base_image, is_included, expected_output_snippets', + [('my-base-image', True, [GENERATED_LICENSE, 'FROM my-base-image'])] +) +def test_component_base_dockerfile_jinja( + base_image: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests readme_jinja, which generates code for a Dockerfile + to be written to the component_base directory. There is one + test case for this function: + 1. Checks for the apache license and the FROM image line. + + Args: + base_image: The image to use in the component base dockerfile. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2' + component_base_dockerfile = render_jinja( + template_path=template_file, + base_image=base_image, + generated_license=GENERATED_LICENSE + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in component_base_dockerfile + elif not is_included: + assert snippet not in component_base_dockerfile + + +@pytest.mark.parametrize( + 'custom_code_contents, kfp_spec_bool, is_included, expected_output_snippets', + [ + ( + 'this is some custom code', True, True, + [GENERATED_LICENSE, + 'this is some custom code', + 'def main():'] + ), + ( + 'this is some custom code', False, True, + [GENERATED_LICENSE, + 'this is some custom code', + 'def main():', + 'import kfp', + 'from kfp.v2.dsl import *'] + ), + ( + 'this is some custom code', True, False, + ['import kfp', + 'from kfp.v2.dsl import *'] + ) + ] +) +def test_component_base_task_file_jinja( + custom_code_contents: str, + kfp_spec_bool: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests component_base_task_file_jinja, which generates code + for the task.py file to be written to the component_base/src directory. + There are three test cases for this function: + 1. Checks for the apache license, the custom_code_contents, and a main function when using kfp spec (kfp spec comes with kfp imports by default). + 2. Checks for the apache license, the custom_code_contents, a main function, and kfp imports when not using kfp spec. + 3. Checks that the kfp imports are not included in the string when using kfp spec (kfp spec comes with kfp imports by default). + + Args: + custom_code_contents: Code inside of the component, specified by the user. + kfp_spec_bool: Boolean that specifies whether components are defined using kfp. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2' + component_base_task_file = render_jinja( + template_path=template_file, + custom_code_contents=custom_code_contents, + generated_license=GENERATED_LICENSE, + kfp_spec_bool=kfp_spec_bool) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in component_base_task_file + elif not is_included: + assert snippet not in component_base_task_file + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_pipeline_runner_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_runner_jinja, which generates code for the pipeline_runner.py + file to be written to the pipelines directory. There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2' + pipeline_runner_py = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in pipeline_runner_py + elif not is_included: + assert snippet not in pipeline_runner_py + + +@pytest.mark.parametrize( + '''components_list, custom_training_job_specs, pipeline_scaffold_contents, project_id,''' + '''is_included, expected_output_snippets''', + [ + ( + ['componentA','componentB','componentC'], + [ + { + 'component_spec': 'componentB', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': '1', + } + ], + 'Pipeline definition goes here', 'my-project', True, + [GENERATED_LICENSE, + 'from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', + 'def upload_pipeline_spec', + 'componentA = load_custom_component', + 'componentB = load_custom_component', + 'componentC = load_custom_component', + 'componentB_custom_training_job_specs', + 'Pipeline definition goes here'] + ), + ( + ['componentA','componentB','componentC'], + None, 'Pipeline definition goes here', 'my-project', True, + [GENERATED_LICENSE, + 'def upload_pipeline_spec', + 'componentA = load_custom_component', + 'componentB = load_custom_component', + 'componentC = load_custom_component', + 'Pipeline definition goes here'] + ), + ( + ['componentA','componentB','componentC'], + None, 'Pipeline definition goes here', 'my-project', False, + ['from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', + 'componentB_custom_training_job_specs'] + ), + ] +) +def test_pipeline_jinja( + components_list: list, + custom_training_job_specs: list, + pipeline_scaffold_contents: str, + project_id: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_jinja, which generates code for the pipeline.py + file to be written to the pipelines directory. + There are three test cases for this function: + 1. Checks for the apache license and relevant code elements when custom_training_job_specs is not None. + 2. Checks for the apache license and relevant code elements when custom_training_job_specs is None. + 3. Checks that the output does not contain custom_training_job_specs code elements when custom_training_job_specs is None. + + Args: + components_list: Contains the names or paths of all component yamls in the dir. + custom_training_job_specs: Specifies the specs to run the training job with. + pipeline_scaffold_contents: The contents of the pipeline scaffold file, + which can be found at PIPELINE_CACHE_FILE. + project_id: The project ID. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2' + pipeline_py = render_jinja( + template_path=template_file, + components_list=components_list, + custom_training_job_specs=custom_training_job_specs, + generated_license=GENERATED_LICENSE, + pipeline_scaffold_contents=pipeline_scaffold_contents, + project_id=project_id) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in pipeline_py + elif not is_included: + assert snippet not in pipeline_py + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform'])] +) +def test_pipeline_requirements_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_requirements_jinja, which generates code for a requirements.txt + to be written to the pipelines directory. There is one test case for this function: + 1. Checks for the pinned kfp version, and the google-cloud-aiplatform dep. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2' + pipeline_requirements_py = render_jinja( + template_path=template_file, + pinned_kfp_version=PINNED_KFP_VERSION + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in pipeline_requirements_py + elif not is_included: + assert snippet not in pipeline_requirements_py + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, 'python:3.9-slim', + 'CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app'])] +) +def test_submission_service_dockerfile_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests pipeline_requirements_jinja, which generates code for a Dockerfile to be + written to the serivces/submission_service directory. There is one test case for this function: + 1. Checks for the apache license and relevant dockerfile elements. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2' + submission_service_dockerfile = render_jinja( + template_path=template_file, + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in submission_service_dockerfile + elif not is_included: + assert snippet not in submission_service_dockerfile + + +@pytest.mark.parametrize( + 'pipeline_job_submission_service_type, is_included, expected_output_snippets', + [('cloud-functions', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'google-cloud-storage', 'functions-framework==3.*']), + ('cloud-functions', False, ['gunicorn']), + ('cloud-run', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'google-cloud-storage', 'gunicorn']), + ('cloud-run', False, ['functions-framework==3.*']),] +) +def test_submission_service_requirements_jinja( + pipeline_job_submission_service_type: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests submission_service_requirements_jinja, which generates code + for a requirements.txt to be written to the serivces/submission_service directory. + There are four test cases for this function: + 1. Checks for the pinned kfp version, the google-cloud-aiplatform, google-cloud-storage and function-framework deps when set to cloud-functions. + 2. Checks that gunicorn dep is not included when set to cloud-functions. + 3. Checks for the pinned kfp version, the google-cloud-aiplatform, google-cloud-storage and gunicorn deps when set to cloud-run. + 4. Checks that functions-framework dep is not included when set to cloud-run. + + Args: + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2' + submission_service_requirements = render_jinja( + template_path=template_file, + pinned_kfp_version=PINNED_KFP_VERSION, + pipeline_job_submission_service_type=pipeline_job_submission_service_type + ) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in submission_service_requirements + elif not is_included: + assert snippet not in submission_service_requirements + + +@pytest.mark.parametrize( + '''naming_prefix, pipeline_root, pipeline_job_runner_service_account, pipeline_job_submission_service_type,''' + '''project_id, setup_model_monitoring, is_included, expected_output_snippets''', + [ + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', + 'my-project', False, True, + [GENERATED_LICENSE, + 'from google.cloud import aiplatform', + 'import functions_framework', + '@functions_framework.http', + 'def process_request(request: flask.Request)', + '''base64_message = request_json['data']['data']'''] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', + 'my-project', False, False, + ['app = flask.Flask', + '''@app.route('/', methods=['POST'])''', + 'request = flask.request', + '''base64_message = request_json['message']['data']''', + '''if __name__ == '__main__':''', + '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))''', + 'from google.cloud import storage', + 'NAMING_PREFIX', + 'def read_gs_auto_retraining_params_file()'] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', + 'my-project', False, True, + [GENERATED_LICENSE, + 'from google.cloud import aiplatform', + 'app = flask.Flask', + '''@app.route('/', methods=['POST'])''', + 'request = flask.request', + '''base64_message = request_json['message']['data']''', + '''if __name__ == '__main__':''', + '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))'''] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', + 'my-project', False, False, + ['import functions_framework', + '@functions_framework.http', + 'def process_request(request: flask.Request)', + '''base64_message = request_json['data']['data']''', + 'from google.cloud import storage', + 'NAMING_PREFIX', + 'def read_gs_auto_retraining_params_file()'] + ), + ( + 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', + 'my-project', True, True, + ['from google.cloud import storage', + 'NAMING_PREFIX', + 'def read_gs_auto_retraining_params_file()', + '''if data_payload['logName'] == f'projects/{PROJECT_ID}/logs/aiplatform.googleapis.com%2Fmodel_monitoring_anomaly':'''] + ), + ] +) +def test_submission_service_main_jinja( + naming_prefix: str, + pipeline_root: str, + pipeline_job_runner_service_account: str, + pipeline_job_submission_service_type: str, + project_id: str, + setup_model_monitoring: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests submission_service_main_jinja, which generates content + for main.py to be written to the serivces/submission_service directory. + There are five test cases for this function: + 1. Checks for functions_framework code elements when set to cloud-functions. + 2. Checks that Flask app code elements are not included when set to cloud-functions. + 3. Checks for Flask app code elements when set to cloud-run. + 4. Checks that functions_framework code elements are not included when set to cloud-run. + 5. Checks that model_monitoring auto retraining code elements exists when setup_model_monitoring is True. + + Args: + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + pipeline_root: GS location where to store metadata from pipeline runs. + pipeline_job_runner_service_account: Service Account to runner PipelineJobs. + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + project_id: The project ID. + setup_model_monitoring: Boolean parameter which specifies whether to set up a Vertex AI Model Monitoring Job. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2' + submission_service_main_py = render_jinja( + template_path=template_file, + generated_license=GENERATED_LICENSE, + naming_prefix=naming_prefix, + pipeline_root=pipeline_root, + pipeline_job_runner_service_account=pipeline_job_runner_service_account, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + project_id=project_id, + setup_model_monitoring=setup_model_monitoring) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in submission_service_main_py + elif not is_included: + assert snippet not in submission_service_main_py diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py b/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py new file mode 100644 index 0000000..a66c74c --- /dev/null +++ b/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py @@ -0,0 +1,318 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for kfp scaffold module.""" + +# pylint: disable=anomalous-backslash-in-string +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring + +from contextlib import nullcontext as does_not_raise +import os +from typing import Callable, List, NamedTuple +from typing import Optional + +import pytest + +from google_cloud_automlops.orchestration.kfp.scaffold import ( + create_component_scaffold, + create_pipeline_scaffold, + get_packages_to_install_command, + get_compile_step, + get_function_parameters, + get_pipeline_decorator, + get_function_return_types, +) +from google_cloud_automlops.utils.constants import DEFAULT_PIPELINE_NAME +import google_cloud_automlops.utils.utils +from google_cloud_automlops.utils.utils import get_function_source_definition, read_yaml_file + + +def add(a: int, b: int) -> NamedTuple('output', [('sum', int)]): + """Testing + + Args: + a (int): Integer a + b (int): Integer b + + Returns: + int: Sum of a and b + """ + return a + b + + +def sub(a, b): + return a - b + + +def div(a: float, b: float): + """Testing + + Args: + a (float): Float a + b (float): Float b + """ + return a/b + + +@pytest.mark.parametrize( + 'func, packages_to_install, expectation, has_return_type', + [ + (add, None, does_not_raise(), True), + (add, ['pandas', 'pytest'], does_not_raise(), True), + (sub, None, pytest.raises(TypeError), False) + ] +) +def test_create_component_scaffold(func: Callable, packages_to_install: list, expectation, has_return_type: bool): + """Tests create_component_scaffold, which creates a tmp component scaffold + which will be used by the formalize function. Code is temporarily stored in + component_spec['implementation']['container']['command']. + + Args: + func (Callable): The python function to create a component from. The function + should have type annotations for all its arguments, indicating how + it is intended to be used (e.g. as an input/output Artifact object, + a plain parameter, or a path to a file). + packages_to_install (list): A list of optional packages to install before + executing func. These will always be installed at component runtime. + expectation: Any corresponding expected errors for each + set of parameters. + has_return_type: boolean indicating if the function has a return type hint. + This is used to determine if an 'outputs' key should exist in the component scaffold. + """ + with expectation: + create_component_scaffold(func=func, + packages_to_install=packages_to_install) + + # Assert the yaml exists + func_path = f'.AutoMLOps-cache/{func.__name__}.yaml' + assert os.path.exists(func_path) + + # Assert yaml contains correct keys + component_spec = read_yaml_file(func_path) + outputs_key = ['outputs'] if has_return_type else [] + assert set(component_spec.keys()) == set(['name', 'description', 'inputs', 'implementation', *outputs_key]) + assert list(component_spec['implementation'].keys()) == ['container'] + assert list(component_spec['implementation']['container'].keys()) == ['image', 'command', 'args'] + + # Remove temporary files + os.remove(func_path) + os.rmdir('.AutoMLOps-cache') + + +@pytest.mark.parametrize( + 'func, packages_to_install', + [ + (add, None), + (add, ['pandas']), + (sub, ['pandas', 'kfp', 'pytest']) + ] +) +def test_get_packages_to_install_command(func: Callable, packages_to_install: list): + """Tests get_packages_to_install_command, which returns a list of + formatted list of commands, including code for tmp storage. + + Args: + func (Callable): The python function to create a component from. The function + should have type annotations for all its arguments, indicating how + it is intended to be used (e.g. as an input/output Artifact object, + a plain parameter, or a path to a file). + packages_to_install (list): A list of optional packages to install before + executing func. These will always be installed at component runtime. + """ + newline = '\n' + if not packages_to_install: + packages_to_install = [] + install_python_packages_script = ( + f'''if ! [ -x "$(command -v pip)" ]; then\n''' + f''' python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\n''' + f'''fi\n''' + f'''PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \{newline}''' + f''' --no-warn-script-location {' '.join([repr(str(package)) for package in packages_to_install])} && "$0" "$@"\n''' + f'''\n''') + assert get_packages_to_install_command(func, packages_to_install) == ['sh', '-c', install_python_packages_script, get_function_source_definition(func=func)] + + +@pytest.mark.parametrize( + 'func, params, expectation', + [ + ( + add, + [ + {'description': 'Integer a', 'name': 'a', 'type': 'Integer'}, + {'description': 'Integer b', 'name': 'b', 'type': 'Integer'} + ], + does_not_raise() + ), + ( + sub, + None, + pytest.raises(TypeError) + ), + ( + div, + [ + {'description': 'Float a', 'name': 'a', 'type': 'Float'}, + {'description': 'Float b', 'name': 'b', 'type': 'Float'} + ], + does_not_raise() + ) + ] +) +def test_get_function_parameters(func: Callable, params: List[dict], expectation): + """Tests get_function_parameters, which returns a formatted list of + parameters. + + Args: + func (Callable): The python function to create a component from. The function + should have type annotations for all its arguments, indicating how + it is intended to be used (e.g. as an input/output Artifact object, + a plain parameter, or a path to a file). + params (List[dict]): Params list with types converted to kubeflow spec. + expectation: Any corresponding expected errors for each + set of parameters. + """ + with expectation: + assert params == get_function_parameters(func=func) + + +@pytest.mark.parametrize( + 'func, name, description', + [ + (add, 'Add', 'This is a test'), + (sub, 'Sub', 'Test 2'), + (div, None, None) + ] +) +def test_create_pipeline_scaffold(mocker, func: Callable, name: str, description: str): + """Tests create_pipeline_scaffold, which creates a temporary pipeline + scaffold which will be used by the formalize function. + + Args: + mocker: Mocker used to patch constants to test in tempoarary + environment. + func (Callable): The python function to create a pipeline from. The + function should have type annotations for all its arguments, + indicating how it is intended to be used (e.g. as an input/output + Artifact object, a plain parameter, or a path to a file). + name (str): The name of the pipeline. + description (str): Short description of what the pipeline does. + """ + mocker.patch.object(google_cloud_automlops.utils.utils, 'CACHE_DIR', '.') + create_pipeline_scaffold(func=func, name=name, description=description) + fold = '.AutoMLOps-cache' + file_path = 'pipeline_scaffold.py' + assert os.path.exists(os.path.join(fold, file_path)) + os.remove(os.path.join(fold, file_path)) + os.rmdir(fold) + + +@pytest.mark.parametrize( + 'name, description', + [ + ('Name1', 'Description1'), + ('Name2', 'Description2'), + (None, None), + ] +) +def test_get_pipeline_decorator(name: str, description: str): + """Tests get_pipeline_decorator, which creates the kfp pipeline decorator. + + Args: + name (str): The name of the pipeline. + description (str): Short description of what the pipeline does. + """ + desc_str = f''' description='{description}',\n''' if description else '' + decorator = ( + f'''@dsl.pipeline''' + f'''(\n name='{DEFAULT_PIPELINE_NAME if not name else name}',\n''' + f'''{desc_str}''' + f''')\n''' + ) + assert decorator == get_pipeline_decorator(name=name, description=description) + + +@pytest.mark.parametrize( + 'func_name', + ['func1', 'func2'] +) +def test_get_compile_step(func_name: str): + """Tests get_compile_step, which creates the compile function call. + + Args: + func_name (str): The name of the pipeline function. + """ + assert get_compile_step(func_name=func_name) == ( + f'\n' + f'compiler.Compiler().compile(\n' + f' pipeline_func={func_name},\n' + f' package_path=pipeline_job_spec_path)\n' + f'\n' + ) + + +@pytest.mark.parametrize( + 'return_annotation, return_types, expectation', + [ + ( + NamedTuple('output', [('sum', int)]), + [{'description': None, 'name': 'sum', 'type': 'Integer'},], + does_not_raise() + ), + ( + NamedTuple('output', [('first', str), ('last', str)]), + [{'description': None, 'name': 'first', 'type': 'String'}, + {'description': None, 'name': 'last', 'type': 'String'},], + does_not_raise() + ), + ( + Optional[NamedTuple('output', [('count', int)])], + None, + pytest.raises(TypeError) + ), + ( + int, + None, + pytest.raises(TypeError) + ),( + None, + None, + pytest.raises(TypeError) + ), + ( + 'NO_ANNOTATION', + None, + does_not_raise() + ) + ] +) +def test_get_function_return_types(return_annotation, return_types: List[dict], expectation): + """Tests get_function_outputs, which returns a formatted list of + return types. + + Args: + annotation (Any): The return type to test. + return_types (List[dict]): The return type converted into the kubeflow output spec. + expectation: Any corresponding expected errors for each + set of parameters. + """ + + def func(): + ... + + if return_annotation != 'NO_ANNOTATION': + func.__annotations__ = {'return' : return_annotation} + + with expectation: + assert return_types == get_function_return_types(func=func) diff --git a/google_cloud_automlops/tests/unit/provisioning/__init__.py b/google_cloud_automlops/tests/unit/provisioning/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py b/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py b/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py new file mode 100644 index 0000000..37e837d --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py @@ -0,0 +1,220 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +from typing import List + +import pytest + +from google_cloud_automlops.provisioning.gcloud.builder import provision_resources_script_jinja + +@pytest.mark.parametrize( + '''artifact_repo_location, artifact_repo_name, artifact_repo_type, build_trigger_location,''' + '''build_trigger_name, deployment_framework, naming_prefix, pipeline_job_runner_service_account,''' + '''pipeline_job_submission_service_location, pipeline_job_submission_service_name, pipeline_job_submission_service_type,''' + '''project_id, pubsub_topic_name,''' + '''required_apis,''' + '''schedule_location, schedule_name, schedule_pattern,''' + '''source_repo_branch, source_repo_name, source_repo_type, storage_bucket_location, storage_bucket_name,''' + '''use_ci, vpc_connector, is_included,''' + '''expected_output_snippets''', + [ + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'my-vpc-connector', True, + ['gcloud artifacts repositories create', 'gcloud iam service-accounts create', + 'gsutil mb -l ${STORAGE_BUCKET_LOCATION} gs://$STORAGE_BUCKET_NAME', 'gcloud iam service-accounts create', + 'gcloud projects add-iam-policy-binding', 'gcloud source repos create', 'gcloud pubsub topics create', + 'gcloud functions deploy', 'gcloud beta builds triggers create', 'gcloud scheduler jobs create pubsub', + '--vpc-connector=my-vpc-connector'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-run', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', True, + ['gcloud artifacts repositories create', 'gcloud iam service-accounts create', + 'gsutil mb -l ${STORAGE_BUCKET_LOCATION} gs://$STORAGE_BUCKET_NAME', 'gcloud iam service-accounts create', + 'gcloud projects add-iam-policy-binding', 'gcloud source repos create', 'gcloud pubsub topics create', + 'gcloud builds submit ${BASE_DIR}services/submission_service', 'gcloud run deploy', 'gcloud pubsub subscriptions create', + 'gcloud beta builds triggers create', 'gcloud scheduler jobs create pubsub'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-run', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'some-other-source-repository', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud source repos create', 'cloud beta builds triggers create'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-run', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', 'No Schedule Specified', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud scheduler jobs create pubsub', '--vpc_connector='] + ), + ( + 'us-central1', 'my-registry', 'some-other-repo-type', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud artifacts repositories create'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'some-other-deployment-framework', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + True, 'No VPC Specified', False, + ['gcloud beta builds triggers create'] + ), + ( + 'us-central1', 'my-registry', 'artifact-registry', 'us-central1', + 'my-trigger', 'cloud-build', 'my-prefix', 'my-service-account@serviceaccount.com', + 'us-central1', 'my-submission-svc', 'cloud-functions', + 'my-project', 'my-topic-name', + ['apiA','apiB','apiC'], + 'us-central1', 'my-schedule', '0 12 * * *', + 'my-branch', 'my-repo', 'cloud-source-repositories', 'us-central1', 'my-bucket', + False, 'No VPC Specified', False, + ['gcloud pubsub topics create', 'gcloud beta builds triggers create', + 'gcloud functions deploy', 'gcloud run deploy', 'gcloud scheduler jobs create pubsub'] + ) + ] +) +def test_provision_resources_script_jinja( + artifact_repo_location: str, + artifact_repo_name: str, + artifact_repo_type: str, + build_trigger_location: str, + build_trigger_name: str, + deployment_framework: str, + naming_prefix: str, + pipeline_job_runner_service_account: str, + pipeline_job_submission_service_location: str, + pipeline_job_submission_service_name: str, + pipeline_job_submission_service_type: str, + project_id: str, + pubsub_topic_name: str, + required_apis: list, + schedule_location: str, + schedule_name: str, + schedule_pattern: str, + source_repo_branch: str, + source_repo_name: str, + source_repo_type: str, + storage_bucket_location: str, + storage_bucket_name: str, + use_ci: bool, + vpc_connector: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests provision_resources_script_jinja, which generates code for + provision_resources.sh which sets up the project's environment. + There are seven test cases for this function: + 1. Checks for relevant gcloud commands when using the following tooling: + artifact-registry, cloud-build, cloud-functions, cloud scheduler, cloud-source-repositories, and a vpc connector + 2. Checks for relevant gcloud commands when using the following tooling: + artifact-registry, cloud-build, cloud-run, cloud scheduler, cloud-source-repositories, and no vpc connector + 3. Checks that gcloud source repo commands are not included when not using cloud-source-repositories. + 4. Checks that gcloud scheduler command and vpc_connector flag are not included when not specifying a vpc connector or schedule. + 5. Checks that gcloud artifacts command is not included when not using artifact-registry. + 6. Checks that gcloud beta builds triggers command is not included when not using cloud-build. + 7. Checks for that CI/CD elements are not included when use_ci=False. + + Args: + artifact_repo_location: Region of the artifact repo (default use with Artifact Registry). + artifact_repo_name: Artifact repo name where components are stored (default use with Artifact Registry). + artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) + build_trigger_location: The location of the build trigger (for cloud build). + build_trigger_name: The name of the build trigger (for cloud build). + deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + pipeline_job_runner_service_account: Service Account to run PipelineJobs. + pipeline_job_submission_service_location: The location of the cloud submission service. + pipeline_job_submission_service_name: The name of the cloud submission service. + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + project_id: The project ID. + pubsub_topic_name: The name of the pubsub topic to publish to. + required_apis: List of APIs that are required to run the service. + schedule_location: The location of the scheduler resource. + schedule_name: The name of the scheduler resource. + schedule_pattern: Cron formatted value used to create a Scheduled retrain job. + source_repo_branch: The branch to use in the source repository. + source_repo_name: The name of the source repository to use. + source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) + storage_bucket_location: Region of the GS bucket. + storage_bucket_name: GS bucket name where pipeline run metadata is stored. + use_ci: Flag that determines whether to use Cloud CI/CD. + vpc_connector: The name of the vpc connector to use. + """ + provision_resources_script = provision_resources_script_jinja( + artifact_repo_location=artifact_repo_location, + artifact_repo_name=artifact_repo_name, + artifact_repo_type=artifact_repo_type, + build_trigger_location=build_trigger_location, + build_trigger_name=build_trigger_name, + deployment_framework=deployment_framework, + naming_prefix=naming_prefix, + pipeline_job_runner_service_account=pipeline_job_runner_service_account, + pipeline_job_submission_service_location=pipeline_job_submission_service_location, + pipeline_job_submission_service_name=pipeline_job_submission_service_name, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + project_id=project_id, + pubsub_topic_name=pubsub_topic_name, + required_apis=required_apis, + schedule_location=schedule_location, + schedule_name=schedule_name, + schedule_pattern=schedule_pattern, + source_repo_branch=source_repo_branch, + source_repo_name=source_repo_name, + source_repo_type=source_repo_type, + storage_bucket_location=storage_bucket_location, + storage_bucket_name=storage_bucket_name, + use_ci=use_ci, + vpc_connector=vpc_connector) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in provision_resources_script + elif not is_included: + assert snippet not in provision_resources_script diff --git a/google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep b/google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep b/google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py b/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py b/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py new file mode 100644 index 0000000..03e5c72 --- /dev/null +++ b/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py @@ -0,0 +1,522 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring +# pylint: disable=missing-module-docstring + +from typing import List + +import pytest + +from google_cloud_automlops.utils.constants import GENERATED_LICENSE +from google_cloud_automlops.provisioning.terraform.builder import ( + create_environment_data_tf_jinja, + create_environment_iam_tf_jinja, + create_environment_main_tf_jinja, + create_environment_outputs_tf_jinja, + create_environment_provider_tf_jinja, + create_environment_variables_tf_jinja, + create_environment_versions_tf_jinja, + create_provision_resources_script_jinja, + create_state_bucket_variables_tf_jinja, + create_state_bucket_main_tf_jinja +) + + +@pytest.mark.parametrize( + 'required_apis, use_ci, is_included, expected_output_snippets', + [ + ( + ['apiA', 'apiB'], True, True, + [GENERATED_LICENSE, 'archive_cloud_functions_submission_service', + 'enable_apis = [\n' + ' "apiA",\n' + ' "apiB",\n' + ' ]' + ] + ), + ( + ['apiA', 'apiB'], False, False, + ['archive_cloud_functions_submission_service'] + ) + ] +) +def test_create_environment_data_tf_jinja( + required_apis: List, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_data_tf_jinja, which generates code for environment/data.tf, + the terraform hcl script that contains terraform remote backend and org project details. + There are two test cases for this function: + 1. Checks for the apache license and relevant terraform blocks. + 2. Checks for that the archive statement is not included when use_ci=False. + + Args: + required_apis: List of APIs that are required to run the service. + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + data_tf_str = create_environment_data_tf_jinja(required_apis, use_ci) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in data_tf_str + elif not is_included: + assert snippet not in data_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_environment_iam_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_iam_tf_jinja, which generates code for environment/iam.tf, the terraform hcl + script that contains service accounts iam bindings for project's environment. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + iam_tf_str = create_environment_iam_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in iam_tf_str + elif not is_included: + assert snippet not in iam_tf_str + + +@pytest.mark.parametrize( + '''artifact_repo_type, deployment_framework, naming_prefix,''' + '''pipeline_job_submission_service_type, schedule_pattern,''' + '''source_repo_type, use_ci, vpc_connector, is_included,''' + '''expected_output_snippets''', + [ + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, 'my-vpc-connector', True, + ['resource "google_artifact_registry_repository"', + 'resource "google_storage_bucket"', + 'resource "google_sourcerepo_repository"', + 'resource "google_pubsub_topic"', + 'resource "google_storage_bucket_object"', + 'resource "google_cloudfunctions_function"', + 'vpc_connector =', + 'resource "google_cloudbuild_trigger"', + 'resource "google_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, 'my-vpc-connector', True, + ['resource "google_artifact_registry_repository"', + 'resource "google_storage_bucket"', + 'resource "google_sourcerepo_repository"', + 'resource "null_resource" "build_and_push_submission_service"', + 'module "cloud_run"', + 'run.googleapis.com/vpc-access-connector', + 'module "pubsub"', + 'resource "google_cloudbuild_trigger"', + 'resource "google_cloud_scheduler_job"'] + ), + ( + 'some-other-repo', 'cloud-build', 'my-prefix', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['resource "google_artifact_registry_repository"', 'vpc_connector ='] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['run.googleapis.com/vpc-access-connector'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['resource "google_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'some-deployment-framework', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, 'No VPC Specified', False, + ['resource "google_cloudbuild_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', True, 'No VPC Specified', False, + ['resource "google_sourcerepo_repository"', 'resource "google_cloudbuild_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', 'my-prefix', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', False, 'No VPC Specified', False, + ['resource "null_resource" "build_and_push_submission_service"', + 'module "cloud_run"', + 'module "pubsub"', + 'resource "google_pubsub_topic"', + 'resource "google_storage_bucket_object"', + 'resource "google_cloudfunctions_function"', + 'resource "google_cloudbuild_trigger"', + 'resource "google_cloud_scheduler_job"'] + ), + ] +) +def test_create_environment_main_tf_jinja( + artifact_repo_type: str, + deployment_framework: str, + naming_prefix: str, + pipeline_job_submission_service_type: str, + schedule_pattern: str, + source_repo_type: str, + use_ci: bool, + vpc_connector: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_main_environment_tf_jinja, which generates code for environment/main.tf, the terraform hcl + script that contains terraform resources configs to deploy resources in the project. + There are eight test cases for this function: + 1. Checks for relevant terraform blocks when using the following tooling: + artifact-registry, cloud-build, cloud-functions, cloud scheduler, cloud-source-repositories, and a vpc connector + 2. Checks for relevant terraform blocks when using the following tooling: + artifact-registry, cloud-build, cloud-run, cloud scheduler, cloud-source-repositories, and a vpc connector + 3. Checks that the artifact-registry terraform block is not included when not using artifact-registry. + 4. Checks that the vpc-connector element is not included when not using a vpc connector. + 5. Checks that the cloud scheduler terraform block is not included when not using a cloud schedule. + 6. Checks that the cloud build trigger terraform block is not included when not using cloud-build. + 7. Checks that the cloud source repositories and cloud build trigger terraform blocks are not included when not using cloud-source-repositories. + 8. Checks for that CI/CD infra terraform blocks are not included when use_ci=False. + + Args: + artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) + deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) + naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + schedule_pattern: Cron formatted value used to create a Scheduled retrain job. + source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) + use_ci: Flag that determines whether to use Cloud CI/CD. + vpc_connector: The name of the vpc connector to use. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + main_tf_str = create_environment_main_tf_jinja( + artifact_repo_type=artifact_repo_type, + deployment_framework=deployment_framework, + naming_prefix=naming_prefix, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + schedule_pattern=schedule_pattern, + source_repo_type=source_repo_type, + use_ci=use_ci, + vpc_connector=vpc_connector) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in main_tf_str + elif not is_included: + assert snippet not in main_tf_str + + +@pytest.mark.parametrize( + '''artifact_repo_type, deployment_framework,''' + '''pipeline_job_submission_service_type, schedule_pattern,''' + '''source_repo_type, use_ci, is_included,''' + '''expected_output_snippets''', + [ + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, True, + ['output "enabled_apis"', + 'output "create_pipeline_job_runner_service_account_email"', + 'output "create_artifact_registry"', + 'output "create_storage_bucket"', + 'output "create_storage_bucket_names"', + 'output "create_cloud_source_repository"', + 'output "create_pubsub_topic"', + 'output "create_cloud_function"', + 'output "create_cloud_build_trigger"', + 'output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, True, + ['output "enabled_apis"', + 'output "create_pipeline_job_runner_service_account_email"', + 'output "create_artifact_registry"', + 'output "create_storage_bucket"', + 'output "create_storage_bucket_names"', + 'output "create_cloud_source_repository"', + 'output "cloud_run_id"', + 'output "create_pubsub_subscription"', + 'output "create_cloud_build_trigger"', + 'output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job"'] + ), + ( + 'some-other-repo', 'cloud-build', + 'cloud-functions', '0 12 * * *', + 'cloud-source-repositories', True, False, + ['output "create_artifact_registry"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-run', '0 12 * * *', + 'cloud-source-repositories', True, False, + ['output "create_cloud_function"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, False, + ['output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job"'] + ), + ( + 'artifact-registry', 'some-deployment-framework', + 'cloud-functions', 'No Schedule Specified', + 'cloud-source-repositories', True, False, + ['output "create_cloud_build_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', True, False, + ['output "create_cloud_source_repository"', + 'output "create_cloud_build_trigger"'] + ), + ( + 'artifact-registry', 'cloud-build', + 'cloud-functions', 'No Schedule Specified', + 'some-other-code-repo', False, False, + ['resource "null_resource" "build_and_push_submission_service"', + 'output "cloud_run_id"' + 'output "create_pubsub_subscription"', + 'output "create_pubsub_topic"', + 'output "create_cloud_function"', + 'output "create_cloud_build_trigger"', + 'output "create_cloud_scheduler_name"', + 'output "create_cloud_scheduler_job" '] + ), + ] +) +def test_create_environment_outputs_tf_jinja( + artifact_repo_type: str, + deployment_framework: str, + pipeline_job_submission_service_type: str, + schedule_pattern: str, + source_repo_type: str, + use_ci: bool, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_outputs_tf_jinja, which gnerates code for environment/outputs.tf, the terraform hcl + script that contains outputs from project's environment. + There are eight test cases for this function: + 1. Checks for relevant terraform output blocks when using the following tooling: + artifact-registry, cloud-build, cloud-functions, cloud scheduler, and cloud-source-repositories + 2. Checks for relevant terraform output blocks when using the following tooling: + artifact-registry, cloud-build, cloud-run, cloud scheduler, and cloud-source-repositories + 3. Checks that the artifact-registry terraform output block is not included when not using artifact-registry. + 4. Checks that the cloud functions terraform output block is not included when using cloud-run. + 5. Checks that the cloud scheduler terraform output blocks are not included when not using a cloud schedule. + 6. Checks that the cloud build trigger terraform output block is not included when not using cloud-build. + 7. Checks that the cloud source repositories and cloud build trigger output blocks are not included when not using cloud-source-repositories. + 8. Checks for that CI/CD infra terraform output blocks are not included when use_ci=False. + + Args: + artifact_repo_type: The type of artifact repository to use (e.g. Artifact Registry, JFrog, etc.) + deployment_framework: The CI tool to use (e.g. cloud build, github actions, etc.) + pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). + schedule_pattern: Cron formatted value used to create a Scheduled retrain job. + source_repo_type: The type of source repository to use (e.g. gitlab, github, etc.) + use_ci: Flag that determines whether to use Cloud CI/CD. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + main_tf_str = create_environment_outputs_tf_jinja( + artifact_repo_type=artifact_repo_type, + deployment_framework=deployment_framework, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + schedule_pattern=schedule_pattern, + source_repo_type=source_repo_type, + use_ci=use_ci) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in main_tf_str + elif not is_included: + assert snippet not in main_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_environment_provider_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_provider_tf_jinja, which generates code for environment/provider.tf, the terraform hcl + script that contains teraform providers used to deploy project's environment. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + provider_tf_str = create_environment_provider_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in provider_tf_str + elif not is_included: + assert snippet not in provider_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_environment_variables_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_variables_tf_jinja, which generates code for environment/variables.tf, + the terraform hcl script that contains variables used to deploy project's environment. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + variables_tf_str = create_environment_variables_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in variables_tf_str + elif not is_included: + assert snippet not in variables_tf_str + + +@pytest.mark.parametrize( + 'storage_bucket_name, is_included, expected_output_snippets', + [('my-storage-bucket', True, [GENERATED_LICENSE, 'bucket = "my-storage-bucket-tfstate"'])] +) +def test_create_environment_versions_tf_jinja( + storage_bucket_name: str, + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_environment_versions_tf_jinja, which generates code for environment/versions.tf, + the terraform hcl script that contains teraform version information. + There is one test case for this function: + 1. Checks for the apache license and state file storage_bucket backend. + + Args: + storage_bucket_name: GS bucket name where pipeline run metadata is stored. + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + versions_tf_str = create_environment_versions_tf_jinja(storage_bucket_name=storage_bucket_name) + + for snippet in expected_output_snippets: + if is_included: + assert snippet in versions_tf_str + elif not is_included: + assert snippet not in versions_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE, '#!/bin/bash'])] +) +def test_create_provision_resources_script_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_provision_resources_script_jinja, which generates code for provision_resources.sh + which sets up the project's environment using terraform. + There is one test case for this function: + 1. Checks for the apache license and the Bash shebang. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + provision_resources_script = create_provision_resources_script_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in provision_resources_script + elif not is_included: + assert snippet not in provision_resources_script + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_state_bucket_variables_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_state_bucket_variables_tf_jinja, which generates code for state_bucket/variables.tf, + the terraform hcl script that contains variables used for the state_bucket. + There is one test case for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + variables_tf_str = create_state_bucket_variables_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in variables_tf_str + elif not is_included: + assert snippet not in variables_tf_str + + +@pytest.mark.parametrize( + 'is_included, expected_output_snippets', + [(True, [GENERATED_LICENSE])] +) +def test_create_state_bucket_main_tf_jinja( + is_included: bool, + expected_output_snippets: List[str]): + """Tests create_main_state_bucket_tf_jinja, which generates code for state_bucket/main.tf, the terraform hcl + script that contains terraform resources configs to create the state_bucket. + There are eight test cases for this function: + 1. Checks for the apache license. + + Args: + is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. + expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. + """ + main_tf_str = create_state_bucket_main_tf_jinja() + + for snippet in expected_output_snippets: + if is_included: + assert snippet in main_tf_str + elif not is_included: + assert snippet not in main_tf_str diff --git a/google_cloud_automlops/tests/unit/utils/__init__.py b/google_cloud_automlops/tests/unit/utils/__init__.py new file mode 100644 index 0000000..70d7dec --- /dev/null +++ b/google_cloud_automlops/tests/unit/utils/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/utils/utils_test.py b/google_cloud_automlops/tests/unit/utils/utils_test.py new file mode 100644 index 0000000..110503f --- /dev/null +++ b/google_cloud_automlops/tests/unit/utils/utils_test.py @@ -0,0 +1,531 @@ +# Copyright 2024 Google LLC. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for utils module.""" + +# pylint: disable=line-too-long +# pylint: disable=missing-function-docstring + +from contextlib import nullcontext as does_not_raise +import os +from typing import Callable, List + +import pandas as pd +import pytest +import pytest_mock +import yaml + +import google_cloud_automlops.utils.utils +from google_cloud_automlops.utils.utils import ( + delete_file, + execute_process, + get_components_list, + get_function_source_definition, + is_component_config, + make_dirs, + read_file, + read_yaml_file, + stringify_job_spec_list, + update_params, + validate_use_ci, + write_and_chmod, + write_file, + write_yaml_file +) + + +# Define simple functions to be used in tests +def func1(x): + return x + 1 + + +def func2(x, y): + return x + y + + +def func3(x, y, z): + return x + y + z + + +def func4(): + + def inner_func(): + res = 1 + 1 + return res + + return inner_func() + + +@pytest.mark.parametrize( + 'directories, existance, expectation', + [ + (['dir1', 'dir2'], [True, True], does_not_raise()), + (['dir1', 'dir1'], [True, False], does_not_raise()), + (['\0', 'dir1'], [True, True], pytest.raises(ValueError)) + ] +) +def test_make_dirs(directories: List[str], existance: List[bool], expectation): + """Tests make_dirs, which creates a list of directories if they do not + already exist. There are three test cases for this function: + 1. Expected outcome, folders created as expected. + 2. Duplicate folder names given, expecting only one folder created. + 3. Invalid folder name given, expects an error. + + Args: + directories (List[str]): List of directories to be created. + existance (List[bool]): List of booleans indicating whether the listed directories to + be created are expected to exist after invoking make_dirs. + expectation: Any corresponding expected errors for each set of parameters. + """ + with expectation: + make_dirs(directories=directories) + for directory, exist in zip(directories, existance): + assert os.path.exists(directory) == exist + if exist: + os.rmdir(directory) + + +@pytest.mark.parametrize( + 'filepath, content1, content2, expectation', + [ + ( + 'test.yaml', + {'key1': 'value1', 'key2': 'value2'}, + None, + does_not_raise() + ), + ( + 'test.yaml', + {'key1': 'value1', False: 'random stuff'}, + r'-A fails', + pytest.raises(yaml.YAMLError) + ) + ] +) +def test_read_yaml_file(filepath: str, content1: dict, content2: str, expectation): + """Tests read_yaml_file, which reads a yaml file and returns the file + contents as a dict. There are two sets of test cases for this function: + 1. Expected outcome, file read in with correct content. + 2. File to be read is not in standard yaml format, expects a yaml error. + + Args: + filepath (str): Path to yaml file to be read. + content1 (dict): First set of content to be included in the yaml at the given + file path. + content2 (str): Second set of content to be included in the yaml at the given + file path. + expectation: Any corresponding expected errors for each set of + parameters. + """ + with open(file=filepath, mode='w', encoding='utf-8') as file: + if content1: + yaml.dump(content1, file) + if content2: + yaml.dump(content2, file) + with expectation: + assert read_yaml_file(filepath=filepath) == content1 + os.remove(path=filepath) + + +@pytest.mark.parametrize( + 'filepath, mode, expectation', + [ + ('test.yaml', 'w', does_not_raise()), + ('/nonexistent/directory', 'w', pytest.raises(FileNotFoundError)), + ('test.yaml', 'r', pytest.raises(IOError)) + ] +) +def test_write_yaml(filepath: str, mode: str, expectation): + """Tests write_yaml_file, which writes a yaml file. There are three sets of + test cases for this function: + 1. Expected outcome, yaml is written correctly. + 2. Invalid file path given, expecting a FileNotFoundError. + 3. Invalid mode given, expecting an IOError. + + Args: + filepath (str): Path for yaml file to be written. + mode (str): Read/write mode to be used. + expectation: Any corresponding expected errors for each set of + parameters. + """ + contents = {'key1': 'value1', 'key2': 'value2'} + with expectation: + write_yaml_file( + filepath=filepath, + contents=contents, + mode=mode + ) + with open(file=filepath, mode='r', encoding='utf-8') as file: + assert yaml.safe_load(file) == contents + os.remove(path=filepath) + + +@pytest.mark.parametrize( + 'filepath, text, write_file_bool, expectation', + [ + ('test.txt', 'This is a text file.', True, does_not_raise()), + ('fail', '', False, pytest.raises(FileNotFoundError)) + ] +) +def test_read_file(filepath: str, text: str, write_file_bool: bool, expectation): + """Tests read_file, which reads a text file in as a string. There are two + sets of test cases for this function: + 1. Expected outcome, file is read correctly. + 2. Invalid file path given (file was not written), expecting a + FileNotFoundError. + + Args: + filepath (str): Path for file to be read from. + text (str): Text expected to be read from the given file. + write_file_bool (bool): Whether or not the file should be written for this + test case. + expectation: Any corresponding expected errors for each set of + parameters. + """ + if write_file_bool: + with open(file=filepath, mode='w', encoding='utf-8') as file: + file.write(text) + with expectation: + assert read_file(filepath=filepath) == text + if os.path.exists(filepath): + os.remove(filepath) + + +@pytest.mark.parametrize( + 'filepath, text, mode, expectation', + [ + ('test.txt', 'This is a test file.', 'w', does_not_raise()), + (15, 'This is a test file.', 'w', pytest.raises(OSError)) + ] +) +def test_write_file(filepath: str, text: str, mode: str, expectation): + """Tests write_file, which writes a string to a text file. There are two + test cases for this function: + 1. Expected outcome, file is written as expected. + 2. Invalid file path given (file was not written), expecting + an OSError. + + Args: + filepath (str): Path for file to be written. + text (str): Content to be written to the file at the given filepath. + mode (str): Read/write mode to be used. + expectation: Any corresponding expected errors for each set of + parameters. + """ + with expectation: + write_file( + filepath=filepath, + text=text, + mode=mode + ) + assert os.path.exists(filepath) + with open(file=filepath, mode='r', encoding='utf-8') as file: + assert text == file.read() + os.remove(filepath) + + +def test_write_and_chmod(): + """Tests write_and_chmod, which writes a file at the specified path + and chmods the file to allow for execution. + """ + # Create a file. + with open(file='test.txt', mode='w', encoding='utf-8') as file: + file.write('This is a test file.') + + # Call the `write_and_chmod` function. + write_and_chmod('test.txt', 'This is a test file.') + + # Assert that the file exists and is executable. + assert os.path.exists('test.txt') + assert os.access('test.txt', os.X_OK) + + # Assert that the contents of the file are correct. + with open(file='test.txt', mode='r', encoding='utf-8') as file: + contents = file.read() + assert contents == 'This is a test file.' + os.remove('test.txt') + + +@pytest.mark.parametrize( + 'file_to_delete, valid_file', + [ + ('test.txt', True), + ('fake.txt', False) + ] +) +def test_delete_file(file_to_delete: str, valid_file: bool): + """Tests delete_file, which deletes a file at the specified path. + There are two test cases for this function: + 1. Create a valid file and call delete_file, which is expected to successfully delete the file. + 2. Pass in a nonexistent file and call delete_file, which is expected to pass. + + Args: + file_to_delete (str): Name of file to delete. + valid_file (bool): Whether or not the file to delete actually exists.""" + if not valid_file: + with does_not_raise(): + delete_file(file_to_delete) + else: + with open(file=file_to_delete, mode='w', encoding='utf-8') as file: + file.write('This is a test file.') + delete_file(file_to_delete) + assert not os.path.exists(file_to_delete) + + +@pytest.mark.parametrize( + 'comp_path, comp_name, patch_cwd, expectation', + [ + (['component.yaml'], ['component'], True, does_not_raise()), + ([], [], True, does_not_raise()), + (['component.yaml'], ['component'], False, pytest.raises(FileNotFoundError)) + ] +) +def test_get_components_list(mocker: pytest_mock.MockerFixture, + comp_path: List[str], + comp_name: List[str], + patch_cwd: bool, + expectation): + """Tests get_components_list, which reads yamls in .AutoMLOps-cache directory, + verifies they are component yamls, and returns the name of the files. There + are three test cases for this function: + 1. Expected outcome, component list is pulled as expected. + 2. Verifies an empty list comes back if no YAMLs are present. + 3. Call function with a nonexistent dir, expecting OSError. + + Args: + mocker: Mocker to patch the cache directory for component files. + comp_path (List[str]): Path(s) to component yamls. + comp_name (List[str]): Name(s) of components. + patch_cwd (bool): Boolean flag indicating whether to patch the current working + directory from CACHE_DIR to root + expectation: Any corresponding expected errors for each set of + parameters. + """ + if patch_cwd: + mocker.patch.object(google_cloud_automlops.utils.utils, 'CACHE_DIR', '.') + if comp_path: + for file in comp_path: + with open(file=file, mode='w', encoding='utf-8') as f: + yaml.dump( + { + 'name': 'value1', + 'inputs': 'value2', + 'implementation': 'value3' + }, + f) + with expectation: + assert get_components_list(full_path=False) == comp_name + assert get_components_list(full_path=True) == [os.path.join('.', file) for file in comp_path] + for file in comp_path: + if os.path.exists(file): + os.remove(file) + + +@pytest.mark.parametrize( + 'yaml_contents, expectation', + [ + ( + { + 'name': 'value1', + 'inputs': 'value2', + 'implementation': 'value3' + }, + True + ), + ( + { + 'name': 'value1', + 'inputs': 'value2' + }, + False + ) + ] +) +def test_is_component_config(yaml_contents: dict, expectation: bool): + """Tests is_component_config, which which checks to see if the given file is + a component yaml. There are two test cases for this function: + 1. A valid component is given, expecting return value True. + 2. An invalid component is given, expecting return value False. + + Args: + yaml_contents (dict): Component configurations to be written to yaml file. + expected (bool): Expectation of whether or not the configuration is valid. + """ + with open(file='component.yaml', mode='w', encoding='utf-8') as f: + yaml.dump(yaml_contents, f) + assert expectation == is_component_config('component.yaml') + os.remove('component.yaml') + + +@pytest.mark.parametrize( + 'command, to_null, expectation', + [ + ('touch test.txt', False, False), + ('not a real command', False, True), + ('echo "howdy"', True, False) + ] +) +def test_execute_process(command: str, to_null: bool, expectation: bool): + """Tests execute_process, which executes an external shell process. There + are two test cases for this function: + 1. A valid command to create a file, which is expected to run successfully. + 2. An invalid command, which is expected to raise a RunTime Error. + 3. A valid command to output a string, which is expected to send output to null + + Args: + command (str): Command that is to be executed. + expectation (bool): Whether or not an error is expected to be raised. + """ + if expectation: + with pytest.raises(RuntimeError): + execute_process(command=command, to_null=to_null) + elif to_null: + assert execute_process(command=command, to_null=to_null) is None + else: + execute_process(command=command, to_null=to_null) + assert os.path.exists('test.txt') + os.remove('test.txt') + + +@pytest.mark.parametrize( + 'sch_pattern, setup_model_monitoring, use_ci, expectation', + [ + ('No Schedule Specified', False, True, does_not_raise()), + ('No Schedule Specified', False, False, does_not_raise()), + ('Schedule', False, False, pytest.raises(ValueError)), + ('Schedule', True, True, does_not_raise()), + ('Schedule', True, False, pytest.raises(ValueError)) + ] +) +def test_validate_use_ci(sch_pattern: str, + setup_model_monitoring: bool, + use_ci: bool, + expectation): + """Tests validate_use_ci, which validates the inputted schedule + parameter and the setup_model_monitoring parameter. There are + five test cases for this function, which tests each + combination of sch_pattern and setup_model_monitoring for the expected results. + + Args: + sch_pattern (str): Cron formatted value used to create a Scheduled retrain job. + setup_model_monitoring (bool): Boolean parameter which specifies whether to set + up a Vertex AI Model Monitoring Job. + use_ci (bool): Flag that determines whether to use Cloud Run CI/CD. + expectation: Any corresponding expected errors for each set of parameters. + """ + with expectation: + validate_use_ci(schedule_pattern=sch_pattern, + setup_model_monitoring=setup_model_monitoring, + use_ci=use_ci) + + +@pytest.mark.parametrize( + 'params, expected_output', + [ + ([{'name': 'param1', 'type': int}], [{'name': 'param1', 'type': 'Integer'}]), + ([{'name': 'param2', 'type': str}], [{'name': 'param2', 'type': 'String'}]), + ([{'name': 'param3', 'type': float}], [{'name': 'param3', 'type': 'Float'}]), + ([{'name': 'param4', 'type': bool}], [{'name': 'param4', 'type': 'Boolean'}]), + ([{'name': 'param5', 'type': list}], [{'name': 'param5', 'type': 'JsonArray'}]), + ([{'name': 'param6', 'type': dict}], [{'name': 'param6', 'type': 'JsonObject'}]), + ([{'name': 'param6', 'type': pd.DataFrame}], None) + ] +) +def test_update_params(params: List[dict], expected_output: List[dict]): + """Tests the update_params function, which reformats the source code type + labels as strings. There are seven test cases for this function, which test + for updating different parameter types. + + Args: + params (List[dict]): Pipeline parameters. A list of dictionaries, each param is a dict containing keys: + 'name': required, str param name. + 'type': required, python primitive type. + 'description': optional, str param desc. + expected_output (List[dict]): Expectation of whether or not the configuration is valid. + """ + if expected_output is not None: + assert expected_output == update_params(params=params) + else: + with pytest.raises(ValueError): + assert update_params(params=params) + + +@pytest.mark.parametrize( + 'func, expected_output', + [ + (func1, 'def func1(x):\n return x + 1\n'), + (func2, 'def func2(x, y):\n return x + y\n'), + (func3, 'def func3(x, y, z):\n return x + y + z\n'), + (func4, 'def func4():\n\n def inner_func():\n res = 1 + 1\n return res\n\n return inner_func()\n') + ] +) +def test_get_function_source_definition(func: Callable, expected_output: str): + """Tests get_function_source_definition, which returns a formatted string of + the source code. + + Args: + func (Callable): Function to pull source definition from. + expected_output (str): Expected source definition of the given function. + """ + assert expected_output == get_function_source_definition(func=func) + + +@pytest.mark.parametrize( + 'job_spec_list, expected_output', + [ + ([{'component_spec': 'train_model', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': 1}], + [{'component_spec': 'train_model', + 'spec_string': + '''{\n''' + ''' "accelerator_count": 1,\n''' + ''' "accelerator_type": "NVIDIA_TESLA_A100",\n''' + ''' "component_spec": train_model,\n''' + ''' "display_name": "train-model-accelerated",\n''' + ''' "machine_type": "a2-highgpu-1g"\n }''' + }]), + ] +) +def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: List[dict]): + """Tests the stringify_job_spec_list function, takes in a list of custom training job spec + dictionaries and turns them into strings. + + Args: + job_spec: Dictionary with job spec info. e.g. + input = [{ + 'component_spec': 'train_model', + 'display_name': 'train-model-accelerated', + 'machine_type': 'a2-highgpu-1g', + 'accelerator_type': 'NVIDIA_TESLA_A100', + 'accelerator_count': 1 + }] + expected_output (List[dict]): Dictionary with key value pair for component_spec, + and a string representation of the full dictionary e.g. + output = [{ + 'component_spec': 'train_model', + 'spec_string': '''{ + "accelerator_count": 1, + "accelerator_type": "NVIDIA_TESLA_A100", + "component_spec": train_model, + "display_name": "train-model-accelerated", + "machine_type": "a2-highgpu-1g" + }''' + }] + """ + + formatted_spec = stringify_job_spec_list(job_spec_list=job_spec_list) + assert formatted_spec == expected_output From 6d7d959bef9ce6b95f5e6b61f2748a4dff72a53d Mon Sep 17 00:00:00 2001 From: Allegra Noto Date: Fri, 23 Feb 2024 14:08:12 -0500 Subject: [PATCH 8/9] Fixed unit testing and incorporated model monitoring changes --- .../orchestration/kfp/builder.py | 39 ++++++--- .../unit/orchestration/kfp/builder_test.py | 45 ++++------ .../unit/provisioning/gcloud/builder_test.py | 71 +++++++++------ .../provisioning/terraform/builder_test.py | 86 +++++++++++++------ .../tests/unit/utils/utils_test.py | 26 ++++++ 5 files changed, 176 insertions(+), 91 deletions(-) diff --git a/google_cloud_automlops/orchestration/kfp/builder.py b/google_cloud_automlops/orchestration/kfp/builder.py index f6a67cd..662e313 100644 --- a/google_cloud_automlops/orchestration/kfp/builder.py +++ b/google_cloud_automlops/orchestration/kfp/builder.py @@ -117,10 +117,32 @@ def build(config: KfpConfig): generated_license=GENERATED_LICENSE, generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, pubsub_topic_name=config.pubsub_topic_name)) + + # If using model monitoring, write correspointing scripts to model_monitoring directory if config.setup_model_monitoring: - write_and_chmod(GENERATED_MODEL_MONITORING_SH_FILE, create_model_monitoring_job_jinja()) - write_file(GENERATED_MODEL_MONITORING_MONITOR_PY_FILE, model_monitoring_monitor_jinja(), 'w') - write_file(GENERATED_MODEL_MONITORING_REQUIREMENTS_FILE, model_monitoring_requirements_jinja(), 'w') + # Writes script create_model_monitoring_job.sh which creates a Vertex AI model monitoring job + write_and_chmod( + filepath=GENERATED_MODEL_MONITORING_SH_FILE, + text=render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'create_model_monitoring_job.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + )) + + # Writes monitor.py to create or update a model monitoring job in Vertex AI for a deployed model endpoint + write_file( + filepath=GENERATED_MODEL_MONITORING_MONITOR_PY_FILE, + text=render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.model_monitoring') / 'monitor.py.j2', + generated_license=GENERATED_LICENSE + ), + mode='w') + + # Writes a requirements.txt to the model_monitoring directory + write_file( + filepath=GENERATED_MODEL_MONITORING_REQUIREMENTS_FILE, + text=render_jinja(template_path=import_files(KFP_TEMPLATES_PATH + '.model_monitoring') / 'requirements.txt.j2'), + mode='w') # Create components and pipelines components_path_list = get_components_list(full_path=True) @@ -136,6 +158,7 @@ def build(config: KfpConfig): f'{BASE_DIR}README.md', render_jinja( template_path=import_files(KFP_TEMPLATES_PATH) / 'README.md.j2', + setup_model_monitoring=config.setup_model_monitoring, use_ci=config.use_ci), 'w') @@ -302,13 +325,6 @@ def build_services(): 'w') # Write main code files for cloud run base and queueing svc - # write_file(f'{submission_service_base}/main.py', submission_service_main_jinja( - # naming_prefix=defaults['gcp']['naming_prefix'], - # pipeline_root=defaults['pipelines']['pipeline_storage_path'], - # pipeline_job_runner_service_account=defaults['gcp']['pipeline_job_runner_service_account'], - # pipeline_job_submission_service_type=defaults['gcp']['pipeline_job_submission_service_type'], - # project_id=defaults['gcp']['project_id'], - # setup_model_monitoring=defaults['gcp']['setup_model_monitoring']), 'w') write_file( f'{submission_service_base}/main.py', render_jinja( @@ -317,7 +333,8 @@ def build_services(): pipeline_root=defaults['pipelines']['pipeline_storage_path'], pipeline_job_runner_service_account=defaults['gcp']['pipeline_job_runner_service_account'], pipeline_job_submission_service_type=defaults['gcp']['pipeline_job_submission_service_type'], - project_id=defaults['gcp']['project_id']), + project_id=defaults['gcp']['project_id'], + setup_model_monitoring=defaults['gcp']['setup_model_monitoring']), 'w') diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py b/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py index 08d0c6e..372e078 100644 --- a/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py +++ b/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py @@ -359,9 +359,8 @@ def test_build_pipeline_spec_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_pipeline_spec.sh.j2' build_pipeline_spec_script = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_pipeline_spec.sh.j2', generated_license=GENERATED_LICENSE, base_dir=BASE_DIR ) @@ -388,9 +387,8 @@ def test_build_components_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_components.sh.j2' build_components_script = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_components.sh.j2', generated_license=GENERATED_LICENSE, base_dir=BASE_DIR ) @@ -417,9 +415,8 @@ def test_run_pipeline_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_pipeline.sh.j2' run_pipeline_script = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_pipeline.sh.j2', generated_license=GENERATED_LICENSE, base_dir=BASE_DIR ) @@ -447,9 +444,8 @@ def test_run_all_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_all.sh.j2' run_all_script = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_all.sh.j2', generated_license=GENERATED_LICENSE, base_dir=BASE_DIR ) @@ -478,9 +474,8 @@ def test_publish_to_topic_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'publish_to_topic.sh.j2' publish_to_topic_script = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'publish_to_topic.sh.j2', base_dir=BASE_DIR, generated_license=GENERATED_LICENSE, generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, @@ -510,9 +505,8 @@ def test_create_model_monitoring_job_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.scripts') / 'create_model_monitoring_job.sh.j2' create_model_monitoring_job_script = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'create_model_monitoring_job.sh.j2', generated_license=GENERATED_LICENSE, base_dir=BASE_DIR ) @@ -579,9 +573,8 @@ def test_readme_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH) / 'README.md.j2' readme_str = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH) / 'README.md.j2', setup_model_monitoring=setup_model_monitoring, use_ci=use_ci ) @@ -611,9 +604,8 @@ def test_component_base_dockerfile_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2' component_base_dockerfile = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2', base_image=base_image, generated_license=GENERATED_LICENSE ) @@ -667,9 +659,8 @@ def test_component_base_task_file_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2' component_base_task_file = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2', custom_code_contents=custom_code_contents, generated_license=GENERATED_LICENSE, kfp_spec_bool=kfp_spec_bool) @@ -696,9 +687,8 @@ def test_pipeline_runner_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2' pipeline_runner_py = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2', generated_license=GENERATED_LICENSE ) @@ -775,9 +765,8 @@ def test_pipeline_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2' pipeline_py = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2', components_list=components_list, custom_training_job_specs=custom_training_job_specs, generated_license=GENERATED_LICENSE, @@ -806,9 +795,8 @@ def test_pipeline_requirements_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2' pipeline_requirements_py = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2', pinned_kfp_version=PINNED_KFP_VERSION ) @@ -835,9 +823,8 @@ def test_submission_service_dockerfile_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2' submission_service_dockerfile = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2', base_dir=BASE_DIR, generated_license=GENERATED_LICENSE ) @@ -873,9 +860,8 @@ def test_submission_service_requirements_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2' submission_service_requirements = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2', pinned_kfp_version=PINNED_KFP_VERSION, pipeline_job_submission_service_type=pipeline_job_submission_service_type ) @@ -975,9 +961,8 @@ def test_submission_service_main_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - template_file = import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2' submission_service_main_py = render_jinja( - template_path=template_file, + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2', generated_license=GENERATED_LICENSE, naming_prefix=naming_prefix, pipeline_root=pipeline_root, diff --git a/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py b/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py index 37e837d..0e7f48f 100644 --- a/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py +++ b/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py @@ -16,11 +16,25 @@ # pylint: disable=missing-function-docstring # pylint: disable=missing-module-docstring +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files + from typing import List import pytest -from google_cloud_automlops.provisioning.gcloud.builder import provision_resources_script_jinja +from google_cloud_automlops.utils.utils import render_jinja + +from google_cloud_automlops.utils.constants import ( + BASE_DIR, + GCLOUD_TEMPLATES_PATH, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH, + IAM_ROLES_RUNNER_SA, +) @pytest.mark.parametrize( '''artifact_repo_location, artifact_repo_name, artifact_repo_type, build_trigger_location,''' @@ -187,31 +201,36 @@ def test_provision_resources_script_jinja( use_ci: Flag that determines whether to use Cloud CI/CD. vpc_connector: The name of the vpc connector to use. """ - provision_resources_script = provision_resources_script_jinja( - artifact_repo_location=artifact_repo_location, - artifact_repo_name=artifact_repo_name, - artifact_repo_type=artifact_repo_type, - build_trigger_location=build_trigger_location, - build_trigger_name=build_trigger_name, - deployment_framework=deployment_framework, - naming_prefix=naming_prefix, - pipeline_job_runner_service_account=pipeline_job_runner_service_account, - pipeline_job_submission_service_location=pipeline_job_submission_service_location, - pipeline_job_submission_service_name=pipeline_job_submission_service_name, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - project_id=project_id, - pubsub_topic_name=pubsub_topic_name, - required_apis=required_apis, - schedule_location=schedule_location, - schedule_name=schedule_name, - schedule_pattern=schedule_pattern, - source_repo_branch=source_repo_branch, - source_repo_name=source_repo_name, - source_repo_type=source_repo_type, - storage_bucket_location=storage_bucket_location, - storage_bucket_name=storage_bucket_name, - use_ci=use_ci, - vpc_connector=vpc_connector) + provision_resources_script = render_jinja( + template_path=import_files(GCLOUD_TEMPLATES_PATH) / 'provision_resources.sh.j2', + artifact_repo_location=artifact_repo_location, + artifact_repo_name=artifact_repo_name, + artifact_repo_type=artifact_repo_type, + base_dir=BASE_DIR, + build_trigger_location=build_trigger_location, + build_trigger_name=build_trigger_name, + deployment_framework=deployment_framework, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + naming_prefix=naming_prefix, + pipeline_job_runner_service_account=pipeline_job_runner_service_account, + pipeline_job_submission_service_location=pipeline_job_submission_service_location, + pipeline_job_submission_service_name=pipeline_job_submission_service_name, + pipeline_job_submission_service_type=pipeline_job_submission_service_type, + project_id=project_id, + pubsub_topic_name=pubsub_topic_name, + required_apis=required_apis, + required_iam_roles=IAM_ROLES_RUNNER_SA, + schedule_location=schedule_location, + schedule_name=schedule_name, + schedule_pattern=schedule_pattern, + source_repo_branch=source_repo_branch, + source_repo_name=source_repo_name, + source_repo_type=source_repo_type, + storage_bucket_location=storage_bucket_location, + storage_bucket_name=storage_bucket_name, + use_ci=use_ci, + vpc_connector=vpc_connector) for snippet in expected_output_snippets: if is_included: diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py b/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py index 03e5c72..744791e 100644 --- a/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py +++ b/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py @@ -16,22 +16,23 @@ # pylint: disable=missing-function-docstring # pylint: disable=missing-module-docstring +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files from typing import List import pytest -from google_cloud_automlops.utils.constants import GENERATED_LICENSE -from google_cloud_automlops.provisioning.terraform.builder import ( - create_environment_data_tf_jinja, - create_environment_iam_tf_jinja, - create_environment_main_tf_jinja, - create_environment_outputs_tf_jinja, - create_environment_provider_tf_jinja, - create_environment_variables_tf_jinja, - create_environment_versions_tf_jinja, - create_provision_resources_script_jinja, - create_state_bucket_variables_tf_jinja, - create_state_bucket_main_tf_jinja +from google_cloud_automlops.utils.utils import render_jinja + +from google_cloud_automlops.utils.constants import ( + BASE_DIR, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH, + IAM_ROLES_RUNNER_SA, + TERRAFORM_TEMPLATES_PATH ) @@ -70,7 +71,13 @@ def test_create_environment_data_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - data_tf_str = create_environment_data_tf_jinja(required_apis, use_ci) + data_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'data.tf.j2', + generated_license=GENERATED_LICENSE, + required_apis=required_apis, + required_iam_roles=IAM_ROLES_RUNNER_SA, + use_ci=use_ci + ) for snippet in expected_output_snippets: if is_included: @@ -95,7 +102,10 @@ def test_create_environment_iam_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - iam_tf_str = create_environment_iam_tf_jinja() + iam_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'iam.tf.j2', + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -220,15 +230,20 @@ def test_create_environment_main_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - main_tf_str = create_environment_main_tf_jinja( + main_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'main.tf.j2', artifact_repo_type=artifact_repo_type, + base_dir=BASE_DIR, deployment_framework=deployment_framework, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, naming_prefix=naming_prefix, pipeline_job_submission_service_type=pipeline_job_submission_service_type, schedule_pattern=schedule_pattern, source_repo_type=source_repo_type, use_ci=use_ci, - vpc_connector=vpc_connector) + vpc_connector=vpc_connector + ) for snippet in expected_output_snippets: if is_included: @@ -355,13 +370,16 @@ def test_create_environment_outputs_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - main_tf_str = create_environment_outputs_tf_jinja( + main_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'outputs.tf.j2', artifact_repo_type=artifact_repo_type, deployment_framework=deployment_framework, + generated_license=GENERATED_LICENSE, pipeline_job_submission_service_type=pipeline_job_submission_service_type, schedule_pattern=schedule_pattern, source_repo_type=source_repo_type, - use_ci=use_ci) + use_ci=use_ci + ) for snippet in expected_output_snippets: if is_included: @@ -386,7 +404,10 @@ def test_create_environment_provider_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - provider_tf_str = create_environment_provider_tf_jinja() + provider_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'provider.tf.j2', + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -411,7 +432,10 @@ def test_create_environment_variables_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - variables_tf_str = create_environment_variables_tf_jinja() + variables_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'variables.tf.j2', + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -438,7 +462,11 @@ def test_create_environment_versions_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - versions_tf_str = create_environment_versions_tf_jinja(storage_bucket_name=storage_bucket_name) + versions_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.environment') / 'versions.tf.j2', + generated_license=GENERATED_LICENSE, + storage_bucket_name=storage_bucket_name + ) for snippet in expected_output_snippets: if is_included: @@ -463,7 +491,11 @@ def test_create_provision_resources_script_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - provision_resources_script = create_provision_resources_script_jinja() + provision_resources_script = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH) / 'provision_resources.sh.j2', + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -488,7 +520,10 @@ def test_create_state_bucket_variables_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - variables_tf_str = create_state_bucket_variables_tf_jinja() + variables_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'variables.tf.j2', + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -513,7 +548,10 @@ def test_create_state_bucket_main_tf_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - main_tf_str = create_state_bucket_main_tf_jinja() + main_tf_str = render_jinja( + template_path=import_files(TERRAFORM_TEMPLATES_PATH + '.state_bucket') / 'main.tf.j2', + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: diff --git a/google_cloud_automlops/tests/unit/utils/utils_test.py b/google_cloud_automlops/tests/unit/utils/utils_test.py index 110503f..f29b797 100644 --- a/google_cloud_automlops/tests/unit/utils/utils_test.py +++ b/google_cloud_automlops/tests/unit/utils/utils_test.py @@ -19,6 +19,7 @@ from contextlib import nullcontext as does_not_raise import os +import tempfile from typing import Callable, List import pandas as pd @@ -36,6 +37,7 @@ make_dirs, read_file, read_yaml_file, + render_jinja, stringify_job_spec_list, update_params, validate_use_ci, @@ -529,3 +531,27 @@ def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: Lis formatted_spec = stringify_job_spec_list(job_spec_list=job_spec_list) assert formatted_spec == expected_output + + +@pytest.mark.parametrize( + 'template_string, template_vars, expected_output', + [ + ('Hello {{ name1 }} my name is {{ name2 }}', {'name1': 'Alice', 'name2': 'John'}, 'Hello Alice my name is John'), + ('The answer is: {{ result }}', {'result': 42}, 'The answer is: 42'), + ] +) +def test_render_jinja(template_string, template_vars, expected_output): + """Tests the render_jinja function using temporary files.""" + + with tempfile.TemporaryDirectory() as tmpdirname: # Creates temp directory + template_path = os.path.join(tmpdirname, 'template.txt.j2') + + # Write the template to the temporary file + with open(template_path, 'w', encoding='utf-8') as f: + f.write(template_string) + + # Call the render_jinja function + result = render_jinja(template_path, **template_vars) + + # Assertion + assert result == expected_output From 2bb364af47178f16b4b475e690e2139d4ce2ae25 Mon Sep 17 00:00:00 2001 From: Allegra Noto Date: Fri, 23 Feb 2024 14:22:09 -0500 Subject: [PATCH 9/9] Fixed test location --- google_cloud_automlops/tests/__init__.py | 13 - google_cloud_automlops/tests/unit/__init__.py | 13 - .../tests/unit/deployments/.gitkeep | 0 .../tests/unit/deployments/__init__.py | 13 - .../unit/deployments/cloudbuild/__init__.py | 13 - .../deployments/github_actions/__init__.py | 13 - .../tests/unit/deployments/gitlab_ci/.gitkeep | 0 .../tests/unit/deployments/jenkins/.gitkeep | 0 .../tests/unit/orchestration/__init__.py | 13 - .../tests/unit/orchestration/airflow/.gitkeep | 0 .../tests/unit/orchestration/argo/.gitkeep | 0 .../tests/unit/orchestration/kfp/__init__.py | 13 - .../unit/orchestration/kfp/builder_test.py | 978 ------------------ .../unit/orchestration/kfp/scaffold_test.py | 318 ------ .../tests/unit/provisioning/__init__.py | 13 - .../unit/provisioning/gcloud/__init__.py | 13 - .../tests/unit/provisioning/pulumi/.gitkeep | 0 .../unit/provisioning/terraform/.gitkeep | 0 .../unit/provisioning/terraform/__init__.py | 13 - .../tests/unit/utils/__init__.py | 13 - .../tests/unit/utils/utils_test.py | 557 ---------- .../deployments/cloudbuild/builder_test.py | 0 .../github_actions/builder_test.py | 0 tests/unit/orchestration/kfp/builder_test.py | 129 ++- .../unit/provisioning/gcloud/builder_test.py | 0 .../provisioning/terraform/builder_test.py | 0 tests/unit/utils/utils_test.py | 1 + 27 files changed, 90 insertions(+), 2036 deletions(-) delete mode 100644 google_cloud_automlops/tests/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/deployments/.gitkeep delete mode 100644 google_cloud_automlops/tests/unit/deployments/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep delete mode 100644 google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep delete mode 100644 google_cloud_automlops/tests/unit/orchestration/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep delete mode 100644 google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep delete mode 100644 google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py delete mode 100644 google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py delete mode 100644 google_cloud_automlops/tests/unit/provisioning/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep delete mode 100644 google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep delete mode 100644 google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/utils/__init__.py delete mode 100644 google_cloud_automlops/tests/unit/utils/utils_test.py rename {google_cloud_automlops/tests => tests}/unit/deployments/cloudbuild/builder_test.py (100%) rename {google_cloud_automlops/tests => tests}/unit/deployments/github_actions/builder_test.py (100%) rename {google_cloud_automlops/tests => tests}/unit/provisioning/gcloud/builder_test.py (100%) rename {google_cloud_automlops/tests => tests}/unit/provisioning/terraform/builder_test.py (100%) diff --git a/google_cloud_automlops/tests/__init__.py b/google_cloud_automlops/tests/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/__init__.py b/google_cloud_automlops/tests/unit/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/.gitkeep b/google_cloud_automlops/tests/unit/deployments/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/google_cloud_automlops/tests/unit/deployments/__init__.py b/google_cloud_automlops/tests/unit/deployments/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/deployments/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py b/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/deployments/cloudbuild/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py b/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/deployments/github_actions/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep b/google_cloud_automlops/tests/unit/deployments/gitlab_ci/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep b/google_cloud_automlops/tests/unit/deployments/jenkins/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/google_cloud_automlops/tests/unit/orchestration/__init__.py b/google_cloud_automlops/tests/unit/orchestration/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/orchestration/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep b/google_cloud_automlops/tests/unit/orchestration/airflow/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep b/google_cloud_automlops/tests/unit/orchestration/argo/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py b/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/orchestration/kfp/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py b/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py deleted file mode 100644 index 372e078..0000000 --- a/google_cloud_automlops/tests/unit/orchestration/kfp/builder_test.py +++ /dev/null @@ -1,978 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# pylint: disable=line-too-long -# pylint: disable=missing-function-docstring -# pylint: disable=missing-module-docstring - -import json -try: - from importlib.resources import files as import_files -except ImportError: - # Try backported to PY<37 `importlib_resources` - from importlib_resources import files as import_files -import os -from typing import List - -import pytest -import pytest_mock - -from google_cloud_automlops.utils.constants import ( - BASE_DIR, - GENERATED_LICENSE, - GENERATED_PARAMETER_VALUES_PATH, - KFP_TEMPLATES_PATH, - PINNED_KFP_VERSION, -) -import google_cloud_automlops.orchestration.kfp.builder -from google_cloud_automlops.orchestration.kfp.builder import ( - build_component, - build_pipeline, - build_services, -) -import google_cloud_automlops.utils.utils -from google_cloud_automlops.utils.utils import ( - make_dirs, - read_yaml_file, - render_jinja, - write_yaml_file -) - - -DEFAULTS = { - 'gcp': { - 'artifact_repo_location': 'us-central1', - 'project_id': 'my_project', - 'artifact_repo_name': 'my_af_registry', - 'naming_prefix': 'my-prefix', - 'pipeline_job_runner_service_account': 'my-service-account@service.com', - 'pipeline_job_submission_service_type': 'cloud-functions', - 'setup_model_monitoring': True - }, - 'pipelines': { - 'gs_pipeline_job_spec_path': 'gs://my-bucket/pipeline_root/my-prefix/pipeline_job.json', - 'pipeline_storage_path': 'gs://my-bucket/pipeline_root/' - } -} - -TEMP_YAML = { - 'name': 'create_dataset', - 'description': 'Custom component that takes in a BQ table and writes it to GCS.', - 'inputs': [ - { - 'name': 'bq_table', - 'description': 'The source biquery table.', - 'type': 'String', - }, - { - 'name': 'data_path', - 'description': 'The gcs location to write the csv.', - 'type': 'String', - }, - { - 'name': 'project_id', - 'description': 'The project ID.', - 'type': 'String'}, - ], - 'implementation': { - 'container': { - 'image': 'TBD', - 'command': [ - 'sh', - '-c', - 'if ! [ -x "$(command -v pip)" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \\\n --no-warn-script-location && "$0" "$@"\n\n', - 'def create_dataset(\n bq_table: str,\n data_path: str,\n project_id: str\n):\n """Custom component that takes in a BQ table and writes it to GCS.\n\n Args:\n bq_table: The source biquery table.\n data_path: The gcs location to write the csv.\n project_id: The project ID.\n """\n from google.cloud import bigquery\n import pandas as pd\n from sklearn import preprocessing\n\n bq_client = bigquery.Client(project=project_id)\n\n def get_query(bq_input_table: str) -> str:\n """Generates BQ Query to read data.\n\n Args:\n bq_input_table: The full name of the bq input table to be read into\n the dataframe (e.g. ..
)\n Returns: A BQ query string.\n """\n return f\'\'\'\n SELECT *\n FROM `{bq_input_table}`\n \'\'\'\n\n def load_bq_data(query: str, client: bigquery.Client) -> pd.DataFrame:\n """Loads data from bq into a Pandas Dataframe for EDA.\n Args:\n query: BQ Query to generate data.\n client: BQ Client used to execute query.\n Returns:\n pd.DataFrame: A dataframe with the requested data.\n """\n df = client.query(query).to_dataframe()\n return df\n\n dataframe = load_bq_data(get_query(bq_table), bq_client)\n le = preprocessing.LabelEncoder()\n dataframe[\'Class\'] = le.fit_transform(dataframe[\'Class\'])\n dataframe.to_csv(data_path, index=False)\n', - ], - 'args': [ - '--executor_input', - {'executorInput': None}, - '--function_to_execute', - 'create_dataset', - ], - } - }, -} - - -@pytest.fixture(name='temp_yaml_dict', params=[TEMP_YAML]) -def fixture_temp_yaml_dict(request: pytest.FixtureRequest, tmpdir: pytest.FixtureRequest): - """Writes temporary yaml file fixture using defaults parameterized - dictionaries during pytest session scope. - - Args: - request: Pytest fixture special object that provides information - about the fixture. - tmpdir: Pytest fixture that provides a temporary directory unique - to the test invocation. - - Returns: - dict: Path of yaml file and dictionary it contains. - """ - yaml_path = tmpdir.join('test.yaml') - write_yaml_file(yaml_path, request.param, 'w') - return {'path': yaml_path, 'vals': request.param} - - -@pytest.fixture(name='defaults_dict', params=[DEFAULTS]) -def fixture_defaults_dict(request: pytest.FixtureRequest, tmpdir: pytest.FixtureRequest): - """Writes temporary yaml file fixture using defaults parameterized - dictionaries during pytest session scope. - - Args: - request: Pytest fixture special object that provides information - about the fixture. - tmpdir: Pytest fixture that provides a temporary directory unique - to the test invocation. - - Returns: - dict: Path of yaml file and dictionary it contains. - """ - yaml_path = tmpdir.join('defaults.yaml') - write_yaml_file(yaml_path, request.param, 'w') - return {'path': yaml_path, 'vals': request.param} - - -@pytest.fixture(name='expected_component_dict') -def fixture_expected_component_dict(): - """Creates the expected component dictionary, which is the temporary yaml - file with a change to the implementation key. - - Returns: - dict: Expected component dictionary generated from the component - builder. - """ - expected = TEMP_YAML - expected['implementation'] = { - 'container': { - 'image': 'us-central1-docker.pkg.dev/my_project/my_af_registry/my-prefix/components/component_base:latest', - 'command': ['python3', '/pipelines/component/src/create_dataset.py'], - 'args': [ - '--executor_input', - {'executorInput': None}, - '--function_to_execute', - 'create_dataset', - ], - } - } - return expected - - -def test_build_component(mocker: pytest_mock.MockerFixture, - tmpdir: pytest.FixtureRequest, - temp_yaml_dict: pytest.FixtureRequest, - defaults_dict: pytest.FixtureRequest, - expected_component_dict: pytest.FixtureRequest): - """Tests build_component, which Constructs and writes component.yaml and - {component_name}.py files. - - Args: - mocker: Mocker to patch directories. - tmpdir: Pytest fixture that provides a temporary directory unique - to the test invocation. - temp_yaml_dict: Locally defined temp_yaml_file Pytest fixture. - defaults_dict: Locally defined defaults_dict Pytest fixture. - expected_component_dict: Locally defined expected_component_dict - Pytest fixture. - """ - # Patch filepath constants to point to test path. - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'BASE_DIR', - f'{tmpdir}/') - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'GENERATED_DEFAULTS_FILE', - defaults_dict['path']) - - # Extract component name, create required directories, run build_component - component_name = TEMP_YAML['name'] - make_dirs([f'{tmpdir}/components/component_base/src']) - build_component(temp_yaml_dict['path']) - - # Ensure correct files are created with build_component call - assert os.path.exists(f'{tmpdir}/components/{component_name}/component.yaml') - assert os.path.exists(f'{tmpdir}/components/component_base/src/{component_name}.py') - - # Load component.yaml file and compare to the expected output in test_data - created_component_dict = read_yaml_file(f'{tmpdir}/components/{component_name}/component.yaml') - assert created_component_dict == expected_component_dict - - -@pytest.mark.parametrize( - 'custom_training_job_specs, pipeline_parameter_values', - [ - ( - [{'component_spec': 'mycomp1', 'other': 'myother'}], - { - 'bq_table': 'automlops-sandbox.test_dataset.dry-beans', - 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 13:00:41.379753', - 'data_path': 'gs://automlops-sandbox-bucket/data.csv', - 'project_id': 'automlops-sandbox', - 'region': 'us-central1' - }, - ), - ( - [ - { - 'component_spec': 'train_model', - 'display_name': 'train-model-accelerated', - 'machine_type': 'a2-highgpu-1g', - 'accelerator_type': 'NVIDIA_TESLA_A100', - 'accelerator_count': '1', - } - ], - { - 'bq_table': 'automlops-sandbox.test_dataset.dry-beans', - 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 13:00:41.379753', - 'data_path': 'gs://automlops-sandbox-bucket/data.csv', - 'project_id': 'automlops-sandbox', - 'region': 'us-central1' - }, - ), - ( - [ - { - 'component_spec': 'test_model', - 'display_name': 'test-model-accelerated', - 'machine_type': 'a2-highgpu-1g', - 'accelerator_type': 'NVIDIA_TESLA_A100', - 'accelerator_count': '1', - } - ], - { - 'bq_table': 'automlops-sandbox.test_dataset.dry-beans2', - 'model_directory': 'gs://automlops-sandbox-bucket/trained_models/2023-05-31 14:00:41.379753', - 'data_path': 'gs://automlops-sandbox-bucket/data2.csv', - 'project_id': 'automlops-sandbox', - 'region': 'us-central1' - }, - ) - ] -) -def test_build_pipeline(mocker: pytest_mock.MockerFixture, - tmpdir: pytest.FixtureRequest, - defaults_dict: pytest.FixtureRequest, - custom_training_job_specs: List[dict], - pipeline_parameter_values: dict): - """Tests build_pipeline, which constructs and writes pipeline.py, - pipeline_runner.py, and pipeline_parameter_values.json files. - - Args: - mocker: Mocker to patch directories. - tmpdir: Pytest fixture that provides a temporary directory unique - to the test invocation. - defaults_dict: Locally defined defaults_dict Pytest fixture. - custom_training_job_specs (List[dict]): Specifies the specs to run the training job with. - pipeline_parameter_values (dict): Dictionary of runtime parameters for the PipelineJob. - """ - # Patch constants and other functions - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'BASE_DIR', - f'{tmpdir}/') - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'GENERATED_DEFAULTS_FILE', - defaults_dict['path']) - mocker.patch.object(google_cloud_automlops.utils.utils, - 'CACHE_DIR', - f'{tmpdir}/.AutoMLOps-cache') - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'PIPELINE_CACHE_FILE', - f'{tmpdir}/.AutoMLOps-cache/pipeline_scaffold.py') - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'GENERATED_PIPELINE_FILE', - f'{tmpdir}/pipelines/pipeline.py') - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'GENERATED_PIPELINE_RUNNER_FILE', - f'{tmpdir}/pipelines/pipeline_runner.py') - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'GENERATED_PIPELINE_REQUIREMENTS_FILE', - f'{tmpdir}/pipelines/requirements.txt') - - # Create required directory and file for build_pipeline - make_dirs([f'{tmpdir}/pipelines/runtime_parameters', f'{tmpdir}/.AutoMLOps-cache']) - os.system(f'touch {tmpdir}/.AutoMLOps-cache/pipeline_scaffold.py') - build_pipeline(custom_training_job_specs, pipeline_parameter_values) - - # Ensure correct files were created - assert os.path.exists(f'{tmpdir}/pipelines/pipeline.py') - assert os.path.exists(f'{tmpdir}/pipelines/pipeline_runner.py') - assert os.path.exists(f'{tmpdir}/pipelines/requirements.txt') - assert os.path.exists(f'{tmpdir}/pipelines/runtime_parameters/pipeline_parameter_values.json') - - # Ensure pipeline_parameter_values.json was created as expected - with open(f'{tmpdir}/pipelines/runtime_parameters/pipeline_parameter_values.json', mode='r', encoding='utf-8') as f: - pipeline_params_dict = json.load(f) - assert pipeline_params_dict == pipeline_parameter_values - - -def test_build_services(mocker: pytest_mock.MockerFixture, - tmpdir: pytest.FixtureRequest, - defaults_dict: pytest.FixtureRequest): - """Tests build_services, which Constructs and writes a Dockerfile, requirements.txt, and - main.py to the services/submission_service directory. - - Args: - mocker: Mocker to patch directories. - tmpdir: Pytest fixture that provides a temporary directory unique - to the test invocation. - defaults_dict: Locally defined defaults_dict Pytest fixture. - """ - # Patch filepath constants to point to test path. - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'BASE_DIR', - f'{tmpdir}/') - mocker.patch.object(google_cloud_automlops.orchestration.kfp.builder, - 'GENERATED_DEFAULTS_FILE', - defaults_dict['path']) - - # create required directories, run build_services - make_dirs([f'{tmpdir}/services/submission_service']) - build_services() - - # Ensure correct files are created with build_services call - assert os.path.exists(f'{tmpdir}/services/submission_service/Dockerfile') - assert os.path.exists(f'{tmpdir}/services/submission_service/requirements.txt') - assert os.path.exists(f'{tmpdir}/services/submission_service/main.py') - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline --config $CONFIG_FILE'])] -) -def test_build_pipeline_spec_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests build_pipeline_spec_jinja, which generates code for build_pipeline_spec.sh - which builds the pipeline specs. There is one test case for this function: - 1. Checks for the apache license and the pipeline compile command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - build_pipeline_spec_script = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_pipeline_spec.sh.j2', - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in build_pipeline_spec_script - elif not is_included: - assert snippet not in build_pipeline_spec_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600'])] -) -def test_build_components_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests build_components_jinja, which generates code for build_components.sh - which builds the components. There is one test case for this function: - 1. Checks for the apache license and the builds submit command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - build_components_script = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_components.sh.j2', - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in build_components_script - elif not is_included: - assert snippet not in build_components_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'python3 -m pipelines.pipeline_runner --config $CONFIG_FILE'])] -) -def test_run_pipeline_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests run_pipeline_jinja, which generates code for run_pipeline.sh - which runs the pipeline locally. There is one test case for this function: - 1. Checks for the apache license and the pipeline runner command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - run_pipeline_script = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_pipeline.sh.j2', - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in run_pipeline_script - elif not is_included: - assert snippet not in run_pipeline_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'gcloud builds submit .. --config cloudbuild.yaml --timeout=3600', - './scripts/build_pipeline_spec.sh', './scripts/run_pipeline.sh'])] -) -def test_run_all_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests run_all_jinja, which generates code for run_all.sh - which builds runs all other shell scripts. There is one test case for this function: - 1. Checks for the apache license and the builds submit, the pipeline compile, and the pipeline runner commands. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - run_all_script = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_all.sh.j2', - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in run_all_script - elif not is_included: - assert snippet not in run_all_script - - -@pytest.mark.parametrize( - 'pubsub_topic_name, is_included, expected_output_snippets', - [('my-topic', True, [GENERATED_LICENSE, 'gcloud pubsub topics publish my-topic'])] -) -def test_publish_to_topic_jinja( - pubsub_topic_name: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests publish_to_topic_jinja, which generates code for publish_to_topic.sh - which submits a message to the pipeline job submission service. - There is one test case for this function: - 1. Checks for the apache license and the pubsub publish command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - publish_to_topic_script = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'publish_to_topic.sh.j2', - base_dir=BASE_DIR, - generated_license=GENERATED_LICENSE, - generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, - pubsub_topic_name=pubsub_topic_name - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in publish_to_topic_script - elif not is_included: - assert snippet not in publish_to_topic_script - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'python3 -m model_monitoring.monitor --config $CONFIG_FILE'])] -) -def test_create_model_monitoring_job_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests create_model_monitoring_job_jinja, which generates code for create_model_monitoring_job.sh - which creates a Model Monitoring Job in Vertex AI for a deployed model endpoint. - There is one test case for this function: - 1. Checks for the apache license and the monitor command. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - create_model_monitoring_job_script = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'create_model_monitoring_job.sh.j2', - generated_license=GENERATED_LICENSE, - base_dir=BASE_DIR - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in create_model_monitoring_job_script - elif not is_included: - assert snippet not in create_model_monitoring_job_script - - -@pytest.mark.parametrize( - 'setup_model_monitoring, use_ci, is_included, expected_output_snippets', - [ - ( - False, True, True, - ['AutoMLOps - Generated Code Directory', - '├── components', - '├── configs', - '├── images', - '├── provision', - '├── scripts', - '├── services', - '├── README.md', - '└── cloudbuild.yaml'] - ), - ( - True, True, True, - ['AutoMLOps - Generated Code Directory', - '├── components', - '├── configs', - '├── images', - '├── model_monitoring', - '├── provision', - '├── scripts', - '├── services', - '├── README.md', - '└── cloudbuild.yaml'] - ), - ( - False, False, False, - ['├── publish_to_topic.sh' - '├── services', - '├── create_model_monitoring_job.sh', - '├── model_monitoring'] - ), - ] -) -def test_readme_jinja( - setup_model_monitoring: bool, - use_ci: bool, - is_included: bool, - expected_output_snippets: List[str]): - """Tests readme_jinja, which generates code for readme.md which - is a readme markdown file to describe the contents of the - generated AutoMLOps code repo. There are three test cases for this function: - 1. Checks that certain directories and files exist when use_ci=True and setup_model_monitoring=False. - 2. Checks that certain directories and files exist when use_ci=True and setup_model_monitoring=True. - 3. Checks that certain directories and files do not exist when use_ci=False. - - Args: - setup_model_monitoring: Boolean parameter which specifies whether to set up a Vertex AI Model Monitoring Job. - use_ci: Flag that determines whether to use Cloud CI/CD. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - readme_str = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH) / 'README.md.j2', - setup_model_monitoring=setup_model_monitoring, - use_ci=use_ci - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in readme_str - elif not is_included: - assert snippet not in readme_str - - -@pytest.mark.parametrize( - 'base_image, is_included, expected_output_snippets', - [('my-base-image', True, [GENERATED_LICENSE, 'FROM my-base-image'])] -) -def test_component_base_dockerfile_jinja( - base_image: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests readme_jinja, which generates code for a Dockerfile - to be written to the component_base directory. There is one - test case for this function: - 1. Checks for the apache license and the FROM image line. - - Args: - base_image: The image to use in the component base dockerfile. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - component_base_dockerfile = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2', - base_image=base_image, - generated_license=GENERATED_LICENSE - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in component_base_dockerfile - elif not is_included: - assert snippet not in component_base_dockerfile - - -@pytest.mark.parametrize( - 'custom_code_contents, kfp_spec_bool, is_included, expected_output_snippets', - [ - ( - 'this is some custom code', True, True, - [GENERATED_LICENSE, - 'this is some custom code', - 'def main():'] - ), - ( - 'this is some custom code', False, True, - [GENERATED_LICENSE, - 'this is some custom code', - 'def main():', - 'import kfp', - 'from kfp.v2.dsl import *'] - ), - ( - 'this is some custom code', True, False, - ['import kfp', - 'from kfp.v2.dsl import *'] - ) - ] -) -def test_component_base_task_file_jinja( - custom_code_contents: str, - kfp_spec_bool: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests component_base_task_file_jinja, which generates code - for the task.py file to be written to the component_base/src directory. - There are three test cases for this function: - 1. Checks for the apache license, the custom_code_contents, and a main function when using kfp spec (kfp spec comes with kfp imports by default). - 2. Checks for the apache license, the custom_code_contents, a main function, and kfp imports when not using kfp spec. - 3. Checks that the kfp imports are not included in the string when using kfp spec (kfp spec comes with kfp imports by default). - - Args: - custom_code_contents: Code inside of the component, specified by the user. - kfp_spec_bool: Boolean that specifies whether components are defined using kfp. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - component_base_task_file = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2', - custom_code_contents=custom_code_contents, - generated_license=GENERATED_LICENSE, - kfp_spec_bool=kfp_spec_bool) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in component_base_task_file - elif not is_included: - assert snippet not in component_base_task_file - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE])] -) -def test_pipeline_runner_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_runner_jinja, which generates code for the pipeline_runner.py - file to be written to the pipelines directory. There is one test case for this function: - 1. Checks for the apache license. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - pipeline_runner_py = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2', - generated_license=GENERATED_LICENSE - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in pipeline_runner_py - elif not is_included: - assert snippet not in pipeline_runner_py - - -@pytest.mark.parametrize( - '''components_list, custom_training_job_specs, pipeline_scaffold_contents, project_id,''' - '''is_included, expected_output_snippets''', - [ - ( - ['componentA','componentB','componentC'], - [ - { - 'component_spec': 'componentB', - 'display_name': 'train-model-accelerated', - 'machine_type': 'a2-highgpu-1g', - 'accelerator_type': 'NVIDIA_TESLA_A100', - 'accelerator_count': '1', - } - ], - 'Pipeline definition goes here', 'my-project', True, - [GENERATED_LICENSE, - 'from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', - 'def upload_pipeline_spec', - 'componentA = load_custom_component', - 'componentB = load_custom_component', - 'componentC = load_custom_component', - 'componentB_custom_training_job_specs', - 'Pipeline definition goes here'] - ), - ( - ['componentA','componentB','componentC'], - None, 'Pipeline definition goes here', 'my-project', True, - [GENERATED_LICENSE, - 'def upload_pipeline_spec', - 'componentA = load_custom_component', - 'componentB = load_custom_component', - 'componentC = load_custom_component', - 'Pipeline definition goes here'] - ), - ( - ['componentA','componentB','componentC'], - None, 'Pipeline definition goes here', 'my-project', False, - ['from google_cloud_pipeline_components.v1.custom_job import create_custom_training_job_op_from_component', - 'componentB_custom_training_job_specs'] - ), - ] -) -def test_pipeline_jinja( - components_list: list, - custom_training_job_specs: list, - pipeline_scaffold_contents: str, - project_id: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_jinja, which generates code for the pipeline.py - file to be written to the pipelines directory. - There are three test cases for this function: - 1. Checks for the apache license and relevant code elements when custom_training_job_specs is not None. - 2. Checks for the apache license and relevant code elements when custom_training_job_specs is None. - 3. Checks that the output does not contain custom_training_job_specs code elements when custom_training_job_specs is None. - - Args: - components_list: Contains the names or paths of all component yamls in the dir. - custom_training_job_specs: Specifies the specs to run the training job with. - pipeline_scaffold_contents: The contents of the pipeline scaffold file, - which can be found at PIPELINE_CACHE_FILE. - project_id: The project ID. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - pipeline_py = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2', - components_list=components_list, - custom_training_job_specs=custom_training_job_specs, - generated_license=GENERATED_LICENSE, - pipeline_scaffold_contents=pipeline_scaffold_contents, - project_id=project_id) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in pipeline_py - elif not is_included: - assert snippet not in pipeline_py - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform'])] -) -def test_pipeline_requirements_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_requirements_jinja, which generates code for a requirements.txt - to be written to the pipelines directory. There is one test case for this function: - 1. Checks for the pinned kfp version, and the google-cloud-aiplatform dep. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - pipeline_requirements_py = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2', - pinned_kfp_version=PINNED_KFP_VERSION - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in pipeline_requirements_py - elif not is_included: - assert snippet not in pipeline_requirements_py - - -@pytest.mark.parametrize( - 'is_included, expected_output_snippets', - [(True, [GENERATED_LICENSE, 'python:3.9-slim', - 'CMD exec gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app'])] -) -def test_submission_service_dockerfile_jinja( - is_included: bool, - expected_output_snippets: List[str]): - """Tests pipeline_requirements_jinja, which generates code for a Dockerfile to be - written to the serivces/submission_service directory. There is one test case for this function: - 1. Checks for the apache license and relevant dockerfile elements. - - Args: - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - submission_service_dockerfile = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2', - base_dir=BASE_DIR, - generated_license=GENERATED_LICENSE - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in submission_service_dockerfile - elif not is_included: - assert snippet not in submission_service_dockerfile - - -@pytest.mark.parametrize( - 'pipeline_job_submission_service_type, is_included, expected_output_snippets', - [('cloud-functions', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'google-cloud-storage', 'functions-framework==3.*']), - ('cloud-functions', False, ['gunicorn']), - ('cloud-run', True, [PINNED_KFP_VERSION, 'google-cloud-aiplatform', 'google-cloud-storage', 'gunicorn']), - ('cloud-run', False, ['functions-framework==3.*']),] -) -def test_submission_service_requirements_jinja( - pipeline_job_submission_service_type: str, - is_included: bool, - expected_output_snippets: List[str]): - """Tests submission_service_requirements_jinja, which generates code - for a requirements.txt to be written to the serivces/submission_service directory. - There are four test cases for this function: - 1. Checks for the pinned kfp version, the google-cloud-aiplatform, google-cloud-storage and function-framework deps when set to cloud-functions. - 2. Checks that gunicorn dep is not included when set to cloud-functions. - 3. Checks for the pinned kfp version, the google-cloud-aiplatform, google-cloud-storage and gunicorn deps when set to cloud-run. - 4. Checks that functions-framework dep is not included when set to cloud-run. - - Args: - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - submission_service_requirements = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2', - pinned_kfp_version=PINNED_KFP_VERSION, - pipeline_job_submission_service_type=pipeline_job_submission_service_type - ) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in submission_service_requirements - elif not is_included: - assert snippet not in submission_service_requirements - - -@pytest.mark.parametrize( - '''naming_prefix, pipeline_root, pipeline_job_runner_service_account, pipeline_job_submission_service_type,''' - '''project_id, setup_model_monitoring, is_included, expected_output_snippets''', - [ - ( - 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', - 'my-project', False, True, - [GENERATED_LICENSE, - 'from google.cloud import aiplatform', - 'import functions_framework', - '@functions_framework.http', - 'def process_request(request: flask.Request)', - '''base64_message = request_json['data']['data']'''] - ), - ( - 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-functions', - 'my-project', False, False, - ['app = flask.Flask', - '''@app.route('/', methods=['POST'])''', - 'request = flask.request', - '''base64_message = request_json['message']['data']''', - '''if __name__ == '__main__':''', - '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))''', - 'from google.cloud import storage', - 'NAMING_PREFIX', - 'def read_gs_auto_retraining_params_file()'] - ), - ( - 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', - 'my-project', False, True, - [GENERATED_LICENSE, - 'from google.cloud import aiplatform', - 'app = flask.Flask', - '''@app.route('/', methods=['POST'])''', - 'request = flask.request', - '''base64_message = request_json['message']['data']''', - '''if __name__ == '__main__':''', - '''app.run(debug=False, host='0.0.0.0', port=int(os.environ.get('PORT', 8080)))'''] - ), - ( - 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', - 'my-project', False, False, - ['import functions_framework', - '@functions_framework.http', - 'def process_request(request: flask.Request)', - '''base64_message = request_json['data']['data']''', - 'from google.cloud import storage', - 'NAMING_PREFIX', - 'def read_gs_auto_retraining_params_file()'] - ), - ( - 'my-prefix', 'gs://my-bucket/pipeline-root', 'my-service-account@service.com', 'cloud-run', - 'my-project', True, True, - ['from google.cloud import storage', - 'NAMING_PREFIX', - 'def read_gs_auto_retraining_params_file()', - '''if data_payload['logName'] == f'projects/{PROJECT_ID}/logs/aiplatform.googleapis.com%2Fmodel_monitoring_anomaly':'''] - ), - ] -) -def test_submission_service_main_jinja( - naming_prefix: str, - pipeline_root: str, - pipeline_job_runner_service_account: str, - pipeline_job_submission_service_type: str, - project_id: str, - setup_model_monitoring: bool, - is_included: bool, - expected_output_snippets: List[str]): - """Tests submission_service_main_jinja, which generates content - for main.py to be written to the serivces/submission_service directory. - There are five test cases for this function: - 1. Checks for functions_framework code elements when set to cloud-functions. - 2. Checks that Flask app code elements are not included when set to cloud-functions. - 3. Checks for Flask app code elements when set to cloud-run. - 4. Checks that functions_framework code elements are not included when set to cloud-run. - 5. Checks that model_monitoring auto retraining code elements exists when setup_model_monitoring is True. - - Args: - naming_prefix: Unique value used to differentiate pipelines and services across AutoMLOps runs. - pipeline_root: GS location where to store metadata from pipeline runs. - pipeline_job_runner_service_account: Service Account to runner PipelineJobs. - pipeline_job_submission_service_type: The tool to host for the cloud submission service (e.g. cloud run, cloud functions). - project_id: The project ID. - setup_model_monitoring: Boolean parameter which specifies whether to set up a Vertex AI Model Monitoring Job. - is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. - expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. - """ - submission_service_main_py = render_jinja( - template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2', - generated_license=GENERATED_LICENSE, - naming_prefix=naming_prefix, - pipeline_root=pipeline_root, - pipeline_job_runner_service_account=pipeline_job_runner_service_account, - pipeline_job_submission_service_type=pipeline_job_submission_service_type, - project_id=project_id, - setup_model_monitoring=setup_model_monitoring) - - for snippet in expected_output_snippets: - if is_included: - assert snippet in submission_service_main_py - elif not is_included: - assert snippet not in submission_service_main_py diff --git a/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py b/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py deleted file mode 100644 index a66c74c..0000000 --- a/google_cloud_automlops/tests/unit/orchestration/kfp/scaffold_test.py +++ /dev/null @@ -1,318 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for kfp scaffold module.""" - -# pylint: disable=anomalous-backslash-in-string -# pylint: disable=line-too-long -# pylint: disable=missing-function-docstring - -from contextlib import nullcontext as does_not_raise -import os -from typing import Callable, List, NamedTuple -from typing import Optional - -import pytest - -from google_cloud_automlops.orchestration.kfp.scaffold import ( - create_component_scaffold, - create_pipeline_scaffold, - get_packages_to_install_command, - get_compile_step, - get_function_parameters, - get_pipeline_decorator, - get_function_return_types, -) -from google_cloud_automlops.utils.constants import DEFAULT_PIPELINE_NAME -import google_cloud_automlops.utils.utils -from google_cloud_automlops.utils.utils import get_function_source_definition, read_yaml_file - - -def add(a: int, b: int) -> NamedTuple('output', [('sum', int)]): - """Testing - - Args: - a (int): Integer a - b (int): Integer b - - Returns: - int: Sum of a and b - """ - return a + b - - -def sub(a, b): - return a - b - - -def div(a: float, b: float): - """Testing - - Args: - a (float): Float a - b (float): Float b - """ - return a/b - - -@pytest.mark.parametrize( - 'func, packages_to_install, expectation, has_return_type', - [ - (add, None, does_not_raise(), True), - (add, ['pandas', 'pytest'], does_not_raise(), True), - (sub, None, pytest.raises(TypeError), False) - ] -) -def test_create_component_scaffold(func: Callable, packages_to_install: list, expectation, has_return_type: bool): - """Tests create_component_scaffold, which creates a tmp component scaffold - which will be used by the formalize function. Code is temporarily stored in - component_spec['implementation']['container']['command']. - - Args: - func (Callable): The python function to create a component from. The function - should have type annotations for all its arguments, indicating how - it is intended to be used (e.g. as an input/output Artifact object, - a plain parameter, or a path to a file). - packages_to_install (list): A list of optional packages to install before - executing func. These will always be installed at component runtime. - expectation: Any corresponding expected errors for each - set of parameters. - has_return_type: boolean indicating if the function has a return type hint. - This is used to determine if an 'outputs' key should exist in the component scaffold. - """ - with expectation: - create_component_scaffold(func=func, - packages_to_install=packages_to_install) - - # Assert the yaml exists - func_path = f'.AutoMLOps-cache/{func.__name__}.yaml' - assert os.path.exists(func_path) - - # Assert yaml contains correct keys - component_spec = read_yaml_file(func_path) - outputs_key = ['outputs'] if has_return_type else [] - assert set(component_spec.keys()) == set(['name', 'description', 'inputs', 'implementation', *outputs_key]) - assert list(component_spec['implementation'].keys()) == ['container'] - assert list(component_spec['implementation']['container'].keys()) == ['image', 'command', 'args'] - - # Remove temporary files - os.remove(func_path) - os.rmdir('.AutoMLOps-cache') - - -@pytest.mark.parametrize( - 'func, packages_to_install', - [ - (add, None), - (add, ['pandas']), - (sub, ['pandas', 'kfp', 'pytest']) - ] -) -def test_get_packages_to_install_command(func: Callable, packages_to_install: list): - """Tests get_packages_to_install_command, which returns a list of - formatted list of commands, including code for tmp storage. - - Args: - func (Callable): The python function to create a component from. The function - should have type annotations for all its arguments, indicating how - it is intended to be used (e.g. as an input/output Artifact object, - a plain parameter, or a path to a file). - packages_to_install (list): A list of optional packages to install before - executing func. These will always be installed at component runtime. - """ - newline = '\n' - if not packages_to_install: - packages_to_install = [] - install_python_packages_script = ( - f'''if ! [ -x "$(command -v pip)" ]; then\n''' - f''' python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\n''' - f'''fi\n''' - f'''PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet \{newline}''' - f''' --no-warn-script-location {' '.join([repr(str(package)) for package in packages_to_install])} && "$0" "$@"\n''' - f'''\n''') - assert get_packages_to_install_command(func, packages_to_install) == ['sh', '-c', install_python_packages_script, get_function_source_definition(func=func)] - - -@pytest.mark.parametrize( - 'func, params, expectation', - [ - ( - add, - [ - {'description': 'Integer a', 'name': 'a', 'type': 'Integer'}, - {'description': 'Integer b', 'name': 'b', 'type': 'Integer'} - ], - does_not_raise() - ), - ( - sub, - None, - pytest.raises(TypeError) - ), - ( - div, - [ - {'description': 'Float a', 'name': 'a', 'type': 'Float'}, - {'description': 'Float b', 'name': 'b', 'type': 'Float'} - ], - does_not_raise() - ) - ] -) -def test_get_function_parameters(func: Callable, params: List[dict], expectation): - """Tests get_function_parameters, which returns a formatted list of - parameters. - - Args: - func (Callable): The python function to create a component from. The function - should have type annotations for all its arguments, indicating how - it is intended to be used (e.g. as an input/output Artifact object, - a plain parameter, or a path to a file). - params (List[dict]): Params list with types converted to kubeflow spec. - expectation: Any corresponding expected errors for each - set of parameters. - """ - with expectation: - assert params == get_function_parameters(func=func) - - -@pytest.mark.parametrize( - 'func, name, description', - [ - (add, 'Add', 'This is a test'), - (sub, 'Sub', 'Test 2'), - (div, None, None) - ] -) -def test_create_pipeline_scaffold(mocker, func: Callable, name: str, description: str): - """Tests create_pipeline_scaffold, which creates a temporary pipeline - scaffold which will be used by the formalize function. - - Args: - mocker: Mocker used to patch constants to test in tempoarary - environment. - func (Callable): The python function to create a pipeline from. The - function should have type annotations for all its arguments, - indicating how it is intended to be used (e.g. as an input/output - Artifact object, a plain parameter, or a path to a file). - name (str): The name of the pipeline. - description (str): Short description of what the pipeline does. - """ - mocker.patch.object(google_cloud_automlops.utils.utils, 'CACHE_DIR', '.') - create_pipeline_scaffold(func=func, name=name, description=description) - fold = '.AutoMLOps-cache' - file_path = 'pipeline_scaffold.py' - assert os.path.exists(os.path.join(fold, file_path)) - os.remove(os.path.join(fold, file_path)) - os.rmdir(fold) - - -@pytest.mark.parametrize( - 'name, description', - [ - ('Name1', 'Description1'), - ('Name2', 'Description2'), - (None, None), - ] -) -def test_get_pipeline_decorator(name: str, description: str): - """Tests get_pipeline_decorator, which creates the kfp pipeline decorator. - - Args: - name (str): The name of the pipeline. - description (str): Short description of what the pipeline does. - """ - desc_str = f''' description='{description}',\n''' if description else '' - decorator = ( - f'''@dsl.pipeline''' - f'''(\n name='{DEFAULT_PIPELINE_NAME if not name else name}',\n''' - f'''{desc_str}''' - f''')\n''' - ) - assert decorator == get_pipeline_decorator(name=name, description=description) - - -@pytest.mark.parametrize( - 'func_name', - ['func1', 'func2'] -) -def test_get_compile_step(func_name: str): - """Tests get_compile_step, which creates the compile function call. - - Args: - func_name (str): The name of the pipeline function. - """ - assert get_compile_step(func_name=func_name) == ( - f'\n' - f'compiler.Compiler().compile(\n' - f' pipeline_func={func_name},\n' - f' package_path=pipeline_job_spec_path)\n' - f'\n' - ) - - -@pytest.mark.parametrize( - 'return_annotation, return_types, expectation', - [ - ( - NamedTuple('output', [('sum', int)]), - [{'description': None, 'name': 'sum', 'type': 'Integer'},], - does_not_raise() - ), - ( - NamedTuple('output', [('first', str), ('last', str)]), - [{'description': None, 'name': 'first', 'type': 'String'}, - {'description': None, 'name': 'last', 'type': 'String'},], - does_not_raise() - ), - ( - Optional[NamedTuple('output', [('count', int)])], - None, - pytest.raises(TypeError) - ), - ( - int, - None, - pytest.raises(TypeError) - ),( - None, - None, - pytest.raises(TypeError) - ), - ( - 'NO_ANNOTATION', - None, - does_not_raise() - ) - ] -) -def test_get_function_return_types(return_annotation, return_types: List[dict], expectation): - """Tests get_function_outputs, which returns a formatted list of - return types. - - Args: - annotation (Any): The return type to test. - return_types (List[dict]): The return type converted into the kubeflow output spec. - expectation: Any corresponding expected errors for each - set of parameters. - """ - - def func(): - ... - - if return_annotation != 'NO_ANNOTATION': - func.__annotations__ = {'return' : return_annotation} - - with expectation: - assert return_types == get_function_return_types(func=func) diff --git a/google_cloud_automlops/tests/unit/provisioning/__init__.py b/google_cloud_automlops/tests/unit/provisioning/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/provisioning/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py b/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/provisioning/gcloud/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep b/google_cloud_automlops/tests/unit/provisioning/pulumi/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep b/google_cloud_automlops/tests/unit/provisioning/terraform/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py b/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/provisioning/terraform/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/utils/__init__.py b/google_cloud_automlops/tests/unit/utils/__init__.py deleted file mode 100644 index 70d7dec..0000000 --- a/google_cloud_automlops/tests/unit/utils/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/google_cloud_automlops/tests/unit/utils/utils_test.py b/google_cloud_automlops/tests/unit/utils/utils_test.py deleted file mode 100644 index f29b797..0000000 --- a/google_cloud_automlops/tests/unit/utils/utils_test.py +++ /dev/null @@ -1,557 +0,0 @@ -# Copyright 2024 Google LLC. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Unit tests for utils module.""" - -# pylint: disable=line-too-long -# pylint: disable=missing-function-docstring - -from contextlib import nullcontext as does_not_raise -import os -import tempfile -from typing import Callable, List - -import pandas as pd -import pytest -import pytest_mock -import yaml - -import google_cloud_automlops.utils.utils -from google_cloud_automlops.utils.utils import ( - delete_file, - execute_process, - get_components_list, - get_function_source_definition, - is_component_config, - make_dirs, - read_file, - read_yaml_file, - render_jinja, - stringify_job_spec_list, - update_params, - validate_use_ci, - write_and_chmod, - write_file, - write_yaml_file -) - - -# Define simple functions to be used in tests -def func1(x): - return x + 1 - - -def func2(x, y): - return x + y - - -def func3(x, y, z): - return x + y + z - - -def func4(): - - def inner_func(): - res = 1 + 1 - return res - - return inner_func() - - -@pytest.mark.parametrize( - 'directories, existance, expectation', - [ - (['dir1', 'dir2'], [True, True], does_not_raise()), - (['dir1', 'dir1'], [True, False], does_not_raise()), - (['\0', 'dir1'], [True, True], pytest.raises(ValueError)) - ] -) -def test_make_dirs(directories: List[str], existance: List[bool], expectation): - """Tests make_dirs, which creates a list of directories if they do not - already exist. There are three test cases for this function: - 1. Expected outcome, folders created as expected. - 2. Duplicate folder names given, expecting only one folder created. - 3. Invalid folder name given, expects an error. - - Args: - directories (List[str]): List of directories to be created. - existance (List[bool]): List of booleans indicating whether the listed directories to - be created are expected to exist after invoking make_dirs. - expectation: Any corresponding expected errors for each set of parameters. - """ - with expectation: - make_dirs(directories=directories) - for directory, exist in zip(directories, existance): - assert os.path.exists(directory) == exist - if exist: - os.rmdir(directory) - - -@pytest.mark.parametrize( - 'filepath, content1, content2, expectation', - [ - ( - 'test.yaml', - {'key1': 'value1', 'key2': 'value2'}, - None, - does_not_raise() - ), - ( - 'test.yaml', - {'key1': 'value1', False: 'random stuff'}, - r'-A fails', - pytest.raises(yaml.YAMLError) - ) - ] -) -def test_read_yaml_file(filepath: str, content1: dict, content2: str, expectation): - """Tests read_yaml_file, which reads a yaml file and returns the file - contents as a dict. There are two sets of test cases for this function: - 1. Expected outcome, file read in with correct content. - 2. File to be read is not in standard yaml format, expects a yaml error. - - Args: - filepath (str): Path to yaml file to be read. - content1 (dict): First set of content to be included in the yaml at the given - file path. - content2 (str): Second set of content to be included in the yaml at the given - file path. - expectation: Any corresponding expected errors for each set of - parameters. - """ - with open(file=filepath, mode='w', encoding='utf-8') as file: - if content1: - yaml.dump(content1, file) - if content2: - yaml.dump(content2, file) - with expectation: - assert read_yaml_file(filepath=filepath) == content1 - os.remove(path=filepath) - - -@pytest.mark.parametrize( - 'filepath, mode, expectation', - [ - ('test.yaml', 'w', does_not_raise()), - ('/nonexistent/directory', 'w', pytest.raises(FileNotFoundError)), - ('test.yaml', 'r', pytest.raises(IOError)) - ] -) -def test_write_yaml(filepath: str, mode: str, expectation): - """Tests write_yaml_file, which writes a yaml file. There are three sets of - test cases for this function: - 1. Expected outcome, yaml is written correctly. - 2. Invalid file path given, expecting a FileNotFoundError. - 3. Invalid mode given, expecting an IOError. - - Args: - filepath (str): Path for yaml file to be written. - mode (str): Read/write mode to be used. - expectation: Any corresponding expected errors for each set of - parameters. - """ - contents = {'key1': 'value1', 'key2': 'value2'} - with expectation: - write_yaml_file( - filepath=filepath, - contents=contents, - mode=mode - ) - with open(file=filepath, mode='r', encoding='utf-8') as file: - assert yaml.safe_load(file) == contents - os.remove(path=filepath) - - -@pytest.mark.parametrize( - 'filepath, text, write_file_bool, expectation', - [ - ('test.txt', 'This is a text file.', True, does_not_raise()), - ('fail', '', False, pytest.raises(FileNotFoundError)) - ] -) -def test_read_file(filepath: str, text: str, write_file_bool: bool, expectation): - """Tests read_file, which reads a text file in as a string. There are two - sets of test cases for this function: - 1. Expected outcome, file is read correctly. - 2. Invalid file path given (file was not written), expecting a - FileNotFoundError. - - Args: - filepath (str): Path for file to be read from. - text (str): Text expected to be read from the given file. - write_file_bool (bool): Whether or not the file should be written for this - test case. - expectation: Any corresponding expected errors for each set of - parameters. - """ - if write_file_bool: - with open(file=filepath, mode='w', encoding='utf-8') as file: - file.write(text) - with expectation: - assert read_file(filepath=filepath) == text - if os.path.exists(filepath): - os.remove(filepath) - - -@pytest.mark.parametrize( - 'filepath, text, mode, expectation', - [ - ('test.txt', 'This is a test file.', 'w', does_not_raise()), - (15, 'This is a test file.', 'w', pytest.raises(OSError)) - ] -) -def test_write_file(filepath: str, text: str, mode: str, expectation): - """Tests write_file, which writes a string to a text file. There are two - test cases for this function: - 1. Expected outcome, file is written as expected. - 2. Invalid file path given (file was not written), expecting - an OSError. - - Args: - filepath (str): Path for file to be written. - text (str): Content to be written to the file at the given filepath. - mode (str): Read/write mode to be used. - expectation: Any corresponding expected errors for each set of - parameters. - """ - with expectation: - write_file( - filepath=filepath, - text=text, - mode=mode - ) - assert os.path.exists(filepath) - with open(file=filepath, mode='r', encoding='utf-8') as file: - assert text == file.read() - os.remove(filepath) - - -def test_write_and_chmod(): - """Tests write_and_chmod, which writes a file at the specified path - and chmods the file to allow for execution. - """ - # Create a file. - with open(file='test.txt', mode='w', encoding='utf-8') as file: - file.write('This is a test file.') - - # Call the `write_and_chmod` function. - write_and_chmod('test.txt', 'This is a test file.') - - # Assert that the file exists and is executable. - assert os.path.exists('test.txt') - assert os.access('test.txt', os.X_OK) - - # Assert that the contents of the file are correct. - with open(file='test.txt', mode='r', encoding='utf-8') as file: - contents = file.read() - assert contents == 'This is a test file.' - os.remove('test.txt') - - -@pytest.mark.parametrize( - 'file_to_delete, valid_file', - [ - ('test.txt', True), - ('fake.txt', False) - ] -) -def test_delete_file(file_to_delete: str, valid_file: bool): - """Tests delete_file, which deletes a file at the specified path. - There are two test cases for this function: - 1. Create a valid file and call delete_file, which is expected to successfully delete the file. - 2. Pass in a nonexistent file and call delete_file, which is expected to pass. - - Args: - file_to_delete (str): Name of file to delete. - valid_file (bool): Whether or not the file to delete actually exists.""" - if not valid_file: - with does_not_raise(): - delete_file(file_to_delete) - else: - with open(file=file_to_delete, mode='w', encoding='utf-8') as file: - file.write('This is a test file.') - delete_file(file_to_delete) - assert not os.path.exists(file_to_delete) - - -@pytest.mark.parametrize( - 'comp_path, comp_name, patch_cwd, expectation', - [ - (['component.yaml'], ['component'], True, does_not_raise()), - ([], [], True, does_not_raise()), - (['component.yaml'], ['component'], False, pytest.raises(FileNotFoundError)) - ] -) -def test_get_components_list(mocker: pytest_mock.MockerFixture, - comp_path: List[str], - comp_name: List[str], - patch_cwd: bool, - expectation): - """Tests get_components_list, which reads yamls in .AutoMLOps-cache directory, - verifies they are component yamls, and returns the name of the files. There - are three test cases for this function: - 1. Expected outcome, component list is pulled as expected. - 2. Verifies an empty list comes back if no YAMLs are present. - 3. Call function with a nonexistent dir, expecting OSError. - - Args: - mocker: Mocker to patch the cache directory for component files. - comp_path (List[str]): Path(s) to component yamls. - comp_name (List[str]): Name(s) of components. - patch_cwd (bool): Boolean flag indicating whether to patch the current working - directory from CACHE_DIR to root - expectation: Any corresponding expected errors for each set of - parameters. - """ - if patch_cwd: - mocker.patch.object(google_cloud_automlops.utils.utils, 'CACHE_DIR', '.') - if comp_path: - for file in comp_path: - with open(file=file, mode='w', encoding='utf-8') as f: - yaml.dump( - { - 'name': 'value1', - 'inputs': 'value2', - 'implementation': 'value3' - }, - f) - with expectation: - assert get_components_list(full_path=False) == comp_name - assert get_components_list(full_path=True) == [os.path.join('.', file) for file in comp_path] - for file in comp_path: - if os.path.exists(file): - os.remove(file) - - -@pytest.mark.parametrize( - 'yaml_contents, expectation', - [ - ( - { - 'name': 'value1', - 'inputs': 'value2', - 'implementation': 'value3' - }, - True - ), - ( - { - 'name': 'value1', - 'inputs': 'value2' - }, - False - ) - ] -) -def test_is_component_config(yaml_contents: dict, expectation: bool): - """Tests is_component_config, which which checks to see if the given file is - a component yaml. There are two test cases for this function: - 1. A valid component is given, expecting return value True. - 2. An invalid component is given, expecting return value False. - - Args: - yaml_contents (dict): Component configurations to be written to yaml file. - expected (bool): Expectation of whether or not the configuration is valid. - """ - with open(file='component.yaml', mode='w', encoding='utf-8') as f: - yaml.dump(yaml_contents, f) - assert expectation == is_component_config('component.yaml') - os.remove('component.yaml') - - -@pytest.mark.parametrize( - 'command, to_null, expectation', - [ - ('touch test.txt', False, False), - ('not a real command', False, True), - ('echo "howdy"', True, False) - ] -) -def test_execute_process(command: str, to_null: bool, expectation: bool): - """Tests execute_process, which executes an external shell process. There - are two test cases for this function: - 1. A valid command to create a file, which is expected to run successfully. - 2. An invalid command, which is expected to raise a RunTime Error. - 3. A valid command to output a string, which is expected to send output to null - - Args: - command (str): Command that is to be executed. - expectation (bool): Whether or not an error is expected to be raised. - """ - if expectation: - with pytest.raises(RuntimeError): - execute_process(command=command, to_null=to_null) - elif to_null: - assert execute_process(command=command, to_null=to_null) is None - else: - execute_process(command=command, to_null=to_null) - assert os.path.exists('test.txt') - os.remove('test.txt') - - -@pytest.mark.parametrize( - 'sch_pattern, setup_model_monitoring, use_ci, expectation', - [ - ('No Schedule Specified', False, True, does_not_raise()), - ('No Schedule Specified', False, False, does_not_raise()), - ('Schedule', False, False, pytest.raises(ValueError)), - ('Schedule', True, True, does_not_raise()), - ('Schedule', True, False, pytest.raises(ValueError)) - ] -) -def test_validate_use_ci(sch_pattern: str, - setup_model_monitoring: bool, - use_ci: bool, - expectation): - """Tests validate_use_ci, which validates the inputted schedule - parameter and the setup_model_monitoring parameter. There are - five test cases for this function, which tests each - combination of sch_pattern and setup_model_monitoring for the expected results. - - Args: - sch_pattern (str): Cron formatted value used to create a Scheduled retrain job. - setup_model_monitoring (bool): Boolean parameter which specifies whether to set - up a Vertex AI Model Monitoring Job. - use_ci (bool): Flag that determines whether to use Cloud Run CI/CD. - expectation: Any corresponding expected errors for each set of parameters. - """ - with expectation: - validate_use_ci(schedule_pattern=sch_pattern, - setup_model_monitoring=setup_model_monitoring, - use_ci=use_ci) - - -@pytest.mark.parametrize( - 'params, expected_output', - [ - ([{'name': 'param1', 'type': int}], [{'name': 'param1', 'type': 'Integer'}]), - ([{'name': 'param2', 'type': str}], [{'name': 'param2', 'type': 'String'}]), - ([{'name': 'param3', 'type': float}], [{'name': 'param3', 'type': 'Float'}]), - ([{'name': 'param4', 'type': bool}], [{'name': 'param4', 'type': 'Boolean'}]), - ([{'name': 'param5', 'type': list}], [{'name': 'param5', 'type': 'JsonArray'}]), - ([{'name': 'param6', 'type': dict}], [{'name': 'param6', 'type': 'JsonObject'}]), - ([{'name': 'param6', 'type': pd.DataFrame}], None) - ] -) -def test_update_params(params: List[dict], expected_output: List[dict]): - """Tests the update_params function, which reformats the source code type - labels as strings. There are seven test cases for this function, which test - for updating different parameter types. - - Args: - params (List[dict]): Pipeline parameters. A list of dictionaries, each param is a dict containing keys: - 'name': required, str param name. - 'type': required, python primitive type. - 'description': optional, str param desc. - expected_output (List[dict]): Expectation of whether or not the configuration is valid. - """ - if expected_output is not None: - assert expected_output == update_params(params=params) - else: - with pytest.raises(ValueError): - assert update_params(params=params) - - -@pytest.mark.parametrize( - 'func, expected_output', - [ - (func1, 'def func1(x):\n return x + 1\n'), - (func2, 'def func2(x, y):\n return x + y\n'), - (func3, 'def func3(x, y, z):\n return x + y + z\n'), - (func4, 'def func4():\n\n def inner_func():\n res = 1 + 1\n return res\n\n return inner_func()\n') - ] -) -def test_get_function_source_definition(func: Callable, expected_output: str): - """Tests get_function_source_definition, which returns a formatted string of - the source code. - - Args: - func (Callable): Function to pull source definition from. - expected_output (str): Expected source definition of the given function. - """ - assert expected_output == get_function_source_definition(func=func) - - -@pytest.mark.parametrize( - 'job_spec_list, expected_output', - [ - ([{'component_spec': 'train_model', - 'display_name': 'train-model-accelerated', - 'machine_type': 'a2-highgpu-1g', - 'accelerator_type': 'NVIDIA_TESLA_A100', - 'accelerator_count': 1}], - [{'component_spec': 'train_model', - 'spec_string': - '''{\n''' - ''' "accelerator_count": 1,\n''' - ''' "accelerator_type": "NVIDIA_TESLA_A100",\n''' - ''' "component_spec": train_model,\n''' - ''' "display_name": "train-model-accelerated",\n''' - ''' "machine_type": "a2-highgpu-1g"\n }''' - }]), - ] -) -def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: List[dict]): - """Tests the stringify_job_spec_list function, takes in a list of custom training job spec - dictionaries and turns them into strings. - - Args: - job_spec: Dictionary with job spec info. e.g. - input = [{ - 'component_spec': 'train_model', - 'display_name': 'train-model-accelerated', - 'machine_type': 'a2-highgpu-1g', - 'accelerator_type': 'NVIDIA_TESLA_A100', - 'accelerator_count': 1 - }] - expected_output (List[dict]): Dictionary with key value pair for component_spec, - and a string representation of the full dictionary e.g. - output = [{ - 'component_spec': 'train_model', - 'spec_string': '''{ - "accelerator_count": 1, - "accelerator_type": "NVIDIA_TESLA_A100", - "component_spec": train_model, - "display_name": "train-model-accelerated", - "machine_type": "a2-highgpu-1g" - }''' - }] - """ - - formatted_spec = stringify_job_spec_list(job_spec_list=job_spec_list) - assert formatted_spec == expected_output - - -@pytest.mark.parametrize( - 'template_string, template_vars, expected_output', - [ - ('Hello {{ name1 }} my name is {{ name2 }}', {'name1': 'Alice', 'name2': 'John'}, 'Hello Alice my name is John'), - ('The answer is: {{ result }}', {'result': 42}, 'The answer is: 42'), - ] -) -def test_render_jinja(template_string, template_vars, expected_output): - """Tests the render_jinja function using temporary files.""" - - with tempfile.TemporaryDirectory() as tmpdirname: # Creates temp directory - template_path = os.path.join(tmpdirname, 'template.txt.j2') - - # Write the template to the temporary file - with open(template_path, 'w', encoding='utf-8') as f: - f.write(template_string) - - # Call the render_jinja function - result = render_jinja(template_path, **template_vars) - - # Assertion - assert result == expected_output diff --git a/google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py b/tests/unit/deployments/cloudbuild/builder_test.py similarity index 100% rename from google_cloud_automlops/tests/unit/deployments/cloudbuild/builder_test.py rename to tests/unit/deployments/cloudbuild/builder_test.py diff --git a/google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py b/tests/unit/deployments/github_actions/builder_test.py similarity index 100% rename from google_cloud_automlops/tests/unit/deployments/github_actions/builder_test.py rename to tests/unit/deployments/github_actions/builder_test.py diff --git a/tests/unit/orchestration/kfp/builder_test.py b/tests/unit/orchestration/kfp/builder_test.py index 67f3485..372e078 100644 --- a/tests/unit/orchestration/kfp/builder_test.py +++ b/tests/unit/orchestration/kfp/builder_test.py @@ -17,43 +17,39 @@ # pylint: disable=missing-module-docstring import json +try: + from importlib.resources import files as import_files +except ImportError: + # Try backported to PY<37 `importlib_resources` + from importlib_resources import files as import_files import os from typing import List import pytest import pytest_mock +from google_cloud_automlops.utils.constants import ( + BASE_DIR, + GENERATED_LICENSE, + GENERATED_PARAMETER_VALUES_PATH, + KFP_TEMPLATES_PATH, + PINNED_KFP_VERSION, +) import google_cloud_automlops.orchestration.kfp.builder from google_cloud_automlops.orchestration.kfp.builder import ( build_component, build_pipeline, build_services, -<<<<<<< HEAD -======= - build_pipeline_spec_jinja, - build_components_jinja, - create_model_monitoring_job_jinja, - run_pipeline_jinja, - run_all_jinja, - publish_to_topic_jinja, - readme_jinja, - component_base_dockerfile_jinja, - component_base_task_file_jinja, - pipeline_runner_jinja, - pipeline_jinja, - pipeline_requirements_jinja, - submission_service_dockerfile_jinja, - submission_service_requirements_jinja, - submission_service_main_jinja ->>>>>>> origin/main ) import google_cloud_automlops.utils.utils from google_cloud_automlops.utils.utils import ( make_dirs, read_yaml_file, + render_jinja, write_yaml_file ) + DEFAULTS = { 'gcp': { 'artifact_repo_location': 'us-central1', @@ -346,8 +342,6 @@ def test_build_services(mocker: pytest_mock.MockerFixture, assert os.path.exists(f'{tmpdir}/services/submission_service/Dockerfile') assert os.path.exists(f'{tmpdir}/services/submission_service/requirements.txt') assert os.path.exists(f'{tmpdir}/services/submission_service/main.py') -<<<<<<< HEAD -======= @pytest.mark.parametrize( @@ -365,7 +359,11 @@ def test_build_pipeline_spec_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - build_pipeline_spec_script = build_pipeline_spec_jinja() + build_pipeline_spec_script = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_pipeline_spec.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) for snippet in expected_output_snippets: if is_included: @@ -389,7 +387,11 @@ def test_build_components_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - build_components_script = build_components_jinja() + build_components_script = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'build_components.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) for snippet in expected_output_snippets: if is_included: @@ -413,7 +415,11 @@ def test_run_pipeline_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - run_pipeline_script = run_pipeline_jinja() + run_pipeline_script = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_pipeline.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) for snippet in expected_output_snippets: if is_included: @@ -438,7 +444,11 @@ def test_run_all_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - run_all_script = run_all_jinja() + run_all_script = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'run_all.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) for snippet in expected_output_snippets: if is_included: @@ -464,7 +474,13 @@ def test_publish_to_topic_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - publish_to_topic_script = publish_to_topic_jinja(pubsub_topic_name=pubsub_topic_name) + publish_to_topic_script = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'publish_to_topic.sh.j2', + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE, + generated_parameter_values_path=GENERATED_PARAMETER_VALUES_PATH, + pubsub_topic_name=pubsub_topic_name + ) for snippet in expected_output_snippets: if is_included: @@ -489,7 +505,11 @@ def test_create_model_monitoring_job_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - create_model_monitoring_job_script = create_model_monitoring_job_jinja() + create_model_monitoring_job_script = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.scripts') / 'create_model_monitoring_job.sh.j2', + generated_license=GENERATED_LICENSE, + base_dir=BASE_DIR + ) for snippet in expected_output_snippets: if is_included: @@ -553,7 +573,11 @@ def test_readme_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - readme_str = readme_jinja(setup_model_monitoring=setup_model_monitoring, use_ci=use_ci) + readme_str = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH) / 'README.md.j2', + setup_model_monitoring=setup_model_monitoring, + use_ci=use_ci + ) for snippet in expected_output_snippets: if is_included: @@ -580,7 +604,11 @@ def test_component_base_dockerfile_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - component_base_dockerfile = component_base_dockerfile_jinja(base_image) + component_base_dockerfile = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base') / 'Dockerfile.j2', + base_image=base_image, + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -631,7 +659,11 @@ def test_component_base_task_file_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - component_base_task_file = component_base_task_file_jinja(custom_code_contents, kfp_spec_bool) + component_base_task_file = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.components.component_base.src') / 'task.py.j2', + custom_code_contents=custom_code_contents, + generated_license=GENERATED_LICENSE, + kfp_spec_bool=kfp_spec_bool) for snippet in expected_output_snippets: if is_included: @@ -655,7 +687,10 @@ def test_pipeline_runner_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - pipeline_runner_py = pipeline_runner_jinja() + pipeline_runner_py = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline_runner.py.j2', + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -730,11 +765,13 @@ def test_pipeline_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - pipeline_py = pipeline_jinja( - components_list, - custom_training_job_specs, - pipeline_scaffold_contents, - project_id) + pipeline_py = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'pipeline.py.j2', + components_list=components_list, + custom_training_job_specs=custom_training_job_specs, + generated_license=GENERATED_LICENSE, + pipeline_scaffold_contents=pipeline_scaffold_contents, + project_id=project_id) for snippet in expected_output_snippets: if is_included: @@ -758,7 +795,10 @@ def test_pipeline_requirements_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - pipeline_requirements_py = pipeline_requirements_jinja() + pipeline_requirements_py = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.pipelines') / 'requirements.txt.j2', + pinned_kfp_version=PINNED_KFP_VERSION + ) for snippet in expected_output_snippets: if is_included: @@ -783,7 +823,11 @@ def test_submission_service_dockerfile_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - submission_service_dockerfile = submission_service_dockerfile_jinja() + submission_service_dockerfile = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'Dockerfile.j2', + base_dir=BASE_DIR, + generated_license=GENERATED_LICENSE + ) for snippet in expected_output_snippets: if is_included: @@ -816,7 +860,11 @@ def test_submission_service_requirements_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - submission_service_requirements = submission_service_requirements_jinja(pipeline_job_submission_service_type=pipeline_job_submission_service_type) + submission_service_requirements = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'requirements.txt.j2', + pinned_kfp_version=PINNED_KFP_VERSION, + pipeline_job_submission_service_type=pipeline_job_submission_service_type + ) for snippet in expected_output_snippets: if is_included: @@ -913,7 +961,9 @@ def test_submission_service_main_jinja( is_included: Boolean that determines whether to check if the expected_output_snippets exist in the string or not. expected_output_snippets: Strings that are expected to be included (or not) based on the is_included boolean. """ - submission_service_main_py = submission_service_main_jinja( + submission_service_main_py = render_jinja( + template_path=import_files(KFP_TEMPLATES_PATH + '.services.submission_service') / 'main.py.j2', + generated_license=GENERATED_LICENSE, naming_prefix=naming_prefix, pipeline_root=pipeline_root, pipeline_job_runner_service_account=pipeline_job_runner_service_account, @@ -926,4 +976,3 @@ def test_submission_service_main_jinja( assert snippet in submission_service_main_py elif not is_included: assert snippet not in submission_service_main_py ->>>>>>> origin/main diff --git a/google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py b/tests/unit/provisioning/gcloud/builder_test.py similarity index 100% rename from google_cloud_automlops/tests/unit/provisioning/gcloud/builder_test.py rename to tests/unit/provisioning/gcloud/builder_test.py diff --git a/google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py b/tests/unit/provisioning/terraform/builder_test.py similarity index 100% rename from google_cloud_automlops/tests/unit/provisioning/terraform/builder_test.py rename to tests/unit/provisioning/terraform/builder_test.py diff --git a/tests/unit/utils/utils_test.py b/tests/unit/utils/utils_test.py index 5e09964..f29b797 100644 --- a/tests/unit/utils/utils_test.py +++ b/tests/unit/utils/utils_test.py @@ -532,6 +532,7 @@ def test_stringify_job_spec_list(job_spec_list: List[dict], expected_output: Lis formatted_spec = stringify_job_spec_list(job_spec_list=job_spec_list) assert formatted_spec == expected_output + @pytest.mark.parametrize( 'template_string, template_vars, expected_output', [