diff --git a/.pipelines/diabetes_regression-batchscoring-ci.yml b/.pipelines/diabetes_regression-batchscoring-ci.yml index 79a7f46f..1392fddb 100644 --- a/.pipelines/diabetes_regression-batchscoring-ci.yml +++ b/.pipelines/diabetes_regression-batchscoring-ci.yml @@ -1,12 +1,27 @@ # Continuous Integration (CI) pipeline that orchestrates the batch scoring of the diabetes_regression model. +# Runtime parameters to select artifacts +parameters: +- name : artifactBuildId + displayName: Model Train CI Build ID. Default is 'latest'. + type: string + default: latest + +pr: none + +# Trigger this pipeline on model-train pipeline completion resources: containers: - container: mlops image: mcr.microsoft.com/mlops/python:latest - + pipelines: + - pipeline: model-train-ci + source: Model-Train-Register-CI # Name of the triggering pipeline + trigger: + branches: + include: + - master -pr: none trigger: branches: include: @@ -34,7 +49,13 @@ stages: timeoutInMinutes: 0 steps: - template: code-quality-template.yml + - template: diabetes_regression-get-model-id-artifact-template.yml + parameters: + projectId: '$(resources.pipeline.model-train-ci.projectID)' + pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' + artifactBuildId: ${{ parameters.artifactBuildId }} - task: AzureCLI@1 + displayName: "Publish Batch Scoring Pipeline" name: publish_batchscore inputs: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' @@ -45,14 +66,18 @@ stages: export SUBSCRIPTION_ID=$(az account show --query id -o tsv) # Invoke the Python building and publishing a training pipeline python -m ml_service.pipelines.diabetes_regression_build_parallel_batchscore_pipeline - + env: + SCORING_DATASTORE_ACCESS_KEY: $(SCORING_DATASTORE_ACCESS_KEY) + - job: "Run_Batch_Score_Pipeline" displayName: "Run Batch Scoring Pipeline" - dependsOn: "Build_Batch_Scoring_Pipeline" + dependsOn: ["Build_Batch_Scoring_Pipeline"] timeoutInMinutes: 240 pool: server variables: pipeline_id: $[ dependencies.Build_Batch_Scoring_Pipeline.outputs['publish_batchscore.pipeline_id']] + model_name: $[ dependencies.Build_Batch_Scoring_Pipeline.outputs['get_model.MODEL_NAME']] + model_version: $[ dependencies.Build_Batch_Scoring_Pipeline.outputs['get_model.MODEL_VERSION']] steps: - task: ms-air-aiagility.vss-services-azureml.azureml-restApi-task.MLPublishedPipelineRestAPITask@0 displayName: 'Invoke Batch Scoring pipeline' @@ -60,5 +85,5 @@ stages: azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' PipelineId: '$(pipeline_id)' ExperimentName: '$(EXPERIMENT_NAME)' - PipelineParameters: '"ParameterAssignments": {"model_name": "$(MODEL_NAME)"}' + PipelineParameters: '"ParameterAssignments": {"model_name": "$(model_name)", "model_version": "$(model_version)"}' \ No newline at end of file diff --git a/.pipelines/diabetes_regression-cd.yml b/.pipelines/diabetes_regression-cd.yml new file mode 100644 index 00000000..16bc4724 --- /dev/null +++ b/.pipelines/diabetes_regression-cd.yml @@ -0,0 +1,161 @@ +# Continuous Integration (CI) pipeline that orchestrates the deployment of the diabetes_regression model. + +# Runtime parameters to select artifacts +parameters: +- name : artifactBuildId + displayName: Model Train CI Build ID. Default is 'latest'. + type: string + default: latest + +pr: none + +# Trigger this pipeline on model-train pipeline completion +trigger: none +resources: + containers: + - container: mlops + image: mcr.microsoft.com/mlops/python:latest + pipelines: + - pipeline: model-train-ci + source: Model-Train-Register-CI # Name of the triggering pipeline + trigger: + branches: + include: + - master + +variables: +- template: diabetes_regression-variables-template.yml +- group: devopsforai-aml-vg + +stages: +- stage: 'Deploy_ACI' + displayName: 'Deploy to ACI' + condition: variables['ACI_DEPLOYMENT_NAME'] + jobs: + - job: "Deploy_ACI" + displayName: "Deploy to ACI" + container: mlops + timeoutInMinutes: 0 + steps: + - download: none + - template: diabetes_regression-get-model-id-artifact-template.yml + parameters: + projectId: '$(resources.pipeline.model-train-ci.projectID)' + pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' + artifactBuildId: ${{ parameters.artifactBuildId }} + - task: AzureCLI@1 + displayName: 'Install AzureML CLI' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: 'az extension add -n azure-cli-ml' + - task: AzureCLI@1 + displayName: "Deploy to ACI (CLI)" + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring + inlineScript: | + set -e # fail on error + + az ml model deploy --name $(ACI_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(MODEL_VERSION)' \ + --ic inference_config.yml \ + --dc deployment_config_aci.yml \ + -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \ + --overwrite -v + - task: AzureCLI@1 + displayName: 'Smoke test' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)" + +- stage: 'Deploy_AKS' + displayName: 'Deploy to AKS' + dependsOn: Deploy_ACI + condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME']) + jobs: + - job: "Deploy_AKS" + displayName: "Deploy to AKS" + container: mlops + timeoutInMinutes: 0 + steps: + - template: diabetes_regression-get-model-id-artifact-template.yml + parameters: + projectId: '$(resources.pipeline.model-train-ci.projectID)' + pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' + artifactBuildId: ${{ parameters.artifactBuildId }} + - task: AzureCLI@1 + displayName: 'Install AzureML CLI' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: 'az extension add -n azure-cli-ml' + - task: AzureCLI@1 + displayName: "Deploy to AKS (CLI)" + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring + inlineScript: | + set -e # fail on error + + az ml model deploy --name $(AKS_DEPLOYMENT_NAME) --model '$(MODEL_NAME):$(MODEL_VERSION)' \ + --compute-target $(AKS_COMPUTE_NAME) \ + --ic inference_config.yml \ + --dc deployment_config_aks.yml \ + -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) \ + --overwrite -v + - task: AzureCLI@1 + displayName: 'Smoke test' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)" + +- stage: 'Deploy_Webapp' + displayName: 'Deploy to Webapp' + condition: variables['WEBAPP_DEPLOYMENT_NAME'] + jobs: + - job: "Deploy_Webapp" + displayName: "Package and deploy model" + container: mlops + timeoutInMinutes: 0 + steps: + - template: diabetes_regression-get-model-id-artifact-template.yml + parameters: + projectId: '$(resources.pipeline.model-train-ci.projectID)' + pipelineId: '$(resources.pipeline.model-train-ci.pipelineID)' + artifactBuildId: ${{ parameters.artifactBuildId }} + - template: diabetes_regression-package-model-template.yml + parameters: + modelId: $(MODEL_NAME):$(MODEL_VERSION) + scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/score.py' + condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml' + - script: echo $(IMAGE_LOCATION) >image_location.txt + displayName: "Write image location file" + - task: AzureWebAppContainer@1 + name: WebAppDeploy + displayName: 'Azure Web App on Container Deploy' + inputs: + azureSubscription: '$(AZURE_RM_SVC_CONNECTION)' + appName: '$(WEBAPP_DEPLOYMENT_NAME)' + resourceGroupName: '$(RESOURCE_GROUP)' + imageName: '$(IMAGE_LOCATION)' + - task: AzureCLI@1 + displayName: 'Smoke test' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e # fail on error + export SUBSCRIPTION_ID=$(az account show --query id -o tsv) + python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score" diff --git a/.pipelines/diabetes_regression-ci-image.yml b/.pipelines/diabetes_regression-ci-image.yml index 6282fd31..d7c925bf 100644 --- a/.pipelines/diabetes_regression-ci-image.yml +++ b/.pipelines/diabetes_regression-ci-image.yml @@ -30,14 +30,9 @@ variables: value: 'scoring/scoreB.py' steps: -- task: AzureCLI@1 - inputs: - azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' - scriptLocation: inlineScript - workingDirectory: $(Build.SourcesDirectory) - inlineScript: | - set -e - export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python3 -m ml_service.util.create_scoring_image - displayName: 'Create Scoring Image' +- template: diabetes_regression-package-model-template.yml + parameters: + modelId: $(MODEL_NAME):$(MODEL_VERSION) + scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/$(SCORE_SCRIPT)' + condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml' diff --git a/.pipelines/diabetes_regression-ci.yml b/.pipelines/diabetes_regression-ci.yml index 56258d50..5a539af0 100644 --- a/.pipelines/diabetes_regression-ci.yml +++ b/.pipelines/diabetes_regression-ci.yml @@ -1,4 +1,4 @@ -# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, registration, deployment, and testing of the diabetes_regression model. +# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model. resources: containers: @@ -27,7 +27,6 @@ pool: stages: - stage: 'Model_CI' displayName: 'Model CI' - condition: not(variables['MODEL_BUILD_ID']) jobs: - job: "Model_CI_Pipeline" displayName: "Model CI Pipeline" @@ -48,8 +47,8 @@ stages: displayName: 'Publish Azure Machine Learning Pipeline' - stage: 'Trigger_AML_Pipeline' - displayName: 'Train model' - condition: and(succeeded(), not(variables['MODEL_BUILD_ID'])) + displayName: 'Train and evaluate model' + condition: succeeded() variables: BUILD_URI: '$(SYSTEM.COLLECTIONURI)$(SYSTEM.TEAMPROJECT)/_build/results?buildId=$(BUILD.BUILDID)' jobs: @@ -91,116 +90,8 @@ stages: - job: "Training_Run_Report" dependsOn: "Run_ML_Pipeline" condition: always() - displayName: "Determine if evaluation succeeded and new model is registered" + displayName: "Publish artifact if new model was registered" container: mlops timeoutInMinutes: 0 steps: - - template: diabetes_regression-get-model-version-template.yml - -- stage: 'Deploy_ACI' - displayName: 'Deploy to ACI' - dependsOn: Trigger_AML_Pipeline - condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['ACI_DEPLOYMENT_NAME']) - jobs: - - job: "Deploy_ACI" - displayName: "Deploy to ACI" - container: mlops - timeoutInMinutes: 0 - steps: - - template: diabetes_regression-get-model-version-template.yml - - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 - displayName: 'Azure ML Model Deploy' - inputs: - azureSubscription: $(WORKSPACE_SVC_CONNECTION) - modelSourceType: manualSpec - modelName: '$(MODEL_NAME)' - modelVersion: $(MODEL_VERSION) - inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml' - deploymentTarget: ACI - deploymentName: $(ACI_DEPLOYMENT_NAME) - deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aci.yml' - overwriteExistingDeployment: true - - task: AzureCLI@1 - displayName: 'Smoke test' - inputs: - azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' - scriptLocation: inlineScript - inlineScript: | - set -e # fail on error - export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)" - -- stage: 'Deploy_AKS' - displayName: 'Deploy to AKS' - dependsOn: Deploy_ACI - condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME']) - jobs: - - job: "Deploy_AKS" - displayName: "Deploy to AKS" - container: mlops - timeoutInMinutes: 0 - steps: - - template: diabetes_regression-get-model-version-template.yml - - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0 - displayName: 'Azure ML Model Deploy' - inputs: - azureSubscription: $(WORKSPACE_SVC_CONNECTION) - modelSourceType: manualSpec - modelName: '$(MODEL_NAME)' - modelVersion: $(MODEL_VERSION) - inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml' - deploymentTarget: AKS - aksCluster: $(AKS_COMPUTE_NAME) - deploymentName: $(AKS_DEPLOYMENT_NAME) - deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aks.yml' - overwriteExistingDeployment: true - - task: AzureCLI@1 - displayName: 'Smoke test' - inputs: - azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' - scriptLocation: inlineScript - inlineScript: | - set -e # fail on error - export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)" - -- stage: 'Deploy_Webapp' - displayName: 'Deploy to Webapp' - dependsOn: Trigger_AML_Pipeline - condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['WEBAPP_DEPLOYMENT_NAME']) - jobs: - - job: "Deploy_Webapp" - displayName: "Deploy to Webapp" - container: mlops - timeoutInMinutes: 0 - steps: - - template: diabetes_regression-get-model-version-template.yml - - task: AzureCLI@1 - displayName: 'Create scoring image and set IMAGE_LOCATION variable' - inputs: - azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' - scriptLocation: inlineScript - inlineScript: | - set -e # fail on error - export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt - # Output image location to Azure DevOps job - IMAGE_LOCATION="$(cat image_location.txt)" - echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" - - task: AzureWebAppContainer@1 - name: WebAppDeploy - displayName: 'Azure Web App on Container Deploy' - inputs: - azureSubscription: '$(AZURE_RM_SVC_CONNECTION)' - appName: '$(WEBAPP_DEPLOYMENT_NAME)' - resourceGroupName: '$(RESOURCE_GROUP)' - imageName: '$(IMAGE_LOCATION)' - - task: AzureCLI@1 - displayName: 'Smoke test' - inputs: - azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' - scriptLocation: inlineScript - inlineScript: | - set -e # fail on error - export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score" + - template: diabetes_regression-publish-model-artifact-template.yml diff --git a/.pipelines/diabetes_regression-get-model-id-artifact-template.yml b/.pipelines/diabetes_regression-get-model-id-artifact-template.yml new file mode 100644 index 00000000..b9e61306 --- /dev/null +++ b/.pipelines/diabetes_regression-get-model-id-artifact-template.yml @@ -0,0 +1,48 @@ +# Pipeline template that gets the model name and version from a previous build's artifact + +parameters: +- name: projectId + type: string + default: '' +- name: pipelineId + type: string + default: '' +- name: artifactBuildId + type: string + default: latest + +steps: + - download: none + - task: DownloadPipelineArtifact@2 + displayName: Download Pipeline Artifacts + inputs: + source: 'specific' + project: '${{ parameters.projectId }}' + pipeline: '${{ parameters.pipelineId }}' + preferTriggeringPipeline: true + ${{ if eq(parameters.artifactBuildId, 'latest') }}: + buildVersionToDownload: 'latestFromBranch' + ${{ if ne(parameters.artifactBuildId, 'latest') }}: + buildVersionToDownload: 'specific' + runId: '${{ parameters.artifactBuildId }}' + runBranch: '$(Build.SourceBranch)' + path: $(Build.SourcesDirectory)/bin + - task: Bash@3 + name: get_model + displayName: Parse Json for Model Name and Version + inputs: + targetType: 'inline' + script: | + # Print JSON + cat $(Build.SourcesDirectory)/bin/model/model.json | jq '.' + + # Set model name and version variables + MODEL_NAME=$(jq -r '.name' <$(Build.SourcesDirectory)/bin/model/model.json) + MODEL_VERSION=$(jq -r '.version' <$(Build.SourcesDirectory)/bin/model/model.json) + + echo "Model Name: $MODEL_NAME" + echo "Model Version: $MODEL_VERSION" + + # Set environment variables + echo "##vso[task.setvariable variable=MODEL_VERSION;isOutput=true]$MODEL_VERSION" + echo "##vso[task.setvariable variable=MODEL_NAME;isOutput=true]$MODEL_NAME" diff --git a/.pipelines/diabetes_regression-get-model-version-template.yml b/.pipelines/diabetes_regression-get-model-version-template.yml deleted file mode 100644 index 870985a6..00000000 --- a/.pipelines/diabetes_regression-get-model-version-template.yml +++ /dev/null @@ -1,15 +0,0 @@ -# Pipeline template that attempts to get the latest model version and adds it to the environment for subsequent tasks to use. -steps: -- task: AzureCLI@1 - inputs: - azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' - scriptLocation: inlineScript - inlineScript: | - set -e # fail on error - export SUBSCRIPTION_ID=$(az account show --query id -o tsv) - python -m ml_service.pipelines.diabetes_regression_verify_train_pipeline --build_id $(modelbuildid) --output_model_version_file "model_version.txt" - # Output model version to Azure DevOps job - MODEL_VERSION="$(cat model_version.txt)" - echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION" - name: 'getversion' - displayName: "Determine if evaluation succeeded and new model is registered" diff --git a/.pipelines/diabetes_regression-package-model-template.yml b/.pipelines/diabetes_regression-package-model-template.yml new file mode 100644 index 00000000..7725b19c --- /dev/null +++ b/.pipelines/diabetes_regression-package-model-template.yml @@ -0,0 +1,42 @@ +# Pipeline template that creates a model package and adds the package location to the environment for subsequent tasks to use. +parameters: +- name: modelId + type: string + default: '' +- name: scoringScriptPath + type: string + default: '' +- name: condaFilePath + type: string + default: '' + +steps: + - task: AzureCLI@1 + displayName: 'Install AzureML CLI' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: 'az extension add -n azure-cli-ml' + - task: AzureCLI@1 + displayName: 'Create model package and set IMAGE_LOCATION variable' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + inlineScript: | + set -e # fail on error + + # Create model package using CLI + az ml model package --workspace-name $(WORKSPACE_NAME) -g $(RESOURCE_GROUP) \ + --model '${{ parameters.modelId }}' \ + --entry-script '${{ parameters.scoringScriptPath }}' \ + --cf '${{ parameters.condaFilePath }}' \ + -v \ + --rt python --query 'location' -o tsv > image_logs.txt + + # Show logs + cat image_logs.txt + + # Set environment variable using the last line of logs that has the package location + IMAGE_LOCATION=$(tail -n 1 image_logs.txt) + echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION" diff --git a/.pipelines/diabetes_regression-publish-model-artifact-template.yml b/.pipelines/diabetes_regression-publish-model-artifact-template.yml new file mode 100644 index 00000000..00e45105 --- /dev/null +++ b/.pipelines/diabetes_regression-publish-model-artifact-template.yml @@ -0,0 +1,29 @@ +# Pipeline template to check if a model was registered for the build and publishes an artifact with the model JSON +steps: +- task: AzureCLI@1 + displayName: 'Install AzureML CLI' + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: 'az extension add -n azure-cli-ml' +- task: AzureCLI@1 + inputs: + azureSubscription: '$(WORKSPACE_SVC_CONNECTION)' + scriptLocation: inlineScript + workingDirectory: $(Build.SourcesDirectory) + inlineScript: | + set -e # fail on error + + # Get the model using the build ID tag + FOUND_MODEL=$(az ml model list -g $(RESOURCE_GROUP) --workspace-name $(WORKSPACE_NAME) --tag BuildId=$(Build.BuildId) --query '[0]') + + # If the variable is empty, print and fail + [[ -z "$FOUND_MODEL" ]] && { echo "Model was not registered for this run." ; exit 1; } + + # Write to a file + echo $FOUND_MODEL >model.json + name: 'getversion' + displayName: "Determine if evaluation succeeded and new model is registered (CLI)" +- publish: model.json + artifact: model diff --git a/.pipelines/diabetes_regression-variables-template.yml b/.pipelines/diabetes_regression-variables-template.yml index cf4ef9cf..502753fb 100644 --- a/.pipelines/diabetes_regression-variables-template.yml +++ b/.pipelines/diabetes_regression-variables-template.yml @@ -65,10 +65,6 @@ variables: # - name: ALLOW_RUN_CANCEL # value: "true" - # For debugging deployment issues. Specify a build id with the MODEL_BUILD_ID pipeline variable at queue time - # to skip training and deploy a model registered by a previous build. - - name: modelbuildid - value: $[coalesce(variables['MODEL_BUILD_ID'], variables['Build.BuildId'])] # Flag to allow rebuilding the AML Environment after it was built for the first time. This enables dependency updates from conda_dependencies.yaml. # - name: AML_REBUILD_ENVIRONMENT # value: "false" diff --git a/bootstrap/bootstrap.py b/bootstrap/bootstrap.py index cbc49e18..9e52af55 100644 --- a/bootstrap/bootstrap.py +++ b/bootstrap/bootstrap.py @@ -84,11 +84,13 @@ def replace_project_name(project_dir, project_name, rename_name): files = [r".env.example", r".pipelines/code-quality-template.yml", r".pipelines/pr.yml", + r".pipelines/diabetes_regression-cd.yml", r".pipelines/diabetes_regression-ci.yml", r".pipelines/abtest.yml", r".pipelines/diabetes_regression-ci-image.yml", + r".pipelines/diabetes_regression-publish-model-artifact-template.yml", # NOQA: E501 + r".pipelines/diabetes_regression-get-model-id-artifact-template.yml", # NOQA: E501 r".pipelines/diabetes_regression-batchscoring-ci.yml", - r".pipelines/diabetes_regression-get-model-version-template.yml", # NOQA: E501 r".pipelines/diabetes_regression-variables-template.yml", r"environment_setup/Dockerfile", r"environment_setup/install_requirements.sh", diff --git a/diabetes_regression/ci_dependencies.yml b/diabetes_regression/ci_dependencies.yml index 52bdeb44..c54f3e32 100644 --- a/diabetes_regression/ci_dependencies.yml +++ b/diabetes_regression/ci_dependencies.yml @@ -12,6 +12,7 @@ dependencies: - r=3.6.0 - r-essentials=3.6.0 + - conda-forge::jq - pip=20.0.* - pip: diff --git a/diabetes_regression/evaluate/evaluate_model.py b/diabetes_regression/evaluate/evaluate_model.py index 125a16a5..5a69addb 100644 --- a/diabetes_regression/evaluate/evaluate_model.py +++ b/diabetes_regression/evaluate/evaluate_model.py @@ -26,7 +26,7 @@ from azureml.core import Run import argparse import traceback -from util.model_helper import get_latest_model +from util.model_helper import get_model run = Run.get_context() @@ -45,7 +45,7 @@ # sources_dir = 'diabetes_regression' # path_to_util = os.path.join(".", sources_dir, "util") # sys.path.append(os.path.abspath(path_to_util)) # NOQA: E402 -# from model_helper import get_latest_model +# from model_helper import get_model # workspace_name = os.environ.get("WORKSPACE_NAME") # experiment_name = os.environ.get("EXPERIMENT_NAME") # resource_group = os.environ.get("RESOURCE_GROUP") @@ -108,8 +108,11 @@ firstRegistration = False tag_name = 'experiment_name' - model = get_latest_model( - model_name, tag_name, exp.name, ws) + model = get_model( + model_name=model_name, + tag_name=tag_name, + tag_value=exp.name, + aml_workspace=ws) if (model is not None): production_model_mse = 10000 diff --git a/diabetes_regression/scoring/parallel_batchscore.py b/diabetes_regression/scoring/parallel_batchscore.py index aef6f3fb..cd42c79c 100644 --- a/diabetes_regression/scoring/parallel_batchscore.py +++ b/diabetes_regression/scoring/parallel_batchscore.py @@ -29,7 +29,8 @@ import joblib import sys from typing import List -from util.model_helper import get_latest_model +from util.model_helper import get_model +from azureml.core import Model model = None @@ -59,6 +60,18 @@ def parse_args() -> List[str]: model_name = model_name_param[0][1] + model_version_param = [ + (sys.argv[idx], sys.argv[idx + 1]) + for idx, itm in enumerate(sys.argv) + if itm == "--model_version" + ] + model_version = ( + None + if len(model_version_param) < 1 + or len(model_version_param[0][1].strip()) == 0 # NOQA: E501 + else model_version_param[0][1] + ) + model_tag_name_param = [ (sys.argv[idx], sys.argv[idx + 1]) for idx, itm in enumerate(sys.argv) @@ -83,7 +96,7 @@ def parse_args() -> List[str]: else model_tag_value_param[0][1] ) - return [model_name, model_tag_name, model_tag_value] + return [model_name, model_version, model_tag_name, model_tag_value] def init(): @@ -94,13 +107,18 @@ def init(): try: print("Initializing batch scoring script...") + # Get the model using name/version/tags filter model_filter = parse_args() - amlmodel = get_latest_model( - model_filter[0], model_filter[1], model_filter[2] - ) # NOQA: E501 + amlmodel = get_model( + model_name=model_filter[0], + model_version=model_filter[1], + tag_name=model_filter[2], + tag_value=model_filter[3]) + # Load the model using name/version found global model - modelpath = amlmodel.get_model_path(model_name=model_filter[0]) + modelpath = Model.get_model_path( + model_name=amlmodel.name, version=amlmodel.version) model = joblib.load(modelpath) print("Loaded model {}".format(model_filter[0])) except Exception as ex: diff --git a/diabetes_regression/util/model_helper.py b/diabetes_regression/util/model_helper.py index ceceff41..f90237e5 100644 --- a/diabetes_regression/util/model_helper.py +++ b/diabetes_regression/util/model_helper.py @@ -22,8 +22,9 @@ def get_current_workspace() -> Workspace: return experiment.workspace -def get_latest_model( +def get_model( model_name: str, + model_version: int = None, # If none, return latest model tag_name: str = None, tag_value: str = None, aml_workspace: Workspace = None @@ -35,53 +36,29 @@ def get_latest_model( Parameters: aml_workspace (Workspace): aml.core Workspace that the model lives. model_name (str): name of the model we are looking for + (optional) model_version (str): model version. Latest if not provided. (optional) tag (str): the tag value & name the model was registered under. Return: A single aml model from the workspace that matches the name and tag. """ - try: - # Validate params. cannot be None. - if model_name is None: - raise ValueError("model_name[:str] is required") + if aml_workspace is None: + print("No workspace defined - using current experiment workspace.") + aml_workspace = get_current_workspace() - if aml_workspace is None: - print("No workspace defined - using current experiment workspace.") - aml_workspace = get_current_workspace() - - model_list = None - tag_ext = "" - - # Get lastest model - # True: by name and tags - if tag_name is not None and tag_value is not None: - model_list = AMLModel.list( - aml_workspace, name=model_name, - tags=[[tag_name, tag_value]], latest=True - ) - tag_ext = f"tag_name: {tag_name}, tag_value: {tag_value}." - # False: Only by name - else: - model_list = AMLModel.list( - aml_workspace, name=model_name, latest=True) - - # latest should only return 1 model, but if it does, - # then maybe sdk or source code changed. - - # define the error messages - too_many_model_message = ("Found more than one latest model. " - f"Models found: {model_list}. " - f"{tag_ext}") - - no_model_found_message = (f"No Model found with name: {model_name}. " - f"{tag_ext}") - - if len(model_list) > 1: - raise ValueError(too_many_model_message) - if len(model_list) == 1: - return model_list[0] - else: - print(no_model_found_message) - return None - except Exception: - raise + if tag_name is not None and tag_value is not None: + model = AMLModel( + aml_workspace, + name=model_name, + version=model_version, + tags=[[tag_name, tag_value]]) + elif (tag_name is None and tag_value is not None) or ( + tag_value is None and tag_name is not None + ): + raise ValueError( + "model_tag_name and model_tag_value should both be supplied" + + "or excluded" # NOQA: E501 + ) + else: + model = AMLModel(aml_workspace, name=model_name, version=model_version) # NOQA: E501 + return model diff --git a/docs/getting_started.md b/docs/getting_started.md index cba7e424..c3abed02 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -140,30 +140,31 @@ Create a new service connection to your Azure ML Workspace using the [Machine Le **Note:** Similar to the Azure Resource Manager service connection you created earlier, creating a service connection with Azure Machine Learning workspace scope requires 'Owner' or 'User Access Administrator' permissions on the Workspace. You'll need sufficient permissions to register an application with your Azure AD tenant, or you can get the ID and secret of a service principal from your Azure AD Administrator. That principal must have Contributor permissions on the Azure ML Workspace. -## Set up Build, Release Trigger, and Release Multi-Stage Pipeline +## Set up Build, Release Trigger, and Release Multi-Stage Pipelines -Now that you've provisioned all the required Azure resources and service connections, you can set up the pipelines for deploying your machine learning model to production. +Now that you've provisioned all the required Azure resources and service connections, you can set up the pipelines for training (CI) and deploying (CD) your machine learning model to production. Additionally, you can set up a pipeline for batch scoring. -**There are two main Azure pipelines - one to handle model training and another to handle batch scoring of the model.** - -### **Azure [pipeline](../.pipelines/diabetes_regression-ci.yml) for model training and deployment** -This pipeline has a sequence of stages for: - -1. **Model Code Continuous Integration:** triggered on code changes to master branch on GitHub. Runs linting, unit tests, code coverage and publishes a training pipeline. -1. **Train Model**: invokes the Azure ML service to trigger the published training pipeline to train, evaluate, and register a model. -1. **Release Deployment:** deploys a model to either [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/), [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service), or [Azure App Service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-app-service) environments. For simplicity, you're going to initially focus on Azure Container Instances. See [Further Exploration](#further-exploration) for other deployment types. +1. **Model CI, training, evaluation, and registration** - triggered on code changes to master branch on GitHub. Runs linting, unit tests, code coverage, and publishes and runs the training pipeline. If a new model is registered after evaluation, it creates a build artifact containing the JSON metadata of the model. Definition: [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml). +1. **Release deployment** - consumes the artifact of the previous pipeline and deploys a model to either [Azure Container Instances (ACI)](https://azure.microsoft.com/en-us/services/container-instances/), [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/services/kubernetes-service), or [Azure App Service](https://docs.microsoft.com/en-us/azure/machine-learning/service/how-to-deploy-app-service) environments. See [Further Exploration](#further-exploration) for other deployment types. Definition: [diabetes_regression-cd.yml](../.pipelines/diabetes_regression-cd.yml). 1. **Note:** Edit the pipeline definition to remove unused stages. For example, if you're deploying to Azure Container Instances and Azure Kubernetes Service only, delete the unused `Deploy_Webapp` stage. +1. **Batch Scoring Code Continuous Integration** - consumes the artifact of the model training pipeline. Runs linting, unit tests, code coverage, publishes a batch scoring pipeline, and invokes the published batch scoring pipeline to score a model. + +These pipelines use a Docker container on the Azure Pipelines agents to accomplish the pipeline steps. The container image ***mcr.microsoft.com/mlops/python:latest*** is built with [this Dockerfile](../environment_setup/Dockerfile) and has all the necessary dependencies installed for MLOpsPython and ***diabetes_regression***. This image is an example of a custom Docker image with a pre-baked environment. The environment is guaranteed to be the same on any building agent, VM, or local machine. In your project, you'll want to build your own Docker image that only contains the dependencies and tools required for your use case. Your image will probably be smaller and faster, and it will be maintained by your team. -### Set up the Training Pipeline +### Set up the Model CI, training, evaluation, and registration pipeline In your Azure DevOps project, create and run a new build pipeline based on the [diabetes_regression-ci.yml](../.pipelines/diabetes_regression-ci.yml) pipeline definition in your forked repository. -![Configure CI build pipeline](./images/ci-build-pipeline-configure.png) +If you plan to use the release deployment pipeline (in the next section), you will need to rename this pipeline to `Model-Train-Register-CI`. Once the pipeline is finished, check the execution result: -![Build](./images/multi-stage-aci.png) +![Build](./images/model-train-register.png) + +And the pipeline artifacts: + +![Build](./images/model-train-register-artifacts.png) Also check the published training pipeline in the **mlops-AML-WS** workspace in [Azure Portal](https://portal.azure.com/): @@ -171,6 +172,12 @@ Also check the published training pipeline in the **mlops-AML-WS** workspace in Great, you now have the build pipeline for training set up which automatically triggers every time there's a change in the master branch! +After the pipeline is finished, you'll see a new model in the **ML Workspace**: + +![Trained model](./images/trained-model.png) + +To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci.yml` pipeline to `false`. You can also override the variable at runtime execution of the pipeline. + The pipeline stages are summarized below: #### Model CI @@ -186,32 +193,62 @@ The pipeline stages are summarized below: - This is an **agentless** job. The CI pipeline can wait for ML pipeline completion for hours or even days without using agent resources. - Determine if a new model was registered by the _ML Training Pipeline_. - If the model evaluation determines that the new model doesn't perform any better than the previous one, the new model won't register and the _ML Training Pipeline_ will be **canceled**. In this case, you'll see a message in the 'Train Model' job under the 'Determine if evaluation succeeded and new model is registered' step saying '**Model was not registered for this run.**' - - See [evaluate_model.py](../diabetes_regression/evaluate/evaluate_model.py#L118) for the evaluation logic and [diabetes_regression_verify_train_pipeline.py](../ml_service/pipelines/diabetes_regression_verify_train_pipeline.py#L54) for the ML pipeline reporting logic. + - See [evaluate_model.py](../diabetes_regression/evaluate/evaluate_model.py#L118) for the evaluation logic. - [Additional Variables and Configuration](#additional-variables-and-configuration) for configuring this and other behavior. -#### Deploy to ACI +#### Create pipeline artifact -- Deploy the model to the QA environment in [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/). -- Smoke test - - The test sends a sample query to the scoring web service and verifies that it returns the expected response. Have a look at the [smoke test code](../ml_service/util/smoke_test_scoring_service.py) for an example. +- Get the info about the registered model +- Create a pipeline artifact called `model` that contains a `model.json` file containing the model information. -The pipeline uses a Docker container on the Azure Pipelines agents to accomplish the pipeline steps. The container image **_mcr.microsoft.com/mlops/python:latest_** is built with [this Dockerfile](../environment_setup/Dockerfile) and has all the necessary dependencies installed for MLOpsPython and **_diabetes_regression_**. This image is an example of a custom Docker image with a pre-baked environment. The environment is guaranteed to be the same on any building agent, VM, or local machine. In your project, you'll want to build your own Docker image that only contains the dependencies and tools required for your use case. Your image will probably be smaller and faster, and it will be maintained by your team. +### Set up the Release Deployment and/or Batch Scoring pipelines -After the pipeline is finished, you'll see a new model in the **ML Workspace**: +--- +**PREREQUISITE** -![Trained model](./images/trained-model.png) +In order to use these pipelines: + +1. Follow the steps to set up the Model CI, training, evaluation, and registration pipeline. +1. You **must** rename your model CI/train/eval/register pipeline to `Model-Train-Register-CI`. + +These pipelines rely on the model CI pipeline and reference it by name. -To disable the automatic trigger of the training pipeline, change the `auto-trigger-training` variable as listed in the `.pipelines\diabetes_regression-ci.yml` pipeline to `false`. You can also override the variable at runtime execution of the pipeline. +--- -To skip model training and registration, and deploy a model successfully registered by a previous build (for testing changes to the score file or inference configuration), add the variable `MODEL_BUILD_ID` when the pipeline is queued, and set the value to the ID of the previous build. +These pipelines have the following behaviors: +- The pipeline will **automatically trigger** on completion of the Model-Train-Register-CI pipeline for the master branch. +- The pipeline will default to using the latest successful build of the Model-Train-Register-CI pipeline. It will deploy the model produced by that build. +- You can specify a `Model-Train-Register-CI` build ID when running the pipeline manually. You can find this in the url of the build, and the model registered from that build will also be tagged with the build ID. This is useful to skip model training and registration, and deploy/score a model successfully registered by a `Model-Train-Register-CI` build. -### **Azure [pipeline](../.pipelines/diabetes_regression-batchscoring-ci.yml) for batch scoring** -This pipeline has a sequence of stages for: -1. **Batch Scoring Code Continuous Integration:** triggered on code changes to master branch on GitHub. Runs linting, unit tests, code coverage and publishes a batch scoring pipeline. -1. **Run Batch Scoring**: invokes the published batch scoring pipeline to score a model. +### Set up the Release Deployment pipeline + +In your Azure DevOps project, create and run a new build pipeline based on the [diabetes_regression-cd.yml](../.pipelines/diabetes_regression-cd.yml) +pipeline definition in your forked repository. + +Your first run will use the latest model created by the `Model-Train-Register-CI` pipeline. + +Once the pipeline is finished, check the execution result: + +![Build](./images/model-deploy-result.png) + +To specify a particular build's model, set the `Model Train CI Build Id` parameter to the build Id you would like to use. + +![Build](./images/model-deploy-configure.png) + +Once your pipeline run begins, you can see the model name and version downloaded from the `Model-Train-Register-CI` pipeline. + +![Build](./images/model-deploy-artifact-logs.png) + +The pipeline has the following stage: + +#### Deploy to ACI + +- Deploy the model to the QA environment in [Azure Container Instances](https://azure.microsoft.com/en-us/services/container-instances/). +- Smoke test + - The test sends a sample query to the scoring web service and verifies that it returns the expected response. Have a look at the [smoke test code](../ml_service/util/smoke_test_scoring_service.py) for an example. -### Set up the Batch Scoring Pipeline +### Set up the Batch Scoring pipeline In your Azure DevOps project, create and run a new build pipeline based on the [diabetes_regression-batchscoring-ci.yml](../.pipelines/diabetes_regression-batchscoring-ci.yml) pipeline definition in your forked repository. @@ -247,7 +284,7 @@ The pipeline stages are summarized below: ## Further Exploration -You should now have a working pipeline that can get you started with MLOpsPython. Below are some additional features offered that might suit your scenario. +You should now have a working set of pipelines that can get you started with MLOpsPython. Below are some additional features offered that might suit your scenario. ### Deploy the model to Azure Kubernetes Service diff --git a/docs/images/model-deploy-artifact-logs.PNG b/docs/images/model-deploy-artifact-logs.PNG new file mode 100644 index 00000000..2dfee305 Binary files /dev/null and b/docs/images/model-deploy-artifact-logs.PNG differ diff --git a/docs/images/model-deploy-configure.png b/docs/images/model-deploy-configure.png new file mode 100644 index 00000000..fcd87750 Binary files /dev/null and b/docs/images/model-deploy-configure.png differ diff --git a/docs/images/model-deploy-result.png b/docs/images/model-deploy-result.png new file mode 100644 index 00000000..cd3d166e Binary files /dev/null and b/docs/images/model-deploy-result.png differ diff --git a/docs/images/model-train-register-artifacts.png b/docs/images/model-train-register-artifacts.png new file mode 100644 index 00000000..0d3eed26 Binary files /dev/null and b/docs/images/model-train-register-artifacts.png differ diff --git a/docs/images/model-train-register.png b/docs/images/model-train-register.png new file mode 100644 index 00000000..5ce4ef41 Binary files /dev/null and b/docs/images/model-train-register.png differ diff --git a/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py b/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py index d7acbf46..ac3d3407 100644 --- a/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_build_parallel_batchscore_pipeline.py @@ -33,65 +33,15 @@ Workspace, Dataset, Datastore, - Model, RunConfiguration, ) from azureml.pipeline.core import Pipeline, PipelineData, PipelineParameter from azureml.core.compute import ComputeTarget from azureml.data.datapath import DataPath from azureml.pipeline.steps import PythonScriptStep -from argparse import ArgumentParser, Namespace from typing import Tuple -def parse_args() -> Namespace: - """ - Parse arguments supplied to the pipeline creation script. - The only allowed arguments are model_tag_name and model_tag_value - specifying a custom tag/value pair to help locate a specific model. - - - :returns: Namespace with two attributes model_tag_name and model_tag_value - and corresponding values - - """ - parser = ArgumentParser() - parser.add_argument("--model_tag_name", default=None, type=str) - parser.add_argument("--model_tag_value", default=None, type=str) - args = parser.parse_args() - return args - - -def get_model( - ws: Workspace, env: Env, tagname: str = None, tagvalue: str = None -) -> Model: - """ - Gets a model from the models registered with the AML workspace. - If a tag/value pair is supplied, uses it to filter. - - :param ws: Current AML workspace - :param env: Environment variables - :param tagname: Optional tag name, default is None - :param tagvalue: Optional tag value, default is None - - :returns: Model - - :raises: ValueError - """ - if tagname is not None and tagvalue is not None: - model = Model(ws, name=env.model_name, tags=[[tagname, tagvalue]]) - elif (tagname is None and tagvalue is not None) or ( - tagvalue is None and tagname is not None - ): - raise ValueError( - "model_tag_name and model_tag_value should both be supplied" - + "or excluded" # NOQA: E501 - ) - else: - model = Model(ws, name=env.model_name) - return model - - def get_or_create_datastore( datastorename: str, ws: Workspace, env: Env, input: bool = True ) -> Datastore: @@ -331,7 +281,6 @@ def get_run_configs( def get_scoring_pipeline( - model: Model, scoring_dataset: Dataset, output_loc: PipelineData, score_run_config: ParallelRunConfig, @@ -343,7 +292,6 @@ def get_scoring_pipeline( """ Creates the scoring pipeline. - :param model: The model to use for scoring :param scoring_dataset: Data to score :param output_loc: Location to save the scoring results :param score_run_config: Parallel Run configuration to support @@ -362,6 +310,9 @@ def get_scoring_pipeline( model_name_param = PipelineParameter( "model_name", default_value=env.model_name ) # NOQA: E501 + model_version_param = PipelineParameter( + "model_version", default_value=env.model_version + ) # NOQA: E501 model_tag_name_param = PipelineParameter( "model_tag_name", default_value=" " ) # NOQA: E501 @@ -376,6 +327,8 @@ def get_scoring_pipeline( arguments=[ "--model_name", model_name_param, + "--model_version", + model_version_param, "--model_tag_name", model_tag_name_param, "--model_tag_value", @@ -425,8 +378,6 @@ def build_batchscore_pipeline(): try: env = Env() - args = parse_args() - # Get Azure machine learning workspace aml_workspace = Workspace.get( name=env.workspace_name, @@ -450,12 +401,7 @@ def build_batchscore_pipeline(): aml_workspace, aml_compute_score, env ) - trained_model = get_model( - aml_workspace, env, args.model_tag_name, args.model_tag_value - ) - scoring_pipeline = get_scoring_pipeline( - trained_model, input_dataset, output_location, scoring_runconfig, diff --git a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py index 306f2259..28511f9b 100644 --- a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py +++ b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py @@ -3,7 +3,7 @@ import os from azureml.core import Run, Experiment, Workspace from ml_service.util.env_variables import Env -from diabetes_regression.util.model_helper import get_latest_model +from diabetes_regression.util.model_helper import get_model def main(): @@ -53,8 +53,12 @@ def main(): try: tag_name = 'BuildId' - model = get_latest_model( - model_name, tag_name, build_id, exp.workspace) + model = get_model( + model_name=model_name, + tag_name=tag_name, + tag_value=build_id, + aml_workspace=exp.workspace) + if (model is not None): print("Model was registered for this build.") if (model is None): diff --git a/ml_service/pipelines/run_parallel_batchscore_pipeline.py b/ml_service/pipelines/run_parallel_batchscore_pipeline.py index ec6cebae..c046eb9c 100644 --- a/ml_service/pipelines/run_parallel_batchscore_pipeline.py +++ b/ml_service/pipelines/run_parallel_batchscore_pipeline.py @@ -115,6 +115,7 @@ def run_batchscore_pipeline(): scoringpipeline, pipeline_parameters={ "model_name": env.model_name, + "model_version": env.model_version, "model_tag_name": " ", "model_tag_value": " ", },