Multi-stage pipeline + scoring observability (#122)

.pipelines/azdo-ci-build-train.yml

@@ -8,6 +8,7 @@ trigger:
     - docs/
     - environment_setup/
     - ml_service/util/create_scoring_image.*
+    - ml_service/util/smoke_test_scoring_service.py
 
 variables:
 - template: azdo-variables.yml
@@ -36,8 +37,9 @@ stages:
           # Invoke the Python building and publishing a training pipeline
           python $(Build.SourcesDirectory)/ml_service/pipelines/${{ variables.BUILD_TRAIN_SCRIPT }}
       displayName: 'Publish Azure Machine Learning Pipeline'
+
 - stage: 'Trigger_AML_Pipeline'
-  displayName: 'Train, evaluate, register model via previously published AML pipeline'
+  displayName: 'Train model'
   jobs:
   - job: "Get_Pipeline_ID"
     condition: and(succeeded(), eq(coalesce(variables['auto-trigger-training'], 'true'), 'true'))
@@ -87,32 +89,121 @@ stages:
   - job: "Training_Run_Report"
     dependsOn: "Run_ML_Pipeline"
     displayName: "Determine if evaluation succeeded and new model is registered"
+    pool:
+        vmImage: 'ubuntu-latest'
+    container: mcr.microsoft.com/mlops/python:latest
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-template-get-model-version.yml
+- stage: 'Deploy_ACI'
+  displayName: 'Deploy to ACI'
+  dependsOn: Trigger_AML_Pipeline
+  condition: and(succeeded(), variables['ACI_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_ACI"
+    displayName: "Deploy to ACI"
+    pool:
+      vmImage: 'ubuntu-latest'
+    container: mcr.microsoft.com/mlops/python:latest
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-template-get-model-version.yml
+    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
+      displayName: 'Azure ML Model Deploy'
+      inputs:
+        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
+        modelSourceType: manualSpec
+        modelName: '$(MODEL_NAME)'
+        modelVersion: $(MODEL_VERSION)
+        inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
+        deploymentTarget: ACI
+        deploymentName: $(ACI_DEPLOYMENT_NAME)
+        deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aci.yml'
+        overwriteExistingDeployment: true
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
+      inputs:
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python ml_service/util/smoke_test_scoring_service.py --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_AKS'
+  displayName: 'Deploy to AKS'
+  dependsOn: Deploy_ACI
+  condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_AKS"
+    displayName: "Deploy to AKS"
     pool:
       vmImage: 'ubuntu-latest'
     container: mcr.microsoft.com/mlops/python:latest
     timeoutInMinutes: 0
     steps:
+    - template: azdo-template-get-model-version.yml
+    - task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
+      displayName: 'Azure ML Model Deploy'
+      inputs:
+        azureSubscription: $(WORKSPACE_SVC_CONNECTION)
+        modelSourceType: manualSpec
+        modelName: '$(MODEL_NAME)'
+        modelVersion: $(MODEL_VERSION)
+        inferencePath: '$(Build.SourcesDirectory)/code/scoring/inference_config.yml'
+        deploymentTarget: AKS
+        aksCluster: $(AKS_COMPUTE_NAME)
+        deploymentName: $(AKS_DEPLOYMENT_NAME)
+        deployConfig: '$(Build.SourcesDirectory)/code/scoring/deployment_config_aks.yml'
+        overwriteExistingDeployment: true
     - task: AzureCLI@1
+      displayName: 'Smoke test'
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
         inlineScript: |
           set -e # fail on error
           export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
-          python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId)
-      displayName: "Determine if evaluation succeeded and new model is registered"
-    - task: CopyFiles@2
-      displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)'
+          python ml_service/util/smoke_test_scoring_service.py --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
+
+- stage: 'Deploy_Webapp'
+  displayName: 'Deploy to Webapp'
+  dependsOn: Trigger_AML_Pipeline
+  condition: and(succeeded(), variables['WEBAPP_DEPLOYMENT_NAME'])
+  jobs:
+  - job: "Deploy_Webapp"
+    displayName: "Deploy to Webapp"
+    pool:
+      vmImage: 'ubuntu-latest'
+    container: mcr.microsoft.com/mlops/python:latest
+    timeoutInMinutes: 0
+    steps:
+    - template: azdo-template-get-model-version.yml
+    - task: AzureCLI@1
+      displayName: 'Create scoring image and set IMAGE_LOCATION variable'
       inputs:
-        SourceFolder: '$(Build.SourcesDirectory)'
-        TargetFolder: '$(Build.ArtifactStagingDirectory)'
-        Contents: |
-          code/scoring/**
-          ml_service/util/**
-    - task: PublishBuildArtifacts@1
-      displayName: 'Publish Artifact'
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python ml_service/util/create_scoring_image.py --output_image_location_file image_location.txt
+          # Output image location to Azure DevOps job
+          IMAGE_LOCATION="$(cat image_location.txt)"
+          echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
+    - task: AzureWebAppContainer@1
+      name: WebAppDeploy
+      displayName: 'Azure Web App on Container Deploy'
+      inputs:
+        azureSubscription: 'AzureResourceConnection'
+        appName: '$(WEBAPP_DEPLOYMENT_NAME)'
+        containers: '$(IMAGE_LOCATION)'
+    - task: AzureCLI@1
+      displayName: 'Smoke test'
       inputs:
-        ArtifactName: 'mlops-pipelines'
-        publishLocation: 'container'
-        pathtoPublish: '$(Build.ArtifactStagingDirectory)' 
-        TargetPath: '$(Build.ArtifactStagingDirectory)'
+        azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -e # fail on error
+          export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+          python ml_service/util/smoke_test_scoring_service.py --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"

.pipelines/azdo-template-get-model-version.yml

@@ -0,0 +1,14 @@
+steps:
+- task: AzureCLI@1
+  inputs:
+    azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
+    scriptLocation: inlineScript
+    inlineScript: |
+      set -e # fail on error
+      export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
+      python $(Build.SourcesDirectory)/ml_service/pipelines/verify_train_pipeline.py --build_id $(Build.BuildId) --output_model_version_file "model_version.txt"
+      # Output model version to Azure DevOps job
+      MODEL_VERSION="$(cat model_version.txt)"
+      echo "##vso[task.setvariable variable=MODEL_VERSION]$MODEL_VERSION"
+  name: 'getversion'
+  displayName: "Determine if evaluation succeeded and new model is registered"

code/scoring/score.py

@@ -38,16 +38,33 @@ def init():
     model = joblib.load(model_path)
 
 
-def run(raw_data):
+def run(raw_data, request_headers):
     data = json.loads(raw_data)["data"]
     data = numpy.array(data)
     result = model.predict(data)
+
+    # Demonstrate how we can log custom data into the Application Insights
+    # traces collection.
+    # The 'X-Ms-Request-id' value is generated internally and can be used to
+    # correlate a log entry with the Application Insights requests collection.
+    # The HTTP 'traceparent' header may be set by the caller to implement
+    # distributed tracing (per the W3C Trace Context proposed specification)
+    # and can be used to correlate the request to external systems.
+    print(('{{"RequestId":"{0}", '
+           '"TraceParent":"{1}", '
+           '"NumberOfPredictions":{2}}}'
+           ).format(
+               request_headers.get("X-Ms-Request-Id", ""),
+               request_headers.get("Traceparent", ""),
+               len(result)
+           ))
+
     return {"result": result.tolist()}
 
 
 if __name__ == "__main__":
     # Test scoring
     init()
     test_row = '{"data":[[1,2,3,4,5,6,7,8,9,10],[10,9,8,7,6,5,4,3,2,1]]}'
-    prediction = run(test_row)
+    prediction = run(test_row, {})
     print("Test result: ", prediction)