From 8fb12afe706c3408618e92fffcb65a9a894d9fd0 Mon Sep 17 00:00:00 2001 From: Stephanie Visser <42637613+zoekdestep@users.noreply.github.com> Date: Tue, 5 May 2020 20:16:56 +0200 Subject: [PATCH] Add Terraform option to environment_setup (#268) * setup basic folder and file structure * add tf backend file and bash script to create state storage * basic pipeline for infrastructure with tf - yaml, tf, bash * naming and deleting unnecessary bash script * updated documentation * added to the get_started.md guide * added terraform plan step --- docs/code_description.md | 9 ++- docs/getting_started.md | 6 +- ...> iac-create-environment-pipeline-arm.yml} | 0 .../iac-create-environment-pipeline-tf.yml | 72 +++++++++++++++++++ environment_setup/tf-templates/backend.tf | 4 ++ environment_setup/tf-templates/main.tf | 71 ++++++++++++++++++ 6 files changed, 158 insertions(+), 4 deletions(-) rename environment_setup/{iac-create-environment-pipeline.yml => iac-create-environment-pipeline-arm.yml} (100%) create mode 100644 environment_setup/iac-create-environment-pipeline-tf.yml create mode 100644 environment_setup/tf-templates/backend.tf create mode 100644 environment_setup/tf-templates/main.tf diff --git a/docs/code_description.md b/docs/code_description.md index 351b4c34..8dc3c756 100644 --- a/docs/code_description.md +++ b/docs/code_description.md @@ -18,7 +18,8 @@ High level directory structure for this repository: │ ├── util <- Python script for various utility operations specific to this ML project. ├── docs <- Extensive markdown documentation for entire project. ├── environment_setup <- The top-level folder for everything related to infrastructure. -│ ├── arm-templates <- Azure Resource Manager(ARM) templates to build infrastructure needed for this project. +│ ├── arm-templates <- Azure Resource Manager(ARM) templates to build infrastructure needed for this project. +│ ├── tf-templates <- Terraform templates to build infrastructure needed for this project. ├── experimentation <- Jupyter notebooks with ML experimentation code. ├── ml_service <- The top-level folder for all Azure Machine Learning resources. │ ├── pipelines <- Python script that builds Azure Machine Learning pipelines. @@ -35,7 +36,11 @@ The repository provides a template with folders structure suitable for maintaini - `environment_setup/install_requirements.sh` : This script prepares a local conda environment i.e. install the Azure ML SDK and the packages specified in environment definitions. -- `environment_setup/iac-*.yml, arm-templates` : Infrastructure as Code piplines to create and delete required resources along with corresponding arm-templates. +- `environment_setup/iac-*-arm.yml, arm-templates` : Infrastructure as Code piplines to create required resources using ARM, along with corresponding arm-templates. Infrastructure as Code can be deployed with this template or with the Terraform template. + +- `environment_setup/iac-*-tf.yml, tf-templates` : Infrastructure as Code piplines to create required resources using Terraform, along with corresponding tf-templates. Infrastructure as Code can be deployed with this template or with the ARM template. + +- `environment_setup/iac-remove-environment.yml` : Infrastructure as Code piplines to delete the created required resources. - `environment_setup/Dockerfile` : Dockerfile of a build agent containing Python 3.6 and all required packages. diff --git a/docs/getting_started.md b/docs/getting_started.md index 86eb73df..2c63fa94 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -81,7 +81,7 @@ More variables are available for further tweaking, but the above variables are a ## Provisioning resources using Azure Pipelines -The easiest way to create all required Azure resources (Resource Group, Azure ML Workspace, Container Registry, and others) is to use the **Infrastructure as Code (IaC)** [pipeline in this repository](../environment_setup/iac-create-environment-pipeline.yml). The pipeline takes care of setting up all required resources based on these [Azure Resource Manager templates](../environment_setup/arm-templates/cloud-environment.json). +The easiest way to create all required Azure resources (Resource Group, Azure ML Workspace, Container Registry, and others) is to use the **Infrastructure as Code (IaC)** [pipeline with ARM templates](../environment_setup/iac-create-environment-pipeline-arm.yml) or the [pipeline with Terraform templates](../environment_setup/iac-create-environment-pipeline-tf.yml). The pipeline takes care of setting up all required resources based on these [Azure Resource Manager templates](../environment_setup/arm-templates/cloud-environment.json), or based on these [Terraform templates](../environment_setup/tf-templates). ### Create an Azure DevOps Service Connection for the Azure Resource Manager @@ -100,10 +100,12 @@ In your Azure DevOps project, create a build pipeline from your forked repositor ![Build connect step](./images/build-connect.png) -Select the **Existing Azure Pipelines YAML file** option and set the path to [/environment_setup/iac-create-environment-pipeline.yml](../environment_setup/iac-create-environment-pipeline.yml): +Select the **Existing Azure Pipelines YAML file** option and set the path to [/environment_setup/iac-create-environment-pipeline-arm.yml](../environment_setup/iac-create-environment-pipeline-arm.yml) or to [/environment_setup/iac-create-environment-pipeline-tf.yml](../environment_setup/iac-create-environment-pipeline-tf.yml), depending on if you want to deploy your infrastructure using ARM templates or Terraform: ![Configure step](./images/select-iac-pipeline.png) +If you decide to use Terraform, make sure the ['Terraform Build & Release Tasks' from Charles Zipp](https://marketplace.visualstudio.com/items?itemName=charleszipp.azure-pipelines-tasks-terraform) is installed. + Having done that, run the pipeline: ![IaC run](./images/run-iac-pipeline.png) diff --git a/environment_setup/iac-create-environment-pipeline.yml b/environment_setup/iac-create-environment-pipeline-arm.yml similarity index 100% rename from environment_setup/iac-create-environment-pipeline.yml rename to environment_setup/iac-create-environment-pipeline-arm.yml diff --git a/environment_setup/iac-create-environment-pipeline-tf.yml b/environment_setup/iac-create-environment-pipeline-tf.yml new file mode 100644 index 00000000..3e94677e --- /dev/null +++ b/environment_setup/iac-create-environment-pipeline-tf.yml @@ -0,0 +1,72 @@ +# CI/PR Pipeline that deploys an TF template to create or update the resources needed by the other pipelines. +trigger: + branches: + include: + - master + paths: + include: + - environment_setup/tf-templates/* +pr: + branches: + include: + - master + paths: + include: + - environment_setup/tf-templates/* + +pool: + vmImage: 'ubuntu-latest' + +variables: +- group: devopsforai-aml-vg + +steps: +- task: charleszipp.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-installer.TerraformInstaller@0 + displayName: 'Use Terraform 0.12.24' + inputs: + terraformVersion: 0.12.24 + +- task: charleszipp.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@0 + displayName: 'TF init - Deploy MLOps resources to Azure' + inputs: + command: init + commandOptions: '-backend=true -backend-config=$(Build.SourcesDirectory)/environment_setup/tf-templates/backend.tf' + workingDirectory: '$(Build.SourcesDirectory)/environment_setup/tf-templates' + backendType: azurerm + backendServiceArm: $(AZURE_RM_SVC_CONNECTION) + ensureBackend: true + backendAzureRmResourceGroupLocation: $(LOCATION) + backendAzureRmResourceGroupName: $(RESOURCE_GROUP) + backendAzureRmStorageAccountName: 'statestor' + backendAzureRmStorageAccountSku: 'Standard_LRS' + backendAzureRmContainerName: 'tfstate-cont' + backendAzureRmKey: 'mlopsinfra.tfstate' + +- task: charleszipp.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@0 + displayName: 'TF validate - Deploy MLOps resources to Azure' + inputs: + command: validate + workingDirectory: '$(Build.SourcesDirectory)/environment_setup/tf-templates' + +- task: charleszipp.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@0 + displayName: 'TF plan - Deploy MLOps resources to Azure' + inputs: + command: plan + workingDirectory: '$(Build.SourcesDirectory)/environment_setup/tf-templates' + environmentServiceName: $(AZURE_RM_SVC_CONNECTION) + env: + TF_VAR_BASE_NAME: $(BASE_NAME) + TF_VAR_RESOURCE_GROUP: $(RESOURCE_GROUP) + TF_VAR_WORKSPACE_NAME: $(WORKSPACE_NAME) + +- task: charleszipp.azure-pipelines-tasks-terraform.azure-pipelines-tasks-terraform-cli.TerraformCLI@0 + displayName: 'TF apply - Deploy MLOps resources to Azure' + inputs: + command: apply + workingDirectory: '$(Build.SourcesDirectory)/environment_setup/tf-templates' + environmentServiceName: $(AZURE_RM_SVC_CONNECTION) + env: + TF_VAR_BASE_NAME: $(BASE_NAME) + TF_VAR_RESOURCE_GROUP: $(RESOURCE_GROUP) + TF_VAR_WORKSPACE_NAME: $(WORKSPACE_NAME) + diff --git a/environment_setup/tf-templates/backend.tf b/environment_setup/tf-templates/backend.tf new file mode 100644 index 00000000..0aec0499 --- /dev/null +++ b/environment_setup/tf-templates/backend.tf @@ -0,0 +1,4 @@ +terraform { + backend "azurerm" { + } +} diff --git a/environment_setup/tf-templates/main.tf b/environment_setup/tf-templates/main.tf new file mode 100644 index 00000000..c57a5a84 --- /dev/null +++ b/environment_setup/tf-templates/main.tf @@ -0,0 +1,71 @@ +provider "azurerm" { + version = "=2.3.0" + features {} +} + +variable BASE_NAME {} +variable RESOURCE_GROUP {} +variable WORKSPACE_NAME {} + +#-------------------------------------------------------------------------------- + +#Set the already-existing resource group +data "azurerm_resource_group" "amlrg" { + name = var.RESOURCE_GROUP +} + +#Set client config for a.o. tenant id +data "azurerm_client_config" "currentconfig" { +} + +#-------------------------------------------------------------------------------- + +# Storage account for AML Service +resource "azurerm_storage_account" "amlstor" { + name = "${var.BASE_NAME}amlsa" + location = data.azurerm_resource_group.amlrg.location + resource_group_name = data.azurerm_resource_group.amlrg.name + account_tier = "Standard" + account_replication_type = "LRS" +} + +# Keyvault for AML Service +resource "azurerm_key_vault" "amlkv" { + name = "${var.BASE_NAME}-AML-KV" + location = data.azurerm_resource_group.amlrg.location + resource_group_name = data.azurerm_resource_group.amlrg.name + tenant_id = data.azurerm_client_config.currentconfig.tenant_id + sku_name = "standard" +} + +# App Insights for AML Service +resource "azurerm_application_insights" "amlai" { + name = "${var.BASE_NAME}-AML-AI" + location = data.azurerm_resource_group.amlrg.location + resource_group_name = data.azurerm_resource_group.amlrg.name + application_type = "web" +} + +# Container registry for AML Service +resource "azurerm_container_registry" "amlacr" { + name = "${var.BASE_NAME}amlcr" + resource_group_name = data.azurerm_resource_group.amlrg.name + location = data.azurerm_resource_group.amlrg.location + sku = "Standard" + admin_enabled = true +} + +# ML Workspace for AML Service, depending on the storage account, Keyvault, App Insights and ACR. +resource "azurerm_machine_learning_workspace" "amlws" { + name = var.WORKSPACE_NAME + location = data.azurerm_resource_group.amlrg.location + resource_group_name = data.azurerm_resource_group.amlrg.name + application_insights_id = azurerm_application_insights.amlai.id + key_vault_id = azurerm_key_vault.amlkv.id + storage_account_id = azurerm_storage_account.amlstor.id + container_registry_id = azurerm_container_registry.amlacr.id + + identity { + type = "SystemAssigned" + } +}