From 3cc929a2f9fa3433765a29b62ff45a8fab975475 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Wed, 9 Sep 2020 19:30:37 +0200 Subject: [PATCH 01/16] Add makefile to plan and apply terraform --- .gitignore | 5 +++++ IaC/Makefile | 17 +++++++++++++++++ IaC/main.tf | 12 ++++++++++++ IaC/variables.tf | 4 ++++ 4 files changed, 38 insertions(+) create mode 100644 .gitignore create mode 100644 IaC/Makefile create mode 100644 IaC/main.tf create mode 100644 IaC/variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b352c75 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +#terraform +.terraform +*.tfstate +*.tfstate.backup +*.tfvars \ No newline at end of file diff --git a/IaC/Makefile b/IaC/Makefile new file mode 100644 index 0000000..1fd4ae1 --- /dev/null +++ b/IaC/Makefile @@ -0,0 +1,17 @@ + + +pre-check: + -gsutil -m mb -l EU -p $(PROJECT_ID) gs://$(BACKEND_TERRAFORM) + +init-terraform: + @terraform init -backend-config="bucket=$(BACKEND_TERRAFORM)" + +apply-terraform: + @terraform apply --quiet -var="project_id=$(PROJECT_ID)" + +plan-terraform: + @terraform plan -var="project_id=$(PROJECT_ID)" + +apply: pre-check init-terraform apply-terraform + +plan: pre-check init-terraform plan-terraform diff --git a/IaC/main.tf b/IaC/main.tf new file mode 100644 index 0000000..c37aab5 --- /dev/null +++ b/IaC/main.tf @@ -0,0 +1,12 @@ +terraform { + backend "gcs" { + } + required_version = "=0.12.29" + required_providers { + google = "~> 3.13" + } +} + +provider "google" { + project = var.project_id +} diff --git a/IaC/variables.tf b/IaC/variables.tf new file mode 100644 index 0000000..8c03cf8 --- /dev/null +++ b/IaC/variables.tf @@ -0,0 +1,4 @@ +variable "project_id" { + description = "GCP project" + type = string +} From ba347a78febd52b3577ab0521d0597a61dec75a4 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Wed, 9 Sep 2020 19:38:31 +0200 Subject: [PATCH 02/16] Add server module with cloud run --- IaC/modules/mlflow/artifacts/main.tf | 0 IaC/modules/mlflow/artifacts/variables.tf | 0 IaC/modules/mlflow/database/main.tf | 0 IaC/modules/mlflow/database/variables.tf | 0 IaC/modules/mlflow/server/main.tf | 53 +++++++++++++++++++++++ IaC/modules/mlflow/server/variables.tf | 21 +++++++++ 6 files changed, 74 insertions(+) create mode 100644 IaC/modules/mlflow/artifacts/main.tf create mode 100644 IaC/modules/mlflow/artifacts/variables.tf create mode 100644 IaC/modules/mlflow/database/main.tf create mode 100644 IaC/modules/mlflow/database/variables.tf create mode 100644 IaC/modules/mlflow/server/main.tf create mode 100644 IaC/modules/mlflow/server/variables.tf diff --git a/IaC/modules/mlflow/artifacts/main.tf b/IaC/modules/mlflow/artifacts/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/IaC/modules/mlflow/artifacts/variables.tf b/IaC/modules/mlflow/artifacts/variables.tf new file mode 100644 index 0000000..e69de29 diff --git a/IaC/modules/mlflow/database/main.tf b/IaC/modules/mlflow/database/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/IaC/modules/mlflow/database/variables.tf b/IaC/modules/mlflow/database/variables.tf new file mode 100644 index 0000000..e69de29 diff --git a/IaC/modules/mlflow/server/main.tf b/IaC/modules/mlflow/server/main.tf new file mode 100644 index 0000000..f2088e2 --- /dev/null +++ b/IaC/modules/mlflow/server/main.tf @@ -0,0 +1,53 @@ +resource "google_cloud_run_service" "default" { + name = var.server_name + location = var.location + + template { + spec { + containers { + image = var.docker_image_name + dynamic "env" { + for_each = var.env_variables + content { + name = env.key + value = env.value + } + } + resources { + limits = { + cpu = var.cpu_limit + memory = var.memory_limit + } + } + } + } + metadata { + annotations = { + "run.googleapis.com/cloudsql-instances" = var.sql_instance_name + } + } + } + + traffic { + percent = 100 + latest_revision = true + } + autogenerate_revision_name = true +} + +data "google_iam_policy" "noauth" { + binding { + role = "roles/run.invoker" + members = [ + "allUsers", + ] + } +} + +resource "google_cloud_run_service_iam_policy" "noauth" { + location = google_cloud_run_service.default.location + project = google_cloud_run_service.default.project + service = google_cloud_run_service.default.name + + policy_data = data.google_iam_policy.noauth.policy_data +} \ No newline at end of file diff --git a/IaC/modules/mlflow/server/variables.tf b/IaC/modules/mlflow/server/variables.tf new file mode 100644 index 0000000..50bfd5f --- /dev/null +++ b/IaC/modules/mlflow/server/variables.tf @@ -0,0 +1,21 @@ +variable "server_name" { + type = string + description = "Name of your cserver" +} +variable "location" { + type = string + description = "Location to deploy your server" + default = "europe-west1" +} +variable "docker_image_name" { + type = string + description = "Name of the docker image" +} +variable "env_variables" { + type = map + description = "Env variable to be used in your container" +} +variable "sql_instance_name" { + type = string + description = "Sql instance name your server needs access to" +} From 2fdf5aa35dc5556aca96923154d413fd49ffce3c Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Thu, 10 Sep 2020 10:41:11 +0200 Subject: [PATCH 03/16] Add database, secret, artifacts, network modules --- IaC/Makefile | 17 ------- IaC/modules/mlflow/artifacts/main.tf | 16 +++++++ IaC/modules/mlflow/artifacts/variables.tf | 24 ++++++++++ IaC/modules/mlflow/database/main.tf | 37 +++++++++++++++ IaC/modules/mlflow/database/outputs.tf | 9 ++++ IaC/modules/mlflow/database/variables.tf | 46 +++++++++++++++++++ IaC/modules/mlflow/secret_manager/main.tf | 19 ++++++++ IaC/modules/mlflow/secret_manager/outputs.tf | 4 ++ .../mlflow/secret_manager/variables.tf | 8 ++++ IaC/modules/network/main.tf | 21 +++++++++ IaC/modules/network/outputs.tf | 8 ++++ IaC/modules/network/variables.tf | 4 ++ 12 files changed, 196 insertions(+), 17 deletions(-) delete mode 100644 IaC/Makefile create mode 100644 IaC/modules/mlflow/database/outputs.tf create mode 100644 IaC/modules/mlflow/secret_manager/main.tf create mode 100644 IaC/modules/mlflow/secret_manager/outputs.tf create mode 100644 IaC/modules/mlflow/secret_manager/variables.tf create mode 100644 IaC/modules/network/main.tf create mode 100644 IaC/modules/network/outputs.tf create mode 100644 IaC/modules/network/variables.tf diff --git a/IaC/Makefile b/IaC/Makefile deleted file mode 100644 index 1fd4ae1..0000000 --- a/IaC/Makefile +++ /dev/null @@ -1,17 +0,0 @@ - - -pre-check: - -gsutil -m mb -l EU -p $(PROJECT_ID) gs://$(BACKEND_TERRAFORM) - -init-terraform: - @terraform init -backend-config="bucket=$(BACKEND_TERRAFORM)" - -apply-terraform: - @terraform apply --quiet -var="project_id=$(PROJECT_ID)" - -plan-terraform: - @terraform plan -var="project_id=$(PROJECT_ID)" - -apply: pre-check init-terraform apply-terraform - -plan: pre-check init-terraform plan-terraform diff --git a/IaC/modules/mlflow/artifacts/main.tf b/IaC/modules/mlflow/artifacts/main.tf index e69de29..0bd2fa6 100644 --- a/IaC/modules/mlflow/artifacts/main.tf +++ b/IaC/modules/mlflow/artifacts/main.tf @@ -0,0 +1,16 @@ +resource "google_storage_bucket" "this" { + name = var.bucket_name + location = var.bucket_location + storage_class = var.storage_class + versioning { + enabled = var.versioning_enabled + } + lifecycle_rule { + condition { + num_newer_versions = var.number_of_version + } + action { + type = "Delete" + } + } +} \ No newline at end of file diff --git a/IaC/modules/mlflow/artifacts/variables.tf b/IaC/modules/mlflow/artifacts/variables.tf index e69de29..5b7e909 100644 --- a/IaC/modules/mlflow/artifacts/variables.tf +++ b/IaC/modules/mlflow/artifacts/variables.tf @@ -0,0 +1,24 @@ +variable "bucket_name" { + description = "Name of the bucket." + type = string +} +variable "bucket_location" { + description = "Location of the bucket." + type = string + default = "EUROPE-WEST1" +} +variable "versioning_enabled" { + description = "True if you want to version your bucket." + type = bool + default = true +} +variable "number_of_version" { + description = "Number of version you want to keep with the versionning." + type = number + default = 1 +} +variable "storage_class" { + description = "Storage class of your bucket" + type = string + default ="STANDARD" +} diff --git a/IaC/modules/mlflow/database/main.tf b/IaC/modules/mlflow/database/main.tf index e69de29..15b8ce1 100644 --- a/IaC/modules/mlflow/database/main.tf +++ b/IaC/modules/mlflow/database/main.tf @@ -0,0 +1,37 @@ +resource "random_id" "db_name_suffix" { + byte_length = 4 +} + +resource "google_sql_database_instance" "this_instance" { + name = "${var.instance_prefix}-${random_id.db_name_suffix.hex}" + database_version = var.database_version + region = var.region + + depends_on = [var.private_vpc_connection] + + settings { + tier = var.size + ip_configuration { + ipv4_enabled = false + private_network = var.network_link + } + backup_configuration { + enabled = true + } + availability_type = var.availability_type + + } +} + +resource "google_sql_database" "this_database" { + name = var.database_name + instance = google_sql_database_instance.this_instance.name + depends_on = [google_sql_database_instance.this_instance] +} + +resource "google_sql_user" "this_user" { + name = var.username + instance = google_sql_database_instance.this_instance.name + password = var.password + depends_on = [google_sql_database_instance.this_instance] +} \ No newline at end of file diff --git a/IaC/modules/mlflow/database/outputs.tf b/IaC/modules/mlflow/database/outputs.tf new file mode 100644 index 0000000..cb7b4a7 --- /dev/null +++ b/IaC/modules/mlflow/database/outputs.tf @@ -0,0 +1,9 @@ +output "instance_connection_name" { + description = "Connection string used to connect to the instance" + value = google_sql_database_instance.this_instance.connection_name +} + +output "database_name" { + description = "The name of the database" + value = google_sql_database.this_database.name +} diff --git a/IaC/modules/mlflow/database/variables.tf b/IaC/modules/mlflow/database/variables.tf index e69de29..37e8922 100644 --- a/IaC/modules/mlflow/database/variables.tf +++ b/IaC/modules/mlflow/database/variables.tf @@ -0,0 +1,46 @@ +variable "instance_prefix" { + type = string + description = "Name of the database instance you want to deploy" + default = "mlflow" +} +variable "database_version" { + type = string + description = "Version of the database instance you use" + default = "MYSQL_5_6" +} +variable "region" { + type = string + description = "Region of the database instance" + default = "europe-west1" +} +variable "private_vpc_connection" { + type = string + description = "Private connection used to connect your instance with" +} +variable "size" { + type = string + description = "Size of the database instance" + default = "db-f1-micro" +} +variable "network_link" { + type = string + description = "Network link you want to connect your database with" +} +variable "availability_type" { + type = string + description = "Availability of your instance" + default = "ZONAL" +} +variable "database_name" { + type = string + description = "Name of the database created" + default = "mlflow" +} +variable "username" { + type = string + description = "Username to connect to database instance" +} +variable "password" { + type = string + description = "Password to connect to database instance" +} diff --git a/IaC/modules/mlflow/secret_manager/main.tf b/IaC/modules/mlflow/secret_manager/main.tf new file mode 100644 index 0000000..8b82ef4 --- /dev/null +++ b/IaC/modules/mlflow/secret_manager/main.tf @@ -0,0 +1,19 @@ +resource "google_secret_manager_secret" "secret" { + provider = google-beta + + secret_id = var.secret_id + + replication { + automatic = true + } +} + + +resource "google_secret_manager_secret_version" "secret-version" { + provider = google-beta + + secret = google_secret_manager_secret.secret.id + + secret_data = var.secret_value + depends_on = [google_secret_manager_secret.secret] +} diff --git a/IaC/modules/mlflow/secret_manager/outputs.tf b/IaC/modules/mlflow/secret_manager/outputs.tf new file mode 100644 index 0000000..2ffa9b4 --- /dev/null +++ b/IaC/modules/mlflow/secret_manager/outputs.tf @@ -0,0 +1,4 @@ +output "secret_value" { + description = "Value of the created secret" + value = google_secret_manager_secret_version.secret-version.secret_data +} diff --git a/IaC/modules/mlflow/secret_manager/variables.tf b/IaC/modules/mlflow/secret_manager/variables.tf new file mode 100644 index 0000000..fc6abf0 --- /dev/null +++ b/IaC/modules/mlflow/secret_manager/variables.tf @@ -0,0 +1,8 @@ +variable "secret_id" { + type = string + description = "Name of the secret you want to create" +} +variable "secret_value" { + type = string + description = "value of the secret you want to create" +} \ No newline at end of file diff --git a/IaC/modules/network/main.tf b/IaC/modules/network/main.tf new file mode 100644 index 0000000..0abb834 --- /dev/null +++ b/IaC/modules/network/main.tf @@ -0,0 +1,21 @@ +resource "google_compute_network" "vpc" { + name = var.vpc_name + routing_mode = "GLOBAL" + auto_create_subnetworks = true +} + +resource "google_compute_global_address" "private_ip_address" { + provider = google-beta + + name = "private-ip-address" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = google_compute_network.vpc.self_link +} + +resource "google_service_networking_connection" "private_vpc_connection" { + network = google_compute_network.vpc.self_link + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_ip_address.name] +} \ No newline at end of file diff --git a/IaC/modules/network/outputs.tf b/IaC/modules/network/outputs.tf new file mode 100644 index 0000000..9aade67 --- /dev/null +++ b/IaC/modules/network/outputs.tf @@ -0,0 +1,8 @@ +output "network_link" { + description = "Link of the created network" + value = google_compute_network.vpc.self_link +} +output "private_vpc_connection" { + description = "Private vpc connection to servicenetworking" + value = google_service_networking_connection.private_vpc_connection +} \ No newline at end of file diff --git a/IaC/modules/network/variables.tf b/IaC/modules/network/variables.tf new file mode 100644 index 0000000..4177413 --- /dev/null +++ b/IaC/modules/network/variables.tf @@ -0,0 +1,4 @@ +variable "vpc_name" { + type = string + description = "Name of the network you want to create" +} \ No newline at end of file From 74b097151c483ef78100ad4e00a0b5d3416aebc9 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Thu, 10 Sep 2020 10:52:00 +0200 Subject: [PATCH 04/16] Adding mlflow modules calls --- IaC/main.tf | 6 +++++ IaC/modules/mlflow/main.tf | 36 ++++++++++++++++++++++++++ IaC/modules/mlflow/server/variables.tf | 2 +- IaC/modules/mlflow/variables.tf | 0 4 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 IaC/modules/mlflow/main.tf create mode 100644 IaC/modules/mlflow/variables.tf diff --git a/IaC/main.tf b/IaC/main.tf index c37aab5..0eb2b15 100644 --- a/IaC/main.tf +++ b/IaC/main.tf @@ -10,3 +10,9 @@ terraform { provider "google" { project = var.project_id } + + +module "network" { + source = "modules/network" + vpc_name = var.network_name +} \ No newline at end of file diff --git a/IaC/modules/mlflow/main.tf b/IaC/modules/mlflow/main.tf new file mode 100644 index 0000000..20fef59 --- /dev/null +++ b/IaC/modules/mlflow/main.tf @@ -0,0 +1,36 @@ +module "artifacts" { + source = "./artifacts" + bucket_name = var.artifacts_bucket_name + bucket_location = var.artifacts_bucket_location + number_of_version = var.artifacts_number_of_version + storage_class = var.artifacts_storage_class +} + +module "db_secret" { + source = "./secret_manager" + secret_id = db_password_name + secret_value = db_password_value +} + +module "database" { + source = "./database" + instance_prefix = var.db_instance_prefix + database_version = var.db_version + region = var.db_region + private_vpc_connection = var.private_vpc_connection + size = var.db_size + network_link = var.network_link + availability_type = var.db_availability_type + database_name = var.db_name + username = var.db_username + password = module.db_secret.secret_value +} + +module "server" { + source = "./server" + server_name = var.mlflow_server + location = var.server_location + docker_image_name = var.server_docker_image + env_variables = var.server_env_variables + sql_instance_name = module.database.instance_connection_name +} \ No newline at end of file diff --git a/IaC/modules/mlflow/server/variables.tf b/IaC/modules/mlflow/server/variables.tf index 50bfd5f..2fc6d31 100644 --- a/IaC/modules/mlflow/server/variables.tf +++ b/IaC/modules/mlflow/server/variables.tf @@ -1,6 +1,6 @@ variable "server_name" { type = string - description = "Name of your cserver" + description = "Name of your server" } variable "location" { type = string diff --git a/IaC/modules/mlflow/variables.tf b/IaC/modules/mlflow/variables.tf new file mode 100644 index 0000000..e69de29 From fc035cb4757d42867294a6cd59e14f5ebaf09611 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Thu, 10 Sep 2020 11:19:55 +0200 Subject: [PATCH 05/16] Add main module to call network and mlflow --- IaC/main.tf | 9 ++++ IaC/modules/mlflow/main.tf | 4 +- IaC/modules/mlflow/variables.tf | 90 +++++++++++++++++++++++++++++++++ bin/Makefile | 17 +++++++ bin/one-click-mlflow.py | 2 - 5 files changed, 118 insertions(+), 4 deletions(-) create mode 100644 bin/Makefile delete mode 100644 bin/one-click-mlflow.py diff --git a/IaC/main.tf b/IaC/main.tf index 0eb2b15..911ef86 100644 --- a/IaC/main.tf +++ b/IaC/main.tf @@ -15,4 +15,13 @@ provider "google" { module "network" { source = "modules/network" vpc_name = var.network_name +} + +module "mlflow" { + source = "modules/mlflow" + artifacts_bucket_name = var.artifacts_bucket + db_password_value = var.db_password_value + private_vpc_connection = module.network.private_vpc_connection + network_link = module.network.network_link + server_docker_image = var.mlflow_docker_image } \ No newline at end of file diff --git a/IaC/modules/mlflow/main.tf b/IaC/modules/mlflow/main.tf index 20fef59..8580341 100644 --- a/IaC/modules/mlflow/main.tf +++ b/IaC/modules/mlflow/main.tf @@ -8,8 +8,8 @@ module "artifacts" { module "db_secret" { source = "./secret_manager" - secret_id = db_password_name - secret_value = db_password_value + secret_id = var.db_password_name + secret_value = var.db_password_value } module "database" { diff --git a/IaC/modules/mlflow/variables.tf b/IaC/modules/mlflow/variables.tf index e69de29..0080542 100644 --- a/IaC/modules/mlflow/variables.tf +++ b/IaC/modules/mlflow/variables.tf @@ -0,0 +1,90 @@ +variable "artifacts_bucket_name" { + description = "Name of the mlflow bucket created to store artifacts" + type = string +} +variable "artifacts_bucket_location" { + description = "Location of the mlflow artifact bucket deployed" + type = string + default = "EUROPE-WEST1" +} +variable "artifacts_number_of_version" { + description = "Number of file version keeped in your artifacts bucket" + type = number + default = 1 +} +variable "artifacts_storage_class" { + description = "Storage class of your artifact bucket" + type = string + default = "STANDARD" +} +variable "db_password_name" { + description = "Name of the database password stored in secret manager" + type = string + default = "mlflow-db-pwd" +} +variable "db_password_value" { + description = "Value of the database password stored in secret manager" + type = string +} +variable "db_username" { + description = "Value of the database username" + type = string + default = "mlflowuser" +} +variable "db_instance_prefix" { + description = "prefix used as database instance name" + type = string + default = "mlflow" +} +variable "db_version" { + description = "Databse instance version in GCP" + type = string + default = "MYSQL_5_6" +} +variable "db_region" { + description = "Database region" + type = string + default = "europe-west1" +} +variable "private_vpc_connection" { + description = "Vpc connection with the database" + type = any +} +variable "db_size" { + description = "Database instance size" + type = string + default = "db-f1-micro" +} +variable "network_link" { + description = "Link to your network" + type = string +} +variable "db_availability_type" { + description = "Availability of your database" + type = string + default = "ZONAL" +} +variable "db_name" { + description = "Name of the database created inside the instance" + type = string + default = "mlflow" +} +variable "mlflow_server" { + description = "Name of the mlflow server deployed to cloud run" + type = string + default = "mlflow" +} +variable "server_location" { + description = "Location to deploy cloud run server" + type = string + default = "europe-west1" +} +variable "server_docker_image" { + description = "Docker image name of your mlflow server" + type = string +} +variable "server_env_variables" { + description = "Env variables used inside your container" + type = map + default = {} +} diff --git a/bin/Makefile b/bin/Makefile new file mode 100644 index 0000000..1fd4ae1 --- /dev/null +++ b/bin/Makefile @@ -0,0 +1,17 @@ + + +pre-check: + -gsutil -m mb -l EU -p $(PROJECT_ID) gs://$(BACKEND_TERRAFORM) + +init-terraform: + @terraform init -backend-config="bucket=$(BACKEND_TERRAFORM)" + +apply-terraform: + @terraform apply --quiet -var="project_id=$(PROJECT_ID)" + +plan-terraform: + @terraform plan -var="project_id=$(PROJECT_ID)" + +apply: pre-check init-terraform apply-terraform + +plan: pre-check init-terraform plan-terraform diff --git a/bin/one-click-mlflow.py b/bin/one-click-mlflow.py deleted file mode 100644 index ed6bfc2..0000000 --- a/bin/one-click-mlflow.py +++ /dev/null @@ -1,2 +0,0 @@ -from lib.entrypoint import run - From 5e1f4fee4098dd1fda899d398e52c9604aac392f Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Thu, 10 Sep 2020 11:23:01 +0200 Subject: [PATCH 06/16] Create cloud run service account and grant access to secret manager and cloud sql --- IaC/modules/mlflow/server/main.tf | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/IaC/modules/mlflow/server/main.tf b/IaC/modules/mlflow/server/main.tf index f2088e2..9e9cdf3 100644 --- a/IaC/modules/mlflow/server/main.tf +++ b/IaC/modules/mlflow/server/main.tf @@ -1,9 +1,15 @@ +resource "google_service_account" "service_account_cloud_run" { + account_id = format("cloud-run-%s", server_name) + display_name = "Cloud run service account used" +} + resource "google_cloud_run_service" "default" { name = var.server_name location = var.location template { spec { + service_account_name = google_service_account.service_account_cloud_run.email containers { image = var.docker_image_name dynamic "env" { @@ -35,6 +41,18 @@ resource "google_cloud_run_service" "default" { autogenerate_revision_name = true } +resource "google_project_iam_member" "cloudsql" { + project = google_cloud_run_service.default.project + role = "roles/cloudsql.client" + member = format("serviceAccount:%s", google_service_account.service_account_cloud_run.email) +} + +resource "google_project_iam_member" "secret" { + project = google_cloud_run_service.default.project + role = "roles/secretmanager.secretAccessor" + member = format("serviceAccount:%s", google_service_account.service_account_cloud_run.email) +} + data "google_iam_policy" "noauth" { binding { role = "roles/run.invoker" From f37779a3bbf345161cafe400e00a696d9e0f0547 Mon Sep 17 00:00:00 2001 From: Alexis VIALARET Date: Thu, 10 Sep 2020 11:40:57 +0200 Subject: [PATCH 07/16] add: dockerfile --- tracking_server/run_tracking.sh | 21 +++++++++++++++++++ tracking_server/tracking.Dockerfile | 32 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 tracking_server/run_tracking.sh create mode 100644 tracking_server/tracking.Dockerfile diff --git a/tracking_server/run_tracking.sh b/tracking_server/run_tracking.sh new file mode 100644 index 0000000..82a6dc7 --- /dev/null +++ b/tracking_server/run_tracking.sh @@ -0,0 +1,21 @@ +GCP_PROJECT=$(curl "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") +FUNCTION_REGION=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/region" -H "Metadata-Flavor: Google") + +echo "/mlflow/cloud_sql_proxy -instances=${GCP_PROJECT}:${FUNCTION_REGION}:${INSTANCE}=tcp:3306 &" + +/mlflow/cloud_sql_proxy -instances=${GCP_PROJECT}:${FUNCTION_REGION}:${INSTANCE}=tcp:3306 & + +sleep 10 + +echo "Artifact Root is ${ARTIFACT_ROOT}" + +DB_PASSWORD=$(gcloud beta secrets versions access 1 --secret="db_password") +BACKEND_URI=mysql+pymysql://root:${DB_PASSWORD}@127.0.0.1:3306/mlflow_store + + +mlflow db upgrade ${BACKEND_URI} + +mlflow server \ + --backend-store-uri ${BACKEND_URI} \ + --default-artifact-root ${ARTIFACT_ROOT} \ + --host 0.0.0.0 \ No newline at end of file diff --git a/tracking_server/tracking.Dockerfile b/tracking_server/tracking.Dockerfile new file mode 100644 index 0000000..dce96bc --- /dev/null +++ b/tracking_server/tracking.Dockerfile @@ -0,0 +1,32 @@ +FROM continuumio/miniconda3:4.7.10 + +WORKDIR /mlflow/ + +ARG MLFLOW_VERSION=1.2.0 +RUN mkdir -p /mlflow/ \ + && apt-get update \ + && apt-get -y install --no-install-recommends apt-transport-https ca-certificates gnupg default-libmysqlclient-dev libpq-dev build-essential curl \ + && pip install \ + mlflow==$MLFLOW_VERSION \ + sqlalchemy \ + boto3 \ + google-cloud-storage \ + psycopg2 \ + mysql \ + pymysql + +EXPOSE 5000 + +RUN curl -sSL https://sdk.cloud.google.com | bash +ENV PATH $PATH:/root/google-cloud-sdk/bin +RUN gcloud components install beta -q + +RUN curl -o cloud_sql_proxy https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 +RUN chmod +x cloud_sql_proxy + +COPY run_tracking.sh . +RUN chmod +x run_tracking.sh + +ENV BACKEND_URI /mlflow/store +ENV ARTIFACT_ROOT /mlflow/mlflow-artifacts +CMD /mlflow/run_tracking.sh From b51727e3a1c9a5c5019cb87dba81d296997ea5b7 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Thu, 10 Sep 2020 12:19:13 +0200 Subject: [PATCH 08/16] Add permission to push and read to gcs --- IaC/modules/mlflow/server/main.tf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/IaC/modules/mlflow/server/main.tf b/IaC/modules/mlflow/server/main.tf index 9e9cdf3..ffbfba3 100644 --- a/IaC/modules/mlflow/server/main.tf +++ b/IaC/modules/mlflow/server/main.tf @@ -53,6 +53,12 @@ resource "google_project_iam_member" "secret" { member = format("serviceAccount:%s", google_service_account.service_account_cloud_run.email) } +resource "google_project_iam_member" "gcs" { + project = google_cloud_run_service.default.project + role = "roles/storage.objectAdmin" + member = format("serviceAccount:%s", google_service_account.service_account_cloud_run.email) +} + data "google_iam_policy" "noauth" { binding { role = "roles/run.invoker" From 4d92968b11e103b52ce8d23187c5c8862ba6c7eb Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Fri, 11 Sep 2020 08:35:50 +0200 Subject: [PATCH 09/16] Add serverless connector and fix few connecting issue between db and cloud run --- IaC/main.tf | 31 ++++++++++- IaC/modules/mlflow/artifacts/outputs.tf | 4 ++ IaC/modules/mlflow/artifacts/variables.tf | 4 ++ IaC/modules/mlflow/database/main.tf | 2 +- IaC/modules/mlflow/database/outputs.tf | 5 +- IaC/modules/mlflow/database/variables.tf | 8 ++- IaC/modules/mlflow/main.tf | 8 +++ .../mlflow/secret_manager/variables.tf | 6 ++- IaC/modules/mlflow/server/main.tf | 54 ++++++++++++------- IaC/modules/mlflow/server/variables.tf | 42 +++++++++++++++ IaC/modules/mlflow/variables.tf | 14 ++++- IaC/modules/network/main.tf | 9 +++- IaC/modules/network/outputs.tf | 6 ++- IaC/modules/network/variables.tf | 5 ++ IaC/modules/services/main.tf | 6 +++ IaC/modules/services/variables.tf | 8 +++ IaC/variables.tf | 18 +++++++ 17 files changed, 202 insertions(+), 28 deletions(-) create mode 100644 IaC/modules/mlflow/artifacts/outputs.tf create mode 100644 IaC/modules/services/main.tf create mode 100644 IaC/modules/services/variables.tf diff --git a/IaC/main.tf b/IaC/main.tf index 911ef86..4061092 100644 --- a/IaC/main.tf +++ b/IaC/main.tf @@ -11,17 +11,44 @@ provider "google" { project = var.project_id } +provider "google-beta" { + project = var.project_id +} + +module "services" { + source = "./modules/services" + project_id = var.project_id + services = ["container.googleapis.com", "servicenetworking.googleapis.com", + "stackdriver.googleapis.com", "vpcaccess.googleapis.com", "run.googleapis.com", + "sqladmin.googleapis.com", "iap.googleapis.com", "secretmanager.googleapis.com"] +} + +resource "null_resource" "docker" { + triggers = { + always_run = "${timestamp()}" + } + provisioner "local-exec" { + command = < Date: Fri, 11 Sep 2020 08:38:27 +0200 Subject: [PATCH 10/16] Delete cloud sql proxy from run tracking --- Makefile | 20 ++++++++++++++++++++ bin/Makefile | 17 ----------------- tracking_server/run_tracking.sh | 21 +++++---------------- tracking_server/tracking.Dockerfile | 10 +++------- 4 files changed, 28 insertions(+), 40 deletions(-) create mode 100644 Makefile delete mode 100644 bin/Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..db04e2f --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +pre-check: + -gsutil -m mb -l EU -p $(PROJECT_ID) gs://$(BACKEND_TERRAFORM) + +init-terraform: + @cd Iac && terraform init -backend-config="bucket=$(BACKEND_TERRAFORM)" + +apply-terraform: + @cd Iac && terraform apply -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=eu.gcr.io/$(PROJECT_ID)/mlflow:latest" + +plan-terraform: + @cd Iac && terraform plan -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=eu.gcr.io/$(PROJECT_ID)/mlflow:latest" + +destroy-terraform: + @cd Iac && terraform destroy -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=eu.gcr.io/$(PROJECT_ID)/mlflow:latest" + +apply: pre-check init-terraform apply-terraform + +plan: pre-check init-terraform plan-terraform + +destroy: pre-check init-terraform destroy-terraform \ No newline at end of file diff --git a/bin/Makefile b/bin/Makefile deleted file mode 100644 index 1fd4ae1..0000000 --- a/bin/Makefile +++ /dev/null @@ -1,17 +0,0 @@ - - -pre-check: - -gsutil -m mb -l EU -p $(PROJECT_ID) gs://$(BACKEND_TERRAFORM) - -init-terraform: - @terraform init -backend-config="bucket=$(BACKEND_TERRAFORM)" - -apply-terraform: - @terraform apply --quiet -var="project_id=$(PROJECT_ID)" - -plan-terraform: - @terraform plan -var="project_id=$(PROJECT_ID)" - -apply: pre-check init-terraform apply-terraform - -plan: pre-check init-terraform plan-terraform diff --git a/tracking_server/run_tracking.sh b/tracking_server/run_tracking.sh index 82a6dc7..2662524 100644 --- a/tracking_server/run_tracking.sh +++ b/tracking_server/run_tracking.sh @@ -1,21 +1,10 @@ -GCP_PROJECT=$(curl "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google") -FUNCTION_REGION=$(curl "http://metadata.google.internal/computeMetadata/v1/instance/region" -H "Metadata-Flavor: Google") - -echo "/mlflow/cloud_sql_proxy -instances=${GCP_PROJECT}:${FUNCTION_REGION}:${INSTANCE}=tcp:3306 &" - -/mlflow/cloud_sql_proxy -instances=${GCP_PROJECT}:${FUNCTION_REGION}:${INSTANCE}=tcp:3306 & - -sleep 10 - -echo "Artifact Root is ${ARTIFACT_ROOT}" - -DB_PASSWORD=$(gcloud beta secrets versions access 1 --secret="db_password") -BACKEND_URI=mysql+pymysql://root:${DB_PASSWORD}@127.0.0.1:3306/mlflow_store - +DB_PASSWORD=$(gcloud beta secrets versions access --project=${GCP_PROJECT} --secret=${DB_PASSWORD_NAME} latest) +BACKEND_URI=mysql+pymysql://${DB_USERNAME}:${DB_PASSWORD}@${DB_PRIVATE_IP}/${DB_NAME} mlflow db upgrade ${BACKEND_URI} mlflow server \ --backend-store-uri ${BACKEND_URI} \ - --default-artifact-root ${ARTIFACT_ROOT} \ - --host 0.0.0.0 \ No newline at end of file + --default-artifact-root ${GCS_BACKEND} \ + --host 0.0.0.0 \ + --port $PORT diff --git a/tracking_server/tracking.Dockerfile b/tracking_server/tracking.Dockerfile index dce96bc..4c4b90a 100644 --- a/tracking_server/tracking.Dockerfile +++ b/tracking_server/tracking.Dockerfile @@ -15,18 +15,14 @@ RUN mkdir -p /mlflow/ \ mysql \ pymysql -EXPOSE 5000 +EXPOSE 8080 RUN curl -sSL https://sdk.cloud.google.com | bash ENV PATH $PATH:/root/google-cloud-sdk/bin RUN gcloud components install beta -q -RUN curl -o cloud_sql_proxy https://dl.google.com/cloudsql/cloud_sql_proxy.linux.amd64 -RUN chmod +x cloud_sql_proxy - COPY run_tracking.sh . RUN chmod +x run_tracking.sh -ENV BACKEND_URI /mlflow/store -ENV ARTIFACT_ROOT /mlflow/mlflow-artifacts -CMD /mlflow/run_tracking.sh + +CMD /mlflow/run_tracking.sh \ No newline at end of file From 4409a88465de4522a11e21750fb99688190151c3 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Fri, 11 Sep 2020 08:39:30 +0200 Subject: [PATCH 11/16] Add shebang --- tracking_server/run_tracking.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tracking_server/run_tracking.sh b/tracking_server/run_tracking.sh index 2662524..5d432f0 100644 --- a/tracking_server/run_tracking.sh +++ b/tracking_server/run_tracking.sh @@ -1,3 +1,5 @@ +#!/bin/bash + DB_PASSWORD=$(gcloud beta secrets versions access --project=${GCP_PROJECT} --secret=${DB_PASSWORD_NAME} latest) BACKEND_URI=mysql+pymysql://${DB_USERNAME}:${DB_PASSWORD}@${DB_PRIVATE_IP}/${DB_NAME} From 321c8f1944b7f6ebf58735a9931f9b94703f7b08 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Fri, 11 Sep 2020 08:54:30 +0200 Subject: [PATCH 12/16] Add EOL --- IaC/main.tf | 2 +- IaC/modules/mlflow/artifacts/main.tf | 2 +- IaC/modules/mlflow/artifacts/variables.tf | 2 +- IaC/modules/mlflow/database/main.tf | 2 +- IaC/modules/mlflow/database/variables.tf | 2 +- IaC/modules/mlflow/main.tf | 2 +- IaC/modules/mlflow/server/main.tf | 2 +- IaC/modules/mlflow/variables.tf | 2 +- IaC/modules/network/variables.tf | 2 +- IaC/modules/services/main.tf | 2 +- IaC/modules/services/variables.tf | 2 +- IaC/variables.tf | 2 +- tracking_server/tracking.Dockerfile | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/IaC/main.tf b/IaC/main.tf index 4061092..2a2e4b9 100644 --- a/IaC/main.tf +++ b/IaC/main.tf @@ -51,4 +51,4 @@ module "mlflow" { project_id = var.project_id vpc_connector = module.network.vpc_connector module_depends_on = null_resource.docker -} \ No newline at end of file +} diff --git a/IaC/modules/mlflow/artifacts/main.tf b/IaC/modules/mlflow/artifacts/main.tf index 0bd2fa6..e7c0fbd 100644 --- a/IaC/modules/mlflow/artifacts/main.tf +++ b/IaC/modules/mlflow/artifacts/main.tf @@ -13,4 +13,4 @@ resource "google_storage_bucket" "this" { type = "Delete" } } -} \ No newline at end of file +} diff --git a/IaC/modules/mlflow/artifacts/variables.tf b/IaC/modules/mlflow/artifacts/variables.tf index 7ce2d86..ec992d4 100644 --- a/IaC/modules/mlflow/artifacts/variables.tf +++ b/IaC/modules/mlflow/artifacts/variables.tf @@ -25,4 +25,4 @@ variable "storage_class" { variable "module_depends_on" { type = any default = null -} \ No newline at end of file +} diff --git a/IaC/modules/mlflow/database/main.tf b/IaC/modules/mlflow/database/main.tf index 0a27397..79b4046 100644 --- a/IaC/modules/mlflow/database/main.tf +++ b/IaC/modules/mlflow/database/main.tf @@ -34,4 +34,4 @@ resource "google_sql_user" "this_user" { instance = google_sql_database_instance.this_instance.name password = var.password depends_on = [google_sql_database_instance.this_instance] -} \ No newline at end of file +} diff --git a/IaC/modules/mlflow/database/variables.tf b/IaC/modules/mlflow/database/variables.tf index 51a50fd..e77d728 100644 --- a/IaC/modules/mlflow/database/variables.tf +++ b/IaC/modules/mlflow/database/variables.tf @@ -47,4 +47,4 @@ variable "password" { variable "module_depends_on" { type = any default = null -} \ No newline at end of file +} diff --git a/IaC/modules/mlflow/main.tf b/IaC/modules/mlflow/main.tf index 0c75ea1..88344ab 100644 --- a/IaC/modules/mlflow/main.tf +++ b/IaC/modules/mlflow/main.tf @@ -41,4 +41,4 @@ module "server" { gcs_backend = module.artifacts.url vpc_connector = var.vpc_connector module_depends_on = var.module_depends_on -} \ No newline at end of file +} diff --git a/IaC/modules/mlflow/server/main.tf b/IaC/modules/mlflow/server/main.tf index 3f2f3b8..0144e62 100644 --- a/IaC/modules/mlflow/server/main.tf +++ b/IaC/modules/mlflow/server/main.tf @@ -92,4 +92,4 @@ resource "google_cloud_run_service_iam_policy" "noauth" { service = google_cloud_run_service.default.name policy_data = data.google_iam_policy.noauth.policy_data -} \ No newline at end of file +} diff --git a/IaC/modules/mlflow/variables.tf b/IaC/modules/mlflow/variables.tf index c4b8bb1..49cd5e0 100644 --- a/IaC/modules/mlflow/variables.tf +++ b/IaC/modules/mlflow/variables.tf @@ -99,4 +99,4 @@ variable "vpc_connector" { variable "module_depends_on" { type = any default = null -} \ No newline at end of file +} diff --git a/IaC/modules/network/variables.tf b/IaC/modules/network/variables.tf index 99395b6..9e896ab 100644 --- a/IaC/modules/network/variables.tf +++ b/IaC/modules/network/variables.tf @@ -6,4 +6,4 @@ variable "region" { type = string description = "Region to deploy your vpc connector" default = "europe-west1" -} \ No newline at end of file +} diff --git a/IaC/modules/services/main.tf b/IaC/modules/services/main.tf index d847a98..d817bbe 100644 --- a/IaC/modules/services/main.tf +++ b/IaC/modules/services/main.tf @@ -3,4 +3,4 @@ resource "google_project_service" "project" { project = var.project_id service = var.services[count.index] disable_dependent_services = true -} \ No newline at end of file +} diff --git a/IaC/modules/services/variables.tf b/IaC/modules/services/variables.tf index ce1bea6..d6d99bd 100644 --- a/IaC/modules/services/variables.tf +++ b/IaC/modules/services/variables.tf @@ -5,4 +5,4 @@ variable "project_id" { variable "services" { description = "List of url of the service you want to activate" type = list(string) -} \ No newline at end of file +} diff --git a/IaC/variables.tf b/IaC/variables.tf index 66e34b3..7acd7f1 100644 --- a/IaC/variables.tf +++ b/IaC/variables.tf @@ -19,4 +19,4 @@ variable "network_name" { description = "Network used" type = string default = "default-private" -} \ No newline at end of file +} diff --git a/tracking_server/tracking.Dockerfile b/tracking_server/tracking.Dockerfile index 4c4b90a..f4600f5 100644 --- a/tracking_server/tracking.Dockerfile +++ b/tracking_server/tracking.Dockerfile @@ -25,4 +25,4 @@ COPY run_tracking.sh . RUN chmod +x run_tracking.sh -CMD /mlflow/run_tracking.sh \ No newline at end of file +CMD /mlflow/run_tracking.sh From 2cd4d03ceb0f190d21e22c37ad4f83b8311788d1 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Fri, 11 Sep 2020 11:49:49 +0200 Subject: [PATCH 13/16] Fix typos --- IaC/modules/mlflow/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IaC/modules/mlflow/variables.tf b/IaC/modules/mlflow/variables.tf index 49cd5e0..122dda8 100644 --- a/IaC/modules/mlflow/variables.tf +++ b/IaC/modules/mlflow/variables.tf @@ -8,7 +8,7 @@ variable "artifacts_bucket_location" { default = "EUROPE-WEST1" } variable "artifacts_number_of_version" { - description = "Number of file version keeped in your artifacts bucket" + description = "Number of file version kept in your artifacts bucket" type = number default = 1 } @@ -37,7 +37,7 @@ variable "db_instance_prefix" { default = "mlflow" } variable "db_version" { - description = "Databse instance version in GCP" + description = "Database instance version in GCP" type = string default = "MYSQL_5_7" } From 3f8b40148a7c3742d3e6611d9c1ce66f6d7bb2b1 Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Wed, 16 Sep 2020 09:16:59 +0200 Subject: [PATCH 14/16] Add terraform to make prerequesites --- IaC/prerequesites/main.tf | 26 ++++++++++++++++++++++++++ IaC/prerequesites/variables.tf | 28 ++++++++++++++++++++++++++++ Makefile | 4 ++-- README.md | 9 +++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 IaC/prerequesites/main.tf create mode 100644 IaC/prerequesites/variables.tf diff --git a/IaC/prerequesites/main.tf b/IaC/prerequesites/main.tf new file mode 100644 index 0000000..60102b4 --- /dev/null +++ b/IaC/prerequesites/main.tf @@ -0,0 +1,26 @@ +terraform { + required_version = "=0.12.29" + required_providers { + google = "~> 3.13" + } +} + +provider "google" { + project = var.project_id +} + +module "services" { + source = "./../modules/services" + project_id = var.project_id + services = ["container.googleapis.com", "servicenetworking.googleapis.com", + "stackdriver.googleapis.com", "vpcaccess.googleapis.com", "run.googleapis.com", + "sqladmin.googleapis.com", "secretmanager.googleapis.com"] +} + +module "bucket_backend" { + source = "./../modules/mlflow/artifacts" + bucket_name = var.backend_bucket + bucket_location = var.backend_bucket_location + number_of_version = var.backend_bucket_number_of_version + storage_class = var.backend_bucket_storage_class +} \ No newline at end of file diff --git a/IaC/prerequesites/variables.tf b/IaC/prerequesites/variables.tf new file mode 100644 index 0000000..86b2805 --- /dev/null +++ b/IaC/prerequesites/variables.tf @@ -0,0 +1,28 @@ +variable "project_id" { + description = "GCP project" + type = string +} +variable "backend_bucket" { + description = "Name of the bucket." + type = string +} +variable "backend_bucket_location" { + description = "Location of the bucket." + type = string + default = "EUROPE-WEST1" +} +variable "versioning_enabled" { + description = "True if you want to version your bucket." + type = bool + default = true +} +variable "backend_bucket_number_of_version" { + description = "Number of version you want to keep with the versionning." + type = number + default = 3 +} +variable "backend_bucket_storage_class" { + description = "Storage class of your bucket" + type = string + default ="STANDARD" +} \ No newline at end of file diff --git a/Makefile b/Makefile index db04e2f..198345b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -pre-check: - -gsutil -m mb -l EU -p $(PROJECT_ID) gs://$(BACKEND_TERRAFORM) +pre-requesites: + @cd Iac/prerequesites && terraform init && terraform apply init-terraform: @cd Iac && terraform init -backend-config="bucket=$(BACKEND_TERRAFORM)" diff --git a/README.md b/README.md index b0c51a2..f3ce2c0 100644 --- a/README.md +++ b/README.md @@ -10,3 +10,12 @@ The project's deliverables are - Terraformed infrastructure - A list of all the GCP APIs that need to be enabled - A list of all the necessary GCP permissions to run the deployment + + +# Prerequesites + + +- Create backend bucket to store terraform state +- Enable google apis (serverless vpc access) + +To do so, just run make pre-requesites \ No newline at end of file From 8e3a2ebcbd95ddb70a54df127c383a7b8c0b60bd Mon Sep 17 00:00:00 2001 From: Thomas Griseau Date: Wed, 16 Sep 2020 16:12:52 +0200 Subject: [PATCH 15/16] Fixing PR comments --- IaC/main.tf | 21 --------------------- IaC/modules/mlflow/server/main.tf | 1 - IaC/modules/mlflow/server/variables.tf | 5 +++++ IaC/modules/network/main.tf | 2 +- IaC/modules/network/variables.tf | 7 +++++++ Makefile | 18 +++++++++++++----- README.md | 25 ++++++++++++++++--------- tracking_server/requirements.txt | 7 +++++++ tracking_server/tracking.Dockerfile | 20 ++++++-------------- 9 files changed, 55 insertions(+), 51 deletions(-) create mode 100644 tracking_server/requirements.txt diff --git a/IaC/main.tf b/IaC/main.tf index 2a2e4b9..df350e7 100644 --- a/IaC/main.tf +++ b/IaC/main.tf @@ -15,26 +15,6 @@ provider "google-beta" { project = var.project_id } -module "services" { - source = "./modules/services" - project_id = var.project_id - services = ["container.googleapis.com", "servicenetworking.googleapis.com", - "stackdriver.googleapis.com", "vpcaccess.googleapis.com", "run.googleapis.com", - "sqladmin.googleapis.com", "iap.googleapis.com", "secretmanager.googleapis.com"] -} - -resource "null_resource" "docker" { - triggers = { - always_run = "${timestamp()}" - } - provisioner "local-exec" { - command = < Date: Thu, 17 Sep 2020 14:25:09 +0200 Subject: [PATCH 16/16] change docker tag --- Makefile | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 8182272..0b8db72 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,28 @@ +DOCKER_REPO := eu.gcr.io +DOCKER_NAME := mlflow +DOCKER_TAG := 0.1 + + pre-requesites: @cd Iac/prerequesites && terraform init && terraform apply -var="project_id=$(PROJECT_ID)" build-docker: - @cd tracking_server && docker build -t eu.gcr.io/$(PROJECT_ID)/mlflow:latest -f tracking.Dockerfile . + @cd tracking_server && docker build -t $(DOCKER_REPO)/$(PROJECT_ID)/$(DOCKER_NAME):$(DOCKER_TAG) -f tracking.Dockerfile . push-docker: - @docker push eu.gcr.io/$(PROJECT_ID)/mlflow:latest + @docker push $(DOCKER_REPO)/$(PROJECT_ID)/$(DOCKER_NAME):$(DOCKER_TAG) init-terraform: @cd Iac && terraform init -backend-config="bucket=$(BACKEND_TERRAFORM)" apply-terraform: - @cd Iac && terraform apply -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=eu.gcr.io/$(PROJECT_ID)/mlflow:latest" + @cd Iac && terraform apply -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=$(DOCKER_REPO)/$(PROJECT_ID)/$(DOCKER_NAME):$(DOCKER_TAG)" plan-terraform: - @cd Iac && terraform plan -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=eu.gcr.io/$(PROJECT_ID)/mlflow:latest" + @cd Iac && terraform plan -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=$(DOCKER_REPO)/$(PROJECT_ID)/$(DOCKER_NAME):$(DOCKER_TAG)" destroy-terraform: - @cd Iac && terraform destroy -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=eu.gcr.io/$(PROJECT_ID)/mlflow:latest" + @cd Iac && terraform destroy -var="project_id=$(PROJECT_ID)" -var="mlflow_docker_image=$(DOCKER_REPO)/$(PROJECT_ID)/$(DOCKER_NAME):$(DOCKER_TAG)" apply: init-terraform apply-terraform