Merge pull request #17 from artefactory/dev

0.0.1

.gitignore

@@ -0,0 +1,5 @@
+#terraform
+.terraform
+*.tfstate
+*.tfstate.backup
+*.tfvars

IaC/main.tf

@@ -0,0 +1,33 @@
+terraform {
+  backend "gcs" {
+  }
+  required_version = "=0.12.29"
+  required_providers {
+    google = "~> 3.13"
+  }
+}
+
+provider "google" {
+  project = var.project_id
+}
+
+provider "google-beta" {
+  project = var.project_id
+}
+
+
+module "network" {
+  source = "./modules/network"
+  vpc_name = var.network_name
+}
+
+module "mlflow" {
+  source = "./modules/mlflow"
+  artifacts_bucket_name = var.artifacts_bucket
+  db_password_value = var.db_password_value
+  private_vpc_connection = module.network.private_vpc_connection
+  network_link = module.network.network_link
+  server_docker_image = var.mlflow_docker_image
+  project_id = var.project_id
+  vpc_connector = module.network.vpc_connector
+}

IaC/modules/mlflow/artifacts/main.tf

@@ -0,0 +1,16 @@
+resource "google_storage_bucket" "this" {
+  name          = var.bucket_name
+  location      = var.bucket_location
+  storage_class = var.storage_class
+  versioning {
+    enabled = var.versioning_enabled
+  }
+  lifecycle_rule {
+    condition {
+      num_newer_versions = var.number_of_version
+    }
+    action {
+      type = "Delete"
+    }
+  }
+}

IaC/modules/mlflow/artifacts/outputs.tf

@@ -0,0 +1,4 @@
+output "url" {
+  description = "gcs uri"
+  value       = google_storage_bucket.this.url
+}

IaC/modules/mlflow/artifacts/variables.tf

@@ -0,0 +1,28 @@
+variable "bucket_name" {
+    description = "Name of the bucket."
+    type        = string
+}
+variable "bucket_location" {
+    description = "Location of the bucket."
+    type        = string
+    default     = "EUROPE-WEST1"
+}
+variable "versioning_enabled" {
+    description = "True if you want to version your bucket."
+    type        = bool
+    default     = true
+}
+variable "number_of_version" {
+    description = "Number of version you want to keep with the versionning."
+    type        = number
+    default     = 1
+}
+variable "storage_class" {
+    description = "Storage class of your bucket"
+    type        = string
+    default     ="STANDARD"
+}
+variable "module_depends_on" {
+  type    = any
+  default = null
+}

IaC/modules/mlflow/database/main.tf

@@ -0,0 +1,37 @@
+resource "random_id" "db_name_suffix" {
+  byte_length = 5
+}
+
+resource "google_sql_database_instance" "this_instance" {
+  name             = "${var.instance_prefix}-${random_id.db_name_suffix.hex}"
+  database_version = var.database_version
+  region           = var.region
+
+  depends_on = [var.private_vpc_connection]
+
+  settings {
+    tier = var.size
+    ip_configuration {
+      ipv4_enabled    = false
+      private_network = var.network_link
+    }
+    backup_configuration {
+      enabled = true
+    }
+    availability_type = var.availability_type
+
+  }
+}
+
+resource "google_sql_database" "this_database" {
+  name     = var.database_name
+  instance = google_sql_database_instance.this_instance.name
+  depends_on = [google_sql_database_instance.this_instance]
+}
+
+resource "google_sql_user" "this_user" {
+  name     = var.username
+  instance = google_sql_database_instance.this_instance.name
+  password = var.password
+  depends_on = [google_sql_database_instance.this_instance]
+}

IaC/modules/mlflow/database/outputs.tf

@@ -0,0 +1,12 @@
+output "instance_connection_name" {
+  description = "Connection string used to connect to the instance"
+  value       = google_sql_database_instance.this_instance.connection_name
+}
+output "private_ip" {
+  description = "Private ip connect to the instance"
+  value       = google_sql_database_instance.this_instance.private_ip_address
+}
+output "database_name" {
+  description = "The name of the database"
+  value       = google_sql_database.this_database.name
+}

IaC/modules/mlflow/database/variables.tf

@@ -0,0 +1,50 @@
+variable "instance_prefix" {
+    type = string
+    description = "Name of the database instance you want to deploy"
+    default = "mlflow"
+}
+variable "database_version" {
+    type = string
+    description = "Version of the database instance you use"
+    default = "MYSQL_5_7"
+}
+variable "region" {
+    type = string
+    description = "Region of the database instance"
+    default = "europe-west1"
+}
+variable "private_vpc_connection" {
+    type = any
+    description = "Private connection used to connect your instance with"
+}
+variable "size" {
+    type = string
+    description = "Size of the database instance"
+    default = "db-f1-micro"
+}
+variable "network_link" {
+    type = string
+    description = "Network link you want to connect your database with" 
+}
+variable "availability_type" {
+    type = string
+    description = "Availability of your instance" 
+    default = "ZONAL"
+}
+variable "database_name" {
+    type = string
+    description = "Name of the database created"
+    default = "mlflow"
+}
+variable "username" {
+    type = string
+    description = "Username to connect to database instance"
+}
+variable "password" {
+    type = string
+    description = "Password to connect to database instance" 
+}
+variable "module_depends_on" {
+  type    = any
+  default = null
+}

IaC/modules/mlflow/main.tf

@@ -0,0 +1,44 @@
+module "artifacts" {
+    source = "./artifacts"
+    bucket_name = var.artifacts_bucket_name
+    bucket_location = var.artifacts_bucket_location
+    number_of_version = var.artifacts_number_of_version
+    storage_class = var.artifacts_storage_class
+}
+
+module "db_secret" {
+    source = "./secret_manager"
+    secret_id = var.db_password_name
+    secret_value = var.db_password_value
+}
+
+module "database" {
+    source = "./database"
+    instance_prefix = var.db_instance_prefix
+    database_version = var.db_version
+    region = var.db_region
+    private_vpc_connection = var.private_vpc_connection
+    size = var.db_size
+    network_link = var.network_link
+    availability_type = var.db_availability_type
+    database_name = var.db_name
+    username = var.db_username
+    password = module.db_secret.secret_value
+}
+
+module "server" {
+    source = "./server"
+    server_name = var.mlflow_server
+    location = var.server_location
+    docker_image_name = var.server_docker_image
+    env_variables = var.server_env_variables
+    sql_instance_name = module.database.instance_connection_name
+    db_private_ip = module.database.private_ip
+    project_id = var.project_id
+    db_password_name = var.db_password_name
+    db_username = var.db_username
+    db_name = var.db_name
+    gcs_backend = module.artifacts.url
+    vpc_connector = var.vpc_connector
+    module_depends_on = var.module_depends_on
+}

IaC/modules/mlflow/secret_manager/main.tf

@@ -0,0 +1,19 @@
+resource "google_secret_manager_secret" "secret" {
+  provider = google-beta
+
+  secret_id = var.secret_id
+
+  replication {
+    automatic = true
+  }
+}
+
+
+resource "google_secret_manager_secret_version" "secret-version" {
+  provider = google-beta
+
+  secret = google_secret_manager_secret.secret.id
+
+  secret_data = var.secret_value
+  depends_on = [google_secret_manager_secret.secret]
+}

IaC/modules/mlflow/secret_manager/outputs.tf

@@ -0,0 +1,4 @@
+output "secret_value" {
+  description = "Value of the created secret"
+  value       = google_secret_manager_secret_version.secret-version.secret_data
+}

IaC/modules/mlflow/secret_manager/variables.tf

@@ -0,0 +1,12 @@
+variable "secret_id" {
+    type        = string
+    description = "Name of the secret you want to create"
+}
+variable "secret_value" {
+    type        = string
+    description = "value of the secret you want to create"
+}
+variable "module_depends_on" {
+  type    = any
+  default = null
+}

IaC/modules/mlflow/server/main.tf

@@ -0,0 +1,94 @@
+locals {
+  env_variables = merge(
+    {
+      "GCP_PROJECT"=var.project_id,
+      "DB_PASSWORD_NAME"=var.db_password_name,
+      "DB_USERNAME"=var.db_username,
+      "DB_NAME"=var.db_name,
+      "DB_PRIVATE_IP"=var.db_private_ip,
+      "GCS_BACKEND"=var.gcs_backend
+    }, var.env_variables)
+}
+
+
+resource "google_service_account" "service_account_cloud_run" {
+  account_id   = format("cloud-run-%s", var.server_name)
+  display_name = "Cloud run service account used"
+}
+
+resource "google_project_iam_member" "cloudsql" {
+  project = google_service_account.service_account_cloud_run.project
+  role    = "roles/cloudsql.client"
+  member  = format("serviceAccount:%s", google_service_account.service_account_cloud_run.email)
+}
+
+resource "google_project_iam_member" "secret" {
+  project = google_service_account.service_account_cloud_run.project
+  role    = "roles/secretmanager.secretAccessor"
+  member  = format("serviceAccount:%s", google_service_account.service_account_cloud_run.email)
+}
+
+resource "google_project_iam_member" "gcs" {
+  project = google_service_account.service_account_cloud_run.project
+  role    = "roles/storage.objectAdmin"
+  member  = format("serviceAccount:%s", google_service_account.service_account_cloud_run.email)
+}
+
+
+resource "google_cloud_run_service" "default" {
+  name     = var.server_name
+  location = var.location
+
+  template {
+    spec {
+      service_account_name = google_service_account.service_account_cloud_run.email
+      containers {
+        image = var.docker_image_name
+        dynamic "env" {
+            for_each = local.env_variables
+            content {
+            name = env.key
+            value = env.value
+            }
+        }
+        resources {
+            limits = {
+                cpu = var.cpu_limit
+                memory = var.memory_limit
+            }
+        }
+      }
+    }
+    metadata {
+      annotations = {
+        "run.googleapis.com/cloudsql-instances" = var.sql_instance_name
+        "run.googleapis.com/vpc-access-connector" = var.vpc_connector
+      }
+    }
+  }
+
+  traffic {
+    percent         = 100
+    latest_revision = true
+  }
+  autogenerate_revision_name = true
+  depends_on = [google_project_iam_member.cloudsql, google_project_iam_member.secret, google_project_iam_member.gcs, var.module_depends_on]
+}
+
+
+data "google_iam_policy" "noauth" {
+  binding {
+    role = "roles/run.invoker"
+    members = [
+      "allUsers",
+    ]
+  }
+}
+
+resource "google_cloud_run_service_iam_policy" "noauth" {
+  location    = google_cloud_run_service.default.location
+  project     = google_cloud_run_service.default.project
+  service     = google_cloud_run_service.default.name
+
+  policy_data = data.google_iam_policy.noauth.policy_data
+}