From 8227447ded5b79cbc0994bb62a98d046598d318f Mon Sep 17 00:00:00 2001 From: Kasper Primdal Lauritzen Date: Wed, 24 Aug 2022 10:36:47 +0200 Subject: [PATCH 1/6] WIP: migration --- CHANGELOG.md | 9 +++++++++ mlflow-terraform/main.tf | 24 +++++++++++++----------- mlflow-terraform/variables.tf | 10 ++++++++++ 3 files changed, 32 insertions(+), 11 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..0a9e5e9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,9 @@ +# Changelog +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] +### Changed +- Changed from KIAM auth to IRSA. See https://wiki.dfds.cloud/en/teams/devex/operations/guides/kiam-to-irsa-migration \ No newline at end of file diff --git a/mlflow-terraform/main.tf b/mlflow-terraform/main.tf index 3ccfafa..2cca8de 100644 --- a/mlflow-terraform/main.tf +++ b/mlflow-terraform/main.tf @@ -15,21 +15,23 @@ resource "aws_s3_bucket" "mlflow_bucket" { } // Create the IAM role to be used by MLFlow to connect to the S3 backend resource "aws_iam_role" "mlflow_server_role" { - assume_role_policy = data.aws_iam_policy_document.kiam_trust_policy.json + assume_role_policy = data.aws_iam_policy_document.irsa_trust_policy.json } -data "aws_iam_policy_document" "kiam_trust_policy" { +} +data "aws_iam_policy_document" "irsa_trust_policy" { statement { - sid = "" - - effect = "Allow" + sid = "" + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] principals { - type = "AWS" - identifiers = ["arn:aws:iam::${var.kubernetes_account_number}:role/eks-hellman-kiam-server"] + type = "Federated" + identifiers = ["arn:aws:iam::${var.kubernetes_account_number}:oidc-provider/oidc.eks.eu-west-1.amazonaws.com/id/B182759F93D251942CB146063F57036B"] + } + condition { + test = "StringEquals" + variable = "oidc.eks.eu-west-1.amazonaws.com/id/B182759F93D251942CB146063F57036B:sub" + values = ["system:serviceaccount:${var.kubernetes_namespace}:${var.service_account}"] } - - actions = [ - "sts:AssumeRole" - ] } } resource "aws_iam_role_policy_attachment" "mlflow_policy_attachment" { diff --git a/mlflow-terraform/variables.tf b/mlflow-terraform/variables.tf index b5409eb..ec5f0b1 100644 --- a/mlflow-terraform/variables.tf +++ b/mlflow-terraform/variables.tf @@ -3,3 +3,13 @@ variable "kubernetes_account_number" { type = string description = "The account number of the kubernetes cluster that has to assume a role in your capability" } + +variable "kubernetes_namespace" { + type = string + description = "The namespace of the kubernetes capability" +} + +variable "service_account" { + type = string + default = "mlflow" +} \ No newline at end of file From 7c9f3dbcf920538c73aba37b0663c52ff92beaa4 Mon Sep 17 00:00:00 2001 From: Kasper Primdal Lauritzen Date: Wed, 24 Aug 2022 11:03:37 +0200 Subject: [PATCH 2/6] fix: typo --- mlflow-terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mlflow-terraform/main.tf b/mlflow-terraform/main.tf index 2cca8de..e50a619 100644 --- a/mlflow-terraform/main.tf +++ b/mlflow-terraform/main.tf @@ -17,7 +17,7 @@ resource "aws_s3_bucket" "mlflow_bucket" { resource "aws_iam_role" "mlflow_server_role" { assume_role_policy = data.aws_iam_policy_document.irsa_trust_policy.json } -} + data "aws_iam_policy_document" "irsa_trust_policy" { statement { sid = "" From b00432a80b5ab5846953fe13882dc643dddc68c9 Mon Sep 17 00:00:00 2001 From: Kasper Primdal Lauritzen Date: Wed, 24 Aug 2022 11:03:49 +0200 Subject: [PATCH 3/6] docs: Add additional variables --- mlflow-terraform/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mlflow-terraform/README.md b/mlflow-terraform/README.md index 7daff6c..9d43d20 100644 --- a/mlflow-terraform/README.md +++ b/mlflow-terraform/README.md @@ -3,5 +3,6 @@ is provisioned centrally in this case. # Variables -- kubernetes_account_number: The account number to trust to assume your role (ie. account number of - KIAM) +- kubernetes_account_number: The account number of your kubernetes namespace. +- kubernetes_namespace: The name of the kubernetes namespace. +- service_account: OPTIONAL. The name of the service account used in the kubernetes deployment. \ No newline at end of file From 6138273a87d0099382ec9956c24e2631f6e8c271 Mon Sep 17 00:00:00 2001 From: Kasper Primdal Lauritzen Date: Wed, 24 Aug 2022 14:37:17 +0200 Subject: [PATCH 4/6] docs: Update CHANGELOG.md --- CHANGELOG.md | 7 ++++++- mlflow-terraform/README.md | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a9e5e9..95d60a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,5 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [4.0.0] - 2022-08-24 ### Changed -- Changed from KIAM auth to IRSA. See https://wiki.dfds.cloud/en/teams/devex/operations/guides/kiam-to-irsa-migration \ No newline at end of file +- BREAKING: Changed from KIAM auth to IRSA. + See https://wiki.dfds.cloud/en/teams/devex/operations/guides/kiam-to-irsa-migration. + KIAM is deprecated and will be removed in the future. We are now using IRSA and ServiceAccounts to assume roles in + AWS. diff --git a/mlflow-terraform/README.md b/mlflow-terraform/README.md index 9d43d20..4babf3f 100644 --- a/mlflow-terraform/README.md +++ b/mlflow-terraform/README.md @@ -1,4 +1,4 @@ -This is an opinionated collection of resouces to be used for the mlflow service. Assuming a database +This is an opinionated collection of resources to be used for the mlflow service. Assuming a database is provisioned centrally in this case. # Variables From b8f4297665af07902e7e5d469ad846188973c9e0 Mon Sep 17 00:00:00 2001 From: Kasper Primdal Lauritzen Date: Wed, 24 Aug 2022 14:43:22 +0200 Subject: [PATCH 5/6] docs: Improve variable descriptions --- mlflow-terraform/main.tf | 1 + mlflow-terraform/variables.tf | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/mlflow-terraform/main.tf b/mlflow-terraform/main.tf index e50a619..e8ba19a 100644 --- a/mlflow-terraform/main.tf +++ b/mlflow-terraform/main.tf @@ -16,6 +16,7 @@ resource "aws_s3_bucket" "mlflow_bucket" { // Create the IAM role to be used by MLFlow to connect to the S3 backend resource "aws_iam_role" "mlflow_server_role" { assume_role_policy = data.aws_iam_policy_document.irsa_trust_policy.json + name = "mlflow-server-role" } data "aws_iam_policy_document" "irsa_trust_policy" { diff --git a/mlflow-terraform/variables.tf b/mlflow-terraform/variables.tf index ec5f0b1..184564a 100644 --- a/mlflow-terraform/variables.tf +++ b/mlflow-terraform/variables.tf @@ -1,15 +1,16 @@ // Declare input variables variable "kubernetes_account_number" { type = string - description = "The account number of the kubernetes cluster that has to assume a role in your capability" + description = "The account number of the kubernetes capability. E.g. '123456789012'" } variable "kubernetes_namespace" { type = string - description = "The namespace of the kubernetes capability" + description = "The namespace of the kubernetes capability. E.g. 'my-capability-jpoxj'." } variable "service_account" { type = string default = "mlflow" + description = "The service account that assumes the mlflow-server-role Role. E.g. 'mlflow'." } \ No newline at end of file From 2ec3a41db451b5584587dc66e6a79025976149e2 Mon Sep 17 00:00:00 2001 From: Kasper Primdal Lauritzen Date: Wed, 24 Aug 2022 14:52:05 +0200 Subject: [PATCH 6/6] docs: Improve terraform naming --- mlflow-terraform/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mlflow-terraform/main.tf b/mlflow-terraform/main.tf index e8ba19a..f62b5bb 100644 --- a/mlflow-terraform/main.tf +++ b/mlflow-terraform/main.tf @@ -55,7 +55,8 @@ data "aws_iam_policy_document" "mlflow_server_policy" { } } resource "aws_iam_policy" "mlflow_server_policy" { - description = "allows mlflow access to S3" + name = "mlflow-server-policy" + description = "Allows mlflow access to S3" policy = data.aws_iam_policy_document.mlflow_server_policy.json } // Create a random password to be used for the mlflow webserver