From 9a2b452d0ab8a1ff201f5af1df4f9583197b67e0 Mon Sep 17 00:00:00 2001 From: Xavier Rakotomamonjy Date: Thu, 22 Jun 2023 15:49:47 +0200 Subject: [PATCH] feat(example): devops-stack on scaleway chore(scaleway): first example --- .gitignore | 13 +- examples/scaleway/README.md | 36 +++++ examples/scaleway/inputs.tfvars | 38 +++++ examples/scaleway/loki.tf | 159 ++++++++++++++++++++ examples/scaleway/main.tf | 257 ++++++++++++++------------------ examples/scaleway/outputs.tf | 26 ++-- examples/scaleway/providers.tf | 35 +++-- examples/scaleway/terraform.tf | 6 +- examples/scaleway/variables.tf | 108 ++++++++++++++ 9 files changed, 504 insertions(+), 174 deletions(-) create mode 100644 examples/scaleway/README.md create mode 100644 examples/scaleway/inputs.tfvars create mode 100644 examples/scaleway/loki.tf create mode 100644 examples/scaleway/variables.tf diff --git a/.gitignore b/.gitignore index 6734e3d6dd..4a8f14ed61 100644 --- a/.gitignore +++ b/.gitignore @@ -1,12 +1,19 @@ ### Unwanted Terraform files when using the examples ### - examples/*/.terraform examples/*/terraform.tfstate examples/*/terraform.tfstate.* examples/*/.terraform.tfstate.lock.info examples/*/*-config examples/*/.terraform.lock.hcl - +examples/*/secrets.yml +examples/*/terraform.tfstate* +examples/*/.terraform.lock.hcl +examples/*/kubeconfig.yml +examples/*/issue.txt +examples/*/log.txt +examples/*/*.png +examples/*/*.html +examples/*/issuers.yml ### Files and folders when building the website and documentation locally ### @@ -55,3 +62,5 @@ ehthumbs_vista.db # Folder config file [Dd]esktop.ini +.DS_STORE +**/*.swp diff --git a/examples/scaleway/README.md b/examples/scaleway/README.md new file mode 100644 index 0000000000..38f93714d7 --- /dev/null +++ b/examples/scaleway/README.md @@ -0,0 +1,36 @@ +## Introduction +The terraform project will instanciated a devops-stack on Scaleway. + +## Installation + +Add your credentials to launch the project. At least the following environement variables are required: `SCW_ACCESS_KEY,SCW_ACCESS_KEY,SCW_DEFAULT_ORGANIZATION_ID,SCW_DEFAULT_PROJECT_ID,SCW_DEFAULT_PROJECT_ID`. + +We also use an environement variable for the variable PROJECT\_ID called `TF_VAR_PROJECT_ID` + +Configure the stack by modifying `inputs.tfvars` (e.g: cluster\_name) and launch the terraform apply with: + +If you want to create a Kapsule cluster, you will have to use the scaleway provider in version 2.33.0. + +If you modify the base\_domain, be sure to add a new star record that points to the load balancer ip address created by the stack in your domain. + +```bash +terraform init +terraform apply -var-file inputs.tfvars +``` + +## Usage +Get the kubeconfig file and the domain name with the following commands: + +```bash +terraform output -raw kubeconfig_file > kubeconfig.json +terraform output base_domain +``` + +Your application are available at the following address: $APP\_NAME.apps.$CLUSTER\_NAME.$BASE\_DOMAIN. +e.g: prometheus.apps.devops-stack.51-51-52-52.np.io + +For authentication on oidc, users and password are available in the output: +```bash +terraform output passwords +``` + diff --git a/examples/scaleway/inputs.tfvars b/examples/scaleway/inputs.tfvars new file mode 100644 index 0000000000..817866516f --- /dev/null +++ b/examples/scaleway/inputs.tfvars @@ -0,0 +1,38 @@ +# ################################################### +# Input for module which creates the scaleway cluster +# ################################################### +cluster_name = "devops-stack" +cluster_description = "Devops-stack on cloud provider scaleway" +cluster_tags = ["demo", "dev", "devops-stack", "test", ] +cluster_type = "multicloud" +kubernetes_version = "1.29.1" +admission_plugins = ["PodNodeSelector", ] +node_pools = { + config1 = { + node_type = "DEV1-L" + size = 2 + min_size = 2 + max_size = 2 + autoscaling = true + autohealing = true + container_runtime = "containerd" + wait_for_pool_ready = true + } +} + +# ######################### +# Additional cluster config +# ######################### +base_domain = "gs-fr-dev.camptocamp.com" +lb_name = "devops-stack" +zone = "fr-par-1" +lb_type = "LB-S" + +# Ingress +ingress_enable_service_monitor = false + +# Keycloak +cluster_issuer = "ca-issuer" + +# Cert-manager +cert_manager_enable_service_monitor = false diff --git a/examples/scaleway/loki.tf b/examples/scaleway/loki.tf new file mode 100644 index 0000000000..037ec2baa1 --- /dev/null +++ b/examples/scaleway/loki.tf @@ -0,0 +1,159 @@ +data "scaleway_account_project" "devops_stack" { + project_id = var.project_id +} + +resource "scaleway_object_bucket" "loki" { + name = "devops-stack-loki-logs" + tags = { + line = "devops-stack" + platform = "scw-devops-stack-example" + } +} + +resource "scaleway_iam_application" "loki" { + name = "devops-stack-example-loki" + description = "Loki access to S3 buckets from Devops Stack example" +} + +resource "scaleway_iam_policy" "loki" { + name = "devops-stack-example-loki" + description = "Loki access to S3 buckets from Devops Stack example" + application_id = scaleway_iam_application.loki.id + + rule { + project_ids = [data.scaleway_account_project.devops_stack.id] + permission_set_names = [ + "ObjectStorageObjectsDelete", + "ObjectStorageObjectsRead", + "ObjectStorageObjectsWrite", + "ObjectStorageBucketsRead", + ] + } +} + +resource "scaleway_iam_api_key" "loki" { + application_id = scaleway_iam_application.loki.id + description = "Loki credentials for Devops Stack example" + default_project_id = data.scaleway_account_project.devops_stack.id +} + +locals { + loki_common_settings = { + extraEnv = [ + { + name = "AWS_ACCESS_KEY_ID" + valueFrom = { + secretKeyRef = { + name = kubernetes_secret.credentials_loki_s3.metadata.0.name + key = "AWS_ACCESS_KEY_ID" + } + } + }, + { + name = "AWS_SECRET_ACCESS_KEY" + valueFrom = { + secretKeyRef = { + name = kubernetes_secret.credentials_loki_s3.metadata.0.name + key = "AWS_SECRET_ACCESS_KEY" + } + } + }, + ] + } +} + +module "loki" { + source = "git::https://github.com/camptocamp/devops-stack-module-loki-stack.git?ref=v8.1.0" + + app_autosync = {} + + retention = "9000h" + ingress = { + hosts = ["loki.apps.${var.cluster_name}.${var.base_domain}"] + cluster_issuer = var.cluster_issuer + } + + helm_values = [{ + loki-distributed = { + loki = merge({ + structuredConfig = { + auth_enabled = false + compactor = { + retention_delete_delay = "1h" + retention_enabled = false + } + ingester = { + lifecycler = { + ring = { + replication_factor = 1 + } + } + } + } + schemaConfig = { + configs = [ + { + from = "2023-04-28", + store = "boltdb-shipper" + object_store = "s3" + schema = "v11" + index = { + prefix = "index_" + period = "24h" + } + } + ] + } + storageConfig = { + aws = { + bucketnames = scaleway_object_bucket.loki.id + endpoint = scaleway_object_bucket.loki.endpoint + s3forcepathstyle = true + #region = "fr-par" + sse_encryption = false + signature_version = "v2" + } + boltdb_shipper = { + shared_store = "s3" + cache_ttl = "24h" + } + } + }, local.loki_common_settings) + indexGateway = local.loki_common_settings + ingester = merge({ replicas = 1 }, local.loki_common_settings) + compactor = local.loki_common_settings + queryFrontend = local.loki_common_settings + querier = local.loki_common_settings + distributor = local.loki_common_settings + } + promtail = { + updateStrategy = { + type = "RollingUpdate" + rollingUpdate = { + maxUnavailable = 3 + } + } + config = { + clients = [ + { + url = "http://loki-distributor:3100/loki/api/v1/push" + #tenant_id = 1 + } + ] + } + } + }] +} + +resource "kubernetes_secret" "credentials_loki_s3" { + metadata { + namespace = "loki-stack" + name = "credentials-loki-s3" + } + + data = { + AWS_ACCESS_KEY_ID = scaleway_iam_api_key.loki.access_key + AWS_SECRET_ACCESS_KEY = scaleway_iam_api_key.loki.secret_key + } + +} diff --git a/examples/scaleway/main.tf b/examples/scaleway/main.tf index b1364d2916..7f4e51321a 100644 --- a/examples/scaleway/main.tf +++ b/examples/scaleway/main.tf @@ -1,54 +1,42 @@ -locals { - cluster_name = "scaleway-test" - cluster_region = "fr-par" - cluster_zone = "fr-par-1" - tags = ["test", "${local.cluster_name}"] -} - -module "cluster" { - source = "git::https://github.com/camptocamp/devops-stack.git//modules/scaleway?ref=v1-alpha" - - kubernetes_version = "1.24.3" - - cluster_type = "kapsule" - cluster_name = local.cluster_name - cluster_tags = local.tags - region = local.cluster_region - zone = local.cluster_zone - lb_type = "LB-S" - +# ########################### +# INFRA + K8s PHASE +# ########################### +module "scaleway" { + source = "git@github.com:camptocamp/devops-stack-module-cluster-scaleway.git" + + base_domain = var.base_domain + cluster_name = var.cluster_name + cluster_description = var.cluster_description + cluster_tags = var.cluster_tags + cluster_type = var.cluster_type + kubernetes_version = var.kubernetes_version + lb_name = var.lb_name + lb_type = var.lb_type + zone = var.zone + node_pools = var.node_pools } +# ########################### +# BOOTSPRAP APPLICATION PHASE +# ########################### module "argocd_bootstrap" { - source = "git::https://github.com/camptocamp/devops-stack-module-argocd.git//bootstrap?ref=v1-alpha" - cluster_name = local.cluster_name - base_domain = module.cluster.base_domain - cluster_issuer = "letsencrypt-prod" - - argocd = { - admin_enabled = "true" - } - - depends_on = [ - module.cluster, - ] + source = "git::https://github.com/camptocamp/devops-stack-module-argocd.git//bootstrap?ref=v4.4.0" } +module "ingress_controller" { + source = "git::https://github.com/camptocamp/devops-stack-module-traefik.git?ref=v5.0.0" -module "ingress" { - source = "git::https://github.com/camptocamp/devops-stack-module-traefik.git//scaleway?ref=v1-alpha" - - cluster_name = local.cluster_name - argocd_namespace = module.argocd_bootstrap.argocd_namespace - base_domain = module.cluster.base_domain + cluster_name = var.cluster_name + base_domain = module.scaleway.base_domain + enable_service_monitor = var.ingress_enable_service_monitor helm_values = [{ traefik = { service = { type = "LoadBalancer" annotations = { - "service.beta.kubernetes.io/scw-loadbalancer-id" = module.cluster.lb_id + "service.beta.kubernetes.io/scw-loadbalancer-id" = module.scaleway.lb_id } } } @@ -60,131 +48,104 @@ module "ingress" { } module "cert-manager" { - source = "git::https://github.com/camptocamp/devops-stack-module-cert-manager.git//scaleway?ref=remove-read-only-attribut" - - cluster_name = local.cluster_name - argocd_namespace = module.argocd_bootstrap.argocd_namespace - base_domain = module.cluster.base_domain + source = "git::https://github.com/camptocamp/devops-stack-module-cert-manager.git//self-signed?ref=v8.1.0" - helm_values = [{ - cert-manager = { - clusterIssuers = { - letsencrypt = { - enabled = true - } - acme = { - solvers = [ - { - http01 = { - ingress = {} - } - } - ] - } - } - } - }] + enable_service_monitor = var.cert_manager_enable_service_monitor dependency_ids = { argocd = module.argocd_bootstrap.id } } -module "argocd" { - source = "git::https://github.com/camptocamp/devops-stack-module-argocd.git?ref=v1-alpha" +module "authentication_with_keycloak" { + source = "git::https://github.com/camptocamp/devops-stack-module-keycloak.git?ref=v2.0.1" - bootstrap_values = module.argocd_bootstrap.bootstrap_values + cluster_name = var.cluster_name argocd_namespace = module.argocd_bootstrap.argocd_namespace + base_domain = var.base_domain + cluster_issuer = var.cluster_issuer - oidc = {} + dependency_ids = { + ingress_controller = module.ingress_controller.id + cert-manager = module.cert-manager.id + } +} - helm_values = [{ - argo-cd = { - global = { - image = { - repository = "camptocamp/argocd" - tag = "v2.3.4_c2c.3" - } - } - server = { - config = { - configManagementPlugins = <<-EOT - - name: kustomized-helm - init: - command: ["/bin/sh", "-c"] - args: ["helm dependency build || true"] - generate: - command: ["/bin/sh", "-c"] - args: ["echo \"$HELM_VALUES\" | helm template . --name-template $ARGOCD_APP_NAME --namespace $ARGOCD_APP_NAMESPACE $HELM_ARGS -f - --include-crds > all.yaml && kustomize build"] - - name: helmfile - init: - command: ["argocd-helmfile"] - args: ["init"] - generate: - command: ["argocd-helmfile"] - args: ["generate"] - lockRepo: true - EOT - } +module "authorization_with_keycloak" { + source = "git::https://github.com/camptocamp/devops-stack-module-keycloak.git//oidc_bootstrap?ref=v2.0.1" + + cluster_name = var.cluster_name + base_domain = var.base_domain + cluster_issuer = var.cluster_issuer + user_map = { + jdoe = { + username = "jdoe" + email = "john.doe@camptocamp.com" + first_name = "John" + last_name = "Doe" + } + } + dependency_ids = { + keycloak = module.authentication_with_keycloak.id + } +} + + +module "kube-prometheus-stack" { + source = "git::https://github.com/camptocamp/devops-stack-module-kube-prometheus-stack?ref=v9.2.0" + + cluster_name = var.cluster_name + base_domain = module.scaleway.base_domain + cluster_issuer = var.cluster_issuer + + metrics_storage_main = null + + prometheus = { + oidc = module.authorization_with_keycloak.oidc + } + alertmanager = { + oidc = module.authorization_with_keycloak.oidc + } + grafana = { + oidc = module.authorization_with_keycloak.oidc + } + + dependency_ids = { + ingress_controller = module.ingress_controller.id + cert-manager = module.cert-manager.id + oidc = module.authentication_with_keycloak.id + } +} + +module "argocd" { + source = "git::https://github.com/camptocamp/devops-stack-module-argocd.git?ref=v4.4.0" + + base_domain = module.scaleway.base_domain + cluster_name = var.cluster_name + cluster_issuer = var.cluster_issuer + server_secretkey = module.argocd_bootstrap.argocd_server_secretkey + accounts_pipeline_tokens = module.argocd_bootstrap.argocd_accounts_pipeline_tokens + + admin_enabled = true + #app_autosync = {} + + oidc = { + name = "OIDC" + issuer = module.authorization_with_keycloak.oidc.issuer_url + clientID = module.authorization_with_keycloak.oidc.client_id + clientSecret = module.authorization_with_keycloak.oidc.client_secret + requestedIDTokenClaims = { + groups = { + essential = true } } - }] + } dependency_ids = { - argocd = module.argocd_bootstrap.id - cert_manager = module.cert-manager.id + ingress_controller = module.ingress_controller.id + cert-manager = module.cert-manager.id + oidc = module.authorization_with_keycloak.id + # kube-prometheus-stack = module.kube-prometheus-stack.id } } -#module "monitoring" { -# source = "git::https://github.com/camptocamp/devops-stack-module-kube-prometheus-stack.git?ref=v1-alpha" -# -# cluster_name = local.cluster_name -# -# prometheus = { -# oidc = { -# issuer_url = module.oidc.issuer_url -# api_url = "${module.oidc.issuer_url}/healthz" -# client_id = module.oidc.clients.prometheus.id -# client_secret = module.oidc.clients.prometheus.secret -# -# oauth2_proxy_extra_args = [ -# ] -# } -# } -# -# alertmanager = { -# oidc = { -# issuer_url = module.oidc.issuer_url -# api_url = "${module.oidc.issuer_url}/healthz" -# client_id = module.oidc.clients.alertmanager.id -# client_secret = module.oidc.clients.alertmanager.secret -# -# oauth2_proxy_extra_args = [ -# ] -# } -# } -# -# grafana = { -# oidc = { -# oauth_url = "${module.oidc.issuer_url}/auth" -# token_url = "${module.oidc.issuer_url}/token" -# api_url = "${module.oidc.issuer_url}/userinfo" -# client_id = module.oidc.clients.grafana.id -# client_secret = module.oidc.clients.grafana.secret -# -# oauth2_proxy_extra_args = [ -# ] -# } -# } -# -# argocd_namespace = module.argocd_bootstrap.argocd_namespace -# base_domain = module.cluster.base_domain -# cluster_issuer = "letsencrypt-prod" -# metrics_archives = {} -# -# dependency_ids = { -# argocd = module.argocd_bootstrap.id -# oidc = module.oidc.id -# } -#} diff --git a/examples/scaleway/outputs.tf b/examples/scaleway/outputs.tf index 398701e3cd..86c3ebaa44 100644 --- a/examples/scaleway/outputs.tf +++ b/examples/scaleway/outputs.tf @@ -1,14 +1,18 @@ -output "argocd_server_admin_password" { - description = "Argocd admin password" - sensitive = true - value = module.argocd_bootstrap.argocd_server_admin_password +output "kubeconfig_file" { + sensitive = true + value = module.scaleway.kubeconfig_file } -#output "lb_ip_address" { -# value = scaleway_lb_ip.this.ip_address -#} -# -#output "lb_id" { -# value = scaleway_lb.this.id -#} +output "base_domain" { + value = module.scaleway.base_domain +} + +output "ca" { + value = module.scaleway.kubeconfig[0].cluster_ca_certificate + sensitive = true +} +output "passwords" { + sensitive = true + value = module.authorization_with_keycloak.devops_stack_users_passwords +} diff --git a/examples/scaleway/providers.tf b/examples/scaleway/providers.tf index d11665ab2e..d6b73c4af3 100644 --- a/examples/scaleway/providers.tf +++ b/examples/scaleway/providers.tf @@ -1,29 +1,40 @@ +provider "scaleway" { + region = "fr-par" +} + provider "helm" { kubernetes { - host = module.cluster.kube_admin_config.host - token = module.cluster.kube_admin_config.token - cluster_ca_certificate = module.cluster.kube_admin_config.cluster_ca_certificate + host = module.scaleway.kubeconfig.0.host + token = module.scaleway.kubeconfig.0.token + cluster_ca_certificate = base64decode(module.scaleway.kubeconfig.0.cluster_ca_certificate) } } provider "kubernetes" { - host = module.cluster.kube_admin_config.host - token = module.cluster.kube_admin_config.token - cluster_ca_certificate = module.cluster.kube_admin_config.cluster_ca_certificate + host = module.scaleway.kubeconfig.0.host + token = module.scaleway.kubeconfig.0.token + cluster_ca_certificate = base64decode(module.scaleway.kubeconfig.0.cluster_ca_certificate) } + provider "argocd" { - server_addr = "127.0.0.1:8080" + port_forward_with_namespace = module.argocd_bootstrap.argocd_namespace auth_token = module.argocd_bootstrap.argocd_auth_token insecure = true plain_text = true - port_forward = true - port_forward_with_namespace = module.argocd_bootstrap.argocd_namespace kubernetes { - host = module.cluster.kube_admin_config.host - token = module.cluster.kube_admin_config.token - cluster_ca_certificate = module.cluster.kube_admin_config.cluster_ca_certificate + host = module.scaleway.kubeconfig.0.host + token = module.scaleway.kubeconfig.0.token + cluster_ca_certificate = base64decode(module.scaleway.kubeconfig.0.cluster_ca_certificate) } } +provider "keycloak" { + client_id = "admin-cli" + username = module.authentication_with_keycloak.admin_credentials.username + password = module.authentication_with_keycloak.admin_credentials.password + url = "https://keycloak.apps.${var.cluster_name}.${var.base_domain}" + initial_login = false # Do no try to setup the provider before Keycloak is provisioned. + tls_insecure_skip_verify = true # Since we are in a testing environment, do not verify the authenticity of SSL certificates. +} diff --git a/examples/scaleway/terraform.tf b/examples/scaleway/terraform.tf index a47dcc5a97..6fb3c576b9 100644 --- a/examples/scaleway/terraform.tf +++ b/examples/scaleway/terraform.tf @@ -5,7 +5,11 @@ terraform { } argocd = { - source = "oboukili/argocd" + source = "oboukili/argocd" + version = "6.0.3" + } + keycloak = { + source = "mrparkers/keycloak" } } } diff --git a/examples/scaleway/variables.tf b/examples/scaleway/variables.tf new file mode 100644 index 0000000000..06b8ceb7c5 --- /dev/null +++ b/examples/scaleway/variables.tf @@ -0,0 +1,108 @@ +variable "project_id" { + type = string + description = "Scaleway project ID where the resources will be created" +} + +# ####################################################### +# Proxy input for the particuleio/kapsule/scaleway module +# ####################################################### + +variable "cluster_name" { + type = string + description = "The name for the Kubernetes cluster" +} + +variable "cluster_type" { + description = "The type of cluster" + type = string + default = "kapsule" + validation { + condition = contains(["kapsule", "multicloud"], var.cluster_type) + error_message = "Values can only be \"kapsule\" or \"multicloud\"." + } +} + +variable "cluster_description" { + type = string + description = "A description for the Kubernetes cluster" + default = null +} + +variable "cluster_tags" { + type = list(any) + default = [] + description = "The tags associated with the Kubernetes cluster" +} + +variable "tags" { + type = list(string) + default = [] + description = "Tags applied to all ressources." +} + +variable "kubernetes_version" { + default = "1.24.5" + type = string + description = "The version of the Kubernetes cluster" +} + +variable "admission_plugins" { + type = list(string) + default = [] + description = "The list of admission plugins to enable on the cluster" +} + +# ############################################### +# Variable for additional resources/configuration +# ############################################### +variable "base_domain" { + description = "A DNS zone if any" + default = null + type = string +} + +variable "lb_type" { + description = "The type of LB to deploy." + type = string +} + +variable "lb_name" { + description = "Name of the load balancer" + type = string +} + +variable "zone" { + description = "Zone in the region" + type = string +} + +variable "node_pools" { + description = "The node pools to create." + type = any + default = null +} +# +# ############################################### +# Variable for Ingress configuration +# ############################################### +variable "ingress_enable_service_monitor" { + description = "Enable Prometheus ServiceMonitor in the Helm chart." + type = bool +} + +# ############################################### +# Variable for keycloak configuration +# ############################################### +variable "cluster_issuer" { + description = "Cluster issuer" + type = string +} + +# ############################################### +# Variable for Cert-manager configuration +# ############################################### +variable "cert_manager_enable_service_monitor" { + description = "Argocd prometheus conf" + default = false + type = bool +}