From c5447074fbc646b744661370e1b34eda4a9c3056 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Tue, 19 Sep 2023 17:17:55 -0300 Subject: [PATCH 01/31] update yaml file for gitlab --- build/gitlab-ci.yml | 32 +++++++++ build/gitlab/gitlab-build-image-ci.yml | 39 ----------- build/gitlab/gitlab-tf-apply.yaml | 91 -------------------------- 3 files changed, 32 insertions(+), 130 deletions(-) delete mode 100644 build/gitlab/gitlab-build-image-ci.yml delete mode 100644 build/gitlab/gitlab-tf-apply.yaml diff --git a/build/gitlab-ci.yml b/build/gitlab-ci.yml index a8367ff9c..e982f5ec4 100644 --- a/build/gitlab-ci.yml +++ b/build/gitlab-ci.yml @@ -17,6 +17,33 @@ stages: - validate - apply +build-image: + stage: build + tags: + - runner2 + image: + name: docker:stable + services: + - name: docker:dind + alias: dockerhost + + variables: + DOCKER_HOST: tcp://dockerhost:2375/ + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + + script: + # - docker build -t $CI_REGISTRY_IMAGE . + # - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin + # - docker push $CI_REGISTRY_IMAGE + - docker build -t $CI_REGISTRY/terraform-gcloud:latest . + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin + - docker push $CI_REGISTRY/terraform-gcloud:latest + - docker logout + + only: + - image + # All branches need to be protected, otherwise the value of environment variables won't be read. image: @@ -26,6 +53,8 @@ image: terraform-plan: stage: validate + tags: + - runner2 id_tokens: GITLAB_OIDC_TOKEN: aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" @@ -41,6 +70,8 @@ terraform-plan: terraform-apply: stage: apply + tags: + - runner2 id_tokens: GITLAB_OIDC_TOKEN: aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" @@ -58,3 +89,4 @@ terraform-apply: - development - production - non-production + when: manual diff --git a/build/gitlab/gitlab-build-image-ci.yml b/build/gitlab/gitlab-build-image-ci.yml deleted file mode 100644 index 50123620e..000000000 --- a/build/gitlab/gitlab-build-image-ci.yml +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -stages: - - build - -build-image: - stage: build - image: - name: docker:stable - services: - - name: docker:dind - alias: dockerhost - - variables: - DOCKER_HOST: tcp://dockerhost:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - - script: - - docker info # Optional: Display Docker info for debugging - - docker build -t $CI_REGISTRY/terraform-gcloud:latest . - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker push $CI_REGISTRY/terraform-gcloud:latest - - docker logout - - only: - - main diff --git a/build/gitlab/gitlab-tf-apply.yaml b/build/gitlab/gitlab-tf-apply.yaml deleted file mode 100644 index 3564d110e..000000000 --- a/build/gitlab/gitlab-tf-apply.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -stages: - - build - - deploy - -build-image: - stage: build - tags: - - runner2 - image: - name: docker:stable - services: - - name: docker:dind - alias: dockerhost - - variables: - DOCKER_HOST: tcp://dockerhost:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - - script: - - env # Shows env variables - - docker info # Optional: Display Docker info for debugging - - docker build -t $CI_REGISTRY_IMAGE . - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin - - docker push $CI_REGISTRY_IMAGE - - docker logout - - only: - - main - -# All branches need to be protected, otherwise the value of environment variables won't be read. - -image: - name: $CI_REGISTRY_IMAGE - entrypoint: [""] - -terraform-plan: - stage: deploy - tags: - - runner2 - script: - - bash run_gcp_sts.sh - - gcloud auth list - - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $CI_PROJECT_NAME - - git status - - git clean -ffdx - - git status - - only: - - plan - -terraform-apply: - stage: deploy - tags: - - runner2 - script: - - bash run_gcp_sts.sh - - gcloud auth list - - pwd - - ls -la - - echo "ci-commit-ref-name ${CI_COMMIT_REF_NAME}" - - echo "ci-builds-dir ${CI_BUILDS_DIR}" - - echo "ci project dir ${CI_PROJECT_DIR}" - - echo "ci-project-name ${CI_PROJECT_NAME}" - - ls -la - - bash tf-wrapper.sh init $CI_COMMIT_REF_NAME - - bash tf-wrapper.sh plan $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $CI_PROJECT_NAME - - bash tf-wrapper.sh validate $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $CI_PROJECT_NAME - - bash tf-wrapper.sh apply $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $CI_PROJECT_NAME - # - git status - # - git clean -ffdx - # - git status - - only: - - development - - production - - non-production From 86e69f34961228002c83fff4bdd079d2e95b2960 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Thu, 21 Sep 2023 09:56:51 -0300 Subject: [PATCH 02/31] update yml file --- build/gitlab-ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/build/gitlab-ci.yml b/build/gitlab-ci.yml index e982f5ec4..f55de86d6 100644 --- a/build/gitlab-ci.yml +++ b/build/gitlab-ci.yml @@ -33,12 +33,9 @@ build-image: DOCKER_TLS_CERTDIR: "" script: - # - docker build -t $CI_REGISTRY_IMAGE . - # - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin - # - docker push $CI_REGISTRY_IMAGE - - docker build -t $CI_REGISTRY/terraform-gcloud:latest . - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin - - docker push $CI_REGISTRY/terraform-gcloud:latest + - docker build -t $CI_REGISTRY_IMAGE/terraform-gcloud:latest . + - docker push $CI_REGISTRY_IMAGE/terraform-gcloud:latest - docker logout only: @@ -50,7 +47,6 @@ image: name: $CI_REGISTRY/terraform-gcloud:latest entrypoint: [""] - terraform-plan: stage: validate tags: From 393f30a4b3a37ed1333e7d7581be35afd22b738e Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Thu, 21 Sep 2023 11:29:40 -0300 Subject: [PATCH 03/31] WIP - adds steps to create runner instance for gitlab --- 0-bootstrap/gitlab.tf | 94 ++++++++++++++++++++++++++++++++ 0-bootstrap/outputs.tf | 21 +++++++ 0-bootstrap/scripts/gl_runner.sh | 27 +++++++++ 0-bootstrap/variables.tf | 39 +++++++++++++ 4 files changed, 181 insertions(+) create mode 100755 0-bootstrap/scripts/gl_runner.sh diff --git a/0-bootstrap/gitlab.tf b/0-bootstrap/gitlab.tf index b8fa1b614..ea649696c 100644 --- a/0-bootstrap/gitlab.tf +++ b/0-bootstrap/gitlab.tf @@ -66,6 +66,10 @@ locals { gl_vars = { for v in concat(local.sa_vars, local.vars_list) : "${v.config}.${v.name}" => v } + network_name = var.create_network ? google_compute_network.gl-network[0].self_link : var.network_name + subnet_name = var.create_network ? google_compute_subnetwork.gl-subnetwork[0].self_link : var.subnet_name + service_account = var.service_account == "" ? google_service_account.runner_service_account[0].email : var.service_account + } provider "gitlab" { @@ -122,3 +126,93 @@ module "cicd_project_wif_iam_member" { parent_id = local.cicd_project_id roles = each.value } + +#### +/***************************************** + Optional Runner Networking + *****************************************/ +resource "google_compute_network" "gl-network" { + count = var.create_network ? 1 : 0 + name = var.network_name + project = var.project_id + auto_create_subnetworks = false +} +resource "google_compute_subnetwork" "gl-subnetwork" { + count = var.create_network ? 1 : 0 + project = var.project_id + name = var.subnet_name + ip_cidr_range = var.subnet_ip + region = var.region + network = google_compute_network.gl-network[0].name +} + +resource "google_compute_router" "default" { + count = var.create_network ? 1 : 0 + name = "${var.network_name}-router" + network = google_compute_network.gl-network[0].self_link + region = var.region + project = var.project_id +} + +resource "google_compute_router_nat" "nat" { + count = var.create_network ? 1 : 0 + project = var.project_id + name = "${var.network_name}-nat" + router = google_compute_router.default[0].name + region = google_compute_router.default[0].region + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" +} + +/***************************************** + IAM Bindings GCE SVC + *****************************************/ + +resource "google_service_account" "runner_service_account" { + count = var.service_account == "" ? 1 : 0 + project = var.project_id + account_id = "runner-service-account" + display_name = "GitLab Runner GCE Service Account" +} + +# allow GCE to pull images from GCR +resource "google_project_iam_binding" "gce" { + count = var.service_account == "" ? 1 : 0 + project = var.project_id + role = "roles/storage.objectViewer" + members = [ + "serviceAccount:${local.service_account}", + ] +} + +resource "google_compute_instance" "gitlab_runner" { + name = "gl-runner-instance" + project = var.project_id + zone = "us-central1-a" + machine_type = "e2-medium" + can_ip_forward = true + + boot_disk { + initialize_params { + image = "debian-cloud/debian-11" + } + } + tags = ["http-server", "https-server"] + metadata_startup_script = file("${abspath(path.module)}/scripts/gl_runner.sh") + + network_interface { + subnetwork = google_compute_subnetwork.gl-subnetwork[0].name + network_ip = "10.10.10.8" + subnetwork_project = var.project_id + } + + service_account { + email = local.service_account + scopes = ["cloud-platform"] + } + + depends_on = [ + google_compute_network.gl-network, + google_compute_subnetwork.gl-subnetwork + ] +} diff --git a/0-bootstrap/outputs.tf b/0-bootstrap/outputs.tf index 2a75b8d52..00e2ea9b8 100644 --- a/0-bootstrap/outputs.tf +++ b/0-bootstrap/outputs.tf @@ -196,3 +196,24 @@ output "projects_gcs_bucket_tfstate" { description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." value = module.seed_bootstrap.gcs_bucket_tfstate } + +### +output "gitlab_instance_network_name" { + description = "Network name used by GitLab runner." + value = module.gitlab_cicd.gitlab_instance_network_name +} + +output "gitlab_instance_subnet_name" { + description = "Subnet name used by GitLab runner." + value = module.gitlab_cicd.gitlab_instance_subnet_name +} + +output "gitlab_router" { + description = "Router used by GitLab runner network." + value = module.gitlab_cicd.gitlab_router +} + +output "gce_service_account" { + description = "Service account used by GitLab runner." + value = module.gitlab_cicd.gce_service_account +} diff --git a/0-bootstrap/scripts/gl_runner.sh b/0-bootstrap/scripts/gl_runner.sh new file mode 100755 index 000000000..fb704ddaf --- /dev/null +++ b/0-bootstrap/scripts/gl_runner.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +sudo apt update + +sudo apt install docker.io -y +sudo systemctl status docker +docker --version + +curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash +sudo apt-get install gitlab-runner +sudo gitlab-runner register --non-interactive --url https://gitlab.com --token "${RUNNER_TOKEN}" --name gl-fdt-runner --executer docker --docker-image "docker:dind" --docker-privileged +sudo gitlab-runner run & diff --git a/0-bootstrap/variables.tf b/0-bootstrap/variables.tf index 0a028d576..301284a01 100644 --- a/0-bootstrap/variables.tf +++ b/0-bootstrap/variables.tf @@ -281,3 +281,42 @@ variable "gitlab_token" { type = string sensitive = true } + +/* ---------------------------------------- + Specific to gitlab_bootstrap + ---------------------------------------- */ + +variable "network_name" { + type = string + description = "Name for the VPC network" + default = "gl-runner-network" +} + +variable "create_network" { + type = bool + description = "When set to true, VPC,router and NAT will be auto created" + default = true +} + +variable "subnetwork_project" { + type = string + description = "The ID of the project in which the subnetwork belongs. If it is not provided, the project_id is used." + #default = "" +} + +variable "subnet_ip" { + type = string + description = "IP range for the subnet" + default = "10.10.10.0/24" +} +variable "subnet_name" { + type = string + description = "Name for the subnet" + default = "gl-runner-subnet" +} + +# variable "service_account" { +# description = "Service account email address" +# type = string +# #default = "xxxx" +# } From b696ecf70cc474a5e93fd747c16248e8d63a72cc Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Fri, 29 Sep 2023 17:29:51 -0300 Subject: [PATCH 04/31] changes image path --- build/gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build/gitlab-ci.yml b/build/gitlab-ci.yml index f55de86d6..55cbf9359 100644 --- a/build/gitlab-ci.yml +++ b/build/gitlab-ci.yml @@ -42,9 +42,10 @@ build-image: - image # All branches need to be protected, otherwise the value of environment variables won't be read. +# Needs to disable -> projeto -> settings -> cicd -> token access -> limit access to this project image: - name: $CI_REGISTRY/terraform-gcloud:latest + name: registry.gitlab.com/renatojr_ciandt_group2/project_group2/terraform-gcloud:latest entrypoint: [""] terraform-plan: From dc438e81f0867655f5996e7f66f9a41a0a00a6a0 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Fri, 29 Sep 2023 18:24:14 -0300 Subject: [PATCH 05/31] wip - changes for runner config using mig --- 0-bootstrap/gitlab.tf | 161 ++++++++++++++++++++++++++++++++---------- 1 file changed, 125 insertions(+), 36 deletions(-) diff --git a/0-bootstrap/gitlab.tf b/0-bootstrap/gitlab.tf index ea649696c..7cf8e7082 100644 --- a/0-bootstrap/gitlab.tf +++ b/0-bootstrap/gitlab.tf @@ -138,12 +138,13 @@ resource "google_compute_network" "gl-network" { auto_create_subnetworks = false } resource "google_compute_subnetwork" "gl-subnetwork" { - count = var.create_network ? 1 : 0 + count = var.create_subnetwork ? 1 : 0 project = var.project_id name = var.subnet_name ip_cidr_range = var.subnet_ip region = var.region - network = google_compute_network.gl-network[0].name + network = local.network_name + #network = google_compute_network.gl-network[0].name } resource "google_compute_router" "default" { @@ -175,44 +176,132 @@ resource "google_service_account" "runner_service_account" { display_name = "GitLab Runner GCE Service Account" } -# allow GCE to pull images from GCR -resource "google_project_iam_binding" "gce" { - count = var.service_account == "" ? 1 : 0 - project = var.project_id - role = "roles/storage.objectViewer" - members = [ - "serviceAccount:${local.service_account}", - ] -} +/***************************************** + Runner Secrets + *****************************************/ +# resource "google_secret_manager_secret" "gl-secret" { +# provider = google-beta +# project = var.project_id +# secret_id = "gl-token" -resource "google_compute_instance" "gitlab_runner" { - name = "gl-runner-instance" - project = var.project_id - zone = "us-central1-a" - machine_type = "e2-medium" - can_ip_forward = true +# labels = { +# label = "gl-token" +# } - boot_disk { - initialize_params { - image = "debian-cloud/debian-11" - } - } - tags = ["http-server", "https-server"] - metadata_startup_script = file("${abspath(path.module)}/scripts/gl_runner.sh") +# replication { +# user_managed { +# replicas { +# location = var.region +# } +# } +# } +# } +# resource "google_secret_manager_secret_version" "gl-secret-version" { +# provider = google-beta +# secret = google_secret_manager_secret.gl-secret.id +# secret_data = jsonencode({ +# "REPO_NAME" = var.repo_name +# "REPO_OWNER" = var.repo_owner +# "GITLAB_TOKEN" = var.gl_token +# "LABELS" = join(",", var.gl_runner_labels) +# }) +# } - network_interface { - subnetwork = google_compute_subnetwork.gl-subnetwork[0].name - network_ip = "10.10.10.8" - subnetwork_project = var.project_id - } - service_account { - email = local.service_account - scopes = ["cloud-platform"] - } +# resource "google_secret_manager_secret_iam_member" "gl-secret-member" { +# provider = google-beta +# project = var.project_id +# secret_id = google_secret_manager_secret.gl-secret.id +# role = "roles/secretmanager.secretAccessor" +# member = "serviceAccount:${local.service_account}" +# } - depends_on = [ - google_compute_network.gl-network, - google_compute_subnetwork.gl-subnetwork +/***************************************** + Runner GCE Instance Template + *****************************************/ +locals { + instance_name = "gl-runner-vm" +} + +module "mig_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 7.0" + project_id = var.project_id + machine_type = var.machine_type + network = local.network_name + subnetwork = local.subnet_name + region = var.region + subnetwork_project = var.subnetwork_project != "" ? var.subnetwork_project : var.project_id + service_account = { + email = local.service_account + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + disk_size_gb = 100 + disk_type = "pd-ssd" + auto_delete = true + name_prefix = "gl-runner" + source_image_family = var.source_image_family + source_image_project = var.source_image_project + startup_script = local.startup_script + source_image = var.source_image + metadata = merge({ + "secret-id" = google_secret_manager_secret_version.gl-secret-version.name + }, { + "shutdown-script" = local.shutdown_script + }, var.custom_metadata) + tags = [ + "gl-runner-vm" ] } +/***************************************** + Runner MIG + *****************************************/ +module "mig" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 7.0" + project_id = var.project_id + subnetwork_project = var.project_id + hostname = local.instance_name + region = var.region + instance_template = module.mig_template.self_link + + /* autoscaler */ + autoscaling_enabled = true + min_replicas = var.min_replicas + max_replicas = var.max_replicas + cooldown_period = var.cooldown_period +} + +# resource "google_compute_instance" "gitlab_runner" { +# name = "gl-runner-instance" +# project = var.project_id +# zone = "us-central1-a" +# machine_type = "e2-medium" +# can_ip_forward = true + +# boot_disk { +# initialize_params { +# image = "debian-cloud/debian-11" +# } +# } +# tags = ["http-server", "https-server"] +# metadata_startup_script = file("${abspath(path.module)}/scripts/gl_runner.sh") + +# network_interface { +# subnetwork = google_compute_subnetwork.gl-subnetwork[0].name +# network_ip = "10.10.10.8" +# subnetwork_project = var.project_id +# } + +# service_account { +# email = local.service_account +# scopes = ["cloud-platform"] +# } + +# depends_on = [ +# google_compute_network.gl-network, +# google_compute_subnetwork.gl-subnetwork +# ] +# } From e608d64cbdfc57ccd4bfd9b31ee574ce78e96530 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Wed, 11 Oct 2023 18:19:47 -0300 Subject: [PATCH 06/31] Adds gitlab runner submodule --- 0-bootstrap/gitlab.tf | 187 +-------------- 0-bootstrap/modules/gitlab-runner/main.tf | 212 ++++++++++++++++++ 0-bootstrap/modules/gitlab-runner/outputs.tf | 45 ++++ .../modules/gitlab-runner/startup_script.sh | 64 ++++++ .../modules/gitlab-runner/variables.tf | 158 +++++++++++++ 0-bootstrap/modules/gitlab-runner/versions.tf | 27 +++ 0-bootstrap/outputs.tf | 211 ++--------------- ...le.tfvars => terraform.example.tfvars.bkp} | 0 0-bootstrap/variables.tf | 5 + 9 files changed, 533 insertions(+), 376 deletions(-) create mode 100644 0-bootstrap/modules/gitlab-runner/main.tf create mode 100644 0-bootstrap/modules/gitlab-runner/outputs.tf create mode 100755 0-bootstrap/modules/gitlab-runner/startup_script.sh create mode 100644 0-bootstrap/modules/gitlab-runner/variables.tf create mode 100644 0-bootstrap/modules/gitlab-runner/versions.tf rename 0-bootstrap/{terraform.example.tfvars => terraform.example.tfvars.bkp} (100%) diff --git a/0-bootstrap/gitlab.tf b/0-bootstrap/gitlab.tf index 7cf8e7082..2d8d38487 100644 --- a/0-bootstrap/gitlab.tf +++ b/0-bootstrap/gitlab.tf @@ -66,10 +66,6 @@ locals { gl_vars = { for v in concat(local.sa_vars, local.vars_list) : "${v.config}.${v.name}" => v } - network_name = var.create_network ? google_compute_network.gl-network[0].self_link : var.network_name - subnet_name = var.create_network ? google_compute_subnetwork.gl-subnetwork[0].self_link : var.subnet_name - service_account = var.service_account == "" ? google_service_account.runner_service_account[0].email : var.service_account - } provider "gitlab" { @@ -127,181 +123,10 @@ module "cicd_project_wif_iam_member" { roles = each.value } -#### -/***************************************** - Optional Runner Networking - *****************************************/ -resource "google_compute_network" "gl-network" { - count = var.create_network ? 1 : 0 - name = var.network_name - project = var.project_id - auto_create_subnetworks = false -} -resource "google_compute_subnetwork" "gl-subnetwork" { - count = var.create_subnetwork ? 1 : 0 - project = var.project_id - name = var.subnet_name - ip_cidr_range = var.subnet_ip - region = var.region - network = local.network_name - #network = google_compute_network.gl-network[0].name -} - -resource "google_compute_router" "default" { - count = var.create_network ? 1 : 0 - name = "${var.network_name}-router" - network = google_compute_network.gl-network[0].self_link - region = var.region - project = var.project_id -} - -resource "google_compute_router_nat" "nat" { - count = var.create_network ? 1 : 0 - project = var.project_id - name = "${var.network_name}-nat" - router = google_compute_router.default[0].name - region = google_compute_router.default[0].region - nat_ip_allocate_option = "AUTO_ONLY" - source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" -} - -/***************************************** - IAM Bindings GCE SVC - *****************************************/ - -resource "google_service_account" "runner_service_account" { - count = var.service_account == "" ? 1 : 0 - project = var.project_id - account_id = "runner-service-account" - display_name = "GitLab Runner GCE Service Account" -} - -/***************************************** - Runner Secrets - *****************************************/ -# resource "google_secret_manager_secret" "gl-secret" { -# provider = google-beta -# project = var.project_id -# secret_id = "gl-token" - -# labels = { -# label = "gl-token" -# } - -# replication { -# user_managed { -# replicas { -# location = var.region -# } -# } -# } -# } -# resource "google_secret_manager_secret_version" "gl-secret-version" { -# provider = google-beta -# secret = google_secret_manager_secret.gl-secret.id -# secret_data = jsonencode({ -# "REPO_NAME" = var.repo_name -# "REPO_OWNER" = var.repo_owner -# "GITLAB_TOKEN" = var.gl_token -# "LABELS" = join(",", var.gl_runner_labels) -# }) -# } - - -# resource "google_secret_manager_secret_iam_member" "gl-secret-member" { -# provider = google-beta -# project = var.project_id -# secret_id = google_secret_manager_secret.gl-secret.id -# role = "roles/secretmanager.secretAccessor" -# member = "serviceAccount:${local.service_account}" -# } - -/***************************************** - Runner GCE Instance Template - *****************************************/ -locals { - instance_name = "gl-runner-vm" -} - -module "mig_template" { - source = "terraform-google-modules/vm/google//modules/instance_template" - version = "~> 7.0" - project_id = var.project_id - machine_type = var.machine_type - network = local.network_name - subnetwork = local.subnet_name - region = var.region - subnetwork_project = var.subnetwork_project != "" ? var.subnetwork_project : var.project_id - service_account = { - email = local.service_account - scopes = [ - "https://www.googleapis.com/auth/cloud-platform", - ] - } - disk_size_gb = 100 - disk_type = "pd-ssd" - auto_delete = true - name_prefix = "gl-runner" - source_image_family = var.source_image_family - source_image_project = var.source_image_project - startup_script = local.startup_script - source_image = var.source_image - metadata = merge({ - "secret-id" = google_secret_manager_secret_version.gl-secret-version.name - }, { - "shutdown-script" = local.shutdown_script - }, var.custom_metadata) - tags = [ - "gl-runner-vm" - ] -} -/***************************************** - Runner MIG - *****************************************/ -module "mig" { - source = "terraform-google-modules/vm/google//modules/mig" - version = "~> 7.0" - project_id = var.project_id - subnetwork_project = var.project_id - hostname = local.instance_name - region = var.region - instance_template = module.mig_template.self_link - - /* autoscaler */ - autoscaling_enabled = true - min_replicas = var.min_replicas - max_replicas = var.max_replicas - cooldown_period = var.cooldown_period -} - -# resource "google_compute_instance" "gitlab_runner" { -# name = "gl-runner-instance" -# project = var.project_id -# zone = "us-central1-a" -# machine_type = "e2-medium" -# can_ip_forward = true - -# boot_disk { -# initialize_params { -# image = "debian-cloud/debian-11" -# } -# } -# tags = ["http-server", "https-server"] -# metadata_startup_script = file("${abspath(path.module)}/scripts/gl_runner.sh") - -# network_interface { -# subnetwork = google_compute_subnetwork.gl-subnetwork[0].name -# network_ip = "10.10.10.8" -# subnetwork_project = var.project_id -# } - -# service_account { -# email = local.service_account -# scopes = ["cloud-platform"] -# } +module "gitlab_runner" { + source = "./modules/gitlab-runner" -# depends_on = [ -# google_compute_network.gl-network, -# google_compute_subnetwork.gl-subnetwork -# ] -# } + repo_owner = var.repo_owner + #gl_token = var.gitlab_token + gitlab_token = var.gitlab_token +} \ No newline at end of file diff --git a/0-bootstrap/modules/gitlab-runner/main.tf b/0-bootstrap/modules/gitlab-runner/main.tf new file mode 100644 index 000000000..785e37673 --- /dev/null +++ b/0-bootstrap/modules/gitlab-runner/main.tf @@ -0,0 +1,212 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cicd_project_id = "../gitlab-oidc.project_id" + network_name = var.create_network ? google_compute_network.gl_network[0].self_link : var.network_name + subnet_name = var.create_network ? google_compute_subnetwork.gl_subnetwork[0].self_link : var.subnet_name + service_account = var.service_account == "" ? google_service_account.runner_service_account[0].email : var.service_account +} + +/***************************************** + Optional Runner Networking + *****************************************/ +resource "google_compute_network" "gl_network" { + count = var.create_network ? 1 : 0 + + name = var.network_name + project = local.cicd_project_id + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "gl_subnetwork" { + count = var.create_subnetwork ? 1 : 0 + + project = local.cicd_project_id + name = var.subnet_name + ip_cidr_range = var.subnet_ip + region = var.region + network = local.network_name + #network = google_compute_network.gl_network[0].name +} + +# module "peered_network" { +# source = "terraform-google-modules/network/google" +# version = "~> 5.2" + +# count = var.private_worker_pool.create_peered_network ? 1 : 0 + +# project_id = var.project_id +# network_name = local.network_name +# delete_default_internet_gateway_routes = "true" + +# subnets = [ +# { +# subnet_name = "sb-b-cbpools-${var.private_worker_pool.region}" +# subnet_ip = var.private_worker_pool.peered_network_subnet_ip +# subnet_region = var.private_worker_pool.region +# subnet_private_access = "true" +# subnet_flow_logs = "true" +# description = "Peered subnet for Cloud Build private pool" +# } +# ] + +# } + +resource "google_compute_router" "default" { + count = var.create_network ? 1 : 0 + + name = "${var.network_name}-router" + network = google_compute_network.gl_network[0].self_link + region = var.region + project = local.cicd_project_id +} + +// Nat is being used here since internet access is required for the Runner Network. Other internet access can be setup instead of NAT resource (e.g: Secure Web Proxy) +resource "google_compute_router_nat" "nat" { + count = var.create_network ? 1 : 0 + + project = local.cicd_project_id + name = "${var.network_name}-nat" + router = google_compute_router.default[0].name + region = google_compute_router.default[0].region + nat_ip_allocate_option = "AUTO_ONLY" + source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" +} + +resource "google_dns_policy" "default_policy" { + project = local.cicd_project_id + #name = "dp-${local.vpc_name}-default-policy" + name = "dp-${local.network_name}-default-policy" + enable_inbound_forwarding = true + enable_logging = true + + networks { + #network_url = module.network.network_self_link + network_url = local.network_name + } +} + +/***************************************** + IAM Bindings GCE SVC + *****************************************/ + +resource "google_service_account" "runner_service_account" { + count = var.service_account == "" ? 1 : 0 + + project = local.cicd_project_id + account_id = "runner-service-account" + display_name = "GitLab Runner GCE Service Account" +} + +/***************************************** + Runner Secrets + *****************************************/ +resource "google_secret_manager_secret" "gl-secret" { + provider = google-beta + + project = local.cicd_project_id + secret_id = "gl-token" + + labels = { + label = "gl-token" + } + + replication { + user_managed { + replicas { + location = var.region + } + } + } +} +resource "google_secret_manager_secret_version" "gl-secret-version" { + provider = google-beta + + secret = google_secret_manager_secret.gl-secret.id + secret_data = jsonencode({ + "REPO_NAME" = var.repo_name + "REPO_OWNER" = var.repo_owner + "GITLAB_TOKEN" = var.gitlab_token + "LABELS" = join(",", var.gl_runner_labels) + }) +} + +resource "google_secret_manager_secret_iam_member" "gl-secret-member" { + provider = google-beta + + project = local.cicd_project_id + secret_id = google_secret_manager_secret.gl-secret.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${local.service_account}" +} + +/***************************************** + Runner GCE Instance Template + *****************************************/ + +module "mig_template" { + source = "terraform-google-modules/vm/google//modules/instance_template" + version = "~> 7.0" + + project_id = local.cicd_project_id + machine_type = var.machine_type + network_ip = var.network_ip + network = local.network_name + subnetwork = local.subnet_name + region = var.region + subnetwork_project = local.cicd_project_id + service_account = { + email = local.service_account + scopes = [ + "https://www.googleapis.com/auth/cloud-platform", + ] + } + disk_size_gb = 100 + disk_type = "pd-ssd" + auto_delete = true + name_prefix = "gl-runner" + source_image_family = var.source_image_family + source_image_project = var.source_image_project + startup_script = file("${abspath(path.module)}/startup_script.sh") + source_image = var.source_image + metadata = merge({ + "secret-id" = google_secret_manager_secret_version.gl-secret-version.name + }, var.custom_metadata) + tags = [ + "gl-runner-vm" + ] +} +/***************************************** + Runner MIG + *****************************************/ +module "mig" { + source = "terraform-google-modules/vm/google//modules/mig" + version = "~> 7.0" + + project_id = local.cicd_project_id + subnetwork_project = local.cicd_project_id + hostname = var.instance_name + region = var.region + instance_template = module.mig_template.self_link + + /* autoscaler */ + autoscaling_enabled = true + min_replicas = var.min_replicas + max_replicas = var.max_replicas + cooldown_period = var.cooldown_period +} + diff --git a/0-bootstrap/modules/gitlab-runner/outputs.tf b/0-bootstrap/modules/gitlab-runner/outputs.tf new file mode 100644 index 000000000..ab9fa54f3 --- /dev/null +++ b/0-bootstrap/modules/gitlab-runner/outputs.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +# output "mig_instance_group" { +# description = "The instance group url of the created MIG" +# value = module.mig.instance_group +# } + +# output "mig_name" { +# description = "The name of the MIG" +# value = local.instance_name +# } + +# output "mig_instance_template" { +# description = "The name of the MIG Instance Template" +# value = module.mig_template.name +# } + +# output "network_name" { +# description = "Name of VPC" +# value = local.network_name +# } + +# output "subnet_name" { +# description = "Name of VPC" +# value = local.subnet_name +# } + +# output "service_account" { +# description = "Service account email for GCE" +# value = local.service_account +# } diff --git a/0-bootstrap/modules/gitlab-runner/startup_script.sh b/0-bootstrap/modules/gitlab-runner/startup_script.sh new file mode 100755 index 000000000..2f3c52dd1 --- /dev/null +++ b/0-bootstrap/modules/gitlab-runner/startup_script.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#install jq +apt-get update +apt-get -y install jq + +secretUri=$(curl -sS "http://metadata.google.internal/computeMetadata/v1/instance/attributes/secret-id" -H "Metadata-Flavor: Google") +#secrets URI is of the form projects/$PROJECT_NUMBER/secrets/$SECRET_NAME/versions/$SECRET_VERSION +#split into array based on `/` delimeter +IFS="/" read -r -a secretsConfig <<<"$secretUri" +#get SECRET_NAME and SECRET_VERSION +SECRET_NAME=${secretsConfig[3]} +SECRET_VERSION=${secretsConfig[5]} +#access secret from secretsmanager +secrets=$(gcloud secrets versions access "$SECRET_VERSION" --secret="$SECRET_NAME") +#set secrets as env vars +# shellcheck disable=SC2046 +# we want to use wordsplitting +export $(echo "$secrets" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]") +#github runner version +GH_RUNNER_VERSION="2.283.2" +#get actions binary +curl -o actions.tar.gz --location "https://github.com/actions/runner/releases/download/v${GH_RUNNER_VERSION}/actions-runner-linux-x64-${GH_RUNNER_VERSION}.tar.gz" +mkdir /runner +mkdir /runner-tmp +tar -zxf actions.tar.gz --directory /runner +rm -f actions.tar.gz +/runner/bin/installdependencies.sh +#get actions token +# shellcheck disable=SC2034 +# ACTIONS_RUNNER_INPUT_NAME is used by config.sh +ACTIONS_RUNNER_INPUT_NAME=$HOSTNAME +if [[ -z $REPO_NAME ]]; then + # Add action runner for an organisation + POST_URL="https://api.github.com/orgs/${REPO_OWNER}/actions/runners/registration-token" + GH_URL="https://github.com/${REPO_OWNER}" +else + # Add action runner for a repo + POST_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/actions/runners/registration-token" + GH_URL="https://github.com/${REPO_OWNER}/${REPO_NAME}" +fi + +# Register runner +ACTIONS_RUNNER_INPUT_TOKEN="$(curl -sS --request POST --url "$POST_URL" --header "authorization: Bearer ${GITHUB_TOKEN}" --header 'content-type: application/json' | jq -r .token)" +#configure runner +RUNNER_ALLOW_RUNASROOT=1 /runner/config.sh --unattended --replace --work "/runner-tmp" --url "$GH_URL" --token "$ACTIONS_RUNNER_INPUT_TOKEN" --labels "$LABELS" + +#install and start runner service +cd /runner || exit +./svc.sh install +./svc.sh start diff --git a/0-bootstrap/modules/gitlab-runner/variables.tf b/0-bootstrap/modules/gitlab-runner/variables.tf new file mode 100644 index 000000000..48ace1431 --- /dev/null +++ b/0-bootstrap/modules/gitlab-runner/variables.tf @@ -0,0 +1,158 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "region" { + type = string + description = "The GCP region to deploy instances into" + default = "us-east4" +} + +variable "network_name" { + type = string + description = "Name for the VPC network" + default = "gl-runner-network" +} + +variable "create_network" { + type = bool + description = "When set to true, VPC,router and NAT will be auto created" + default = true +} + +variable "subnetwork_project" { + type = string + description = "The ID of the project in which the subnetwork belongs. If it is not provided, the project_id is used." + default = "" +} + +variable "subnet_ip" { + type = string + description = "IP range for the subnet" + default = "10.10.10.0/24" +} + +variable "create_subnetwork" { + type = bool + description = "Whether to create subnetwork or use the one provided via subnet_name" + default = true +} + +variable "subnet_name" { + type = string + description = "Name for the subnet" + default = "gl-runner-subnet" +} + +variable "repo_name" { + type = string + description = "Name of the repo for the Github Action" + default = "" +} + +variable "repo_owner" { + type = string + description = "Owner of the repo for the Github Action" +} + +variable "gl_runner_labels" { + type = set(string) + description = "GitHub runner labels to attach to the runners. Docs: https://docs.github.com/en/actions/hosting-your-own-runners/using-labels-with-self-hosted-runners" + default = [] +} + +variable "min_replicas" { + type = number + description = "Minimum number of runner instances" + default = 2 +} + +variable "max_replicas" { + type = number + default = 10 + description = "Maximum number of runner instances" +} + +variable "gitlab_token" { + type = string + sensitive = true + description = "Gitlab token that is used for generating Self Hosted Runner Token" +} + +variable "service_account" { + description = "Service account email address" + type = string + default = "" +} + +variable "instance_name" { + description = "Instance name used by Gitlab Runner" + type = string + default = "gl-runner-vm" +} + +variable "network_ip" { + description = "Private IP address to assign to the instance" + type = string + default = "10.10.10.8" +} + +variable "machine_type" { + type = string + description = "The GCP machine type to deploy" + default = "n1-standard-1" +} + +variable "source_image_family" { + type = string + description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public Ubuntu image." + default = "ubuntu-1804-lts" +} + +variable "source_image_project" { + type = string + description = "Project where the source image comes from" + default = "ubuntu-os-cloud" +} + +variable "source_image" { + type = string + description = "Source disk image. If neither source_image nor source_image_family is specified, defaults to the latest public CentOS image." + default = "" +} + +variable "startup_script" { + type = string + description = "User startup script to run when instances spin up" + default = "" +} + +variable "shutdown_script" { + type = string + description = "User shutdown script to run when instances shutdown" + default = "" +} + +variable "custom_metadata" { + type = map(any) + description = "User provided custom metadata" + default = {} +} + +variable "cooldown_period" { + description = "The number of seconds that the autoscaler should wait before it starts collecting information from a new instance." + type = number + default = 60 +} diff --git a/0-bootstrap/modules/gitlab-runner/versions.tf b/0-bootstrap/modules/gitlab-runner/versions.tf new file mode 100644 index 000000000..a7ceb97f0 --- /dev/null +++ b/0-bootstrap/modules/gitlab-runner/versions.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.13" + required_providers { + + google = { + source = "hashicorp/google" + version = ">= 3.64, < 5.0.0" + } + } + +} diff --git a/0-bootstrap/outputs.tf b/0-bootstrap/outputs.tf index 00e2ea9b8..1ed2bca09 100644 --- a/0-bootstrap/outputs.tf +++ b/0-bootstrap/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2021 Google LLC + * Copyright 2020 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,206 +14,27 @@ * limitations under the License. */ -output "seed_project_id" { - description = "Project where service accounts and core APIs will be enabled." - value = module.seed_bootstrap.seed_project_id +output "mig_instance_group" { + description = "The instance group url of the created MIG" + value = module.mig.instance_group } -output "bootstrap_step_terraform_service_account_email" { - description = "Bootstrap Step Terraform Account" - value = google_service_account.terraform-env-sa["bootstrap"].email +output "mig_instance_template" { + description = "The name of the MIG Instance Template" + value = module.mig_template.name } -output "projects_step_terraform_service_account_email" { - description = "Projects Step Terraform Account" - value = google_service_account.terraform-env-sa["proj"].email +output "network_name" { + description = "Name of VPC" + value = local.network_name } -output "networks_step_terraform_service_account_email" { - description = "Networks Step Terraform Account" - value = google_service_account.terraform-env-sa["net"].email +output "subnet_name" { + description = "Name of VPC" + value = local.subnet_name } -output "environment_step_terraform_service_account_email" { - description = "Environment Step Terraform Account" - value = google_service_account.terraform-env-sa["env"].email -} - -output "organization_step_terraform_service_account_email" { - description = "Organization Step Terraform Account" - value = google_service_account.terraform-env-sa["org"].email -} - -output "gcs_bucket_tfstate" { - description = "Bucket used for storing terraform state for Foundations Pipelines in Seed Project." - value = module.seed_bootstrap.gcs_bucket_tfstate -} - -output "common_config" { - description = "Common configuration data to be used in other steps." - value = { - org_id = var.org_id, - parent_folder = var.parent_folder, - billing_account = var.billing_account, - default_region = var.default_region, - project_prefix = var.project_prefix, - folder_prefix = var.folder_prefix - parent_id = local.parent - bootstrap_folder_name = google_folder.bootstrap.name - } -} - -output "group_org_admins" { - description = "Google Group for GCP Organization Administrators." - value = var.groups.create_groups == true ? module.required_group["group_org_admins"].id : var.group_org_admins -} - -output "group_billing_admins" { - description = "Google Group for GCP Billing Administrators." - value = var.groups.create_groups == true ? module.required_group["group_billing_admins"].id : var.group_billing_admins -} - -output "required_groups" { - description = "List of Google Groups created that are required by the Example Foundation steps." - value = var.groups.create_groups == true ? module.required_group : {} -} - -output "optional_groups" { - description = "List of Google Groups created that are optional to the Example Foundation steps." - value = var.groups.create_groups == true ? module.optional_group : {} -} - -/* ---------------------------------------- - Specific to cloudbuild_module - ---------------------------------------- */ -# Comment-out the cloudbuild_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build -# output "cloudbuild_project_id" { -# description = "Project where Cloud Build configuration and terraform container image will reside." -# value = module.tf_source.cloudbuild_project_id -# } - -# output "gcs_bucket_cloudbuild_artifacts" { -# description = "Bucket used to store Cloud Build artifacts in cicd project." -# value = { for key, value in module.tf_workspace : key => replace(value.artifacts_bucket, local.bucket_self_link_prefix, "") } -# } - -# output "gcs_bucket_cloudbuild_logs" { -# description = "Bucket used to store Cloud Build logs in cicd project." -# value = { for key, value in module.tf_workspace : key => replace(value.logs_bucket, local.bucket_self_link_prefix, "") } -# } - -# output "projects_gcs_bucket_tfstate" { -# description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." -# value = module.gcp_projects_state_bucket.bucket.name -# } - -# output "cloud_builder_artifact_repo" { -# description = "Artifact Registry (AR) Repository created to store TF Cloud Builder images." -# value = "projects/${module.tf_source.cloudbuild_project_id}/locations/${var.default_region}/repositories/${module.tf_cloud_builder.artifact_repo}" -# } - -# output "csr_repos" { -# description = "List of Cloud Source Repos created by the module, linked to Cloud Build triggers." -# value = { for k, v in module.tf_source.csr_repos : k => { -# "id" = v.id, -# "name" = v.name, -# "project" = v.project, -# "url" = v.url, -# } -# } -# } - -# output "cloud_build_private_worker_pool_id" { -# description = "ID of the Cloud Build private worker pool." -# value = module.tf_private_pool.private_worker_pool_id - -# } - -# output "cloud_build_worker_range_id" { -# description = "The Cloud Build private worker IP range ID." -# value = module.tf_private_pool.worker_range_id -# } - -# output "cloud_build_worker_peered_ip_range" { -# description = "The IP range of the peered service network." -# value = module.tf_private_pool.worker_peered_ip_range -# } - -# output "cloud_build_peered_network_id" { -# description = "The ID of the Cloud Build peered network." -# value = module.tf_private_pool.peered_network_id -# } - -/* ---------------------------------------- - Specific to jenkins_bootstrap module - ---------------------------------------- */ -# # Un-comment the jenkins_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build -# output "cicd_project_id" { -# description = "Project where the [CI/CD Pipeline](/docs/GLOSSARY.md#foundation-cicd-pipeline) (Jenkins Agents and terraform builder container image) reside." -# value = module.jenkins_bootstrap.cicd_project_id -# } - -# output "jenkins_agent_gce_instance_id" { -# description = "Jenkins Agent GCE Instance id." -# value = module.jenkins_bootstrap.jenkins_agent_gce_instance_id -# } - -# output "jenkins_agent_vpc_id" { -# description = "Jenkins Agent VPC name." -# value = module.jenkins_bootstrap.jenkins_agent_vpc_id -# } - -# output "projects_gcs_bucket_tfstate" { -# description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." -# value = module.seed_bootstrap.gcs_bucket_tfstate -# } - -# output "jenkins_agent_sa_email" { -# description = "Email for privileged custom service account for Jenkins Agent GCE instance." -# value = module.jenkins_bootstrap.jenkins_agent_sa_email -# } - -# output "jenkins_agent_sa_name" { -# description = "Fully qualified name for privileged custom service account for Jenkins Agent GCE instance." -# value = module.jenkins_bootstrap.jenkins_agent_sa_name -# } - -# output "gcs_bucket_jenkins_artifacts" { -# description = "Bucket used to store Jenkins artifacts in Jenkins project." -# value = module.jenkins_bootstrap.gcs_bucket_jenkins_artifacts -# } - -/* ---------------------------------------- - Specific to gitlab_bootstrap - ---------------------------------------- */ -# Un-comment gitlab_bootstrap and its outputs if you want to use GitLab CI/CD instead of Cloud Build -output "cicd_project_id" { - description = "Project where the CI/CD infrastructure for GitLab CI/CD resides." - value = module.gitlab_cicd.project_id -} - -output "projects_gcs_bucket_tfstate" { - description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." - value = module.seed_bootstrap.gcs_bucket_tfstate -} - -### -output "gitlab_instance_network_name" { - description = "Network name used by GitLab runner." - value = module.gitlab_cicd.gitlab_instance_network_name -} - -output "gitlab_instance_subnet_name" { - description = "Subnet name used by GitLab runner." - value = module.gitlab_cicd.gitlab_instance_subnet_name -} - -output "gitlab_router" { - description = "Router used by GitLab runner network." - value = module.gitlab_cicd.gitlab_router -} - -output "gce_service_account" { - description = "Service account used by GitLab runner." - value = module.gitlab_cicd.gce_service_account +output "service_account" { + description = "Service account email for GCE" + value = local.service_account } diff --git a/0-bootstrap/terraform.example.tfvars b/0-bootstrap/terraform.example.tfvars.bkp similarity index 100% rename from 0-bootstrap/terraform.example.tfvars rename to 0-bootstrap/terraform.example.tfvars.bkp diff --git a/0-bootstrap/variables.tf b/0-bootstrap/variables.tf index 301284a01..620274607 100644 --- a/0-bootstrap/variables.tf +++ b/0-bootstrap/variables.tf @@ -282,6 +282,11 @@ variable "gitlab_token" { sensitive = true } +variable "repo_owner" { + description = "The owner of Gitlab repository." + type = string +} + /* ---------------------------------------- Specific to gitlab_bootstrap ---------------------------------------- */ From db95074614dee45d950182c485af12abae9bfe54 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Wed, 11 Oct 2023 18:21:03 -0300 Subject: [PATCH 07/31] Adds Runner submodule --- 0-bootstrap/terraform.example.tfvars.bkp | 113 ----------------------- 1 file changed, 113 deletions(-) delete mode 100644 0-bootstrap/terraform.example.tfvars.bkp diff --git a/0-bootstrap/terraform.example.tfvars.bkp b/0-bootstrap/terraform.example.tfvars.bkp deleted file mode 100644 index 447c2ad57..000000000 --- a/0-bootstrap/terraform.example.tfvars.bkp +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -org_id = "REPLACE_ME" # format "000000000000" - -billing_account = "REPLACE_ME" # format "000000-000000-000000" - -group_org_admins = "REPLACE_ME" - -group_billing_admins = "REPLACE_ME" - -# Example of values for the groups -# group_org_admins = "gcp-organization-admins@example.com" -# group_billing_admins = "gcp-billing-admins@example.com" - -default_region = "us-central1" - -// Optional - for an organization with existing projects or for development/validation. -// Uncomment this variable to place all the example foundation resources under -// the provided folder instead of the root organization. -// The variable value is the numeric folder ID -// The folder must already exist. -//parent_folder = "01234567890" - -// Optional - for enabling the automatic groups creation, uncoment the groups -// variable and update the values with the desired group names -//groups = { -// create_groups = true, -// billing_project = "billing-project", -// required_groups = { -// group_org_admins = "group_org_admins_local_test@example.com" -// group_billing_admins = "group_billing_admins_local_test@example.com" -// billing_data_users = "billing_data_users_local_test@example.com" -// audit_data_users = "audit_data_users_local_test@example.com" -// monitoring_workspace_users = "monitoring_workspace_users_local_test@example.com" -// }, -// optional_groups = { -// gcp_platform_viewer = "gcp_platform_viewer_local_test@example.com" -// gcp_security_reviewer = "gcp_security_reviewer_local_test@example.com" -// gcp_network_viewer = "gcp_network_viewer_local_test@example.com" -// gcp_scc_admin = "gcp_scc_admin_local_test@example.com" -// gcp_global_secrets_admin = "gcp_global_secrets_admin_local_test@example.com" -// gcp_audit_viewer = "gcp_audit_viewer_local_test@example.com" -// } -//} -// - - -/* ---------------------------------------- - Specific to jenkins_bootstrap module - ---------------------------------------- */ -// Un-comment the jenkins_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build -//jenkins_agent_gce_subnetwork_cidr_range = "172.16.1.0/24" -// -//jenkins_agent_gce_private_ip_address = "172.16.1.6" -// -//jenkins_agent_gce_ssh_pub_key = "ssh-rsa [KEY_VALUE] [USERNAME]" -// -//jenkins_agent_sa_email = "jenkins-agent-gce" # service_account_prefix will be added -// -//jenkins_controller_subnetwork_cidr_range = ["10.1.0.6/32"] -// -//nat_bgp_asn = "64514" -// -//vpn_shared_secret = "shared_secret" -// -//on_prem_vpn_public_ip_address = "" -// -//on_prem_vpn_public_ip_address2 = "" -// -//router_asn = "64515" -// -//bgp_peer_asn = "64513" -// -//tunnel0_bgp_peer_address = "169.254.1.1" -// -//tunnel0_bgp_session_range = "169.254.1.2/30" -// -//tunnel1_bgp_peer_address = "169.254.2.1" -// -//tunnel1_bgp_session_range = "169.254.2.2/30" - -/* ---------------------------------------- - Specific to gitlab_bootstrap - ---------------------------------------- */ -# Un-comment gitlab_bootstrap and its outputs if you want to use GitLab CI/CD instead of Cloud Build -# gl_repos = { -# owner = "YOUR-GITLAB-USER-OR-GROUP", -# bootstrap = "YOUR-BOOTSTRAP-REPOSITORY", -# organization = "YOUR-ORGANIZATION-REPOSITORY", -# environments = "YOUR-ENVIRONMENTS-REPOSITORY", -# networks = "YOUR-NETWORKS-REPOSITORY", -# projects = "YOUR-PROJECTS-REPOSITORY", -# } -# -# to prevent saving the `gitlab_token` in plain text in this file, -# export the GitLab access token in the command line -# as an environment variable before running terraform. -# Run the following commnad in your shell: -# export TF_VAR_gitlab_token="YOUR-ACCESS-TOKEN" From 078e8be924e5087a5e92f10177ae8cfe5fb4a224 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Wed, 18 Oct 2023 17:32:15 -0300 Subject: [PATCH 08/31] fixes for gitlab-runner module --- 0-bootstrap/gitlab.tf | 10 +- 0-bootstrap/modules/gitlab-runner/main.tf | 104 ++++------ 0-bootstrap/modules/gitlab-runner/outputs.tf | 39 ++-- .../modules/gitlab-runner/variables.tf | 11 +- 0-bootstrap/outputs.tf | 191 ++++++++++++++++-- 0-bootstrap/variables.tf | 5 - 6 files changed, 240 insertions(+), 120 deletions(-) diff --git a/0-bootstrap/gitlab.tf b/0-bootstrap/gitlab.tf index 2d8d38487..3e0ba9ca7 100644 --- a/0-bootstrap/gitlab.tf +++ b/0-bootstrap/gitlab.tf @@ -91,6 +91,9 @@ module "gitlab_cicd" { "cloudresourcemanager.googleapis.com", "iamcredentials.googleapis.com", "sts.googleapis.com", + "dns.googleapis.com", + "secretmanager.googleapis.com", + ] } @@ -126,7 +129,8 @@ module "cicd_project_wif_iam_member" { module "gitlab_runner" { source = "./modules/gitlab-runner" - repo_owner = var.repo_owner - #gl_token = var.gitlab_token + repo_owner = var.repo_owner gitlab_token = var.gitlab_token -} \ No newline at end of file + project_id = local.cicd_project_id +} + diff --git a/0-bootstrap/modules/gitlab-runner/main.tf b/0-bootstrap/modules/gitlab-runner/main.tf index 785e37673..f77c669e3 100644 --- a/0-bootstrap/modules/gitlab-runner/main.tf +++ b/0-bootstrap/modules/gitlab-runner/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,71 +15,48 @@ */ locals { - cicd_project_id = "../gitlab-oidc.project_id" - network_name = var.create_network ? google_compute_network.gl_network[0].self_link : var.network_name - subnet_name = var.create_network ? google_compute_subnetwork.gl_subnetwork[0].self_link : var.subnet_name service_account = var.service_account == "" ? google_service_account.runner_service_account[0].email : var.service_account } /***************************************** Optional Runner Networking *****************************************/ -resource "google_compute_network" "gl_network" { - count = var.create_network ? 1 : 0 - - name = var.network_name - project = local.cicd_project_id - auto_create_subnetworks = false -} - -resource "google_compute_subnetwork" "gl_subnetwork" { - count = var.create_subnetwork ? 1 : 0 - - project = local.cicd_project_id - name = var.subnet_name - ip_cidr_range = var.subnet_ip - region = var.region - network = local.network_name - #network = google_compute_network.gl_network[0].name +module "vpc_network" { + source = "terraform-google-modules/network/google" + version = "~> 7.0" + project_id = var.project_id + network_name = "my-network" + mtu = 1460 + + subnets = [ + { + subnet_name = "subnet-01" + subnet_ip = "10.10.10.0/24" + subnet_region = "us-central1" + # subnet_private_access = "true" + # subnet_flow_logs = "true" + # subnet_flow_logs_interval = "INTERVAL_10_MIN" + # subnet_flow_logs_sampling = 0.7 + # subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" + # subnet_flow_logs_filter = "false" + }, + ] } -# module "peered_network" { -# source = "terraform-google-modules/network/google" -# version = "~> 5.2" - -# count = var.private_worker_pool.create_peered_network ? 1 : 0 - -# project_id = var.project_id -# network_name = local.network_name -# delete_default_internet_gateway_routes = "true" - -# subnets = [ -# { -# subnet_name = "sb-b-cbpools-${var.private_worker_pool.region}" -# subnet_ip = var.private_worker_pool.peered_network_subnet_ip -# subnet_region = var.private_worker_pool.region -# subnet_private_access = "true" -# subnet_flow_logs = "true" -# description = "Peered subnet for Cloud Build private pool" -# } -# ] - -# } - resource "google_compute_router" "default" { count = var.create_network ? 1 : 0 name = "${var.network_name}-router" - network = google_compute_network.gl_network[0].self_link + network = module.vpc_network.network_self_link region = var.region - project = local.cicd_project_id + project = var.project_id } // Nat is being used here since internet access is required for the Runner Network. Other internet access can be setup instead of NAT resource (e.g: Secure Web Proxy) resource "google_compute_router_nat" "nat" { count = var.create_network ? 1 : 0 - project = local.cicd_project_id + project = var.project_id name = "${var.network_name}-nat" router = google_compute_router.default[0].name region = google_compute_router.default[0].region @@ -88,26 +65,23 @@ resource "google_compute_router_nat" "nat" { } resource "google_dns_policy" "default_policy" { - project = local.cicd_project_id - #name = "dp-${local.vpc_name}-default-policy" - name = "dp-${local.network_name}-default-policy" + project = var.project_id + name = "dns-gl-runner-default-policy" enable_inbound_forwarding = true enable_logging = true networks { - #network_url = module.network.network_self_link - network_url = local.network_name + network_url = module.vpc_network.network_self_link } } /***************************************** IAM Bindings GCE SVC *****************************************/ - resource "google_service_account" "runner_service_account" { count = var.service_account == "" ? 1 : 0 - project = local.cicd_project_id + project = var.project_id account_id = "runner-service-account" display_name = "GitLab Runner GCE Service Account" } @@ -118,7 +92,7 @@ resource "google_service_account" "runner_service_account" { resource "google_secret_manager_secret" "gl-secret" { provider = google-beta - project = local.cicd_project_id + project = var.project_id secret_id = "gl-token" labels = { @@ -148,7 +122,7 @@ resource "google_secret_manager_secret_version" "gl-secret-version" { resource "google_secret_manager_secret_iam_member" "gl-secret-member" { provider = google-beta - project = local.cicd_project_id + project = var.project_id secret_id = google_secret_manager_secret.gl-secret.id role = "roles/secretmanager.secretAccessor" member = "serviceAccount:${local.service_account}" @@ -157,18 +131,17 @@ resource "google_secret_manager_secret_iam_member" "gl-secret-member" { /***************************************** Runner GCE Instance Template *****************************************/ - module "mig_template" { source = "terraform-google-modules/vm/google//modules/instance_template" version = "~> 7.0" - project_id = local.cicd_project_id + project_id = var.project_id machine_type = var.machine_type network_ip = var.network_ip - network = local.network_name - subnetwork = local.subnet_name + network = module.vpc_network.network_name + subnetwork = module.vpc_network.subnets_names[0] region = var.region - subnetwork_project = local.cicd_project_id + subnetwork_project = var.project_id service_account = { email = local.service_account scopes = [ @@ -181,8 +154,8 @@ module "mig_template" { name_prefix = "gl-runner" source_image_family = var.source_image_family source_image_project = var.source_image_project - startup_script = file("${abspath(path.module)}/startup_script.sh") - source_image = var.source_image + #startup_script = file("${abspath(path.module)}/startup_script.sh") + source_image = var.source_image metadata = merge({ "secret-id" = google_secret_manager_secret_version.gl-secret-version.name }, var.custom_metadata) @@ -197,8 +170,8 @@ module "mig" { source = "terraform-google-modules/vm/google//modules/mig" version = "~> 7.0" - project_id = local.cicd_project_id - subnetwork_project = local.cicd_project_id + project_id = var.project_id + subnetwork_project = var.project_id hostname = var.instance_name region = var.region instance_template = module.mig_template.self_link @@ -209,4 +182,3 @@ module "mig" { max_replicas = var.max_replicas cooldown_period = var.cooldown_period } - diff --git a/0-bootstrap/modules/gitlab-runner/outputs.tf b/0-bootstrap/modules/gitlab-runner/outputs.tf index ab9fa54f3..5af1b03e9 100644 --- a/0-bootstrap/modules/gitlab-runner/outputs.tf +++ b/0-bootstrap/modules/gitlab-runner/outputs.tf @@ -14,32 +14,17 @@ * limitations under the License. */ -# output "mig_instance_group" { -# description = "The instance group url of the created MIG" -# value = module.mig.instance_group -# } +output "mig_instance_group" { + description = "The instance group url of the created MIG" + value = module.mig.instance_group +} -# output "mig_name" { -# description = "The name of the MIG" -# value = local.instance_name -# } +output "mig_instance_template" { + description = "The name of the MIG Instance Template" + value = module.mig_template.name +} -# output "mig_instance_template" { -# description = "The name of the MIG Instance Template" -# value = module.mig_template.name -# } - -# output "network_name" { -# description = "Name of VPC" -# value = local.network_name -# } - -# output "subnet_name" { -# description = "Name of VPC" -# value = local.subnet_name -# } - -# output "service_account" { -# description = "Service account email for GCE" -# value = local.service_account -# } +output "service_account" { + description = "Service account email for GCE" + value = local.service_account +} diff --git a/0-bootstrap/modules/gitlab-runner/variables.tf b/0-bootstrap/modules/gitlab-runner/variables.tf index 48ace1431..01784eca9 100644 --- a/0-bootstrap/modules/gitlab-runner/variables.tf +++ b/0-bootstrap/modules/gitlab-runner/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ variable "region" { type = string description = "The GCP region to deploy instances into" - default = "us-east4" + default = "us-central1" } variable "network_name" { @@ -118,7 +118,7 @@ variable "machine_type" { variable "source_image_family" { type = string description = "Source image family. If neither source_image nor source_image_family is specified, defaults to the latest public Ubuntu image." - default = "ubuntu-1804-lts" + default = "ubuntu-2204-lts" } variable "source_image_project" { @@ -156,3 +156,8 @@ variable "cooldown_period" { type = number default = 60 } + +variable "project_id" { + type = string + description = "ID of the project where the Gitlab runner will be created." +} diff --git a/0-bootstrap/outputs.tf b/0-bootstrap/outputs.tf index 1ed2bca09..2c8ff5f8d 100644 --- a/0-bootstrap/outputs.tf +++ b/0-bootstrap/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2020 Google LLC + * Copyright 2021 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,27 +14,186 @@ * limitations under the License. */ -output "mig_instance_group" { - description = "The instance group url of the created MIG" - value = module.mig.instance_group +output "seed_project_id" { + description = "Project where service accounts and core APIs will be enabled." + value = module.seed_bootstrap.seed_project_id } -output "mig_instance_template" { - description = "The name of the MIG Instance Template" - value = module.mig_template.name +output "bootstrap_step_terraform_service_account_email" { + description = "Bootstrap Step Terraform Account" + value = google_service_account.terraform-env-sa["bootstrap"].email } -output "network_name" { - description = "Name of VPC" - value = local.network_name +output "projects_step_terraform_service_account_email" { + description = "Projects Step Terraform Account" + value = google_service_account.terraform-env-sa["proj"].email } -output "subnet_name" { - description = "Name of VPC" - value = local.subnet_name +output "networks_step_terraform_service_account_email" { + description = "Networks Step Terraform Account" + value = google_service_account.terraform-env-sa["net"].email } -output "service_account" { - description = "Service account email for GCE" - value = local.service_account +output "environment_step_terraform_service_account_email" { + description = "Environment Step Terraform Account" + value = google_service_account.terraform-env-sa["env"].email } + +output "organization_step_terraform_service_account_email" { + description = "Organization Step Terraform Account" + value = google_service_account.terraform-env-sa["org"].email +} + +output "gcs_bucket_tfstate" { + description = "Bucket used for storing terraform state for Foundations Pipelines in Seed Project." + value = module.seed_bootstrap.gcs_bucket_tfstate +} + +output "common_config" { + description = "Common configuration data to be used in other steps." + value = { + org_id = var.org_id, + parent_folder = var.parent_folder, + billing_account = var.billing_account, + default_region = var.default_region, + project_prefix = var.project_prefix, + folder_prefix = var.folder_prefix + parent_id = local.parent + bootstrap_folder_name = google_folder.bootstrap.name + } +} + +output "group_org_admins" { + description = "Google Group for GCP Organization Administrators." + value = var.groups.create_groups == true ? module.required_group["group_org_admins"].id : var.group_org_admins +} + +output "group_billing_admins" { + description = "Google Group for GCP Billing Administrators." + value = var.groups.create_groups == true ? module.required_group["group_billing_admins"].id : var.group_billing_admins +} + +output "required_groups" { + description = "List of Google Groups created that are required by the Example Foundation steps." + value = var.groups.create_groups == true ? module.required_group : {} +} + +output "optional_groups" { + description = "List of Google Groups created that are optional to the Example Foundation steps." + value = var.groups.create_groups == true ? module.optional_group : {} +} + +/* ---------------------------------------- + Specific to cloudbuild_module + ---------------------------------------- */ +# Comment-out the cloudbuild_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build +# output "cloudbuild_project_id" { +# description = "Project where Cloud Build configuration and terraform container image will reside." +# value = module.tf_source.cloudbuild_project_id +# } + +# output "gcs_bucket_cloudbuild_artifacts" { +# description = "Bucket used to store Cloud Build artifacts in cicd project." +# value = { for key, value in module.tf_workspace : key => replace(value.artifacts_bucket, local.bucket_self_link_prefix, "") } +# } + +# output "gcs_bucket_cloudbuild_logs" { +# description = "Bucket used to store Cloud Build logs in cicd project." +# value = { for key, value in module.tf_workspace : key => replace(value.logs_bucket, local.bucket_self_link_prefix, "") } +# } + +# output "projects_gcs_bucket_tfstate" { +# description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." +# value = module.gcp_projects_state_bucket.bucket.name +# } + +# output "cloud_builder_artifact_repo" { +# description = "Artifact Registry (AR) Repository created to store TF Cloud Builder images." +# value = "projects/${module.tf_source.cloudbuild_project_id}/locations/${var.default_region}/repositories/${module.tf_cloud_builder.artifact_repo}" +# } + +# output "csr_repos" { +# description = "List of Cloud Source Repos created by the module, linked to Cloud Build triggers." +# value = { for k, v in module.tf_source.csr_repos : k => { +# "id" = v.id, +# "name" = v.name, +# "project" = v.project, +# "url" = v.url, +# } +# } +# } + +# output "cloud_build_private_worker_pool_id" { +# description = "ID of the Cloud Build private worker pool." +# value = module.tf_private_pool.private_worker_pool_id + +# } + +# output "cloud_build_worker_range_id" { +# description = "The Cloud Build private worker IP range ID." +# value = module.tf_private_pool.worker_range_id +# } + +# output "cloud_build_worker_peered_ip_range" { +# description = "The IP range of the peered service network." +# value = module.tf_private_pool.worker_peered_ip_range +# } + +# output "cloud_build_peered_network_id" { +# description = "The ID of the Cloud Build peered network." +# value = module.tf_private_pool.peered_network_id +# } + +/* ---------------------------------------- + Specific to jenkins_bootstrap module + ---------------------------------------- */ +# # Un-comment the jenkins_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build +# output "cicd_project_id" { +# description = "Project where the [CI/CD Pipeline](/docs/GLOSSARY.md#foundation-cicd-pipeline) (Jenkins Agents and terraform builder container image) reside." +# value = module.jenkins_bootstrap.cicd_project_id +# } + +# output "jenkins_agent_gce_instance_id" { +# description = "Jenkins Agent GCE Instance id." +# value = module.jenkins_bootstrap.jenkins_agent_gce_instance_id +# } + +# output "jenkins_agent_vpc_id" { +# description = "Jenkins Agent VPC name." +# value = module.jenkins_bootstrap.jenkins_agent_vpc_id +# } + +# output "projects_gcs_bucket_tfstate" { +# description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." +# value = module.seed_bootstrap.gcs_bucket_tfstate +# } + +# output "jenkins_agent_sa_email" { +# description = "Email for privileged custom service account for Jenkins Agent GCE instance." +# value = module.jenkins_bootstrap.jenkins_agent_sa_email +# } + +# output "jenkins_agent_sa_name" { +# description = "Fully qualified name for privileged custom service account for Jenkins Agent GCE instance." +# value = module.jenkins_bootstrap.jenkins_agent_sa_name +# } + +# output "gcs_bucket_jenkins_artifacts" { +# description = "Bucket used to store Jenkins artifacts in Jenkins project." +# value = module.jenkins_bootstrap.gcs_bucket_jenkins_artifacts +# } + +/* ---------------------------------------- + Specific to gitlab_bootstrap + ---------------------------------------- */ +# Un-comment gitlab_bootstrap and its outputs if you want to use GitLab CI/CD instead of Cloud Build +output "cicd_project_id" { + description = "Project where the CI/CD infrastructure for GitLab CI/CD resides." + value = module.gitlab_cicd.project_id +} + +output "projects_gcs_bucket_tfstate" { + description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." + value = module.seed_bootstrap.gcs_bucket_tfstate +} + diff --git a/0-bootstrap/variables.tf b/0-bootstrap/variables.tf index 620274607..83092fa42 100644 --- a/0-bootstrap/variables.tf +++ b/0-bootstrap/variables.tf @@ -320,8 +320,3 @@ variable "subnet_name" { default = "gl-runner-subnet" } -# variable "service_account" { -# description = "Service account email address" -# type = string -# #default = "xxxx" -# } From 2f8c161d82433ea3a87de049223fb187ccea581e Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Mon, 23 Oct 2023 18:00:16 -0300 Subject: [PATCH 09/31] add instructions for gitlab runner --- 0-bootstrap/README-GitLab.md | 18 +++- 0-bootstrap/README.md | 6 ++ .../modules/gitlab-runner/.gitlab-ci.yml | 99 +++++++++++++++++++ .../modules/gitlab-runner/startup_script.sh | 72 +++++--------- 4 files changed, 147 insertions(+), 48 deletions(-) create mode 100644 0-bootstrap/modules/gitlab-runner/.gitlab-ci.yml diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index 146cd695e..5c20aec30 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -25,13 +25,14 @@ Also make sure that you have the following: - Environments - Networks - Projects -A [Personal](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) access token or a [Group](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html) access token configured with the following [scopes](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#personal-access-token-scopes): +- A [Personal](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) access token or a [Group](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html) access token configured with the following [scopes](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#personal-access-token-scopes): - read_api - create_runner - read_repository - write_repository - read_registry - write_registry +- A [Runner](https://docs.gitlab.com/ee/tutorials/create_register_first_runner/) with the name `gl_runner` should be created in your GitLab account. Save the token of the runner once it has been created. It will be used in the next steps. - A Google Cloud [organization](https://cloud.google.com/resource-manager/docs/creating-managing-organization). - A Google Cloud [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account). - Cloud Identity or Google Workspace groups for organization and billing admins. @@ -170,7 +171,6 @@ export the GitLab personal or group access token as an environment variable: ```bash export TF_VAR_gitlab_token="YOUR-PERSONAL-OR-GROUP-ACCESS-TOKEN" ``` - 1. Use the helper script [validate-requirements.sh](../scripts/validate-requirements.sh) to validate your environment: ```bash @@ -207,6 +207,19 @@ export the GitLab personal or group access token as an environment variable: terraform apply bootstrap.tfplan ``` +1. Once `terraform apply` has finished, access the Gitlab instance by ssh that has been created and update following fields in the file `/etc/gitlab-runner/config.toml` using the values showed during the Gitlab Runner creation. + +```bash +name = "xxxx" +token = "glrt-XXX" +``` + +1. Copy the file `gitlab-ci.yml` to the 0-bootstrap directory. +```bash +cp ./modules/gitlab-runner/gitlab-ci.yml . +``` + + 1. Run `terraform output` to get the email address of the terraform service accounts that will be used to run manual steps for `shared` environments in steps `3-networks-dual-svpc`, `3-networks-hub-and-spoke`, and `4-projects`. ```bash @@ -255,6 +268,7 @@ export the GitLab personal or group access token as an environment variable: git push --set-upstream origin plan ``` + TODO **Note 1:** The stages after `0-bootstrap` use `terraform_remote_state` data source to read common configuration like the organization ID from the output of the `0-bootstrap` stage. diff --git a/0-bootstrap/README.md b/0-bootstrap/README.md index 27937a2e9..7ad41c055 100644 --- a/0-bootstrap/README.md +++ b/0-bootstrap/README.md @@ -282,6 +282,7 @@ Each step has instructions for this change. | billing\_account | The ID of the billing account to associate projects with. | `string` | n/a | yes | | bucket\_force\_destroy | When deleting a bucket, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no | | bucket\_prefix | Name prefix to use for state bucket created. | `string` | `"bkt"` | no | +| create\_network | When set to true, VPC,router and NAT will be auto created | `bool` | `true` | no | | default\_region | Default region to create resources where applicable. | `string` | `"us-central1"` | no | | folder\_prefix | Name prefix to use for folders created. Should be the same in all steps. | `string` | `"fldr"` | no | | gitlab\_token | A GitLab personal access token or group access token.
See:
https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html
https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html | `string` | n/a | yes | @@ -290,11 +291,16 @@ Each step has instructions for this change. | group\_org\_admins | Google Group for GCP Organization Administrators | `string` | n/a | yes | | groups | Contain the details of the Groups to be created. | 
object({
    create_groups   = bool
    billing_project = string
    required_groups = object({
      group_org_admins           = string
      group_billing_admins       = string
      billing_data_users         = string
      audit_data_users           = string
      monitoring_workspace_users = string
    })
    optional_groups = object({
      gcp_platform_viewer      = string
      gcp_security_reviewer    = string
      gcp_network_viewer       = string
      gcp_scc_admin            = string
      gcp_global_secrets_admin = string
      gcp_audit_viewer         = string
    })
  })
| 
{
  "billing_project": "",
  "create_groups": false,
  "optional_groups": {
    "gcp_audit_viewer": "",
    "gcp_global_secrets_admin": "",
    "gcp_network_viewer": "",
    "gcp_platform_viewer": "",
    "gcp_scc_admin": "",
    "gcp_security_reviewer": ""
  },
  "required_groups": {
    "audit_data_users": "",
    "billing_data_users": "",
    "group_billing_admins": "",
    "group_org_admins": "",
    "monitoring_workspace_users": ""
  }
}
| no | | initial\_group\_config | Define the group configuration when it is initialized. Valid values are: WITH\_INITIAL\_OWNER, EMPTY and INITIAL\_GROUP\_CONFIG\_UNSPECIFIED. | `string` | `"WITH_INITIAL_OWNER"` | no | +| network\_name | Name for the VPC network | `string` | `"gl-runner-network"` | no | | org\_id | GCP Organization ID | `string` | n/a | yes | | org\_policy\_admin\_role | Additional Org Policy Admin role for admin group. You can use this for testing purposes. | `bool` | `false` | no | | org\_project\_creators | Additional list of members to have project creator role across the organization. Prefix of group: user: or serviceAccount: is required. | `list(string)` | `[]` | no | | parent\_folder | Optional - for an organization with existing projects or for development/validation. It will place all the example foundation resources under the provided folder instead of the root organization. The value is the numeric folder ID. The folder must already exist. | `string` | `""` | no | | project\_prefix | Name prefix to use for projects created. Should be the same in all steps. Max size is 3 characters. | `string` | `"prj"` | no | +| repo\_owner | The owner of Gitlab repository. | `string` | n/a | yes | +| subnet\_ip | IP range for the subnet | `string` | `"10.10.10.0/24"` | no | +| subnet\_name | Name for the subnet | `string` | `"gl-runner-subnet"` | no | +| subnetwork\_project | The ID of the project in which the subnetwork belongs. If it is not provided, the project\_id is used. | `string` | n/a | yes | ## Outputs diff --git a/0-bootstrap/modules/gitlab-runner/.gitlab-ci.yml b/0-bootstrap/modules/gitlab-runner/.gitlab-ci.yml new file mode 100644 index 000000000..febdf1edc --- /dev/null +++ b/0-bootstrap/modules/gitlab-runner/.gitlab-ci.yml @@ -0,0 +1,99 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stages: + - build + - validate + - apply + +build-image: + stage: build + tags: + - gl_runner + image: + name: docker:stable + services: + - name: docker:dind + alias: dockerhost + + variables: + DOCKER_HOST: tcp://dockerhost:2375/ + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + + script: + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin + - docker build -t $CI_REGISTRY_IMAGE/terraform-gcloud:latest . + - docker push $CI_REGISTRY_IMAGE/terraform-gcloud:latest + - docker logout + + only: + - image + +# All branches need to be protected, otherwise the value of environment variables won't be read. +# disable -> projeto -> settings -> cicd -> token access -> limit access to this project + +image: + name: registry.gitlab.com/renatojr_ciandt_group2/project_group2/terraform-gcloud:latest + entrypoint: [""] + +terraform-plan: + stage: validate + tags: + - gl_runner + id_tokens: + GITLAB_OIDC_TOKEN: + aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" + variables: + GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" + script: + - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` + - gcloud config set project "$PROJECT_ID" + - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" + +terraform-plan: + stage: validate + tags: + - gl_runner + + script: + - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` + - gcloud config set project "$PROJECT_ID" + - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" + + only: + - plan + +terraform-apply: + stage: apply + tags: + - gl_runner + id_tokens: + GITLAB_OIDC_TOKEN: + aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" + variables: + GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" + script: + - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` + - gcloud config set project "$PROJECT_ID" + - bash tf-wrapper.sh init $CI_COMMIT_REF_NAME + - bash tf-wrapper.sh plan $CI_COMMIT_REF_NAME + - bash tf-wrapper.sh validate $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $PROJECT_ID "FILESYSTEM" + - bash tf-wrapper.sh apply $CI_COMMIT_REF_NAME + + only: + - development + - production + - non-production + when: manual diff --git a/0-bootstrap/modules/gitlab-runner/startup_script.sh b/0-bootstrap/modules/gitlab-runner/startup_script.sh index 2f3c52dd1..8af377b4f 100755 --- a/0-bootstrap/modules/gitlab-runner/startup_script.sh +++ b/0-bootstrap/modules/gitlab-runner/startup_script.sh @@ -14,51 +14,31 @@ # limitations under the License. #install jq -apt-get update -apt-get -y install jq +#apt-get update +#apt-get -y install jq +curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash +#sudo apt-get -y install gitlab-runner -secretUri=$(curl -sS "http://metadata.google.internal/computeMetadata/v1/instance/attributes/secret-id" -H "Metadata-Flavor: Google") -#secrets URI is of the form projects/$PROJECT_NUMBER/secrets/$SECRET_NAME/versions/$SECRET_VERSION -#split into array based on `/` delimeter -IFS="/" read -r -a secretsConfig <<<"$secretUri" -#get SECRET_NAME and SECRET_VERSION -SECRET_NAME=${secretsConfig[3]} -SECRET_VERSION=${secretsConfig[5]} -#access secret from secretsmanager -secrets=$(gcloud secrets versions access "$SECRET_VERSION" --secret="$SECRET_NAME") -#set secrets as env vars -# shellcheck disable=SC2046 -# we want to use wordsplitting -export $(echo "$secrets" | jq -r "to_entries|map(\"\(.key)=\(.value|tostring)\")|.[]") -#github runner version -GH_RUNNER_VERSION="2.283.2" -#get actions binary -curl -o actions.tar.gz --location "https://github.com/actions/runner/releases/download/v${GH_RUNNER_VERSION}/actions-runner-linux-x64-${GH_RUNNER_VERSION}.tar.gz" -mkdir /runner -mkdir /runner-tmp -tar -zxf actions.tar.gz --directory /runner -rm -f actions.tar.gz -/runner/bin/installdependencies.sh -#get actions token -# shellcheck disable=SC2034 -# ACTIONS_RUNNER_INPUT_NAME is used by config.sh -ACTIONS_RUNNER_INPUT_NAME=$HOSTNAME -if [[ -z $REPO_NAME ]]; then - # Add action runner for an organisation - POST_URL="https://api.github.com/orgs/${REPO_OWNER}/actions/runners/registration-token" - GH_URL="https://github.com/${REPO_OWNER}" -else - # Add action runner for a repo - POST_URL="https://api.github.com/repos/${REPO_OWNER}/${REPO_NAME}/actions/runners/registration-token" - GH_URL="https://github.com/${REPO_OWNER}/${REPO_NAME}" -fi +sudo sed -i '/[runners.docker]/a image = "docker:dind"' /etc/gitlab-runner/conf.toml +sudo sed -i '/image = "docker:dind"/a privileged = true' /etc/gitlab-runner/conf.toml +sudo sed -i '/privileged = true/a tls_verify = false' /etc/gitlab-runner/conf.toml -# Register runner -ACTIONS_RUNNER_INPUT_TOKEN="$(curl -sS --request POST --url "$POST_URL" --header "authorization: Bearer ${GITHUB_TOKEN}" --header 'content-type: application/json' | jq -r .token)" -#configure runner -RUNNER_ALLOW_RUNASROOT=1 /runner/config.sh --unattended --replace --work "/runner-tmp" --url "$GH_URL" --token "$ACTIONS_RUNNER_INPUT_TOKEN" --labels "$LABELS" - -#install and start runner service -cd /runner || exit -./svc.sh install -./svc.sh start +# [[runners]] +# name = "RUNNER-XXX" +# url = "https://gitlab.com/" +# id = 27149207 +# token = "glrt-XXX" +# token_obtained_at = 2023-08-22T13:53:29Z +# token_expires_at = 0001-01-01T00:00:00Z +# executor = "docker" +# [runners.cache] +# MaxUploadedArchiveSize = 0 +# [runners.docker] +# tls_verify = false +# image = "docker:dind" +# privileged = true +# disable_entrypoint_overwrite = false +# oom_kill_disable = false +# disable_cache = false +# volumes = ["/cache"] +# shm_size = 0 From 2340ff7b85440841b052bcc4d6e692a580985caf Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Thu, 26 Oct 2023 10:57:37 -0300 Subject: [PATCH 10/31] updating code for gitlab --- .../gitlab-runner => }/.gitlab-ci.yml | 0 0-bootstrap/README-GitLab.md | 1 + .../modules/gitlab-runner/gitlab-ci.yml | 100 ++++++++++++++++++ 0-bootstrap/modules/gitlab-runner/main.tf | 4 +- .../modules/gitlab-runner/startup_script.sh | 54 +++++----- 5 files changed, 130 insertions(+), 29 deletions(-) rename 0-bootstrap/{modules/gitlab-runner => }/.gitlab-ci.yml (100%) create mode 100644 0-bootstrap/modules/gitlab-runner/gitlab-ci.yml diff --git a/0-bootstrap/modules/gitlab-runner/.gitlab-ci.yml b/0-bootstrap/.gitlab-ci.yml similarity index 100% rename from 0-bootstrap/modules/gitlab-runner/.gitlab-ci.yml rename to 0-bootstrap/.gitlab-ci.yml diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index 5c20aec30..f850cdbd1 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -277,3 +277,4 @@ They will [fail](../docs/TROUBLESHOOTING.md#error-unsupported-attribute) if the **Note 2:** After the deploy, to prevent the project quota error described in the [Troubleshooting guide](../docs/TROUBLESHOOTING.md#project-quota-exceeded), we recommend that you request 50 additional projects for the **projects step service account** created in this step. + diff --git a/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml b/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml new file mode 100644 index 000000000..f531e5a38 --- /dev/null +++ b/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml @@ -0,0 +1,100 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +stages: + - build + - validate + - apply + +build-image: + stage: build + tags: + - gl_runner + image: + name: docker:stable + services: + - name: docker:dind + alias: dockerhost + + variables: + DOCKER_HOST: tcp://dockerhost:2375/ + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + + script: + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin + - docker build -t $CI_REGISTRY_IMAGE/terraform-gcloud:latest . + - docker push $CI_REGISTRY_IMAGE/terraform-gcloud:latest + - docker logout + + only: + - image + +# All branches need to be protected, otherwise the value of environment variables won't be read. +# disable -> projeto -> settings -> cicd -> token access -> limit access to this project + +image: + name: registry.gitlab.com/renatojr_ciandt_group2/project_group2/terraform-gcloud:latest + entrypoint: [""] + +terraform-plan: + stage: validate + tags: + - gl_runner + id_tokens: + GITLAB_OIDC_TOKEN: + aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" + variables: + GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" + script: + - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` + - gcloud config set project "$PROJECT_ID" + - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" + +terraform-plan: + stage: validate + tags: + - gl_runner + + script: + - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` + - gcloud config set project "$PROJECT_ID" + - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" + + only: + - plan + +terraform-apply: + stage: apply + tags: + - gl_runner + id_tokens: + GITLAB_OIDC_TOKEN: + aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" + variables: + GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" + script: + - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` + - gcloud config set project "$PROJECT_ID" + - bash tf-wrapper.sh init $CI_COMMIT_REF_NAME + - bash tf-wrapper.sh plan $CI_COMMIT_REF_NAME + - bash tf-wrapper.sh validate $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $PROJECT_ID "FILESYSTEM" + - bash tf-wrapper.sh apply $CI_COMMIT_REF_NAME + + only: + - development + - production + - non-production + when: manual + diff --git a/0-bootstrap/modules/gitlab-runner/main.tf b/0-bootstrap/modules/gitlab-runner/main.tf index f77c669e3..3bc96fcb9 100644 --- a/0-bootstrap/modules/gitlab-runner/main.tf +++ b/0-bootstrap/modules/gitlab-runner/main.tf @@ -154,8 +154,8 @@ module "mig_template" { name_prefix = "gl-runner" source_image_family = var.source_image_family source_image_project = var.source_image_project - #startup_script = file("${abspath(path.module)}/startup_script.sh") - source_image = var.source_image + startup_script = file("${abspath(path.module)}/startup_script.sh") + source_image = var.source_image metadata = merge({ "secret-id" = google_secret_manager_secret_version.gl-secret-version.name }, var.custom_metadata) diff --git a/0-bootstrap/modules/gitlab-runner/startup_script.sh b/0-bootstrap/modules/gitlab-runner/startup_script.sh index 8af377b4f..45f3355ab 100755 --- a/0-bootstrap/modules/gitlab-runner/startup_script.sh +++ b/0-bootstrap/modules/gitlab-runner/startup_script.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2020 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,32 +13,32 @@ # See the License for the specific language governing permissions and # limitations under the License. -#install jq -#apt-get update -#apt-get -y install jq curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash -#sudo apt-get -y install gitlab-runner +sudo apt-get -y install gitlab-runner -sudo sed -i '/[runners.docker]/a image = "docker:dind"' /etc/gitlab-runner/conf.toml -sudo sed -i '/image = "docker:dind"/a privileged = true' /etc/gitlab-runner/conf.toml -sudo sed -i '/privileged = true/a tls_verify = false' /etc/gitlab-runner/conf.toml +sudo tee /etc/gitlab-runner/config.toml <<'EOF' +concurrent = 20 +check_interval = 0 +shutdown_timeout = 0 + +[session_server] + session_timeout = 1800 + +[[runners]] + name = "gl_runner" + url = "https://gitlab.com/" + token = "glrt-XXX" + executor = "docker" + [runners.cache] + MaxUploadedArchiveSize = 0 + [runners.docker] + tls_verify = false + image = "docker:dind" + privileged = true + disable_entrypoint_overwrite = false + oom_kill_disable = false + disable_cache = false + volumes = ["/cache"] + shm_size = 0 +EOF -# [[runners]] -# name = "RUNNER-XXX" -# url = "https://gitlab.com/" -# id = 27149207 -# token = "glrt-XXX" -# token_obtained_at = 2023-08-22T13:53:29Z -# token_expires_at = 0001-01-01T00:00:00Z -# executor = "docker" -# [runners.cache] -# MaxUploadedArchiveSize = 0 -# [runners.docker] -# tls_verify = false -# image = "docker:dind" -# privileged = true -# disable_entrypoint_overwrite = false -# oom_kill_disable = false -# disable_cache = false -# volumes = ["/cache"] -# shm_size = 0 From 2113d2476100a9972d7aca7da9fd09756c43c339 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Thu, 26 Oct 2023 11:05:40 -0300 Subject: [PATCH 11/31] fix path for gitlab-ci.yml file --- .../.gitlab-ci.yml => build/gitlab/gitlab-build-image-ci.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename 0-bootstrap/.gitlab-ci.yml => build/gitlab/gitlab-build-image-ci.yml (100%) diff --git a/0-bootstrap/.gitlab-ci.yml b/build/gitlab/gitlab-build-image-ci.yml similarity index 100% rename from 0-bootstrap/.gitlab-ci.yml rename to build/gitlab/gitlab-build-image-ci.yml From c228eab63efa6b8433181eb4e1203f10c7c33be3 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Thu, 26 Oct 2023 11:10:45 -0300 Subject: [PATCH 12/31] rename gitlab file --- build/gitlab/{gitlab-build-image-ci.yml => .gitlab-ci.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build/gitlab/{gitlab-build-image-ci.yml => .gitlab-ci.yml} (100%) diff --git a/build/gitlab/gitlab-build-image-ci.yml b/build/gitlab/.gitlab-ci.yml similarity index 100% rename from build/gitlab/gitlab-build-image-ci.yml rename to build/gitlab/.gitlab-ci.yml From f133dbc338f6b4bb1779618d6be32a767b5305f6 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Fri, 27 Oct 2023 10:46:56 -0300 Subject: [PATCH 13/31] update network values --- 0-bootstrap/modules/gitlab-runner/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/0-bootstrap/modules/gitlab-runner/main.tf b/0-bootstrap/modules/gitlab-runner/main.tf index 3bc96fcb9..08e910693 100644 --- a/0-bootstrap/modules/gitlab-runner/main.tf +++ b/0-bootstrap/modules/gitlab-runner/main.tf @@ -25,12 +25,12 @@ module "vpc_network" { source = "terraform-google-modules/network/google" version = "~> 7.0" project_id = var.project_id - network_name = "my-network" + network_name = "gl-network" mtu = 1460 subnets = [ { - subnet_name = "subnet-01" + subnet_name = "gl-subnet" subnet_ip = "10.10.10.0/24" subnet_region = "us-central1" # subnet_private_access = "true" From 60d3694c62f944d5f8128e32d98894b2d1c53a42 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Fri, 27 Oct 2023 18:25:03 -0300 Subject: [PATCH 14/31] update readme instructions --- 0-bootstrap/README-GitLab.md | 142 ++++++++++++++++++++++------------- 1 file changed, 91 insertions(+), 51 deletions(-) diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index f850cdbd1..9c0ed9821 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -60,40 +60,7 @@ that are created, see the organization bootstrap module git clone https://github.com/terraform-google-modules/terraform-example-foundation.git ``` -### Build CI/CD runner image - -1. Clone the private project you created to host the docker configuration for the CI/CD runner at the same level of the `terraform-example-foundation` folder. -You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured with GitLab. - - ```bash - git clone git@gitlab.com:/.git gcp-cicd-runner - ``` - -1. Navigate into the repo. All subsequent - steps assume you are running them from the `gcp-cicd-runner` directory. - If you run them from another directory, adjust your copy paths accordingly. - ```bash - cd gcp-cicd-runner - ``` - -1. Copy contents of foundation to cloned project (modify accordingly based on your current directory). - - ```bash - cp ../terraform-example-foundation/build/gitlab/gitlab-build-image-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/Dockerfile ./Dockerfile - ``` - -1. Save the CI/CD runner configuration to `gcp-cicd-runner` GitLab project: - - ```bash - git add . - git commit -m 'Initialize CI/CD runner project' - git push --set-upstream origin main - ``` - -1. Review the CI/CD Job output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-RUNNER-REPO/-/jobs under name `build-image`. -1. If the CI/CD Job is successful proceed with the next steps ### Deploying step 0-bootstrap @@ -133,9 +100,9 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi cp -RT ../terraform-example-foundation/0-bootstrap/ ./envs/shared cp -RT ../terraform-example-foundation/policy-library/ ./policy-library - cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/run_gcp_sts.sh . - chmod 755 ./run_gcp_sts.sh + cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/gitlab/run_gcp_* . + chmod 755 ./run_gcp_* cp ../terraform-example-foundation/build/tf-wrapper.sh . chmod 755 ./tf-wrapper.sh cd ./envs/shared @@ -174,7 +141,7 @@ export the GitLab personal or group access token as an environment variable: 1. Use the helper script [validate-requirements.sh](../scripts/validate-requirements.sh) to validate your environment: ```bash - ../../../terraform-example-foundation/scripts/validate-requirements.sh -o -b -u -t GitHub + ../../../terraform-example-foundation/scripts/validate-requirements.sh -o -b -u -t GitLab ``` **Note:** The script is not able to validate if the user is in a Cloud Identity or Google Workspace group with the required roles. @@ -207,19 +174,6 @@ export the GitLab personal or group access token as an environment variable: terraform apply bootstrap.tfplan ``` -1. Once `terraform apply` has finished, access the Gitlab instance by ssh that has been created and update following fields in the file `/etc/gitlab-runner/config.toml` using the values showed during the Gitlab Runner creation. - -```bash -name = "xxxx" -token = "glrt-XXX" -``` - -1. Copy the file `gitlab-ci.yml` to the 0-bootstrap directory. -```bash -cp ./modules/gitlab-runner/gitlab-ci.yml . -``` - - 1. Run `terraform output` to get the email address of the terraform service accounts that will be used to run manual steps for `shared` environments in steps `3-networks-dual-svpc`, `3-networks-hub-and-spoke`, and `4-projects`. ```bash @@ -268,6 +222,93 @@ cp ./modules/gitlab-runner/gitlab-ci.yml . git push --set-upstream origin plan ``` +### Build CI/CD runner image + +1. Clone the private project you created to host the docker configuration for the CI/CD runner at the same level of the `terraform-example-foundation` folder. +You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured with GitLab. + + ```bash + git clone git@gitlab.com:/.git gcp-cicd-runner + ``` + +1. Navigate into the repo. All subsequent steps assume you are running them from the `gcp-cicd-runner` directory. + If you run them from another directory, adjust your copy paths accordingly. + + ```bash + cd gcp-cicd-runner + ``` + +1. Create a new branch called `image` which will contains the Docker image using in the Gitlab Pipelines. + + ```bash + git checkout -b image + ``` + +1. Copy contents of foundation to cloned project (modify accordingly based on your current directory). + + ```bash + cp ../terraform-example-foundation/build/gitlab/.gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/gitlab/Dockerfile ./Dockerfile + ``` + +1. Edit the `./.gitlab-ci.yml` file and update the paramenter the following line: + +```bash +image: + name: registry.gitlab.com/EXAMPLE-GITLAB-ACCOUNT/EXAMPLE-GITLAB-REPOSITORY/terraform-gcloud:latest +``` + +To: + +```bash +image: + name: registry.gitlab.com/GITLAB-ACCOUNT/GITLAB-REPOSITORY/terraform-gcloud:latest +``` + +1. Save the CI/CD runner configuration to `gcp-cicd-runner` GitLab project: + + ```bash + git add . + git commit -m 'Initialize CI/CD runner project' + git push --set-upstream origin image + ``` + +1. Review the CI/CD Job output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-RUNNER-REPO/-/jobs under name `build-image`. +1. If the CI/CD Job is successful proceed with the next steps. + +1. Access the Gitlab instance created in the 0-bootstrap step by ssh and update token field in the file `/etc/gitlab-runner/config.toml` using the value showed during the Gitlab Runner creation. To access the instance created you will need to open the SSH port in the firewall: + +```bash +gcloud compute firewall-rules create allow-ssh --allow tcp:22 --network gl-network --project +``` + +Then, update the following field in the GitLab runner config file `sudo vi /etc/gitlab-runner/config.toml`. + +```bash +token = "glrt-XXX" +``` + +Once GitLab runner file is updated, restart your runner: +```bash +sudo gitlab-runner restart +``` + +1. Copy the file `gitlab-ci.yml` to the 0-bootstrap directory. + +```bash +cp ../terraform-example-foundation/build/gitlab/.gitlab-ci.yml ../gcp-bootstrap/.gitlab-ci.yml +``` + +1. Make sure you have enabled the Gitlab Runner to the bootstrap project. You can do this in `https://gitlab.com/YOUR-GITLAB-USER/gcp-cicd-runner/-/settings/ci_cd#js-runners-settings`. + +1. Push the `gitlab-ci.yml` to the 0-bootsrap repository. + + ```bash + cd ../gcp-bootstrap + git add . + git commit -m 'Initialize CICD to the bootstrap project' + git push --set-upstream origin plan + ``` TODO @@ -277,4 +318,3 @@ They will [fail](../docs/TROUBLESHOOTING.md#error-unsupported-attribute) if the **Note 2:** After the deploy, to prevent the project quota error described in the [Troubleshooting guide](../docs/TROUBLESHOOTING.md#project-quota-exceeded), we recommend that you request 50 additional projects for the **projects step service account** created in this step. - From 0af2196c14a7d072391151535a293a11cecf2079 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Fri, 27 Oct 2023 18:25:50 -0300 Subject: [PATCH 15/31] update default value for gitlab subnetwork --- 0-bootstrap/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/0-bootstrap/variables.tf b/0-bootstrap/variables.tf index 83092fa42..d15ab5faf 100644 --- a/0-bootstrap/variables.tf +++ b/0-bootstrap/variables.tf @@ -306,7 +306,7 @@ variable "create_network" { variable "subnetwork_project" { type = string description = "The ID of the project in which the subnetwork belongs. If it is not provided, the project_id is used." - #default = "" + default = "" } variable "subnet_ip" { From c147267f258e4dbf5f4c85a6aaaa2884f5c282de Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Fri, 27 Oct 2023 18:47:17 -0300 Subject: [PATCH 16/31] update image path --- 0-bootstrap/modules/gitlab-runner/gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml b/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml index f531e5a38..8c363e5a7 100644 --- a/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml +++ b/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml @@ -45,7 +45,7 @@ build-image: # disable -> projeto -> settings -> cicd -> token access -> limit access to this project image: - name: registry.gitlab.com/renatojr_ciandt_group2/project_group2/terraform-gcloud:latest + name: registry.gitlab.com/GITLAB-ACCOUNT/GITLAB-REPOSITORY/terraform-gcloud:latest entrypoint: [""] terraform-plan: From 05b3a42869850fe06fdfeb01fcd1d16d039eb527 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Mon, 30 Oct 2023 19:38:44 -0300 Subject: [PATCH 17/31] update readme --- 0-bootstrap/README-GitLab.md | 729 ++++++++++++++++++++++++++++++++++- 1 file changed, 711 insertions(+), 18 deletions(-) diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index 9c0ed9821..df45306ec 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -32,7 +32,6 @@ Also make sure that you have the following: - write_repository - read_registry - write_registry -- A [Runner](https://docs.gitlab.com/ee/tutorials/create_register_first_runner/) with the name `gl_runner` should be created in your GitLab account. Save the token of the runner once it has been created. It will be used in the next steps. - A Google Cloud [organization](https://cloud.google.com/resource-manager/docs/creating-managing-organization). - A Google Cloud [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account). - Cloud Identity or Google Workspace groups for organization and billing admins. @@ -250,6 +249,19 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi cp ../terraform-example-foundation/build/gitlab/.gitlab-ci.yml ./.gitlab-ci.yml cp ../terraform-example-foundation/build/gitlab/Dockerfile ./Dockerfile ``` +1. Create a runner for your GitLab in [Settings -> CICD -> Runners](https://gitlab.com/renatojr_ciandt/fdt-bootstrap/-/settings/ci_cd)called `gl_runner` and set the tag name also for `gl_runner`. + +1. To execute the next step, access the Gitlab instance created in the 0-bootstrap step by ssh and update token field in the file `/etc/gitlab-runner/config.toml` using the value showed during the Gitlab Runner creation. To access the instance created you will need to open the SSH port in the firewall: + +```bash +gcloud compute firewall-rules create allow-ssh --allow tcp:22 --network gl-network --project +``` + +1. After have created the runner on GitLab interface, run the command below in the GCP instance for the GitLab. Remember to change the token `glrt-xxx` for the token generated in the GitLab interface. + +```bash +sudo gitlab-runner register --non-interactive --url https://gitlab.com --name gl_runner --executor docker --docker-image docker:dind --docker-privileged true --token glrt-xxx +``` 1. Edit the `./.gitlab-ci.yml` file and update the paramenter the following line: @@ -274,24 +286,8 @@ image: ``` 1. Review the CI/CD Job output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-RUNNER-REPO/-/jobs under name `build-image`. -1. If the CI/CD Job is successful proceed with the next steps. -1. Access the Gitlab instance created in the 0-bootstrap step by ssh and update token field in the file `/etc/gitlab-runner/config.toml` using the value showed during the Gitlab Runner creation. To access the instance created you will need to open the SSH port in the firewall: - -```bash -gcloud compute firewall-rules create allow-ssh --allow tcp:22 --network gl-network --project -``` - -Then, update the following field in the GitLab runner config file `sudo vi /etc/gitlab-runner/config.toml`. - -```bash -token = "glrt-XXX" -``` - -Once GitLab runner file is updated, restart your runner: -```bash -sudo gitlab-runner restart -``` +1. If the CI/CD Job is successful proceed with the next steps. 1. Copy the file `gitlab-ci.yml` to the 0-bootstrap directory. @@ -318,3 +314,700 @@ They will [fail](../docs/TROUBLESHOOTING.md#error-unsupported-attribute) if the **Note 2:** After the deploy, to prevent the project quota error described in the [Troubleshooting guide](../docs/TROUBLESHOOTING.md#project-quota-exceeded), we recommend that you request 50 additional projects for the **projects step service account** created in this step. + +## Deploying step 1-org + +1. Clone the repository you created to host the `1-org` terraform configuration at the same level of the `terraform-example-foundation` folder. + + ```bash + git clone git@gilab.com:/.git gcp-org + ``` + +1. Navigate into the repo. All subsequent steps assume you are running them from the `gcp-org` directory. + If you run them from another directory, adjust your copy paths accordingly. + + ```bash + cd gcp-org + ``` + +1. Seed the repository if it has not been initialized yet. + + ```bash + git commit --allow-empty -m 'repository seed' + git push --set-upstream origin main + + git checkout -b production + git push --set-upstream origin production + ``` + +1. change to a non-production branch. + + ```bash + git checkout -b plan + ``` + +1. Copy contents of foundation to new repo. + + ```bash + cp -RT ../terraform-example-foundation/1-org/ . + cp -RT ../terraform-example-foundation/policy-library/ ./policy-library + cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/gitlab/run_gcp_* . + chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/tf-wrapper.sh . + chmod 755 ./tf-wrapper.sh + ``` + +1. Rename `./envs/shared/terraform.example.tfvars` to `./envs/shared/terraform.tfvars` + + ```bash + mv ./envs/shared/terraform.example.tfvars ./envs/shared/terraform.tfvars + ``` + +1. Update the file `envs/shared/terraform.tfvars` with values from your GCP environment. +See the shared folder [README.md](../1-org/envs/shared/README.md#inputs) for additional information on the values in the `terraform.tfvars` file. + +1. Un-comment the variable `create_access_context_manager_access_policy = false` if your organization already has an Access Context Manager Policy. + + ```bash + export ORGANIZATION_ID=$(terraform -chdir="../gcp-bootstrap/envs/shared" output -json common_config | jq '.org_id' --raw-output) + + export ACCESS_CONTEXT_MANAGER_ID=$(gcloud access-context-manager policies list --organization ${ORGANIZATION_ID} --format="value(name)") + + echo "access_context_manager_policy_id = ${ACCESS_CONTEXT_MANAGER_ID}" + + if [ ! -z "${ACCESS_CONTEXT_MANAGER_ID}" ]; then sed -i "s=//create_access_context_manager_access_policy=create_access_context_manager_access_policy=" ./envs/shared/terraform.tfvars; fi + ``` + +1. Update the `remote_state_bucket` variable with the backend bucket from step Bootstrap. + + ```bash + export backend_bucket=$(terraform -chdir="../gcp-bootstrap/envs/shared" output -raw gcs_bucket_tfstate) + + echo "remote_state_bucket = ${backend_bucket}" + + sed -i "s/REMOTE_STATE_BUCKET/${backend_bucket}/" ./envs/shared/terraform.tfvars + ``` + +1. Check if a Security Command Center Notification with the default name, **scc-notify**, already exists in your organization. + + ```bash + export ORG_STEP_SA=$(terraform -chdir="../gcp-bootstrap/envs/shared" output -raw organization_step_terraform_service_account_email) + + gcloud scc notifications describe "scc-notify" --format="value(name)" --organization=${ORGANIZATION_ID} --impersonate-service-account=${ORG_STEP_SA} + ``` + +1. If the notification exists the output will be: + + ```text + organizations/ORGANIZATION_ID/notificationConfigs/scc-notify + ``` + +1. If the notification does not exist the output will be: + + ```text + ERROR: (gcloud.scc.notifications.describe) NOT_FOUND: Requested entity was not found. + ``` + +1. If the notification exists, choose a different value for the `scc_notification_name` variable in the `./envs/shared/terraform.tfvars` file. + +1. Commit changes. + + ```bash + git add . + git commit -m 'Initialize org repo' + ``` + +1. Push your plan branch. + + ```bash + git push --set-upstream origin plan + ``` + +**Note 1:** Before open a merge request, make sure the option `Delete source branch` is unchecked. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/merge_requests?scope=all&state=opened from the `plan` branch to the `production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the GitLab pipelines is successful, apply the next environment. + + +## Deploying step 2-environments + +1. Clone the repository you created to host the `2-environments` terraform configuration at the same level of the `terraform-example-foundation` folder. + + ```bash + git clone git@gitlab.com:/.git gcp-environments + ``` + +1. Navigate into the repo. All subsequent + steps assume you are running them from the `gcp-environments` directory. + If you run them from another directory, adjust your copy paths accordingly. + + ```bash + cd gcp-environments + ``` + +1. Seed the repository if it has not been initialized yet. + + ```bash + git commit --allow-empty -m 'repository seed' + git push --set-upstream origin main + + git checkout -b production + git push --set-upstream origin production + + git checkout -b non-production + git push --set-upstream origin non-production + + git checkout -b development + git push --set-upstream origin development + ``` + +1. change to a non-production branch. + + ```bash + git checkout -b plan + ``` + +1. Copy contents of foundation to new repo. + + ```bash + cp -RT ../terraform-example-foundation/2-environments/ . + cp -RT ../terraform-example-foundation/policy-library/ ./policy-library + cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/gitlab/run_gcp_* . + chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/tf-wrapper.sh . + chmod 755 ./tf-wrapper.sh + ``` + +1. Rename `terraform.example.tfvars` to `terraform.tfvars`. + + ```bash + mv terraform.example.tfvars terraform.tfvars + ``` + +1. Update the file with values from your GCP environment. +See any of the envs folder [README.md](../2-environments/envs/production/README.md#inputs) files for additional information on the values in the `terraform.tfvars` file. + +1. Update the `remote_state_bucket` variable with the backend bucket from step Bootstrap. + + ```bash + export backend_bucket=$(terraform -chdir="../gcp-bootstrap/envs/shared" output -raw gcs_bucket_tfstate) + echo "remote_state_bucket = ${backend_bucket}" + + sed -i "s/REMOTE_STATE_BUCKET/${backend_bucket}/" terraform.tfvars + ``` + +1. Commit changes. + + ```bash + git add . + git commit -m 'Initialize environments repo' + ``` + +1. Push your plan branch. + + ```bash + git push --set-upstream origin plan + ``` + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `plan` branch to the `development` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `development` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `development` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `development` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the Gitlab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `development` branch to the `non-production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `non-production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `non-production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `non-production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the GitLab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `non-production` branch to the `production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. + +1. You can now move to the instructions in the network stage. +To use the [Dual Shared VPC](https://cloud.google.com/architecture/security-foundations/networking#vpcsharedvpc-id7-1-shared-vpc-) network mode go to [Deploying step 3-networks-dual-svpc](#deploying-step-3-networks-dual-svpc), +or go to [Deploying step 3-networks-hub-and-spoke](#deploying-step-3-networks-hub-and-spoke) to use the [Hub and Spoke](https://cloud.google.com/architecture/security-foundations/networking#hub-and-spoke) network mode. + +## Deploying step 3-networks-dual-svpc + +1. Clone the repository you created to host the `3-networks-dual-svpc` terraform configuration at the same level of the `terraform-example-foundation` folder. + + ```bash + git clone git@gitlab.com:/.git gcp-networks + ``` + +1. Navigate into the repo. All subsequent steps assume you are running them from the `gcp-networks` directory. + If you run them from another directory, adjust your copy paths accordingly. + + ```bash + cd gcp-networks + ``` + +1. Seed the repository if it has not been initialized yet. + + ```bash + git commit --allow-empty -m 'repository seed' + git push --set-upstream origin main + + git checkout -b production + git push --set-upstream origin production + + git checkout -b non-production + git push --set-upstream origin non-production + + git checkout -b development + git push --set-upstream origin development + ``` + +1. change to a non-production branch. + + ```bash + git checkout -b plan + ``` + +1. Copy contents of foundation to new repo. + + ```bash + cp -RT ../terraform-example-foundation/3-networks-dual-svpc/ . + cp -RT ../terraform-example-foundation/policy-library/ ./policy-library + cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/gitlab/run_gcp_* . + chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/tf-wrapper.sh . + chmod 755 ./tf-wrapper.sh + ``` + +1. Rename `common.auto.example.tfvars` to `common.auto.tfvars`, rename `shared.auto.example.tfvars` to `shared.auto.tfvars` and rename `access_context.auto.example.tfvars` to `access_context.auto.tfvars`. + + ```bash + mv common.auto.example.tfvars common.auto.tfvars + mv shared.auto.example.tfvars shared.auto.tfvars + mv access_context.auto.example.tfvars access_context.auto.tfvars + ``` + +1. Update the file `shared.auto.tfvars` with the values for the `target_name_server_addresses`. +1. Update the file `access_context.auto.tfvars` with the organization's `access_context_manager_policy_id`. + + ```bash + export ORGANIZATION_ID=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -json common_config | jq '.org_id' --raw-output) + + export ACCESS_CONTEXT_MANAGER_ID=$(gcloud access-context-manager policies list --organization ${ORGANIZATION_ID} --format="value(name)") + + echo "access_context_manager_policy_id = ${ACCESS_CONTEXT_MANAGER_ID}" + + sed -i "s/ACCESS_CONTEXT_MANAGER_ID/${ACCESS_CONTEXT_MANAGER_ID}/" ./access_context.auto.tfvars + ``` + +1. Update `common.auto.tfvars` file with values from your GCP environment. +See any of the envs folder [README.md](../3-networks-dual-svpc/envs/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars` file. +1. You must add your user email in the variable `perimeter_additional_members` to be able to see the resources created in the restricted project. +1. Update the `remote_state_bucket` variable with the backend bucket from step Bootstrap in the `common.auto.tfvars` file. + + ```bash + export backend_bucket=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw gcs_bucket_tfstate) + + echo "remote_state_bucket = ${backend_bucket}" + + sed -i "s/REMOTE_STATE_BUCKET/${backend_bucket}/" ./common.auto.tfvars + ``` + +1. Commit changes + + ```bash + git add . + git commit -m 'Initialize networks repo' + ``` + +1. You must manually plan and apply the `shared` environment (only once) since the `development`, `non-production` and `production` environments depend on it. +1. Use `terraform output` to get the CI/CD project ID and the networks step Terraform Service Account from gcp-bootstrap output. +1. The CI/CD project ID will be used in the [validation](https://cloud.google.com/docs/terraform/policy-validation/quickstart) of the Terraform configuration + + ```bash + export CICD_PROJECT_ID=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw cicd_project_id) + echo ${CICD_PROJECT_ID} + ``` + +1. The networks step Terraform Service Account will be used for [Service Account impersonation](https://cloud.google.com/docs/authentication/use-service-account-impersonation) in the following steps. +An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with the Terraform Service Account to enable impersonation. + + ```bash + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw networks_step_terraform_service_account_email) + echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} + ``` + +1. Run `init` and `plan` and review output for environment shared. + + ```bash + ./tf-wrapper.sh init shared + ./tf-wrapper.sh plan shared + ``` + +1. To use the `validate` option of the `tf-wrapper.sh` script, please follow the [instructions](https://cloud.google.com/docs/terraform/policy-validation/validate-policies#install) to install the terraform-tools component. +1. Run `validate` and check for violations. + + ```bash + ./tf-wrapper.sh validate shared $(pwd)/policy-library ${CICD_PROJECT_ID} + ``` + +1. Run `apply` shared. + + ```bash + ./tf-wrapper.sh apply shared + ``` + +1. Push your plan branch. + + ```bash + git push --set-upstream origin plan + ``` + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `plan` branch to the `development` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `development` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `development` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `development` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the Gitlab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `development` branch to the `non-production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `non-production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `non-production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `non-production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the GitLab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `non-production` branch to the `production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. + +1. Before executing the next steps, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` environment variable. + + ```bash + unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT + ``` + +1. You can now move to the instructions in the [4-projects](#deploying-step-4-projects) stage. + +## Deploying step 3-networks-hub-and-spoke + +1. Clone the repository you created to host the `3-networks-hub-and-spoke` terraform configuration at the same level of the `terraform-example-foundation` folder. + + ```bash + git clone git@gitlab.com:/.git gcp-networks + ``` + +1. Navigate into the repo. All subsequent steps assume you are running them from the `gcp-networks` directory. + If you run them from another directory, adjust your copy paths accordingly. + + ```bash + cd gcp-networks + ``` + +1. Seed the repository if it has not been initialized yet. + + ```bash + git commit --allow-empty -m 'repository seed' + git push --set-upstream origin main + + git checkout -b production + git push --set-upstream origin production + + git checkout -b non-production + git push --set-upstream origin non-production + + git checkout -b development + git push --set-upstream origin development + ``` + +1. change to a non-production branch. + + ```bash + git checkout -b plan + ``` + +1. Copy contents of foundation to new repo. + + ```bash + cp -RT ../terraform-example-foundation/3-networks-hub-and-spoke/ . + cp -RT ../terraform-example-foundation/policy-library/ ./policy-library + cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/gitlab/run_gcp_* . + chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/tf-wrapper.sh . + chmod 755 ./tf-wrapper.sh + ``` + +1. Rename `common.auto.example.tfvars` to `common.auto.tfvars`, rename `shared.auto.example.tfvars` to `shared.auto.tfvars` and rename `access_context.auto.example.tfvars` to `access_context.auto.tfvars`. + + ```bash + mv common.auto.example.tfvars common.auto.tfvars + mv shared.auto.example.tfvars shared.auto.tfvars + mv access_context.auto.example.tfvars access_context.auto.tfvars + ``` + +1. Update `common.auto.tfvars` file with values from your GCP environment. +See any of the envs folder [README.md](../3-networks-hub-and-spoke/envs/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars` file. +1. You must add your user email in the variable `perimeter_additional_members` to be able to see the resources created in the restricted project. +1. Update the `remote_state_bucket` variable with the backend bucket from step Bootstrap in the `common.auto.tfvars` file. + + ```bash + export backend_bucket=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw gcs_bucket_tfstate) + + echo "remote_state_bucket = ${backend_bucket}" + + sed -i "s/REMOTE_STATE_BUCKET/${backend_bucket}/" ./common.auto.tfvars + ``` + +1. Commit changes + + ```bash + git add . + git commit -m 'Initialize networks repo' + ``` + +1. You must manually plan and apply the `shared` environment (only once) since the `development`, `non-production` and `production` environments depend on it. +1. Use `terraform output` to get the CI/CD project ID and the networks step Terraform Service Account from gcp-bootstrap output. +1. The CI/CD project ID will be used in the [validation](https://cloud.google.com/docs/terraform/policy-validation/quickstart) of the Terraform configuration + + ```bash + export CICD_PROJECT_ID=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw cicd_project_id) + echo ${CICD_PROJECT_ID} + ``` + +1. The networks step Terraform Service Account will be used for [Service Account impersonation](https://cloud.google.com/docs/authentication/use-service-account-impersonation) in the following steps. +An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with the Terraform Service Account to enable impersonation. + + ```bash + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw networks_step_terraform_service_account_email) + echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} + ``` + +1. Run `init` and `plan` and review output for environment shared. + + ```bash + ./tf-wrapper.sh init shared + ./tf-wrapper.sh plan shared + ``` + +1. To use the `validate` option of the `tf-wrapper.sh` script, please follow the [instructions](https://cloud.google.com/docs/terraform/policy-validation/validate-policies#install) to install the terraform-tools component. +1. Run `validate` and check for violations. + + ```bash + ./tf-wrapper.sh validate shared $(pwd)/policy-library ${CICD_PROJECT_ID} + ``` + +1. Run `apply` shared. + + ```bash + ./tf-wrapper.sh apply shared + ``` + +1. Push your plan branch. + + ```bash + git push --set-upstream origin plan + ``` + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `plan` branch to the `development` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `development` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `development` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `development` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the Gitlab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `development` branch to the `non-production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `non-production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `non-production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `non-production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the GitLab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `non-production` branch to the `production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. + + +1. Before executing the next steps, unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` environment variable. + + ```bash + unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT + ``` + +1. You can now move to the instructions in the [4-projects](#deploying-step-4-projects) stage. + +## Deploying step 4-projects + +1. Clone the repository you created to host the `4-projects` terraform configuration at the same level of the `terraform-example-foundation` folder. + + ```bash + git clone git@gitlab.com:/.git gcp-projects + ``` + +1. Navigate into the repo. All subsequent + steps assume you are running them from the `gcp-projects` directory. + If you run them from another directory, adjust your copy paths accordingly. + + ```bash + cd gcp-projects + ``` + +1. Seed the repository if it has not been initialized yet. + + ```bash + git commit --allow-empty -m 'repository seed' + git push --set-upstream origin main + + git checkout -b production + git push --set-upstream origin production + + git checkout -b non-production + git push --set-upstream origin non-production + + git checkout -b development + git push --set-upstream origin development + ``` + +1. change to a non-production branch. + + ```bash + git checkout -b plan + ``` + +1. Copy contents of foundation to new repo. + + ```bash + cp -RT ../terraform-example-foundation/4-projects/ . + cp -RT ../terraform-example-foundation/policy-library/ ./policy-library + cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/gitlab/run_gcp_* . + chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/tf-wrapper.sh . + chmod 755 ./tf-wrapper.sh + ``` + +1. Rename `auto.example.tfvars` files to `auto.tfvars`. + + ```bash + mv common.auto.example.tfvars common.auto.tfvars + mv shared.auto.example.tfvars shared.auto.tfvars + mv development.auto.example.tfvars development.auto.tfvars + mv non-production.auto.example.tfvars non-production.auto.tfvars + mv production.auto.example.tfvars production.auto.tfvars + ``` + +1. See any of the envs folder [README.md](../4-projects/business_unit_1/production/README.md#inputs) files for additional information on the values in the `common.auto.tfvars`, `development.auto.tfvars`, `non-production.auto.tfvars`, and `production.auto.tfvars` files. +1. See any of the shared folder [README.md](../4-projects/business_unit_1/shared/README.md#inputs) files for additional information on the values in the `shared.auto.tfvars` file. + +1. Use `terraform output` to get the backend bucket value from bootstrap output. + + ```bash + export remote_state_bucket=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw gcs_bucket_tfstate) + echo "remote_state_bucket = ${remote_state_bucket}" + + sed -i "s/REMOTE_STATE_BUCKET/${remote_state_bucket}/" ./common.auto.tfvars + ``` + +1. Commit changes. + + ```bash + git add . + git commit -m 'Initialize projects repo' + ``` + +1. You need to manually plan and apply only once the `business_unit_1/shared` and `business_unit_2/shared` environments since `development`, `non-production`, and `production` depend on them. + +1. Use `terraform output` to get the CI/CD project ID and the projects step Terraform Service Account from gcp-bootstrap output. +1. The CI/CD project ID will be used in the [validation](https://cloud.google.com/docs/terraform/policy-validation/quickstart) of the Terraform configuration + + ```bash + export CICD_PROJECT_ID=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw cicd_project_id) + echo ${CICD_PROJECT_ID} + ``` + +1. The projects step Terraform Service Account will be used for [Service Account impersonation](https://cloud.google.com/docs/authentication/use-service-account-impersonation) in the following steps. +An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with the Terraform Service Account to enable impersonation. + + ```bash + export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(terraform -chdir="../gcp-bootstrap/envs/shared/" output -raw projects_step_terraform_service_account_email) + echo ${GOOGLE_IMPERSONATE_SERVICE_ACCOUNT} + ``` + +1. Run `init` and `plan` and review output for environment shared. + + ```bash + ./tf-wrapper.sh init shared + ./tf-wrapper.sh plan shared + ``` + +1. To use the `validate` option of the `tf-wrapper.sh` script, please follow the [instructions](https://cloud.google.com/docs/terraform/policy-validation/validate-policies#install) to install the terraform-tools component. +1. Run `validate` and check for violations. + + ```bash + ./tf-wrapper.sh validate shared $(pwd)/policy-library ${CICD_PROJECT_ID} + ``` + +1. Run `apply` shared. + + ```bash + ./tf-wrapper.sh apply shared + ``` + +1. Push your plan branch. + + ```bash + git push --set-upstream origin plan + ``` + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `plan` branch to the `development` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `development` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `development` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `development` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the Gitlab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `development` branch to the `non-production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `non-production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `non-production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `non-production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. +1. If the GitLab pipelines is successful, apply the next environment. + +1. Open a merge request in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/merge_requests?scope=all&state=opened from the `non-production` branch to the `production` branch and review the output. +1. The merge request will trigger a GitLab pipelines that will run Terraform `init`/`plan`/`validate` in the `production` environment. +1. Review the GitLab pipelines output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ENVIRONMENTS-REPO/-/pipelines. +1. If the GitLab pipelines is successful, merge the merge request in to the `production` branch. +1. The merge will trigger a GitLab pipelines that will apply the terraform configuration for the `production` environment. +1. Review merge output in GitLab https://gitlab.com/GITLAB-OWNER/GITLAB-ORGANIZATION-REPO/-/pipelines under `tf-apply`. + +1. Unset the `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` environment variable. + + ```bash + unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT + ``` + From 9b86832d333b9fb609d9642bd262e7931baa5ede Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 31 Oct 2023 16:14:14 -0300 Subject: [PATCH 18/31] review fixes --- 0-bootstrap/README-GitLab.md | 18 ++- 0-bootstrap/{cb.tf.example => cb.tf} | 0 0-bootstrap/{gitlab.tf => gitlab.tf.example} | 4 +- .../{ => files}/startup_script.sh | 28 +---- .../modules/gitlab-runner/gitlab-ci.yml | 100 ---------------- 0-bootstrap/modules/gitlab-runner/main.tf | 79 +++---------- .../modules/gitlab-runner/variables.tf | 23 ---- 0-bootstrap/outputs.tf | 108 +++++++++--------- 0-bootstrap/variables.tf | 9 +- 0-bootstrap/versions.tf | 8 +- build/gitlab-ci.yml | 21 +++- 11 files changed, 101 insertions(+), 297 deletions(-) rename 0-bootstrap/{cb.tf.example => cb.tf} (100%) rename 0-bootstrap/{gitlab.tf => gitlab.tf.example} (97%) rename 0-bootstrap/modules/gitlab-runner/{ => files}/startup_script.sh (57%) delete mode 100644 0-bootstrap/modules/gitlab-runner/gitlab-ci.yml diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index df45306ec..32e21c894 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -13,18 +13,19 @@ To run the instructions described in this document, install the following: - [Google Cloud SDK](https://cloud.google.com/sdk/install) version 393.0.0 or later - [terraform-tools](https://cloud.google.com/docs/terraform/policy-validation/validate-policies#install) component - [Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) version 2.28.0 or later -- [GLab](https://gitlab.com/gitlab-org/cli) version 1.32.0 or later -- [Terraform](https://www.terraform.io/downloads.html) version 1.3.0 or later +- [Terraform](https://www.terraform.io/downloads.html) version 1.3.0 or later +- [jq](https://jqlang.github.io/jq/) version 1.6 or later. Also make sure that you have the following: - A [GitLab](https://docs.gitlab.com/ee/user/profile/account/create_accounts.html) account for your User or Group. -- A private GitLab project (repository) for each one of the stages of Foundation: +- A private GitLab project (repository) for each one of the stages of Foundation and one for the GutLab runner Image: - Bootstrap - Organization - Environments - Networks - Projects + - CI/CD Runner - A [Personal](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) access token or a [Group](https://docs.gitlab.com/ee/user/group/settings/group_access_tokens.html) access token configured with the following [scopes](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html#personal-access-token-scopes): - read_api - create_runner @@ -59,8 +60,6 @@ that are created, see the organization bootstrap module git clone https://github.com/terraform-google-modules/terraform-example-foundation.git ``` - - ### Deploying step 0-bootstrap 1. Clone the private project you created to host the `0-bootstrap` terraform configuration at the same level of the `terraform-example-foundation` folder. @@ -74,7 +73,6 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi ```bash gcp-bootstrap/ - gcp-cicd-runner/ terraform-example-foundation/ ``` @@ -100,9 +98,9 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi cp -RT ../terraform-example-foundation/0-bootstrap/ ./envs/shared cp -RT ../terraform-example-foundation/policy-library/ ./policy-library cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/run_gcp_* . - chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/gitlab/run_gcp_auth.sh . cp ../terraform-example-foundation/build/tf-wrapper.sh . + chmod 755 ./run_gcp_auth.sh chmod 755 ./tf-wrapper.sh cd ./envs/shared ``` @@ -249,7 +247,7 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi cp ../terraform-example-foundation/build/gitlab/.gitlab-ci.yml ./.gitlab-ci.yml cp ../terraform-example-foundation/build/gitlab/Dockerfile ./Dockerfile ``` -1. Create a runner for your GitLab in [Settings -> CICD -> Runners](https://gitlab.com/renatojr_ciandt/fdt-bootstrap/-/settings/ci_cd)called `gl_runner` and set the tag name also for `gl_runner`. +1. Create a runner for your GitLab in [Settings -> CICD -> Runners](https://gitlab.com/renatojr_ciandt/fdt-bootstrap/-/settings/ci_cd) called `gl_runner` and set the tag name also for `gl_runner`. 1. To execute the next step, access the Gitlab instance created in the 0-bootstrap step by ssh and update token field in the file `/etc/gitlab-runner/config.toml` using the value showed during the Gitlab Runner creation. To access the instance created you will need to open the SSH port in the firewall: @@ -257,7 +255,7 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi gcloud compute firewall-rules create allow-ssh --allow tcp:22 --network gl-network --project ``` -1. After have created the runner on GitLab interface, run the command below in the GCP instance for the GitLab. Remember to change the token `glrt-xxx` for the token generated in the GitLab interface. +1. After have created the runner on GitLab interface, run the command below in the GCP instance for the GitLab. Remember to change the token `glrt-xxx` for the token generated in the GitLab interface. ```bash sudo gitlab-runner register --non-interactive --url https://gitlab.com --name gl_runner --executor docker --docker-image docker:dind --docker-privileged true --token glrt-xxx diff --git a/0-bootstrap/cb.tf.example b/0-bootstrap/cb.tf similarity index 100% rename from 0-bootstrap/cb.tf.example rename to 0-bootstrap/cb.tf diff --git a/0-bootstrap/gitlab.tf b/0-bootstrap/gitlab.tf.example similarity index 97% rename from 0-bootstrap/gitlab.tf rename to 0-bootstrap/gitlab.tf.example index 3e0ba9ca7..3a706003a 100644 --- a/0-bootstrap/gitlab.tf +++ b/0-bootstrap/gitlab.tf.example @@ -129,8 +129,6 @@ module "cicd_project_wif_iam_member" { module "gitlab_runner" { source = "./modules/gitlab-runner" - repo_owner = var.repo_owner - gitlab_token = var.gitlab_token - project_id = local.cicd_project_id + project_id = local.cicd_project_id } diff --git a/0-bootstrap/modules/gitlab-runner/startup_script.sh b/0-bootstrap/modules/gitlab-runner/files/startup_script.sh similarity index 57% rename from 0-bootstrap/modules/gitlab-runner/startup_script.sh rename to 0-bootstrap/modules/gitlab-runner/files/startup_script.sh index 45f3355ab..97a352edb 100755 --- a/0-bootstrap/modules/gitlab-runner/startup_script.sh +++ b/0-bootstrap/modules/gitlab-runner/files/startup_script.sh @@ -15,30 +15,4 @@ curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash sudo apt-get -y install gitlab-runner - -sudo tee /etc/gitlab-runner/config.toml <<'EOF' -concurrent = 20 -check_interval = 0 -shutdown_timeout = 0 - -[session_server] - session_timeout = 1800 - -[[runners]] - name = "gl_runner" - url = "https://gitlab.com/" - token = "glrt-XXX" - executor = "docker" - [runners.cache] - MaxUploadedArchiveSize = 0 - [runners.docker] - tls_verify = false - image = "docker:dind" - privileged = true - disable_entrypoint_overwrite = false - oom_kill_disable = false - disable_cache = false - volumes = ["/cache"] - shm_size = 0 -EOF - +# TODO install docker diff --git a/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml b/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml deleted file mode 100644 index 8c363e5a7..000000000 --- a/0-bootstrap/modules/gitlab-runner/gitlab-ci.yml +++ /dev/null @@ -1,100 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -stages: - - build - - validate - - apply - -build-image: - stage: build - tags: - - gl_runner - image: - name: docker:stable - services: - - name: docker:dind - alias: dockerhost - - variables: - DOCKER_HOST: tcp://dockerhost:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - - script: - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin - - docker build -t $CI_REGISTRY_IMAGE/terraform-gcloud:latest . - - docker push $CI_REGISTRY_IMAGE/terraform-gcloud:latest - - docker logout - - only: - - image - -# All branches need to be protected, otherwise the value of environment variables won't be read. -# disable -> projeto -> settings -> cicd -> token access -> limit access to this project - -image: - name: registry.gitlab.com/GITLAB-ACCOUNT/GITLAB-REPOSITORY/terraform-gcloud:latest - entrypoint: [""] - -terraform-plan: - stage: validate - tags: - - gl_runner - id_tokens: - GITLAB_OIDC_TOKEN: - aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" - variables: - GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" - script: - - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` - - gcloud config set project "$PROJECT_ID" - - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" - -terraform-plan: - stage: validate - tags: - - gl_runner - - script: - - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` - - gcloud config set project "$PROJECT_ID" - - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" - - only: - - plan - -terraform-apply: - stage: apply - tags: - - gl_runner - id_tokens: - GITLAB_OIDC_TOKEN: - aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" - variables: - GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" - script: - - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` - - gcloud config set project "$PROJECT_ID" - - bash tf-wrapper.sh init $CI_COMMIT_REF_NAME - - bash tf-wrapper.sh plan $CI_COMMIT_REF_NAME - - bash tf-wrapper.sh validate $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $PROJECT_ID "FILESYSTEM" - - bash tf-wrapper.sh apply $CI_COMMIT_REF_NAME - - only: - - development - - production - - non-production - when: manual - diff --git a/0-bootstrap/modules/gitlab-runner/main.tf b/0-bootstrap/modules/gitlab-runner/main.tf index 08e910693..4afb8607b 100644 --- a/0-bootstrap/modules/gitlab-runner/main.tf +++ b/0-bootstrap/modules/gitlab-runner/main.tf @@ -22,30 +22,24 @@ locals { Optional Runner Networking *****************************************/ module "vpc_network" { - source = "terraform-google-modules/network/google" - version = "~> 7.0" + source = "terraform-google-modules/network/google" + version = "~> 7.0" + project_id = var.project_id - network_name = "gl-network" + network_name = var.network_name mtu = 1460 subnets = [ { - subnet_name = "gl-subnet" - subnet_ip = "10.10.10.0/24" - subnet_region = "us-central1" - # subnet_private_access = "true" - # subnet_flow_logs = "true" - # subnet_flow_logs_interval = "INTERVAL_10_MIN" - # subnet_flow_logs_sampling = 0.7 - # subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" - # subnet_flow_logs_filter = "false" + subnet_name = var.subnet_name + subnet_ip = var.subnet_ip + subnet_region = var.region + subnet_private_access = "true" }, ] } resource "google_compute_router" "default" { - count = var.create_network ? 1 : 0 - name = "${var.network_name}-router" network = module.vpc_network.network_self_link region = var.region @@ -54,12 +48,10 @@ resource "google_compute_router" "default" { // Nat is being used here since internet access is required for the Runner Network. Other internet access can be setup instead of NAT resource (e.g: Secure Web Proxy) resource "google_compute_router_nat" "nat" { - count = var.create_network ? 1 : 0 - project = var.project_id name = "${var.network_name}-nat" - router = google_compute_router.default[0].name - region = google_compute_router.default[0].region + router = google_compute_router.default.name + region = google_compute_router.default.region nat_ip_allocate_option = "AUTO_ONLY" source_subnetwork_ip_ranges_to_nat = "ALL_SUBNETWORKS_ALL_IP_RANGES" } @@ -86,48 +78,6 @@ resource "google_service_account" "runner_service_account" { display_name = "GitLab Runner GCE Service Account" } -/***************************************** - Runner Secrets - *****************************************/ -resource "google_secret_manager_secret" "gl-secret" { - provider = google-beta - - project = var.project_id - secret_id = "gl-token" - - labels = { - label = "gl-token" - } - - replication { - user_managed { - replicas { - location = var.region - } - } - } -} -resource "google_secret_manager_secret_version" "gl-secret-version" { - provider = google-beta - - secret = google_secret_manager_secret.gl-secret.id - secret_data = jsonencode({ - "REPO_NAME" = var.repo_name - "REPO_OWNER" = var.repo_owner - "GITLAB_TOKEN" = var.gitlab_token - "LABELS" = join(",", var.gl_runner_labels) - }) -} - -resource "google_secret_manager_secret_iam_member" "gl-secret-member" { - provider = google-beta - - project = var.project_id - secret_id = google_secret_manager_secret.gl-secret.id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:${local.service_account}" -} - /***************************************** Runner GCE Instance Template *****************************************/ @@ -139,7 +89,7 @@ module "mig_template" { machine_type = var.machine_type network_ip = var.network_ip network = module.vpc_network.network_name - subnetwork = module.vpc_network.subnets_names[0] + subnetwork = module.vpc_network.subnets_names region = var.region subnetwork_project = var.project_id service_account = { @@ -154,15 +104,14 @@ module "mig_template" { name_prefix = "gl-runner" source_image_family = var.source_image_family source_image_project = var.source_image_project - startup_script = file("${abspath(path.module)}/startup_script.sh") + startup_script = file("${abspath(path.module)}/files/startup_script.sh") source_image = var.source_image - metadata = merge({ - "secret-id" = google_secret_manager_secret_version.gl-secret-version.name - }, var.custom_metadata) + metadata = var.custom_metadata tags = [ "gl-runner-vm" ] } + /***************************************** Runner MIG *****************************************/ diff --git a/0-bootstrap/modules/gitlab-runner/variables.tf b/0-bootstrap/modules/gitlab-runner/variables.tf index 01784eca9..86287a0ff 100644 --- a/0-bootstrap/modules/gitlab-runner/variables.tf +++ b/0-bootstrap/modules/gitlab-runner/variables.tf @@ -26,12 +26,6 @@ variable "network_name" { default = "gl-runner-network" } -variable "create_network" { - type = bool - description = "When set to true, VPC,router and NAT will be auto created" - default = true -} - variable "subnetwork_project" { type = string description = "The ID of the project in which the subnetwork belongs. If it is not provided, the project_id is used." @@ -56,17 +50,6 @@ variable "subnet_name" { default = "gl-runner-subnet" } -variable "repo_name" { - type = string - description = "Name of the repo for the Github Action" - default = "" -} - -variable "repo_owner" { - type = string - description = "Owner of the repo for the Github Action" -} - variable "gl_runner_labels" { type = set(string) description = "GitHub runner labels to attach to the runners. Docs: https://docs.github.com/en/actions/hosting-your-own-runners/using-labels-with-self-hosted-runners" @@ -85,12 +68,6 @@ variable "max_replicas" { description = "Maximum number of runner instances" } -variable "gitlab_token" { - type = string - sensitive = true - description = "Gitlab token that is used for generating Self Hosted Runner Token" -} - variable "service_account" { description = "Service account email address" type = string diff --git a/0-bootstrap/outputs.tf b/0-bootstrap/outputs.tf index 2c8ff5f8d..e0a3f7797 100644 --- a/0-bootstrap/outputs.tf +++ b/0-bootstrap/outputs.tf @@ -87,62 +87,62 @@ output "optional_groups" { Specific to cloudbuild_module ---------------------------------------- */ # Comment-out the cloudbuild_bootstrap module and its outputs if you want to use Jenkins instead of Cloud Build -# output "cloudbuild_project_id" { -# description = "Project where Cloud Build configuration and terraform container image will reside." -# value = module.tf_source.cloudbuild_project_id -# } +output "cloudbuild_project_id" { + description = "Project where Cloud Build configuration and terraform container image will reside." + value = module.tf_source.cloudbuild_project_id +} -# output "gcs_bucket_cloudbuild_artifacts" { -# description = "Bucket used to store Cloud Build artifacts in cicd project." -# value = { for key, value in module.tf_workspace : key => replace(value.artifacts_bucket, local.bucket_self_link_prefix, "") } -# } +output "gcs_bucket_cloudbuild_artifacts" { + description = "Bucket used to store Cloud Build artifacts in cicd project." + value = { for key, value in module.tf_workspace : key => replace(value.artifacts_bucket, local.bucket_self_link_prefix, "") } +} -# output "gcs_bucket_cloudbuild_logs" { -# description = "Bucket used to store Cloud Build logs in cicd project." -# value = { for key, value in module.tf_workspace : key => replace(value.logs_bucket, local.bucket_self_link_prefix, "") } -# } +output "gcs_bucket_cloudbuild_logs" { + description = "Bucket used to store Cloud Build logs in cicd project." + value = { for key, value in module.tf_workspace : key => replace(value.logs_bucket, local.bucket_self_link_prefix, "") } +} -# output "projects_gcs_bucket_tfstate" { -# description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." -# value = module.gcp_projects_state_bucket.bucket.name -# } +output "projects_gcs_bucket_tfstate" { + description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." + value = module.gcp_projects_state_bucket.bucket.name +} -# output "cloud_builder_artifact_repo" { -# description = "Artifact Registry (AR) Repository created to store TF Cloud Builder images." -# value = "projects/${module.tf_source.cloudbuild_project_id}/locations/${var.default_region}/repositories/${module.tf_cloud_builder.artifact_repo}" -# } +output "cloud_builder_artifact_repo" { + description = "Artifact Registry (AR) Repository created to store TF Cloud Builder images." + value = "projects/${module.tf_source.cloudbuild_project_id}/locations/${var.default_region}/repositories/${module.tf_cloud_builder.artifact_repo}" +} -# output "csr_repos" { -# description = "List of Cloud Source Repos created by the module, linked to Cloud Build triggers." -# value = { for k, v in module.tf_source.csr_repos : k => { -# "id" = v.id, -# "name" = v.name, -# "project" = v.project, -# "url" = v.url, -# } -# } -# } +output "csr_repos" { + description = "List of Cloud Source Repos created by the module, linked to Cloud Build triggers." + value = { for k, v in module.tf_source.csr_repos : k => { + "id" = v.id, + "name" = v.name, + "project" = v.project, + "url" = v.url, + } + } +} -# output "cloud_build_private_worker_pool_id" { -# description = "ID of the Cloud Build private worker pool." -# value = module.tf_private_pool.private_worker_pool_id +output "cloud_build_private_worker_pool_id" { + description = "ID of the Cloud Build private worker pool." + value = module.tf_private_pool.private_worker_pool_id -# } +} -# output "cloud_build_worker_range_id" { -# description = "The Cloud Build private worker IP range ID." -# value = module.tf_private_pool.worker_range_id -# } +output "cloud_build_worker_range_id" { + description = "The Cloud Build private worker IP range ID." + value = module.tf_private_pool.worker_range_id +} -# output "cloud_build_worker_peered_ip_range" { -# description = "The IP range of the peered service network." -# value = module.tf_private_pool.worker_peered_ip_range -# } +output "cloud_build_worker_peered_ip_range" { + description = "The IP range of the peered service network." + value = module.tf_private_pool.worker_peered_ip_range +} -# output "cloud_build_peered_network_id" { -# description = "The ID of the Cloud Build peered network." -# value = module.tf_private_pool.peered_network_id -# } +output "cloud_build_peered_network_id" { + description = "The ID of the Cloud Build peered network." + value = module.tf_private_pool.peered_network_id +} /* ---------------------------------------- Specific to jenkins_bootstrap module @@ -187,13 +187,13 @@ output "optional_groups" { Specific to gitlab_bootstrap ---------------------------------------- */ # Un-comment gitlab_bootstrap and its outputs if you want to use GitLab CI/CD instead of Cloud Build -output "cicd_project_id" { - description = "Project where the CI/CD infrastructure for GitLab CI/CD resides." - value = module.gitlab_cicd.project_id -} +# output "cicd_project_id" { +# description = "Project where the CI/CD infrastructure for GitLab CI/CD resides." +# value = module.gitlab_cicd.project_id +# } -output "projects_gcs_bucket_tfstate" { - description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." - value = module.seed_bootstrap.gcs_bucket_tfstate -} +# output "projects_gcs_bucket_tfstate" { +# description = "Bucket used for storing terraform state for stage 4-projects foundations pipelines in seed project." +# value = module.seed_bootstrap.gcs_bucket_tfstate +# } diff --git a/0-bootstrap/variables.tf b/0-bootstrap/variables.tf index d15ab5faf..bd74ec714 100644 --- a/0-bootstrap/variables.tf +++ b/0-bootstrap/variables.tf @@ -247,10 +247,10 @@ variable "initial_group_config" { # } /* ---------------------------------------- - Specific to github_bootstrap + Specific to gitlab_bootstrap ---------------------------------------- */ -# Un-comment github_bootstrap and its outputs if you want to use GitHub Actions instead of Cloud Build +# Un-comment gitlab_bootstrap and its outputs if you want to use GitLab Pipelines instead of Cloud Build variable "gl_repos" { description = < projeto -> settings -> cicd -> token access -> limit access to this project +# disable -> project -> settings -> cicd -> token access -> limit access to this project image: - name: registry.gitlab.com/renatojr_ciandt_group2/project_group2/terraform-gcloud:latest + name: registry.gitlab.com/GITLAB-ACCOUNT/GITLAB-REPOSITORY/terraform-gcloud:latest entrypoint: [""] terraform-plan: stage: validate tags: - - runner2 + - gl_runner id_tokens: GITLAB_OIDC_TOKEN: aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" @@ -62,13 +62,23 @@ terraform-plan: - gcloud config set project "$PROJECT_ID" - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" +terraform-plan: + stage: validate + tags: + - gl_runner + + script: + - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` + - gcloud config set project "$PROJECT_ID" + - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" + only: - plan terraform-apply: stage: apply tags: - - runner2 + - gl_runner id_tokens: GITLAB_OIDC_TOKEN: aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" @@ -87,3 +97,4 @@ terraform-apply: - production - non-production when: manual + From 74e6b1ff33d8f5ef074ac633b84b5fcb5baff650 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:21:08 -0300 Subject: [PATCH 19/31] Update README-GitLab.md --- 0-bootstrap/README-GitLab.md | 48 ++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index 32e21c894..1c7b993b8 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -97,11 +97,10 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi cp -RT ../terraform-example-foundation/0-bootstrap/ ./envs/shared cp -RT ../terraform-example-foundation/policy-library/ ./policy-library - cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/run_gcp_auth.sh . + cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/run_gcp_auth.sh . cp ../terraform-example-foundation/build/tf-wrapper.sh . - chmod 755 ./run_gcp_auth.sh - chmod 755 ./tf-wrapper.sh + chmod 755 ./*.sh cd ./envs/shared ``` @@ -244,8 +243,8 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi 1. Copy contents of foundation to cloned project (modify accordingly based on your current directory). ```bash - cp ../terraform-example-foundation/build/gitlab/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/Dockerfile ./Dockerfile + cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/0-bootstrap/Dockerfile ./Dockerfile ``` 1. Create a runner for your GitLab in [Settings -> CICD -> Runners](https://gitlab.com/renatojr_ciandt/fdt-bootstrap/-/settings/ci_cd) called `gl_runner` and set the tag name also for `gl_runner`. @@ -290,7 +289,7 @@ image: 1. Copy the file `gitlab-ci.yml` to the 0-bootstrap directory. ```bash -cp ../terraform-example-foundation/build/gitlab/.gitlab-ci.yml ../gcp-bootstrap/.gitlab-ci.yml +cp ../terraform-example-foundation/build/gitlab-ci.yml ../gcp-bootstrap/.gitlab-ci.yml ``` 1. Make sure you have enabled the Gitlab Runner to the bootstrap project. You can do this in `https://gitlab.com/YOUR-GITLAB-USER/gcp-cicd-runner/-/settings/ci_cd#js-runners-settings`. @@ -349,11 +348,10 @@ we recommend that you request 50 additional projects for the **projects step ser ```bash cp -RT ../terraform-example-foundation/1-org/ . cp -RT ../terraform-example-foundation/policy-library/ ./policy-library - cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/run_gcp_* . - chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/run_gcp_auth.sh . cp ../terraform-example-foundation/build/tf-wrapper.sh . - chmod 755 ./tf-wrapper.sh + chmod 755 ./*.sh ``` 1. Rename `./envs/shared/terraform.example.tfvars` to `./envs/shared/terraform.tfvars` @@ -476,11 +474,10 @@ See the shared folder [README.md](../1-org/envs/shared/README.md#inputs) for add ```bash cp -RT ../terraform-example-foundation/2-environments/ . cp -RT ../terraform-example-foundation/policy-library/ ./policy-library - cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/run_gcp_* . - chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/run_gcp_auth.sh . cp ../terraform-example-foundation/build/tf-wrapper.sh . - chmod 755 ./tf-wrapper.sh + chmod 755 ./*.sh ``` 1. Rename `terraform.example.tfvars` to `terraform.tfvars`. @@ -583,11 +580,10 @@ or go to [Deploying step 3-networks-hub-and-spoke](#deploying-step-3-networks-hu ```bash cp -RT ../terraform-example-foundation/3-networks-dual-svpc/ . cp -RT ../terraform-example-foundation/policy-library/ ./policy-library - cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/run_gcp_* . - chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/run_gcp_auth.sh . cp ../terraform-example-foundation/build/tf-wrapper.sh . - chmod 755 ./tf-wrapper.sh + chmod 755 ./*.sh ``` 1. Rename `common.auto.example.tfvars` to `common.auto.tfvars`, rename `shared.auto.example.tfvars` to `shared.auto.tfvars` and rename `access_context.auto.example.tfvars` to `access_context.auto.tfvars`. @@ -747,11 +743,10 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th ```bash cp -RT ../terraform-example-foundation/3-networks-hub-and-spoke/ . cp -RT ../terraform-example-foundation/policy-library/ ./policy-library - cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/run_gcp_* . - chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/run_gcp_auth.sh . cp ../terraform-example-foundation/build/tf-wrapper.sh . - chmod 755 ./tf-wrapper.sh + chmod 755 ./*.sh ``` 1. Rename `common.auto.example.tfvars` to `common.auto.tfvars`, rename `shared.auto.example.tfvars` to `shared.auto.tfvars` and rename `access_context.auto.example.tfvars` to `access_context.auto.tfvars`. @@ -900,11 +895,10 @@ An environment variable `GOOGLE_IMPERSONATE_SERVICE_ACCOUNT` will be set with th ```bash cp -RT ../terraform-example-foundation/4-projects/ . cp -RT ../terraform-example-foundation/policy-library/ ./policy-library - cp ../terraform-example-foundation/build/.gitlab-ci.yml ./.gitlab-ci.yml - cp ../terraform-example-foundation/build/gitlab/run_gcp_* . - chmod 755 ./run_gcp_* + cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml + cp ../terraform-example-foundation/build/run_gcp_auth.sh . cp ../terraform-example-foundation/build/tf-wrapper.sh . - chmod 755 ./tf-wrapper.sh + chmod 755 ./*.sh ``` 1. Rename `auto.example.tfvars` files to `auto.tfvars`. From 7cf9595515f1eb3ce0a30de065aaf6d3d0918ed5 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:23:58 -0300 Subject: [PATCH 20/31] Delete build/gitlab/main.tf --- build/gitlab/main.tf | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 build/gitlab/main.tf diff --git a/build/gitlab/main.tf b/build/gitlab/main.tf deleted file mode 100644 index a43039d56..000000000 --- a/build/gitlab/main.tf +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_providers { - gitlab = { - source = "gitlabhq/gitlab" - version = "~> 16.2.0" - } - } -} - -variable "gitlab_token" { - type = string -} - -provider "gitlab" { - token = var.gitlab_token -} - -provider "null" {} - -resource "null_resource" "gcloud_auth_list" { - provisioner "local-exec" { - command = "gcloud auth list" - } -} From 92481a826e7905f2a7484e9272e27ceb519eb358 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:24:08 -0300 Subject: [PATCH 21/31] Delete build/gitlab/Dockerfile --- build/gitlab/Dockerfile | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 build/gitlab/Dockerfile diff --git a/build/gitlab/Dockerfile b/build/gitlab/Dockerfile deleted file mode 100644 index 9b3025035..000000000 --- a/build/gitlab/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM gcr.io/cloud-builders/gcloud-slim - -# Use ARG so that values can be overriden by user/cloudbuild -ARG TERRAFORM_VERSION=1.3.0 - -ENV ENV_TERRAFORM_VERSION=$TERRAFORM_VERSION - -RUN apt-get update && \ - /builder/google-cloud-sdk/bin/gcloud -q components install alpha beta terraform-tools && \ - apt-get -y install curl jq unzip git ca-certificates gnupg && \ - curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip --output terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip && \ - curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS.sig --output terraform_SHA256SUMS.sig && \ - curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS --output terraform_SHA256SUMS && \ - curl https://keybase.io/hashicorp/pgp_keys.asc --output pgp_keys.asc && \ - gpg --import pgp_keys.asc && \ - gpg --verify terraform_SHA256SUMS.sig terraform_SHA256SUMS && \ - grep terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS | shasum --algorithm 256 --check && \ - unzip terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/terraform && \ - rm -f terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS && \ - apt-get --purge -y autoremove && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -ENV PATH=/builder/terraform/:$PATH -ENTRYPOINT ["terraform"] From 58ec7291e188ef6849523a1e0636158a5ec03ca8 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:25:09 -0300 Subject: [PATCH 22/31] Create run _gcp_auth.sh --- build/run _gcp_auth.sh | 43 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 build/run _gcp_auth.sh diff --git a/build/run _gcp_auth.sh b/build/run _gcp_auth.sh new file mode 100644 index 000000000..d2ed72d51 --- /dev/null +++ b/build/run _gcp_auth.sh @@ -0,0 +1,43 @@ +#!/bin/sh -x + +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# An id_tokens configured for the audience of the Workload Identity Federation provider +# See https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html +OIDC_TOKEN="$1" + +# A Workload Identity Federation provider +# See https://cloud.google.com/iam/docs/workload-identity-federation +WIF_PROVIDER="$2" + +# A Google Cloud Platform Service Account +# See https://cloud.google.com/iam/docs/service-account-overview +SA="$3" + +# System folder to save the temporary files +SAVE_PATH="$4" + +# TODO +echo "${OIDC_TOKEN}" > "${SAVE_PATH}"/.ci_job_token_file + +# TODO +gcloud iam workload-identity-pools \ +create-cred-config "${WIF_PROVIDER}" \ +--service-account="${SA}" \ +--output-file="${SAVE_PATH}"/.gcp_generated_credentials.json \ +--credential-source-file="${SAVE_PATH}"/.ci_job_token_file \ + +# TODO +gcloud auth login --cred-file="${SAVE_PATH}"/.gcp_generated_credentials.json --update-adc From 6d579372ade85f04cea4b8f2ad69246a56fe98b5 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:25:41 -0300 Subject: [PATCH 23/31] Delete build/gitlab directory --- build/gitlab/.gitlab-ci.yml | 99 ------------------------------------ build/gitlab/run_gcp_auth.sh | 43 ---------------- build/gitlab/run_gcp_sts.sh | 51 ------------------- 3 files changed, 193 deletions(-) delete mode 100644 build/gitlab/.gitlab-ci.yml delete mode 100755 build/gitlab/run_gcp_auth.sh delete mode 100755 build/gitlab/run_gcp_sts.sh diff --git a/build/gitlab/.gitlab-ci.yml b/build/gitlab/.gitlab-ci.yml deleted file mode 100644 index febdf1edc..000000000 --- a/build/gitlab/.gitlab-ci.yml +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -stages: - - build - - validate - - apply - -build-image: - stage: build - tags: - - gl_runner - image: - name: docker:stable - services: - - name: docker:dind - alias: dockerhost - - variables: - DOCKER_HOST: tcp://dockerhost:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - - script: - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER $CI_REGISTRY --password-stdin - - docker build -t $CI_REGISTRY_IMAGE/terraform-gcloud:latest . - - docker push $CI_REGISTRY_IMAGE/terraform-gcloud:latest - - docker logout - - only: - - image - -# All branches need to be protected, otherwise the value of environment variables won't be read. -# disable -> projeto -> settings -> cicd -> token access -> limit access to this project - -image: - name: registry.gitlab.com/renatojr_ciandt_group2/project_group2/terraform-gcloud:latest - entrypoint: [""] - -terraform-plan: - stage: validate - tags: - - gl_runner - id_tokens: - GITLAB_OIDC_TOKEN: - aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" - variables: - GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" - script: - - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` - - gcloud config set project "$PROJECT_ID" - - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" - -terraform-plan: - stage: validate - tags: - - gl_runner - - script: - - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` - - gcloud config set project "$PROJECT_ID" - - bash tf-wrapper.sh plan_validate_all $CI_COMMIT_REF_NAME $CI_PROJECT_DIR/policy-library $PROJECT_ID "FILESYSTEM" - - only: - - plan - -terraform-apply: - stage: apply - tags: - - gl_runner - id_tokens: - GITLAB_OIDC_TOKEN: - aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" - variables: - GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" - script: - - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` - - gcloud config set project "$PROJECT_ID" - - bash tf-wrapper.sh init $CI_COMMIT_REF_NAME - - bash tf-wrapper.sh plan $CI_COMMIT_REF_NAME - - bash tf-wrapper.sh validate $CI_COMMIT_REF_NAME "$CI_PROJECT_DIR/policy-library" $PROJECT_ID "FILESYSTEM" - - bash tf-wrapper.sh apply $CI_COMMIT_REF_NAME - - only: - - development - - production - - non-production - when: manual diff --git a/build/gitlab/run_gcp_auth.sh b/build/gitlab/run_gcp_auth.sh deleted file mode 100755 index d2ed72d51..000000000 --- a/build/gitlab/run_gcp_auth.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -x - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# An id_tokens configured for the audience of the Workload Identity Federation provider -# See https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html -OIDC_TOKEN="$1" - -# A Workload Identity Federation provider -# See https://cloud.google.com/iam/docs/workload-identity-federation -WIF_PROVIDER="$2" - -# A Google Cloud Platform Service Account -# See https://cloud.google.com/iam/docs/service-account-overview -SA="$3" - -# System folder to save the temporary files -SAVE_PATH="$4" - -# TODO -echo "${OIDC_TOKEN}" > "${SAVE_PATH}"/.ci_job_token_file - -# TODO -gcloud iam workload-identity-pools \ -create-cred-config "${WIF_PROVIDER}" \ ---service-account="${SA}" \ ---output-file="${SAVE_PATH}"/.gcp_generated_credentials.json \ ---credential-source-file="${SAVE_PATH}"/.ci_job_token_file \ - -# TODO -gcloud auth login --cred-file="${SAVE_PATH}"/.gcp_generated_credentials.json --update-adc diff --git a/build/gitlab/run_gcp_sts.sh b/build/gitlab/run_gcp_sts.sh deleted file mode 100755 index 2b15c2f2d..000000000 --- a/build/gitlab/run_gcp_sts.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/sh -x - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -PAYLOAD=$(cat < .ci_job_jwt_file -gcloud iam workload-identity-pools create-cred-config "${WIF_PROVIDER_NAME}" \ ---service-account="${SERVICE_ACCOUNT_EMAIL}" \ ---output-file=.gcp_temp_cred.json \ ---credential-source-file=.ci_job_jwt_fgcloud conf listile -gcloud auth login --cred-file=.gcp_temp_cred.json From b4029c259f054f38c54225ed028bbd27780592e5 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:27:35 -0300 Subject: [PATCH 24/31] Rename run _gcp_auth.sh to run_gcp_auth.sh --- build/{run _gcp_auth.sh => run_gcp_auth.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename build/{run _gcp_auth.sh => run_gcp_auth.sh} (100%) diff --git a/build/run _gcp_auth.sh b/build/run_gcp_auth.sh similarity index 100% rename from build/run _gcp_auth.sh rename to build/run_gcp_auth.sh From e3e816b9073da9d504680c51d295843549637b75 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:32:17 -0300 Subject: [PATCH 25/31] Update startup_script.sh --- 0-bootstrap/modules/gitlab-runner/files/startup_script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/0-bootstrap/modules/gitlab-runner/files/startup_script.sh b/0-bootstrap/modules/gitlab-runner/files/startup_script.sh index 97a352edb..4d3861f8c 100755 --- a/0-bootstrap/modules/gitlab-runner/files/startup_script.sh +++ b/0-bootstrap/modules/gitlab-runner/files/startup_script.sh @@ -15,4 +15,4 @@ curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash sudo apt-get -y install gitlab-runner -# TODO install docker +sudo apt install docker.io -y From a5c956fdfda79ca116342138be71fa095cd28472 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:32:51 -0300 Subject: [PATCH 26/31] Delete 0-bootstrap/scripts/gl_runner.sh --- 0-bootstrap/scripts/gl_runner.sh | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100755 0-bootstrap/scripts/gl_runner.sh diff --git a/0-bootstrap/scripts/gl_runner.sh b/0-bootstrap/scripts/gl_runner.sh deleted file mode 100755 index fb704ddaf..000000000 --- a/0-bootstrap/scripts/gl_runner.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sudo apt update - -sudo apt install docker.io -y -sudo systemctl status docker -docker --version - -curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash -sudo apt-get install gitlab-runner -sudo gitlab-runner register --non-interactive --url https://gitlab.com --token "${RUNNER_TOKEN}" --name gl-fdt-runner --executer docker --docker-image "docker:dind" --docker-privileged -sudo gitlab-runner run & From a9334f4010f9e3cbbad7c199fbc4f2fb73b52fef Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:40:28 -0300 Subject: [PATCH 27/31] Update variables.tf --- 0-bootstrap/modules/gitlab-runner/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/0-bootstrap/modules/gitlab-runner/variables.tf b/0-bootstrap/modules/gitlab-runner/variables.tf index 86287a0ff..6d426cb9c 100644 --- a/0-bootstrap/modules/gitlab-runner/variables.tf +++ b/0-bootstrap/modules/gitlab-runner/variables.tf @@ -52,7 +52,7 @@ variable "subnet_name" { variable "gl_runner_labels" { type = set(string) - description = "GitHub runner labels to attach to the runners. Docs: https://docs.github.com/en/actions/hosting-your-own-runners/using-labels-with-self-hosted-runners" + description = "GitLab runner labels to attach to the runners." default = [] } From 6643abdc9c6bed7b1f9ca5cbb9c94ac1ade2a27d Mon Sep 17 00:00:00 2001 From: Renato Rudnicki <77694243+renato-rudnicki@users.noreply.github.com> Date: Tue, 31 Oct 2023 18:43:20 -0300 Subject: [PATCH 28/31] Update README-GitLab.md --- 0-bootstrap/README-GitLab.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index 1c7b993b8..e6df72dcd 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -246,7 +246,7 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml cp ../terraform-example-foundation/0-bootstrap/Dockerfile ./Dockerfile ``` -1. Create a runner for your GitLab in [Settings -> CICD -> Runners](https://gitlab.com/renatojr_ciandt/fdt-bootstrap/-/settings/ci_cd) called `gl_runner` and set the tag name also for `gl_runner`. +1. Create a runner for your GitLab in [Settings -> CICD -> Runners](https://gitlab.com/GITLAB-ACCOUNT/BOOTSTRAP-REPOSITORY/-/settings/ci_cd) called `gl_runner` and set the tag name also for `gl_runner`. 1. To execute the next step, access the Gitlab instance created in the 0-bootstrap step by ssh and update token field in the file `/etc/gitlab-runner/config.toml` using the value showed during the Gitlab Runner creation. To access the instance created you will need to open the SSH port in the firewall: From 95222148de8d34ce9b1dbc5c0bdbc6e63503b1c6 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Wed, 1 Nov 2023 19:34:33 -0300 Subject: [PATCH 29/31] fix steps --- 0-bootstrap/README-GitLab.md | 2 +- 0-bootstrap/modules/gitlab-runner/main.tf | 2 +- 0-bootstrap/outputs.tf | 1 - 0-bootstrap/variables.tf | 117 +++++++++++----------- scripts/validate-requirements.sh | 2 +- 5 files changed, 61 insertions(+), 63 deletions(-) diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index e6df72dcd..8a92b5850 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -137,7 +137,7 @@ export the GitLab personal or group access token as an environment variable: 1. Use the helper script [validate-requirements.sh](../scripts/validate-requirements.sh) to validate your environment: ```bash - ../../../terraform-example-foundation/scripts/validate-requirements.sh -o -b -u -t GitLab + ../../../terraform-example-foundation/scripts/validate-requirements.sh -o -b -u ``` **Note:** The script is not able to validate if the user is in a Cloud Identity or Google Workspace group with the required roles. diff --git a/0-bootstrap/modules/gitlab-runner/main.tf b/0-bootstrap/modules/gitlab-runner/main.tf index 4afb8607b..65832b96c 100644 --- a/0-bootstrap/modules/gitlab-runner/main.tf +++ b/0-bootstrap/modules/gitlab-runner/main.tf @@ -89,7 +89,7 @@ module "mig_template" { machine_type = var.machine_type network_ip = var.network_ip network = module.vpc_network.network_name - subnetwork = module.vpc_network.subnets_names + subnetwork = module.vpc_network.subnets_names[0] region = var.region subnetwork_project = var.project_id service_account = { diff --git a/0-bootstrap/outputs.tf b/0-bootstrap/outputs.tf index e0a3f7797..0248e8e03 100644 --- a/0-bootstrap/outputs.tf +++ b/0-bootstrap/outputs.tf @@ -126,7 +126,6 @@ output "csr_repos" { output "cloud_build_private_worker_pool_id" { description = "ID of the Cloud Build private worker pool." value = module.tf_private_pool.private_worker_pool_id - } output "cloud_build_worker_range_id" { diff --git a/0-bootstrap/variables.tf b/0-bootstrap/variables.tf index bd74ec714..e97fae4a1 100644 --- a/0-bootstrap/variables.tf +++ b/0-bootstrap/variables.tf @@ -251,69 +251,68 @@ variable "initial_group_config" { ---------------------------------------- */ # Un-comment gitlab_bootstrap and its outputs if you want to use GitLab Pipelines instead of Cloud Build -variable "gl_repos" { - description = < Date: Fri, 3 Nov 2023 18:41:40 -0300 Subject: [PATCH 30/31] update instructions for readme and troubleshooting --- 0-bootstrap/README-GitLab.md | 20 ++++++++++-------- docs/TROUBLESHOOTING.md | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/0-bootstrap/README-GitLab.md b/0-bootstrap/README-GitLab.md index 8a92b5850..4e66b2021 100644 --- a/0-bootstrap/README-GitLab.md +++ b/0-bootstrap/README-GitLab.md @@ -246,32 +246,32 @@ You must have [SSH keys](https://docs.gitlab.com/ee/user/ssh.html) configured wi cp ../terraform-example-foundation/build/gitlab-ci.yml ./.gitlab-ci.yml cp ../terraform-example-foundation/0-bootstrap/Dockerfile ./Dockerfile ``` -1. Create a runner for your GitLab in [Settings -> CICD -> Runners](https://gitlab.com/GITLAB-ACCOUNT/BOOTSTRAP-REPOSITORY/-/settings/ci_cd) called `gl_runner` and set the tag name also for `gl_runner`. +1. Create a runner for your GitLab CICD Repository in [Settings -> CICD -> Runners](https://gitlab.com/GITLAB-ACCOUNT/BOOTSTRAP-REPOSITORY/-/settings/ci_cd) called `gl_runner` and also set the tag name for `gl_runner`. -1. To execute the next step, access the Gitlab instance created in the 0-bootstrap step by ssh and update token field in the file `/etc/gitlab-runner/config.toml` using the value showed during the Gitlab Runner creation. To access the instance created you will need to open the SSH port in the firewall: +1. To execute the next step, you need to allow the SSH access for your Gitlab instance created in the CICD Project from 0-bootstrap step running the command below: ```bash -gcloud compute firewall-rules create allow-ssh --allow tcp:22 --network gl-network --project +gcloud compute firewall-rules create allow-ssh --allow tcp:22 --network gl-runner-network --project ``` -1. After have created the runner on GitLab interface, run the command below in the GCP instance for the GitLab. Remember to change the token `glrt-xxx` for the token generated in the GitLab interface. +1. Once you have access to your GitLab runner instance, you need to register your runner in the GitLab using the command below. Remember to change the token `glrt-xxx` for the token generated in the GitLab interface. ```bash -sudo gitlab-runner register --non-interactive --url https://gitlab.com --name gl_runner --executor docker --docker-image docker:dind --docker-privileged true --token glrt-xxx +sudo gitlab-runner register --non-interactive --url https://gitlab.com --name gl_runner --executor docker --docker-image docker:dind --docker-privileged=true --token glrt-xxx ``` 1. Edit the `./.gitlab-ci.yml` file and update the paramenter the following line: ```bash image: - name: registry.gitlab.com/EXAMPLE-GITLAB-ACCOUNT/EXAMPLE-GITLAB-REPOSITORY/terraform-gcloud:latest + name: registry.gitlab.com/GITLAB-ACCOUNT/UPDATE_ME/terraform-gcloud:latest ``` To: ```bash image: - name: registry.gitlab.com/GITLAB-ACCOUNT/GITLAB-REPOSITORY/terraform-gcloud:latest + name: registry.gitlab.com/GITLAB-ACCOUNT/GITLAB-IMAGE_REPOSITORY/terraform-gcloud:latest ``` 1. Save the CI/CD runner configuration to `gcp-cicd-runner` GitLab project: @@ -289,7 +289,7 @@ image: 1. Copy the file `gitlab-ci.yml` to the 0-bootstrap directory. ```bash -cp ../terraform-example-foundation/build/gitlab-ci.yml ../gcp-bootstrap/.gitlab-ci.yml +cp .gitlab-ci.yml ../gcp-bootstrap/.gitlab-ci.yml ``` 1. Make sure you have enabled the Gitlab Runner to the bootstrap project. You can do this in `https://gitlab.com/YOUR-GITLAB-USER/gcp-cicd-runner/-/settings/ci_cd#js-runners-settings`. @@ -311,6 +311,10 @@ They will [fail](../docs/TROUBLESHOOTING.md#error-unsupported-attribute) if the **Note 2:** After the deploy, to prevent the project quota error described in the [Troubleshooting guide](../docs/TROUBLESHOOTING.md#project-quota-exceeded), we recommend that you request 50 additional projects for the **projects step service account** created in this step. +**Note 3:** In case GitLab variables are not being created on Gitlab repositories, it may be related to the existence of Access Token in both User/Group profile and in the repo settings. The deploy will [fail](../docs/TROUBLESHOOTING.md#error-repository-not-found). + +**Note 4:** If the GitLab pipelines fail in 0-bootstrep or next steps, you may need to disable `Limit access to this project` in the CICD Runner repository. For more details see [fail](../docs/TROUBLESHOOTING.md#gitlab-pipelines-access-denied) + ## Deploying step 1-org diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 193c16556..dc854e3da 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -24,6 +24,7 @@ See [GLOSSARY.md](./GLOSSARY.md). - [Cannot assign requested address error in Cloud Shell](#cannot-assign-requested-address-error-in-cloud-shell) - [Error: Unsupported attribute](#error-unsupported-attribute) - [Error: Error adding network peering](#error-error-adding-network-peering) +- [Error: Repository not found](#error-repository-not-found) - - - @@ -388,3 +389,42 @@ You can get this information from step `0-bootstrap` by running the following co **Terraform State lock possible causes:** - If you realize that the Terraform State lock was due to a build timeout increase the build timeout on [build configuration](https://github.com/terraform-google-modules/terraform-example-foundation/blob/master/build/cloudbuild-tf-apply.yaml#L15). + +### Terraform deploy fails due GitLab repositories not found + +**Error message:** + +```text +Error: POST https://gitlab.com/api/v4/projects///variables: 404 {message: 404 Project Not Found} + +``` + +**Cause:** + +This message means that you are using a wrong Access Token or you have Access Token created in both Gitlab Account/Group and GitLab Repository. + +Only Personal Access Token under GitLab Account/Group should exist. + +**Solution:** + +Remove any Access Token from the GitLab repositories used by Google Secure Foundation Blueprint. + + +######### +### Gitlab pipelines fails in 0-bootstrap + +**Error message:** + +From the logs of your Pipeline job: +```text +Error response from daemon: pull access denied for registry.gitlab.com///terraform-gcloud, repository does not exist or may require 'docker login': denied: requested access to the resource is denied + +``` + +**Cause:** + +This message means that you GitLab CICD repository may have `Limit access to this project` option enabled. + +**Solution:** + +Disable this option on your `CICD Repo -> Settings -> CI/CD -> Token Access`. From 2d7a15dd0f1a09f1997c2893da961d12930209c4 Mon Sep 17 00:00:00 2001 From: Renato Rudnicki Date: Fri, 10 Nov 2023 16:29:35 -0300 Subject: [PATCH 31/31] service private connect and small changes --- 0-bootstrap/gitlab.tf.example | 1 + 0-bootstrap/modules/gitlab-runner/main.tf | 84 ++++++++++++++++++- .../modules/gitlab-runner/variables.tf | 10 +++ build/gitlab-ci.yml | 5 ++ build/run_gcp_auth.sh | 0 5 files changed, 99 insertions(+), 1 deletion(-) mode change 100644 => 100755 build/run_gcp_auth.sh diff --git a/0-bootstrap/gitlab.tf.example b/0-bootstrap/gitlab.tf.example index 3a706003a..588f2e16c 100644 --- a/0-bootstrap/gitlab.tf.example +++ b/0-bootstrap/gitlab.tf.example @@ -130,5 +130,6 @@ module "gitlab_runner" { source = "./modules/gitlab-runner" project_id = local.cicd_project_id + #service_account = google_service_account.terraform-env-sa["bootstrap"].email } diff --git a/0-bootstrap/modules/gitlab-runner/main.tf b/0-bootstrap/modules/gitlab-runner/main.tf index 65832b96c..1ef3bca5d 100644 --- a/0-bootstrap/modules/gitlab-runner/main.tf +++ b/0-bootstrap/modules/gitlab-runner/main.tf @@ -16,6 +16,7 @@ locals { service_account = var.service_account == "" ? google_service_account.runner_service_account[0].email : var.service_account + private_googleapis_cidr = module.private_service_connect.private_service_connect_ip } /***************************************** @@ -27,7 +28,6 @@ module "vpc_network" { project_id = var.project_id network_name = var.network_name - mtu = 1460 subnets = [ { @@ -67,6 +67,47 @@ resource "google_dns_policy" "default_policy" { } } +/******************************************* + Private service connect and firewall rule + *******************************************/ +resource "google_compute_firewall" "allow_private_api_egress" { + name = "fw-${module.vpc_network.network_name}-65430-e-a-allow-google-apis-all-tcp-443" + network = module.vpc_network.network_name + project = var.project_id + direction = "EGRESS" + priority = 65430 + + dynamic "log_config" { + for_each = var.firewall_enable_logging == true ? [{ + metadata = "INCLUDE_ALL_METADATA" + }] : [] + + content { + metadata = log_config.value.metadata + } + } + + allow { + protocol = "tcp" + ports = ["443"] + } + + destination_ranges = [local.private_googleapis_cidr] + + target_tags = ["gl-runner-vm"] +} + +module "private_service_connect" { + source = "terraform-google-modules/network/google//modules/private-service-connect" + version = "~> 5.2" + + project_id = var.project_id + dns_code = "dz-${module.vpc_network.network_name}" + network_self_link = module.vpc_network.network_self_link + private_service_connect_ip = var.private_service_connect_ip + forwarding_rule_target = "all-apis" +} + /***************************************** IAM Bindings GCE SVC *****************************************/ @@ -78,6 +119,15 @@ resource "google_service_account" "runner_service_account" { display_name = "GitLab Runner GCE Service Account" } +# allow GCE to pull images from GCR +resource "google_project_iam_binding" "gce" { + project = var.project_id + role = "roles/storage.objectViewer" + members = [ + "serviceAccount:${local.service_account}", + ] +} + /***************************************** Runner GCE Instance Template *****************************************/ @@ -131,3 +181,35 @@ module "mig" { max_replicas = var.max_replicas cooldown_period = var.cooldown_period } + + +# resource "google_compute_instance" "gitlab_runner" { +# name = "gl-runner-instance" +# project = var.project_id +# zone = "us-central1-a" +# machine_type = "e2-medium" +# can_ip_forward = true + +# boot_disk { +# initialize_params { +# image = "debian-cloud/debian-11" +# } +# } +# tags = ["https-server", "gl-runner-vm"] +# metadata_startup_script = file("${abspath(path.module)}/files/startup_script.sh") + +# network_interface { +# subnetwork = module.vpc_network.subnets_names[0] +# network_ip = "10.10.10.8" +# subnetwork_project = var.project_id +# } + +# service_account { +# email = local.service_account +# scopes = ["cloud-platform"] +# } + +# depends_on = [ +# module.vpc_network +# ] +# } diff --git a/0-bootstrap/modules/gitlab-runner/variables.tf b/0-bootstrap/modules/gitlab-runner/variables.tf index 6d426cb9c..edb1181a4 100644 --- a/0-bootstrap/modules/gitlab-runner/variables.tf +++ b/0-bootstrap/modules/gitlab-runner/variables.tf @@ -138,3 +138,13 @@ variable "project_id" { type = string description = "ID of the project where the Gitlab runner will be created." } + +variable "private_service_connect_ip" { + type = string + default = "10.10.64.5" +} + +variable "firewall_enable_logging" { + type = bool + default = true +} diff --git a/build/gitlab-ci.yml b/build/gitlab-ci.yml index 31e1f01a4..c64154dac 100644 --- a/build/gitlab-ci.yml +++ b/build/gitlab-ci.yml @@ -66,6 +66,11 @@ terraform-plan: stage: validate tags: - gl_runner + id_tokens: + GITLAB_OIDC_TOKEN: + aud: "//iam.googleapis.com/${WIF_PROVIDER_NAME}" + variables: + GOOGLE_IMPERSONATE_SERVICE_ACCOUNT: "$SERVICE_ACCOUNT_EMAIL" script: - bash run_gcp_auth.sh "${GITLAB_OIDC_TOKEN}" "${WIF_PROVIDER_NAME}" "${SERVICE_ACCOUNT_EMAIL}" `pwd` diff --git a/build/run_gcp_auth.sh b/build/run_gcp_auth.sh old mode 100644 new mode 100755