From 552c4a421b5612e48942d357123c99a7fa167bbd Mon Sep 17 00:00:00 2001 From: kaariger <23217852+kaariger@users.noreply.github.com> Date: Mon, 19 Sep 2022 11:20:04 -0700 Subject: [PATCH] feat: Terraform code to setup Bank of Anthos as a sample deployment on an Anthos cluster (#881) * feat:Terraform code to setup Bank of Anthos as a sample deployment on an Anthos cluster * Add a README file * Update README * Update README * Update README.md * Use the latest version of the Anthos TF modules * update README * Update terraform/anthos-gke/README.md Co-authored-by: Olivier Bourgeois <3271352+bourgeoisor@users.noreply.github.com> * Update terraform/anthos-gke/README.md Co-authored-by: Olivier Bourgeois <3271352+bourgeoisor@users.noreply.github.com> * Update terraform/anthos-gke/README.md Co-authored-by: Olivier Bourgeois <3271352+bourgeoisor@users.noreply.github.com> * Update terraform/anthos-gke/README.md Co-authored-by: Olivier Bourgeois <3271352+bourgeoisor@users.noreply.github.com> * Update terraform/anthos-gke/README.md Co-authored-by: Olivier Bourgeois <3271352+bourgeoisor@users.noreply.github.com> * Rename the TF scripts dir and add release automation for updating the BoA version used Co-authored-by: kaariger Co-authored-by: Christine Kim <35876108+ckim328@users.noreply.github.com> Co-authored-by: Olivier Bourgeois <3271352+bourgeoisor@users.noreply.github.com> --- .gitignore | 24 +++++++++++++ iac/tf-anthos-gke/README.md | 52 +++++++++++++++++++++++++++ iac/tf-anthos-gke/acm.tf | 32 +++++++++++++++++ iac/tf-anthos-gke/asm.tf | 49 ++++++++++++++++++++++++++ iac/tf-anthos-gke/boa.tf | 38 ++++++++++++++++++++ iac/tf-anthos-gke/gke.tf | 56 ++++++++++++++++++++++++++++++ iac/tf-anthos-gke/main.tf | 41 ++++++++++++++++++++++ iac/tf-anthos-gke/outputs.tf | 23 ++++++++++++ iac/tf-anthos-gke/providers.tf | 37 ++++++++++++++++++++ iac/tf-anthos-gke/terraform.tfvars | 23 ++++++++++++ iac/tf-anthos-gke/variables.tf | 55 +++++++++++++++++++++++++++++ release/make-release.sh | 3 ++ 12 files changed, 433 insertions(+) create mode 100644 iac/tf-anthos-gke/README.md create mode 100644 iac/tf-anthos-gke/acm.tf create mode 100644 iac/tf-anthos-gke/asm.tf create mode 100644 iac/tf-anthos-gke/boa.tf create mode 100644 iac/tf-anthos-gke/gke.tf create mode 100644 iac/tf-anthos-gke/main.tf create mode 100644 iac/tf-anthos-gke/outputs.tf create mode 100644 iac/tf-anthos-gke/providers.tf create mode 100644 iac/tf-anthos-gke/terraform.tfvars create mode 100644 iac/tf-anthos-gke/variables.tf diff --git a/.gitignore b/.gitignore index f3c570bc0..16ddf56e1 100644 --- a/.gitignore +++ b/.gitignore @@ -177,3 +177,27 @@ extras/asm-multicluster/asm-cluster-2/* *.crt env multicluster-ingress-https.yaml + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# tf lock file +.terraform.lock.hcl \ No newline at end of file diff --git a/iac/tf-anthos-gke/README.md b/iac/tf-anthos-gke/README.md new file mode 100644 index 000000000..4eecbc92e --- /dev/null +++ b/iac/tf-anthos-gke/README.md @@ -0,0 +1,52 @@ +# Deploy Bank of Anthos on an Anthos cluster + +This page walks you through the steps required to deploy Bank of Anthos on an Anthos cluster using [Terraform](https://www.terraform.io/) and [Anthos Config Management (ACM)](https://cloud.google.com/anthos/config-management). + +## Prerequisites + +Setting up the sample requires that you have a [Google Cloud Platform (GCP) project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#console), connected to your billing account. + +## Deploy Bank of Anthos + +Once you have ensured that all the prerequisites are met, follow the steps below to create an Anthos cluster and deploy Bank of Anthos. + +1. Clone the repo: +`git clone https://github.com/GoogleCloudPlatform/bank-of-anthos.git` +1. Set $TF_VAR_project enviornment variable to a project that has billing enabled: +`export TF_VAR_project=` +1. Enable Terraform to use the default service account. Follow the prompts to login via the URL and enter the verification code: +`gcloud auth application-default login --no-launch-browser` +1. Move into the `iac/tf-anthos-gke` directory that has the installation scripts: +`cd iac/tf-anthos-gke` +1. Initialize Terraform: +`terraform init` +1. See what resources will be created: + `terraform plan` +1. Create the resources and deploy the sample: + `terraform apply` + +## Delete the sample and the cluster + +Once you have finished working with the sample, you can tear down the sample application and the cluster + +1. Run `terraform destroy` from the `iac/tf-anthos-gke` directory. + +Please note that this does not delete the project where the Anthos cluster was created. + +## Troubeshooting + +* Error about a GCP API not enabled e.g.: + + ``` + Error: Error creating Feature: failed to create a diff: failed to retrieve Feature resource: googleapi: Error 403: GKE Hub API has not been used in project {project-number} before or it is disabled. Enable it by visiting https://console.developers.google.com/apis/api/gkehub.googleapis.com/overview?project={project-number} then retry. If you enabled this API recently, wait a few minutes for the action to propagate to our systems and retry. + ``` + + As the error suggests, sometime if an API has just been enabled, the action may not have propogated to all the systems. So wait a few minutes and apply the terraform again. + +* Failure to create GKE Hub membership or feature e.g.: + + ``` + Error creating Feature: Resource already exists. + ``` + + This is likely because you already have the GKE Hub membership or feature enabled. To resolve the error, edit the `acm.tf` and add the appropriate variables `enable_fleet_registration` and `enable_fleet_feature` and set them to `false` to prevent the module from trying to add the resource that already exists. diff --git a/iac/tf-anthos-gke/acm.tf b/iac/tf-anthos-gke/acm.tf new file mode 100644 index 000000000..e44b8de51 --- /dev/null +++ b/iac/tf-anthos-gke/acm.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "acm" { + source = "terraform-google-modules/kubernetes-engine/google//modules/acm" + version = "~> 23.0" + project_id = data.google_project.project.project_id + location = module.gke.location + cluster_name = module.gke.name + configmanagement_version = "1.12.2" + + sync_repo = var.sync_repo + sync_branch = var.sync_branch + sync_revision = var.sync_rev + policy_dir = var.policy_dir + source_format = "unstructured" + + secret_type = "none" +} \ No newline at end of file diff --git a/iac/tf-anthos-gke/asm.tf b/iac/tf-anthos-gke/asm.tf new file mode 100644 index 000000000..f68b94469 --- /dev/null +++ b/iac/tf-anthos-gke/asm.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "asm" { + source = "terraform-google-modules/kubernetes-engine/google//modules/asm" + version = "~> 23.0" + project_id = data.google_project.project.project_id + cluster_name = module.gke.name + cluster_location = module.gke.location + enable_cni = true + enable_fleet_registration = false + enable_mesh_feature = true +} + +module "istio-annotation" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + + project_id = data.google_project.project.project_id + cluster_name = module.gke.name + cluster_location = module.gke.location + module_depends_on = [module.gke] + kubectl_create_command = "kubectl annotate --overwrite namespace default mesh.cloud.google.com/proxy='{\"managed\":\"true\"}'" + kubectl_destroy_command = "kubectl annotate --overwrite namespace default mesh.cloud.google.com/proxy='{\"managed\":\"false\"}'" +} + + +module "istio-injection-label" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + + project_id = data.google_project.project.project_id + cluster_name = module.gke.name + cluster_location = module.gke.location + module_depends_on = [module.gke] + kubectl_create_command = "kubectl label namespace default istio-injection=enabled istio.io/rev- --overwrite" + kubectl_destroy_command = "kubectl label namespace default istio-injection-" +} \ No newline at end of file diff --git a/iac/tf-anthos-gke/boa.tf b/iac/tf-anthos-gke/boa.tf new file mode 100644 index 000000000..935101a15 --- /dev/null +++ b/iac/tf-anthos-gke/boa.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "boa-secret" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + + project_id = data.google_project.project.project_id + cluster_name = module.gke.name + cluster_location = module.gke.location + module_depends_on = [module.gke] + kubectl_create_command = "kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/bank-of-anthos/${var.sync_branch}/extras/jwt/jwt-secret.yaml" + kubectl_destroy_command = "kubectl delete -f https://raw.githubusercontent.com/GoogleCloudPlatform/bank-of-anthos/${var.sync_branch}/extras/jwt/jwt-secret.yaml" +} + +module "boa-istio" { + source = "terraform-google-modules/gcloud/google//modules/kubectl-wrapper" + + project_id = data.google_project.project.project_id + cluster_name = module.gke.name + cluster_location = module.gke.location + module_depends_on = [module.asm.wait] + + kubectl_create_command = "kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/bank-of-anthos/${var.sync_branch}/istio-manifests/frontend-ingress.yaml" + kubectl_destroy_command = "kubectl delete -f https://raw.githubusercontent.com/GoogleCloudPlatform/bank-of-anthos/${var.sync_branch}/istio-manifests/frontend-ingress.yaml" +} \ No newline at end of file diff --git a/iac/tf-anthos-gke/gke.tf b/iac/tf-anthos-gke/gke.tf new file mode 100644 index 000000000..aef5e5d26 --- /dev/null +++ b/iac/tf-anthos-gke/gke.tf @@ -0,0 +1,56 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google" + version = "~> 23.0" + project_id = data.google_project.project.project_id + name = var.cluster_name + region = var.region + zones = [var.zone] + initial_node_count = 1 + remove_default_node_pool = true + network = "default" + subnetwork = "default" + ip_range_pods = "" + ip_range_services = "" + cluster_resource_labels = { + "mesh_id" : "proj-${data.google_project.project.number}", + } + identity_namespace = "${data.google_project.project.project_id}.svc.id.goog" + + node_pools = [ + { + name = "asd-node-pool" + autoscaling = true + node_count = 3 + min_count = 1 + max_count = 10 + auto_upgrade = true + machine_type = "e2-standard-2" + }, + ] + + depends_on = [ + module.enabled_google_apis + ] +} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} \ No newline at end of file diff --git a/iac/tf-anthos-gke/main.tf b/iac/tf-anthos-gke/main.tf new file mode 100644 index 000000000..9bdd69a98 --- /dev/null +++ b/iac/tf-anthos-gke/main.tf @@ -0,0 +1,41 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +data "google_project" "project" { + project_id = var.project +} + +data "google_client_config" "default" {} + +module "enabled_google_apis" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 13.1" + + project_id = data.google_project.project.project_id + disable_services_on_destroy = false + + activate_apis = [ + "compute.googleapis.com", + "anthos.googleapis.com", + "container.googleapis.com", + "gkehub.googleapis.com", + "gkeconnect.googleapis.com", + "anthosconfigmanagement.googleapis.com", + "mesh.googleapis.com", + "meshconfig.googleapis.com", + "meshtelemetry.googleapis.com", + ] +} \ No newline at end of file diff --git a/iac/tf-anthos-gke/outputs.tf b/iac/tf-anthos-gke/outputs.tf new file mode 100644 index 000000000..9cd70d5ce --- /dev/null +++ b/iac/tf-anthos-gke/outputs.tf @@ -0,0 +1,23 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_location" { + value = module.gke.location +} + +output "cluster_name" { + value = module.gke.name +} diff --git a/iac/tf-anthos-gke/providers.tf b/iac/tf-anthos-gke/providers.tf new file mode 100644 index 000000000..5b3f88fb4 --- /dev/null +++ b/iac/tf-anthos-gke/providers.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 0.14" + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } +} +provider "google-beta" { + project = var.project + region = var.region + zone = var.zone +} diff --git a/iac/tf-anthos-gke/terraform.tfvars b/iac/tf-anthos-gke/terraform.tfvars new file mode 100644 index 000000000..031280306 --- /dev/null +++ b/iac/tf-anthos-gke/terraform.tfvars @@ -0,0 +1,23 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +region = "us-central1" +zone = "us-central1-b" +cluster_name = "anthos-sample-cluster1" +sync_repo = "https://github.com/GoogleCloudPlatform/bank-of-anthos" +sync_branch = "v0.5.6" +sync_rev = "" +policy_dir = "/kubernetes-manifests" \ No newline at end of file diff --git a/iac/tf-anthos-gke/variables.tf b/iac/tf-anthos-gke/variables.tf new file mode 100644 index 000000000..6926ef7ff --- /dev/null +++ b/iac/tf-anthos-gke/variables.tf @@ -0,0 +1,55 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project" { + type = string + description = "the GCP project where the cluster will be created" +} + +variable "region" { + type = string + description = "the GCP region where the cluster will be created" +} + +variable "zone" { + type = string + description = "the GCP zone in the region where the cluster will be created" +} + +variable "cluster_name" { + type = string + description = "the name of the Anthos cluster" +} + +variable "sync_repo" { + type = string + description = "git URL for the repo which will be sync'ed into the cluster via Config Management" +} + +variable "sync_branch" { + type = string + description = "the git branch in the repo to sync" +} + +variable "sync_rev" { + type = string + description = "the git revision (tag or hash) to sync" +} + +variable "policy_dir" { + type = string + description = "the root directory in the repo branch that contains the resources." +} \ No newline at end of file diff --git a/release/make-release.sh b/release/make-release.sh index e0c0bffc5..50b949842 100755 --- a/release/make-release.sh +++ b/release/make-release.sh @@ -52,6 +52,9 @@ cp -a "${REPO_ROOT}/dev-kubernetes-manifests/." "${REPO_ROOT}/kubernetes-manifes find "${REPO_ROOT}/kubernetes-manifests" -name '*.yaml' -exec sed -i -e "s'image: \(.*\)'image: ${REPO_PREFIX}\/\1:${NEW_VERSION}'g" {} \; find "${REPO_ROOT}/kubernetes-manifests" -name '*.yaml' -exec sed -i -e "s'value: \"dev\"'value: \"${NEW_VERSION}\"'g" {} \; +# update version in terraform scripts +sed -i -e "s@sync_branch = .*@sync_branch = \"${NEW_VERSION}\"@g" ${REPO_ROOT}/iac/tf-anthos-gke/terraform.tfvars + # remove the region tags so that there are no duplicates find "${REPO_ROOT}/kubernetes-manifests" -name '*.yaml' -exec sed -i -e "s/dev_kubernetes_manifests/boa_kubernetes_manifests/g" {} \;