diff --git a/Makefile b/Makefile index 6359b31fd..a26c8a6d2 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,7 @@ go-lint: .PHONY: terraform-lint terraform-lint: docker run --rm -w /opt/dss -v ./deploy:/opt/dss/deploy -e TF_LOG=TRACE hashicorp/terraform fmt -recursive -check + ./deploy/infrastructure/utils/generate_terraform_variables.sh --lint # This mirrors the hygiene-tests continuous integration workflow job (.github/workflows/ci.yml) .PHONY: hygiene-tests diff --git a/deploy/infrastructure/utils/Dockerfile b/deploy/infrastructure/utils/Dockerfile index 117653ff1..ce34c23a0 100644 --- a/deploy/infrastructure/utils/Dockerfile +++ b/deploy/infrastructure/utils/Dockerfile @@ -1,5 +1,8 @@ FROM python:3.10 + RUN pip install python-hcl2 +RUN ls + WORKDIR /app diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 824443fd7..d5bea1678 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -1,13 +1,23 @@ #!/usr/bin/env bash set -e -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")" +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "${BASEDIR}" docker build -t terraform-variables:latest . docker run \ - -v "${SCRIPT_DIR}/":/app/utils:rw \ - -v "${SCRIPT_DIR}/../dependencies":/app/examples:rw \ - -v "${SCRIPT_DIR}/../modules":/app/modules:rw \ + -v "${BASEDIR}/":/app/utils:rw \ + -v "${BASEDIR}/../dependencies":/app/dependencies:rw \ + -v "${BASEDIR}/../modules":/app/modules:rw \ + -v "${BASEDIR}/../../operations":/operations:rw \ -w /app/utils \ terraform-variables \ - ls ../ && python variables.py \ No newline at end of file + /bin/bash -c "python variables.py $*" diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 0811471da..c0d157ea8 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -8,6 +8,8 @@ from os.path import isfile, join, abspath, dirname, exists from typing import Dict, List, Tuple import hcl2 +import argparse +import sys DEFINITIONS_PATH = join(abspath(dirname(__file__)), "definitions") GENERATED_TFVARS_MD_FILENAME = "TFVARS.gen.md" @@ -20,10 +22,7 @@ # Variables per project # For all */terraform-* -GLOBAL_VARIABLES = [ - "app_hostname", - "crdb_hostname_suffix" -] +GLOBAL_VARIABLES = ["app_hostname", "crdb_hostname_suffix"] # dependencies/terraform-commons-dss COMMONS_DSS_VARIABLES = GLOBAL_VARIABLES + [ @@ -38,14 +37,14 @@ "crdb_cluster_name", "crdb_locality", "crdb_external_nodes", - "kubernetes_namespace" + "kubernetes_namespace", ] # dependencies/terraform-*-kubernetes COMMON_KUBERNETES_VARIABLES = GLOBAL_VARIABLES + [ "cluster_name", "node_count", - "kubernetes_version" + "kubernetes_version", ] # dependencies/terraform-google-kubernetes @@ -70,7 +69,7 @@ "aws_region", "aws_instance_type", "aws_route53_zone_id", - "aws_iam_permissions_boundary" + "aws_iam_permissions_boundary", ] + COMMON_KUBERNETES_VARIABLES # modules/terraform-aws-dss @@ -88,9 +87,7 @@ "../dependencies/terraform-aws-kubernetes": AWS_KUBERNETES_VARIABLES, "../dependencies/terraform-google-kubernetes": GOOGLE_KUBERNETES_VARIABLES, "../dependencies/terraform-commons-dss": COMMONS_DSS_VARIABLES, - "../../operations/ci/aws-1": list( - dict.fromkeys(AWS_MODULE_VARIABLES) - ) + "../../operations/ci/aws-1": list(dict.fromkeys(AWS_MODULE_VARIABLES)), } @@ -175,9 +172,11 @@ def comment(content: str) -> str: return commented_lines -def get_variables_tf_content(variables: List[str], definitions: Dict[str, str]) -> str: +def get_variables_gen_tf_content( + variables: List[str], definitions: Dict[str, str] +) -> str: """ - Generate the content of variables.tf (Terraform definitions) based + Generate the content of variables.gen.tf (Terraform definitions) based on the `variables` list. `variables` contains the variables names to include in the content. `definitions` contains the definitions of all available variables. @@ -234,9 +233,9 @@ def write_files(definitions: Dict[str, str]): for path, variables in PROJECT_VARIABLES.items(): project_name = path.split("/")[-1] - # Generate variables.tf definition + # Generate variables.gen.tf definition var_filename = join(path, GENERATED_VARIABLES_FILENAME) - content = get_variables_tf_content(variables, definitions) + content = get_variables_gen_tf_content(variables, definitions) write_file(var_filename, content) if is_example_project(path): @@ -249,6 +248,73 @@ def write_files(definitions: Dict[str, str]): write_file(tfvars_md_filename, content) +def read_file(file_path: str) -> str: + """ + Reads a file and returns its content + + Arguments: + file_path: the relative path of the file to be read from + Returns: + the content of the file as a str + """ + with open(file_path, "r") as file: + content = file.read() + return content + + +def diff_files(definitions: Dict[str, str]) -> bool: + """ + Generates the `variables.gen.tf` file content in memory and diffs it + against the `variables.get.tf` files found on disk + + Arguments: + definitions: a dict where the keys are the terraform variable names, + and the valuies are the content of that file. + Returns: + true iff the generated content and locally stored content are equal + string values + """ + for path, variables in PROJECT_VARIABLES.items(): + # Generate variables.gen.tf definition + var_filename = join(path, GENERATED_VARIABLES_FILENAME) + generated_content = get_variables_gen_tf_content(variables, definitions) + + try: + actual_content = read_file(var_filename) + except Exception as e: + print(f"Error reading {var_filename}: {e}") + return False + + if generated_content != actual_content: + return False + + return True + + if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--lint", + action="store_true", + help="Evaluates the differences between the generated `variables.gen.tf` files and the ones\ + stored locally without modifiying existing files or writing any\ + results out to disk.\ + Exits with code 0 on success and if there are no differences, else exits with code 1", + ) + args = parser.parse_args() + definitions = load_tf_definitions() - write_files(definitions) + + if args.lint: + if not diff_files(definitions): + print( + "variables.py: generated content was NOT equal to local variables.gen.tf content" + ) + sys.exit(1) + else: + print( + "variables.py: generated content was equal to local variables.gen.tf content" + ) + sys.exit(0) + else: + write_files(definitions)