From 6e2c43661afa29a519cd8e0b45fff7e5665965f8 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Fri, 6 Sep 2024 13:50:29 +0200 Subject: [PATCH 01/21] updated variables.py to diff generated content and content found in the files --- deploy/infrastructure/utils/variables.py | 44 +++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 0811471da..fddfe977b 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -8,6 +8,8 @@ from os.path import isfile, join, abspath, dirname, exists from typing import Dict, List, Tuple import hcl2 +import argparse +import sys DEFINITIONS_PATH = join(abspath(dirname(__file__)), "definitions") GENERATED_TFVARS_MD_FILENAME = "TFVARS.gen.md" @@ -248,7 +250,47 @@ def write_files(definitions: Dict[str, str]): ) write_file(tfvars_md_filename, content) +def read_file(filename: str) -> str: + """ + Reads a file and returns its content as a string + """ + with open(filename, 'r') as file: + content = file.read() + return content + +def diff_files(definitions: Dict[str, str]) -> bool: + """ + Generates the `.gen.tf` in memory and diffs them against the ones found + on disk + """ + for path, variables in PROJECT_VARIABLES.items(): + # Generate variables.tf definition + var_filename = join(path, GENERATED_VARIABLES_FILENAME) + generated_content = get_variables_tf_content(variables, definitions) + + try: + actual_content = read_file(var_filename) + except Exception as e: + print(f'Error reading {var_filename}: {e}') + return False + + if generated_content != actual_content: + return False + + return True if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument('--diff', action='store_true') + args = parser.parse_args() + definitions = load_tf_definitions() - write_files(definitions) + + if args.diff: + if not diff_files(definitions): + sys.exit(1) + else: + sys.exit(0) + else: + print("not activated") + write_files(definitions) From 7078daa36b6ae3255e9037e05a71392dc9a8a507 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Fri, 6 Sep 2024 13:59:16 +0200 Subject: [PATCH 02/21] ran formatter --- deploy/infrastructure/utils/variables.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index fddfe977b..215ec79b8 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -22,10 +22,7 @@ # Variables per project # For all */terraform-* -GLOBAL_VARIABLES = [ - "app_hostname", - "crdb_hostname_suffix" -] +GLOBAL_VARIABLES = ["app_hostname", "crdb_hostname_suffix"] # dependencies/terraform-commons-dss COMMONS_DSS_VARIABLES = GLOBAL_VARIABLES + [ @@ -40,14 +37,14 @@ "crdb_cluster_name", "crdb_locality", "crdb_external_nodes", - "kubernetes_namespace" + "kubernetes_namespace", ] # dependencies/terraform-*-kubernetes COMMON_KUBERNETES_VARIABLES = GLOBAL_VARIABLES + [ "cluster_name", "node_count", - "kubernetes_version" + "kubernetes_version", ] # dependencies/terraform-google-kubernetes @@ -72,7 +69,7 @@ "aws_region", "aws_instance_type", "aws_route53_zone_id", - "aws_iam_permissions_boundary" + "aws_iam_permissions_boundary", ] + COMMON_KUBERNETES_VARIABLES # modules/terraform-aws-dss @@ -90,9 +87,7 @@ "../dependencies/terraform-aws-kubernetes": AWS_KUBERNETES_VARIABLES, "../dependencies/terraform-google-kubernetes": GOOGLE_KUBERNETES_VARIABLES, "../dependencies/terraform-commons-dss": COMMONS_DSS_VARIABLES, - "../../operations/ci/aws-1": list( - dict.fromkeys(AWS_MODULE_VARIABLES) - ) + "../../operations/ci/aws-1": list(dict.fromkeys(AWS_MODULE_VARIABLES)), } @@ -250,14 +245,16 @@ def write_files(definitions: Dict[str, str]): ) write_file(tfvars_md_filename, content) + def read_file(filename: str) -> str: """ Reads a file and returns its content as a string """ - with open(filename, 'r') as file: + with open(filename, "r") as file: content = file.read() return content + def diff_files(definitions: Dict[str, str]) -> bool: """ Generates the `.gen.tf` in memory and diffs them against the ones found @@ -271,7 +268,7 @@ def diff_files(definitions: Dict[str, str]) -> bool: try: actual_content = read_file(var_filename) except Exception as e: - print(f'Error reading {var_filename}: {e}') + print(f"Error reading {var_filename}: {e}") return False if generated_content != actual_content: @@ -279,9 +276,10 @@ def diff_files(definitions: Dict[str, str]) -> bool: return True + if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument('--diff', action='store_true') + parser.add_argument("--diff", action="store_true") args = parser.parse_args() definitions = load_tf_definitions() From 5efd549e2a2fb5cb0b4092c19a23e590b961bd4b Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Fri, 6 Sep 2024 14:16:02 +0200 Subject: [PATCH 03/21] removed redundant print --- deploy/infrastructure/utils/variables.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 215ec79b8..8e4b9d897 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -290,5 +290,4 @@ def diff_files(definitions: Dict[str, str]) -> bool: else: sys.exit(0) else: - print("not activated") write_files(definitions) From 425fb2db37b6b123fbaec8b6c180cfa57096abac Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Fri, 6 Sep 2024 16:46:06 +0200 Subject: [PATCH 04/21] Updated comments, added logs. --- deploy/infrastructure/utils/variables.py | 53 ++++++++++++++++++------ 1 file changed, 40 insertions(+), 13 deletions(-) diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 8e4b9d897..30e703a80 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -172,9 +172,11 @@ def comment(content: str) -> str: return commented_lines -def get_variables_tf_content(variables: List[str], definitions: Dict[str, str]) -> str: +def get_variables_gen_tf_content( + variables: List[str], definitions: Dict[str, str] +) -> str: """ - Generate the content of variables.tf (Terraform definitions) based + Generate the content of variables.gen.tf (Terraform definitions) based on the `variables` list. `variables` contains the variables names to include in the content. `definitions` contains the definitions of all available variables. @@ -231,9 +233,9 @@ def write_files(definitions: Dict[str, str]): for path, variables in PROJECT_VARIABLES.items(): project_name = path.split("/")[-1] - # Generate variables.tf definition + # Generate variables.gen.tf definition var_filename = join(path, GENERATED_VARIABLES_FILENAME) - content = get_variables_tf_content(variables, definitions) + content = get_variables_gen_tf_content(variables, definitions) write_file(var_filename, content) if is_example_project(path): @@ -246,24 +248,36 @@ def write_files(definitions: Dict[str, str]): write_file(tfvars_md_filename, content) -def read_file(filename: str) -> str: +def read_file(file_path: str) -> str: """ - Reads a file and returns its content as a string + Reads a file and returns its content + + Arguments: + file_path: the relative path of the file to be read from + Returns: + the content of the file as a str """ - with open(filename, "r") as file: + with open(file_path, "r") as file: content = file.read() return content def diff_files(definitions: Dict[str, str]) -> bool: """ - Generates the `.gen.tf` in memory and diffs them against the ones found - on disk + Generates the `variables.gen.tf` file content in memory and diffs it + against the `variables.get.tf` files found on disk + + Arguments: + definitions: a dict where the keys are the terraform variable names, + and the valuies are the content of that file. + Returns: + true iff the generated content and locally stored content are equal + string values """ for path, variables in PROJECT_VARIABLES.items(): - # Generate variables.tf definition + # Generate variables.gen.tf definition var_filename = join(path, GENERATED_VARIABLES_FILENAME) - generated_content = get_variables_tf_content(variables, definitions) + generated_content = get_variables_gen_tf_content(variables, definitions) try: actual_content = read_file(var_filename) @@ -278,16 +292,29 @@ def diff_files(definitions: Dict[str, str]) -> bool: if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument("--diff", action="store_true") + parser = argparse.ArgumentParser(usage="python variables") + parser.add_argument( + "--diff", + action="store_true", + help="diffs the generated `variables.gen.tf` files against the ones\ + found locally without modifiying existing files or writing any\ + results out to disk.\ + Exits with code 0 if there are no diffs, else exits with code 1", + ) args = parser.parse_args() definitions = load_tf_definitions() if args.diff: if not diff_files(definitions): + print( + "variables.py: generated content was NOT equal to local variables.gen.tf content" + ) sys.exit(1) else: + print( + "variables.py: generated content was equal to local variables.gen.tf content" + ) sys.exit(0) else: write_files(definitions) From 40d039325e5fcf1dad89511acb9bdd017dc14997 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 09:12:08 +0200 Subject: [PATCH 05/21] added new linting step to Makefile --- Makefile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Makefile b/Makefile index 6359b31fd..aed3c69ff 100644 --- a/Makefile +++ b/Makefile @@ -61,6 +61,17 @@ go-lint: .PHONY: terraform-lint terraform-lint: docker run --rm -w /opt/dss -v ./deploy:/opt/dss/deploy -e TF_LOG=TRACE hashicorp/terraform fmt -recursive -check + docker run --rm -v $(PWD):/app -w /app python:3.9-slim /bin/bash -c "\ + pip install python-hcl2 && \ + cd /app/deploy/infrastructure/utils && \ + python variables.py --diff \ + " + @if [ $$? -eq 0 ]; then \ + echo "Generated code lint succeeded."; \ + else \ + echo "Generated code lint failed!"; \ + exit 1; \ + fi # This mirrors the hygiene-tests continuous integration workflow job (.github/workflows/ci.yml) .PHONY: hygiene-tests From 8c0d6459309a5732192fdb6cd3ec51ab3a700a82 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 10:25:38 +0200 Subject: [PATCH 06/21] bash script now takes argument which is passed to variables.py --- deploy/infrastructure/utils/generate_terraform_variables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 824443fd7..f5745d4a3 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -10,4 +10,4 @@ docker run \ -v "${SCRIPT_DIR}/../modules":/app/modules:rw \ -w /app/utils \ terraform-variables \ - ls ../ && python variables.py \ No newline at end of file + ls ../ && python variables.py $@ From 5707cd5d35c58d52c9b5e0108951ddbeeefda8b7 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 10:26:04 +0200 Subject: [PATCH 07/21] updated how makefile calls terraform lint --- Makefile | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Makefile b/Makefile index aed3c69ff..9198a23dd 100644 --- a/Makefile +++ b/Makefile @@ -61,17 +61,7 @@ go-lint: .PHONY: terraform-lint terraform-lint: docker run --rm -w /opt/dss -v ./deploy:/opt/dss/deploy -e TF_LOG=TRACE hashicorp/terraform fmt -recursive -check - docker run --rm -v $(PWD):/app -w /app python:3.9-slim /bin/bash -c "\ - pip install python-hcl2 && \ - cd /app/deploy/infrastructure/utils && \ - python variables.py --diff \ - " - @if [ $$? -eq 0 ]; then \ - echo "Generated code lint succeeded."; \ - else \ - echo "Generated code lint failed!"; \ - exit 1; \ - fi + cd deploy/infrastructure/utils && ./generate_terraform_variables.sh --diff-only && cd ../../.. # This mirrors the hygiene-tests continuous integration workflow job (.github/workflows/ci.yml) .PHONY: hygiene-tests From 74ff5c24593a00167affb7184fff6380b2ffd89e Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 10:49:38 +0200 Subject: [PATCH 08/21] updated flag name and comments --- deploy/infrastructure/utils/variables.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 30e703a80..8f792d230 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -8,6 +8,7 @@ from os.path import isfile, join, abspath, dirname, exists from typing import Dict, List, Tuple import hcl2 +import difflib import argparse import sys @@ -292,20 +293,20 @@ def diff_files(definitions: Dict[str, str]) -> bool: if __name__ == "__main__": - parser = argparse.ArgumentParser(usage="python variables") + parser = argparse.ArgumentParser() parser.add_argument( - "--diff", + "--diff-only", action="store_true", - help="diffs the generated `variables.gen.tf` files against the ones\ - found locally without modifiying existing files or writing any\ + help="Evaluates the differences between the generated `variables.gen.tf` files and the ones\ + stored locally without modifiying existing files or writing any\ results out to disk.\ - Exits with code 0 if there are no diffs, else exits with code 1", + Exits with code 0 on success and if there are no differences, else exits with code 1", ) args = parser.parse_args() definitions = load_tf_definitions() - if args.diff: + if args.diff_only: if not diff_files(definitions): print( "variables.py: generated content was NOT equal to local variables.gen.tf content" From 4430cc00672adf241b30f1f454ab4d4d57e10561 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 15:01:47 +0200 Subject: [PATCH 09/21] updated flag name to lint --- Makefile | 2 +- deploy/infrastructure/utils/variables.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9198a23dd..aa8fceccc 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ go-lint: .PHONY: terraform-lint terraform-lint: docker run --rm -w /opt/dss -v ./deploy:/opt/dss/deploy -e TF_LOG=TRACE hashicorp/terraform fmt -recursive -check - cd deploy/infrastructure/utils && ./generate_terraform_variables.sh --diff-only && cd ../../.. + cd deploy/infrastructure/utils && ./generate_terraform_variables.sh --lint && cd ../../.. # This mirrors the hygiene-tests continuous integration workflow job (.github/workflows/ci.yml) .PHONY: hygiene-tests diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 8f792d230..28ec35b20 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -295,7 +295,7 @@ def diff_files(definitions: Dict[str, str]) -> bool: if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( - "--diff-only", + "--lint", action="store_true", help="Evaluates the differences between the generated `variables.gen.tf` files and the ones\ stored locally without modifiying existing files or writing any\ @@ -306,7 +306,7 @@ def diff_files(definitions: Dict[str, str]) -> bool: definitions = load_tf_definitions() - if args.diff_only: + if args.lint: if not diff_files(definitions): print( "variables.py: generated content was NOT equal to local variables.gen.tf content" From d840b158b28c66722ac863b688927d77198668d8 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 18:49:36 +0200 Subject: [PATCH 10/21] updated makefile and bash script to remove cd into utils dir --- Makefile | 2 +- .../utils/generate_terraform_variables.sh | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index aa8fceccc..a26c8a6d2 100644 --- a/Makefile +++ b/Makefile @@ -61,7 +61,7 @@ go-lint: .PHONY: terraform-lint terraform-lint: docker run --rm -w /opt/dss -v ./deploy:/opt/dss/deploy -e TF_LOG=TRACE hashicorp/terraform fmt -recursive -check - cd deploy/infrastructure/utils && ./generate_terraform_variables.sh --lint && cd ../../.. + ./deploy/infrastructure/utils/generate_terraform_variables.sh --lint # This mirrors the hygiene-tests continuous integration workflow job (.github/workflows/ci.yml) .PHONY: hygiene-tests diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index f5745d4a3..7de013569 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -1,7 +1,16 @@ #!/usr/bin/env bash set -e -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +set -eo pipefail + +OS=$(uname) +if [[ "$OS" == "Darwin" ]]; then + # OSX uses BSD readlink + BASEDIR="$(dirname "$0")/.." +else + BASEDIR=$(readlink -e "$(dirname "$0")") +fi +cd "${BASEDIR}" docker build -t terraform-variables:latest . docker run \ From f00892b110af8d1f2604a164a6b6d50ebbb0c9cc Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 19:00:28 +0200 Subject: [PATCH 11/21] remove SCRIPT_DIR references from generate_terraform_variables.sh --- deploy/infrastructure/utils/generate_terraform_variables.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 7de013569..68ff85680 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -14,9 +14,9 @@ cd "${BASEDIR}" docker build -t terraform-variables:latest . docker run \ - -v "${SCRIPT_DIR}/":/app/utils:rw \ - -v "${SCRIPT_DIR}/../dependencies":/app/examples:rw \ - -v "${SCRIPT_DIR}/../modules":/app/modules:rw \ + -v "/":/app/utils:rw \ + -v "/../dependencies":/app/examples:rw \ + -v "/../modules":/app/modules:rw \ -w /app/utils \ terraform-variables \ ls ../ && python variables.py $@ From a6f83b7432f98dc93bf3cebdbf4a5e86f1b95ecd Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Mon, 9 Sep 2024 20:53:40 +0200 Subject: [PATCH 12/21] removed difflib dep, added script dir back as . --- .../infrastructure/utils/generate_terraform_variables.sh | 8 +++++--- deploy/infrastructure/utils/variables.py | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 68ff85680..4be988deb 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -12,11 +12,13 @@ else fi cd "${BASEDIR}" +SCRIPT_DIR=. + docker build -t terraform-variables:latest . docker run \ - -v "/":/app/utils:rw \ - -v "/../dependencies":/app/examples:rw \ - -v "/../modules":/app/modules:rw \ + -v "$(SCRIPT_DIR)/":/app/utils:rw \ + -v "$(SCRIPT_DIR)/../dependencies":/app/examples:rw \ + -v "$(SCRIPT_DIR)/../modules":/app/modules:rw \ -w /app/utils \ terraform-variables \ ls ../ && python variables.py $@ diff --git a/deploy/infrastructure/utils/variables.py b/deploy/infrastructure/utils/variables.py index 28ec35b20..c0d157ea8 100755 --- a/deploy/infrastructure/utils/variables.py +++ b/deploy/infrastructure/utils/variables.py @@ -8,7 +8,6 @@ from os.path import isfile, join, abspath, dirname, exists from typing import Dict, List, Tuple import hcl2 -import difflib import argparse import sys From d5ee0702e75408e86fc00c94ab76d90e3b60cab1 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 09:17:34 +0200 Subject: [PATCH 13/21] removed usage of script dir, using base_dir --- .../utils/generate_terraform_variables.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 4be988deb..5050157d4 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -6,19 +6,17 @@ set -eo pipefail OS=$(uname) if [[ "$OS" == "Darwin" ]]; then # OSX uses BSD readlink - BASEDIR="$(dirname "$0")/.." + BASEDIR="$(dirname "$0")" else BASEDIR=$(readlink -e "$(dirname "$0")") fi cd "${BASEDIR}" -SCRIPT_DIR=. - docker build -t terraform-variables:latest . docker run \ - -v "$(SCRIPT_DIR)/":/app/utils:rw \ - -v "$(SCRIPT_DIR)/../dependencies":/app/examples:rw \ - -v "$(SCRIPT_DIR)/../modules":/app/modules:rw \ + -v "${BASE_DIR}/":/app/utils:rw \ + -v "${BASE_DIR}/../dependencies":/app/examples:rw \ + -v "${BASE_DIR}/../modules":/app/modules:rw \ -w /app/utils \ terraform-variables \ ls ../ && python variables.py $@ From 6028babf0a8a573968e71942792e05f3d28caec3 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 09:18:06 +0200 Subject: [PATCH 14/21] fix typo --- deploy/infrastructure/utils/generate_terraform_variables.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 5050157d4..0b32455f0 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -14,9 +14,9 @@ cd "${BASEDIR}" docker build -t terraform-variables:latest . docker run \ - -v "${BASE_DIR}/":/app/utils:rw \ - -v "${BASE_DIR}/../dependencies":/app/examples:rw \ - -v "${BASE_DIR}/../modules":/app/modules:rw \ + -v "${BASEDIR}/":/app/utils:rw \ + -v "${BASEDIR}/../dependencies":/app/examples:rw \ + -v "${BASEDIR}/../modules":/app/modules:rw \ -w /app/utils \ terraform-variables \ ls ../ && python variables.py $@ From 0bcd8afc276b065324951dbf471032e00e7dd094 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 09:23:33 +0200 Subject: [PATCH 15/21] shell lintn --- deploy/infrastructure/utils/generate_terraform_variables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 0b32455f0..1b884c497 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -19,4 +19,4 @@ docker run \ -v "${BASEDIR}/../modules":/app/modules:rw \ -w /app/utils \ terraform-variables \ - ls ../ && python variables.py $@ + ls ../ && python variables.py "$@" From 73d67d070f97b4c5ec921aa1bce395ba3a325efa Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 15:38:41 +0200 Subject: [PATCH 16/21] added --user flag to utils/Dockerfile --- deploy/infrastructure/utils/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/infrastructure/utils/Dockerfile b/deploy/infrastructure/utils/Dockerfile index 117653ff1..2243d1ce6 100644 --- a/deploy/infrastructure/utils/Dockerfile +++ b/deploy/infrastructure/utils/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.10 -RUN pip install python-hcl2 +RUN pip install --user python-hcl2 WORKDIR /app From fecf59ed5fcd30200470bc160a25423f949b61d8 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 15:48:15 +0200 Subject: [PATCH 17/21] updated dockerfile to use pip3 instead of pip --- deploy/infrastructure/utils/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/infrastructure/utils/Dockerfile b/deploy/infrastructure/utils/Dockerfile index 2243d1ce6..2c5d6b545 100644 --- a/deploy/infrastructure/utils/Dockerfile +++ b/deploy/infrastructure/utils/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.10 -RUN pip install --user python-hcl2 +RUN pip3 install python-hcl2 WORKDIR /app From c302a83fd9637eeee71fc0a2a511c64115c78db2 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 15:53:45 +0200 Subject: [PATCH 18/21] added venv to dockerfile --- deploy/infrastructure/utils/Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/deploy/infrastructure/utils/Dockerfile b/deploy/infrastructure/utils/Dockerfile index 2c5d6b545..d7a2afeb7 100644 --- a/deploy/infrastructure/utils/Dockerfile +++ b/deploy/infrastructure/utils/Dockerfile @@ -1,5 +1,8 @@ FROM python:3.10 -RUN pip3 install python-hcl2 - WORKDIR /app + +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" + +RUN pip install python-hcl2 From 1614007a4c3518fa1839675c4873ed3f2bd8599e Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 16:26:06 +0200 Subject: [PATCH 19/21] backup --- deploy/infrastructure/utils/Dockerfile | 8 ++++---- .../infrastructure/utils/generate_terraform_variables.sh | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deploy/infrastructure/utils/Dockerfile b/deploy/infrastructure/utils/Dockerfile index d7a2afeb7..ce34c23a0 100644 --- a/deploy/infrastructure/utils/Dockerfile +++ b/deploy/infrastructure/utils/Dockerfile @@ -1,8 +1,8 @@ FROM python:3.10 -WORKDIR /app - -RUN python -m venv /opt/venv -ENV PATH="/opt/venv/bin:$PATH" RUN pip install python-hcl2 + +RUN ls + +WORKDIR /app diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 1b884c497..1290aed77 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -19,4 +19,5 @@ docker run \ -v "${BASEDIR}/../modules":/app/modules:rw \ -w /app/utils \ terraform-variables \ - ls ../ && python variables.py "$@" + /bin/bash -c "ls ../ && python variables.py $*" + #ls ../ && python3 variables.py "$@" From d847d5a96b2782fe4e5a558b217948a2b537f481 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 16:49:46 +0200 Subject: [PATCH 20/21] updated generate_terraform_variables.sh --- deploy/infrastructure/utils/generate_terraform_variables.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 1290aed77..753b7841e 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -15,9 +15,9 @@ cd "${BASEDIR}" docker build -t terraform-variables:latest . docker run \ -v "${BASEDIR}/":/app/utils:rw \ - -v "${BASEDIR}/../dependencies":/app/examples:rw \ + -v "${BASEDIR}/../dependencies":/app/dependencies:rw \ -v "${BASEDIR}/../modules":/app/modules:rw \ + -v "${BASEDIR}/../../operations":/operations:rw \ -w /app/utils \ terraform-variables \ - /bin/bash -c "ls ../ && python variables.py $*" - #ls ../ && python3 variables.py "$@" + /bin/bash -c "python variables.py $@" From 829f91df96811e997e0370a1d00e1312e653b339 Mon Sep 17 00:00:00 2001 From: Ethan Graham Date: Tue, 10 Sep 2024 16:51:55 +0200 Subject: [PATCH 21/21] shell lint --- deploy/infrastructure/utils/generate_terraform_variables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/infrastructure/utils/generate_terraform_variables.sh b/deploy/infrastructure/utils/generate_terraform_variables.sh index 753b7841e..d5bea1678 100755 --- a/deploy/infrastructure/utils/generate_terraform_variables.sh +++ b/deploy/infrastructure/utils/generate_terraform_variables.sh @@ -20,4 +20,4 @@ docker run \ -v "${BASEDIR}/../../operations":/operations:rw \ -w /app/utils \ terraform-variables \ - /bin/bash -c "python variables.py $@" + /bin/bash -c "python variables.py $*"