feat(modules): set variable default values (#6)

* feat(irsa): Align IRSA variables

* feat: set module vars default values

* feat: add variable syncing

* feat(ci): add pip caching

* feat(ci): add asdf cache

* feat(ci): add pre-commit to pip deps cache

* fix: path empty string defaults

* docs(inputs): update

* feat: review improvements

* fix: integration defaults

* fix: update oidc_assume_role_policy_condition_test default to StringLike

* fix: update module source references

* fix: update module source references to v0.0.12

---------

Co-authored-by: Martin Odstrčilík <[email protected]>

.github/workflows/pre-commit.yaml

@@ -17,5 +17,36 @@ jobs:
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@v4
-      - uses: asdf-vm/actions/install@v3
+
+      - name: Cache ASDF
+        uses: actions/cache@v4
+        id: asdf-cache
+        with:
+          path: ~/.asdf/
+          key: ${{ runner.os }}-asdf-${{ hashFiles('.tool-versions') }}
+          restore-keys: ${{ runner.os }}-asdf-
+
+      - name: Setup ASDF
+        uses: asdf-vm/actions/setup@v3
+        if: ${{ steps.asdf-cache.outputs.cache-hit == 'true' }}
+
+      - name: Install ASDF
+        uses: asdf-vm/actions/install@v3
+        if: ${{ steps.asdf-cache.outputs.cache-hit != 'true' }}
+
+      - name: Reshim installed ASDF tools
+        shell: bash
+        run: asdf reshim
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '.pre-commit-config.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install pip dependencies
+        run: pip install -r requirements.txt
+
       - uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
       - id: trailing-whitespace
       - id: check-merge-conflict
       - id: detect-aws-credentials
-        args: ['--allow-missing-credentials']
+        args: ["--allow-missing-credentials"]
       - id: detect-private-key
       - id: end-of-file-fixer
 
@@ -25,14 +25,24 @@ repos:
       - id: terraform_validate
       - id: terraform_checkov
         args:
-          - '--args=--skip-check CKV_TF_1' #CKV_TF_1: "Ensure Terraform module sources use a commit hash"
+          - "--args=--skip-check CKV_TF_1" #CKV_TF_1: "Ensure Terraform module sources use a commit hash"
       - id: terraform_docs
         args:
-          - '--args=--config=.terraform-docs.yml'
+          - "--args=--config=.terraform-docs.yml"
 
   - repo: https://github.com/Yelp/detect-secrets
     rev: v1.4.0
     hooks:
       - id: detect-secrets
-        args: ['--baseline', '.secrets.baseline']
+        args: ["--baseline", ".secrets.baseline"]
         exclude: terraform.tfstate
+
+  - repo: local
+    hooks:
+      - id: sync-variables
+        name: Sync module variables
+        entry: ./scripts/sync-variables.py
+        language: system
+        types: [python]
+        always_run: true
+        pass_filenames: false

.tool-versions

@@ -4,3 +4,4 @@ tflint 0.50.3
 checkov 3.2.37
 awscli 2.15.29
 pre-commit 3.6.2
+python 3.13.1

addon-irsa.tf

@@ -2,7 +2,7 @@
 module "addon-irsa" {
   for_each = local.addon_irsa
 
-  source = "git::https://github.com/lablabs/terraform-aws-eks-universal-addon.git//modules/addon-irsa?ref=v0.0.8"
+  source = "git::https://github.com/lablabs/terraform-aws-eks-universal-addon.git//modules/addon-irsa?ref=v0.0.12"
 
   enabled = var.enabled
 
@@ -22,9 +22,12 @@ module "addon-irsa" {
   irsa_policy               = var.irsa_policy != null ? var.irsa_policy : try(each.value.irsa_policy, "")
   irsa_assume_role_enabled  = var.irsa_assume_role_enabled != null ? var.irsa_assume_role_enabled : try(each.value.irsa_assume_role_enabled, false)
   irsa_assume_role_arns     = var.irsa_assume_role_arns != null ? var.irsa_assume_role_arns : try(each.value.irsa_assume_role_arns, [])
-  irsa_permissions_boundary = var.irsa_permissions_boundary != null ? var.irsa_permissions_boundary : try(each.value.irsa_permissions_boundary, "")
+  irsa_permissions_boundary = var.irsa_permissions_boundary != null ? var.irsa_permissions_boundary : try(each.value.irsa_permissions_boundary, null)
   irsa_additional_policies  = var.irsa_additional_policies != null ? var.irsa_additional_policies : try(each.value.irsa_additional_policies, tomap({}))
 
+  irsa_assume_role_policy_condition_test   = var.irsa_assume_role_policy_condition_test != null ? var.irsa_assume_role_policy_condition_test : try(each.value.irsa_assume_role_policy_condition_test, "StringEquals")
+  irsa_assume_role_policy_condition_values = var.irsa_assume_role_policy_condition_values != null ? var.irsa_assume_role_policy_condition_values : try(each.value.irsa_assume_role_policy_condition_values, [])
+
   irsa_tags = var.irsa_tags != null ? var.irsa_tags : try(each.value.irsa_tags, tomap({}))
 }
 

addon-oidc.tf

@@ -2,7 +2,7 @@
 module "addon-oidc" {
   for_each = local.addon_oidc
 
-  source = "git::https://github.com/lablabs/terraform-aws-eks-universal-addon.git//modules/addon-oidc?ref=v0.0.9"
+  source = "git::https://github.com/lablabs/terraform-aws-eks-universal-addon.git//modules/addon-oidc?ref=v0.0.12"
 
   enabled = var.enabled
 
@@ -15,7 +15,7 @@ module "addon-oidc" {
   oidc_policy                                = var.oidc_policy != null ? var.oidc_policy : try(each.value.oidc_policy, "")
   oidc_assume_role_enabled                   = var.oidc_assume_role_enabled != null ? var.oidc_assume_role_enabled : try(each.value.oidc_assume_role_enabled, false)
   oidc_assume_role_arns                      = var.oidc_assume_role_arns != null ? var.oidc_assume_role_arns : try(each.value.oidc_assume_role_arns, [])
-  oidc_permissions_boundary                  = var.oidc_permissions_boundary != null ? var.oidc_permissions_boundary : try(each.value.oidc_permissions_boundary, "") # tflint-ignore: aws_iam_role_invalid_permissions_boundary
+  oidc_permissions_boundary                  = var.oidc_permissions_boundary != null ? var.oidc_permissions_boundary : try(each.value.oidc_permissions_boundary, null)
   oidc_additional_policies                   = var.oidc_additional_policies != null ? var.oidc_additional_policies : try(each.value.oidc_additional_policies, tomap({}))
   oidc_openid_client_ids                     = var.oidc_openid_client_ids != null ? var.oidc_openid_client_ids : try(each.value.oidc_openid_client_ids, [])
   oidc_openid_provider_url                   = var.oidc_openid_provider_url != null ? var.oidc_openid_provider_url : try(each.value.oidc_openid_provider_url, "")

addon.tf

@@ -6,7 +6,7 @@ locals {
 }
 
 module "addon" {
-  source = "git::https://github.com/lablabs/terraform-aws-eks-universal-addon.git//modules/addon?ref=v0.0.11"
+  source = "git::https://github.com/lablabs/terraform-aws-eks-universal-addon.git//modules/addon?ref=v0.0.12"
 
   enabled = var.enabled
 

docs/.inputs.md

@@ -1,2 +1,2 @@
 > [!IMPORTANT]
-> Variables defined in [variables-addon.tf](variables-addon.tf) defaults to `null` to have them overridable by the addon configuration defined though the [`local.addon.*`](main.tf) local variable with some default values defined in [addon.tf](addon.tf).
+> Variables defined in [variables-addon[-irsa|oidc].tf](variables-addon.tf) defaults to `null` to have them overridable by the addon configuration defined though the [`local.addon[_irsa|oidc].*`](main.tf) local variable with the default values defined in [addon[-irsa|oidc].tf](addon.tf).

requirements.txt

@@ -0,0 +1,2 @@
+python-hcl2==5.1.1
+Jinja2==3.1.4

scripts/sync-variables.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+import os
+import hcl2
+import re
+import argparse
+import logging
+from jinja2 import Environment, FileSystemLoader
+
+def filter_terraform_type(value):
+    # Currently there is a limition in handling Terraform complex types
+    #   https://github.com/amplify-education/python-hcl2/issues/179
+    #   https://github.com/amplify-education/python-hcl2/issues/172
+    if isinstance(value, str):
+        return re.sub(r'\${(.*)}', r'\1', value)
+    return value
+
+def filter_terraform_default(value):
+    if isinstance(value, bool):
+        return str(value).lower()
+
+    if isinstance(value, str):
+        if value == "":
+            return '\\"\\"'
+
+    if value == None:
+        return 'null'
+
+    return re.sub(r'\'', r'\\"', str(value))
+
+def get_template():
+    env = Environment(loader=FileSystemLoader("."))
+    env.filters['terraform_type'] = filter_terraform_type
+    env.filters['terraform_default'] = filter_terraform_default
+
+    return env.from_string("""# IMPORTANT: This file is synced with the "terraform-aws-eks-universal-addon" module. Any changes to this file might be overwritten upon the next release of that module.
+{%- for variable in variables %}
+{%- for name, spec in variable.items() %}
+{%- if name != 'enabled' %}
+variable "{{ name }}" {
+  type        = {{ spec.type | terraform_type }}
+  default     = null
+  description = "{{ spec.description }}{% if spec.default is defined %} Defaults to `{{ spec.default | terraform_default }}`.{% endif %}"
+}
+{%- endif %}
+{%- endfor %}
+{% endfor %}
+""")
+
+def get_logger(args):
+    log_level = args.log.upper()
+
+    numeric_level = getattr(logging, log_level, None)
+    if not isinstance(numeric_level, int):
+        raise ValueError('Invalid log level `%s`' % log_level)
+
+    logging.basicConfig(format='%(levelname)s: %(message)s', level=numeric_level)
+
+    return logging.getLogger(__name__)
+
+def main(args):
+    log = get_logger(args)
+    log.info("Syncing variables from Terraform modules...")
+    log.warning("Terraform variable complex types are NOT supported!")
+
+    template = get_template()
+
+    for module in os.listdir('.terraform/modules'): # Iterate over all initialized modules
+        if not module.startswith('addon') or module.find(".") != -1: # Skip non-addon modules, ie. nested modules
+            log.info("Skipping module `%s`", module)
+            continue
+
+        log.info("Processing module `%s`", module)
+
+        source = '.terraform/modules/'+module+'/modules/'+module+'/variables.tf'
+        target = 'variables-'+module+'.tf'
+
+        with open(source, 'r') as f:
+            log.info("Reading variables from `%s`", source)
+
+            variables = hcl2.load(f).get('variable')
+            log.info("Collected variables: %d", len(variables))
+            log.debug(variables)
+
+            with open(target, "w") as f:
+                log.info("Writing variables to `%s`", target)
+                f.write(template.render(variables=variables))
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Sync Terraform variables from the local addon modules to proxy variable files')
+    parser.add_argument('--log', default='INFO', help='Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)')
+
+    args = parser.parse_args()
+
+    main(args)

variables-addon-irsa.tf

@@ -1,7 +1,5 @@
 # IMPORTANT: This file is synced with the "terraform-aws-eks-universal-addon" module. Any changes to this file might be overwritten upon the next release of that module.
 
-# ================ IRSA variables (optional) ================
-
 variable "cluster_identity_oidc_issuer" {
   type        = string
   default     = null
@@ -29,13 +27,13 @@ variable "service_account_create" {
 variable "service_account_name" {
   type        = string
   default     = null
-  description = "The Kubernetes Service Account name. Defaults to addon name."
+  description = "The Kubernetes Service Account name. Defaults to the addon name. Defaults to `\"\"`."
 }
 
 variable "service_account_namespace" {
   type        = string
   default     = null
-  description = "The Kubernetes Service Account namespace. Defaults to addon namespace."
+  description = "The Kubernetes Service Account namespace. Defaults to the addon namespace. Defaults to `\"\"`."
 }
 
 variable "irsa_role_create" {
@@ -47,13 +45,13 @@ variable "irsa_role_create" {
 variable "irsa_role_name_prefix" {
   type        = string
   default     = null
-  description = "IRSA role name prefix. Defaults to addon IRSA component name with `irsa` suffix."
+  description = "IRSA role name prefix. Either `irsa_role_name_prefix` or `irsa_role_name` must be set. Defaults to `\"\"`."
 }
 
 variable "irsa_role_name" {
   type        = string
   default     = null
-  description = "IRSA role name. The value is prefixed by `var.irsa_role_name_prefix`. Defaults to addon Helm chart name."
+  description = "IRSA role name. The value is prefixed by `irsa_role_name_prefix`. Either `irsa_role_name` or `irsa_role_name_prefix` must be set. Defaults to `\"\"`."
 }
 
 variable "irsa_policy_enabled" {
@@ -65,7 +63,7 @@ variable "irsa_policy_enabled" {
 variable "irsa_policy" {
   type        = string
   default     = null
-  description = "Policy to be attached to the IRSA role. Applied only if `irsa_policy_enabled` is `true`."
+  description = "AWS IAM policy JSON document to be attached to the IRSA role. Applied only if `irsa_policy_enabled` is `true`. Defaults to `\"\"`."
 }
 
 variable "irsa_assume_role_enabled" {
@@ -77,13 +75,13 @@ variable "irsa_assume_role_enabled" {
 variable "irsa_assume_role_arns" {
   type        = list(string)
   default     = null
-  description = "List of ARNs assumable by the IRSA role. Applied only if `irsa_assume_role_enabled` is `true`."
+  description = "List of ARNs assumable by the IRSA role. Applied only if `irsa_assume_role_enabled` is `true`. Defaults to `[]`."
 }
 
 variable "irsa_permissions_boundary" {
   type        = string
   default     = null
-  description = "ARN of the policy that is used to set the permissions boundary for the IRSA role. Defaults to `\"\"`."
+  description = "ARN of the policy that is used to set the permissions boundary for the IRSA role. Defaults to `null`."
 }
 
 variable "irsa_additional_policies" {
@@ -97,3 +95,15 @@ variable "irsa_tags" {
   default     = null
   description = "IRSA resources tags. Defaults to `{}`."
 }
+
+variable "irsa_assume_role_policy_condition_test" {
+  type        = string
+  default     = null
+  description = "Specifies the condition test to use for the assume role trust policy. Defaults to `StringEquals`."
+}
+
+variable "irsa_assume_role_policy_condition_values" {
+  type        = list(string)
+  default     = null
+  description = "Specifies the values for the assume role trust policy condition. Each entry in this list must follow the required format `system:serviceaccount:$service_account_namespace:$service_account_name`. If this variable is left as the default, `local.irsa_assume_role_policy_condition_values_default` is used instead, which is a list containing a single value. Note that if this list is defined, the `service_account_name` and `service_account_namespace` variables are ignored. Defaults to `[]`."
+}

variables-addon-oidc.tf

@@ -1,29 +1,33 @@
 # IMPORTANT: This file is synced with the "terraform-aws-eks-universal-addon" module. Any changes to this file might be overwritten upon the next release of that module.
 
-# ================ oidc variables (optional) ================
-
 variable "oidc_provider_create" {
   type        = bool
   default     = null
-  description = "Whether to create OIDC provider. Defaults to `true`. Set to false if you want to disable default OIDC provider when oidc_custom_provider_arn is set."
+  description = "Whether to create OIDC provider. Set to `false` if you want to disable default OIDC provider when `oidc_custom_provider_arn` is set. Defaults to `true`."
+}
+
+variable "oidc_custom_provider_arn" {
+  type        = string
+  default     = null
+  description = "Specifies a custom OIDC provider ARN. Defaults to `\"\"`."
 }
 
 variable "oidc_role_create" {
   type        = bool
   default     = null
-  description = "Whether to create OIDC role and annotate Service Account. Defaults to `true`."
+  description = "Whether to create OIDC role. Defaults to `true`."
 }
 
 variable "oidc_role_name_prefix" {
   type        = string
   default     = null
-  description = "OIDC role name prefix. Defaults to addon OIDC component name with `oidc` suffix."
+  description = "OIDC role name prefix. Either `oidc_role_name_prefix` or `oidc_role_name` must be set. Defaults to `\"\"`."
 }
 
 variable "oidc_role_name" {
   type        = string
   default     = null
-  description = "OIDC role name. The value is prefixed by `var.oidc_role_name_prefix`. Defaults to addon Helm chart name."
+  description = "OIDC role name. The value is prefixed by `oidc_role_name_prefix`. Either `oidc_role_name` or `oidc_role_name_prefix` must be set. Defaults to `\"\"`."
 }
 
 variable "oidc_policy_enabled" {
@@ -35,7 +39,7 @@ variable "oidc_policy_enabled" {
 variable "oidc_policy" {
   type        = string
   default     = null
-  description = "Policy to be attached to the OIDC role. Applied only if `oidc_policy_enabled` is `true`."
+  description = "AWS IAM policy JSON document to be attached to the OIDC role. Applied only if `oidc_policy_enabled` is `true`. Defaults to `\"\"`."
 }
 
 variable "oidc_assume_role_enabled" {
@@ -53,7 +57,7 @@ variable "oidc_assume_role_arns" {
 variable "oidc_permissions_boundary" {
   type        = string
   default     = null
-  description = "ARN of the policy that is used to set the permissions boundary for the OIDC role. Defaults to `\"\"`."
+  description = "ARN of the policy that is used to set the permissions boundary for the OIDC role. Defaults to `null`."
 }
 
 variable "oidc_additional_policies" {
@@ -103,9 +107,3 @@ variable "oidc_openid_thumbprints" {
   default     = null
   description = "List of thumbprints of the OIDC provider's server certificate. Defaults to `[]`."
 }
-
-variable "oidc_custom_provider_arn" {
-  type        = string
-  default     = null
-  description = "Specifies a custom OIDC provider ARN. If specified, overrides provider created by this module. If set, it is recommended to disable default OIDC provider creation by setting var.oidc_provider_create to false. Defaults to `\"\"`."
-}