test(modules): implement first tests of this repo

.github/labeler.yml

@@ -0,0 +1,17 @@
+# Add 'feature' label to any PR where the head branch name starts with `feature` or has a `feature` section in the name
+feature:
+  - head-branch: ['^feature', 'feature']
+
+# Add 'test' label to any changes within 'test' folder or any subfolders
+test:
+  - changed-files:
+      - any-glob-to-any-file:
+        - test/**.go
+        - test/**/go.mod
+        - modules/fixtures/**
+
+terraform:
+  - changed-files:
+      - any-glob-to-any-file:
+          - modules/**.tf
+          - .tool-versions

.github/markdown-links.json

@@ -0,0 +1,20 @@
+{
+  "projectBaseUrl":"${workspaceFolder}",
+  "ignorePatterns": [
+    {
+      "pattern": "^http(s?)://localhost"
+    },
+    {
+      "pattern": "^#"
+    }
+  ],
+  "replacementPatterns": [
+  ],
+  "httpHeaders": [
+  ],
+  "timeout": "20s",
+  "retryOn429": true,
+  "retryCount": 5,
+  "fallbackRetryDelay": "30s",
+  "aliveStatusCodes": [200, 206]
+}

.github/renovate.json5

@@ -8,6 +8,7 @@
     ":enablePreCommit",
   ],
   schedule: ["every weekend"],
+  groupName: "mono-update-renovate", // we keep all updates in a single renovate branch in order to save CI tests
   platformAutomerge: false,
   prHourlyLimit: 6,
   prConcurrentLimit: 20,
@@ -31,8 +32,24 @@
   },
   vulnerabilityAlerts: {
     addLabels: ["security"],
-    enabled: true
+    enabled: true,
   },
+  customManagers: [
+    {
+      "customType": "regex",
+      "fileMatch": ["^.tool-versions$"],
+      "matchStrings": [
+        "datasource=(?<datasource>.*?) depName=(?<depName>.*?)( versioning=(?<versioning>.*?))?( extractVersion=(?<extractVersion>.*?))?\\s.*? (?<currentValue>.*)\\s"
+      ],
+    },
+    {
+      "customType": "regex",
+      "fileMatch": ["^justfile$"],
+      "matchStrings": [
+        "datasource=(?<datasource>.*?) depName=(?<depName>.*?)( versioning=(?<versioning>.*?))?( extractVersion=(?<extractVersion>.*?))?\\s(?<originalPackageName>.*) := \"(?<currentValue>.*?)\"\\s"
+      ],
+    },
+  ],
   packageRules: [
     // limit the PR creation for the Renovate pre-commit hook (it's released very frequently)
     {
@@ -52,8 +69,9 @@
     {
       matchDatasources: ["terraform-module"],
       matchPackagePatterns: ["terraform-aws-modules.*"],
-      addLabels: ["group:terraform"],
-      groupName: "Terraform AWS modules",
+      addLabels: ["group:terraform", "automerge"],
+      automerge: true,
+      groupName: "mono-update-renovate-automerge",
       schedule: [
         "every 2 weeks on Saturday and Sunday",
       ],
@@ -63,15 +81,15 @@
       matchDatasources: ["terraform-provider"],
       addLabels: ["group:terraform"],
       schedule: [
-        "every 2 weeks on Saturday and Sunday",
-      ]
+        "every 2 weeks on Saturday and Sunday"
+      ],
     },
     // Terraform patch provider updates
     {
       matchUpdateTypes: ["patch"],
       matchDatasources: ["terraform-provider"],
       addLabels: ["group:terraform", "automerge"],
-      groupName: "Terraform providers",
+      groupName: "mono-update-renovate-automerge",
       automerge: true,
       schedule: [
         "every 2 weeks on Saturday and Sunday",
@@ -82,6 +100,7 @@
       matchUpdateTypes: ["minor", "patch"],
       matchManagers: ["github-actions"],
       addLabels: ["automerge"],
+      groupName: "mono-update-renovate-automerge",
       automerge: true,
     },
     // Patches
@@ -98,6 +117,7 @@
         "terraform-linters/tflint",
       ],
       addLabels: ["automerge"],
+      groupName: "mono-update-renovate-automerge",
       automerge: true,
     },
     // Minor versions
@@ -111,6 +131,7 @@
         "terraform-linters/tflint",
       ],
       addLabels: ["automerge"],
+      groupName: "mono-update-renovate-automerge",
       automerge: true,
     },
     // Create PRs and automerge mentioned components afterward
@@ -121,25 +142,26 @@
         "pre-commit/.+",
       ],
       addLabels: ["automerge"],
+      groupName: "mono-update-renovate-automerge",
       automerge: true,
     },
     {
       matchUpdateTypes: ["major", "minor", "patch"],
       matchManagers: ["pre-commit"],
-      groupName: "pre-commit hooks",
+      groupName: "mono-update-renovate-automerge",
       addLabels: ["automerge"],
       automerge: true
     },
     // For known GitHub repositories that use GitHub tags/releases of format
     // "v1.2.3" and where the asdf plugin ignores the "v" prefix, we also tell
     // Renovate to ignore it via extractVersion when updating .tool-version file
     {
-      matchFileNames: ["**/.tool-versions", "**/*.tf"],
+      matchFileNames: ["**/*.tf"],
       matchPackageNames: [
         "hashicorp/terraform",
         "pre-commit/pre-commit",
       ],
       extractVersion: "^v(?<version>.*)$",
-    },
-  ],
+    }
+  ]
 }

.github/workflows/labeler.yml

@@ -0,0 +1,13 @@
+name: "Pull Request Labeler"
+on:
+  pull_request_target:
+  schedule:
+    - cron: "0 1 * * 1"
+jobs:
+  labeler:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5

.github/workflows/lint.yml

@@ -0,0 +1,19 @@
+---
+# desc: lint sanity check
+name: lint
+
+on:
+  push:
+  workflow_dispatch:
+
+
+jobs:
+  lint:
+    name: pre-commit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+      - name: Install tooling using asdf
+        uses: asdf-vm/actions/install@05e0d2ed97b598bfce82fd30daf324ae0c4570e6 # v3
+
+      - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1

.github/workflows/nightly_cleanup.yml

@@ -0,0 +1,79 @@
+---
+name: Nightly Cleanup
+
+on:
+  schedule:
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+
+
+env:
+  AWS_PROFILE: "infex"
+  # remember to also update tests.yml!
+  AWS_REGION: "eu-west-2"
+  CLEANUP_NEWER_THAN: "6 hours"
+  CLOUD_NUKE_VERSION: "v0.35.0"
+
+jobs:
+  aws-nightly-cleanup:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4
+
+      - name: Import Secrets
+        id: secrets
+        uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3
+        with:
+          url: ${{ secrets.VAULT_ADDR }}
+          method: approle
+          roleId: ${{ secrets.VAULT_ROLE_ID }}
+          secretId: ${{ secrets.VAULT_SECRET_ID }}
+          exportEnv: false
+          secrets: |
+            secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY;
+            secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY;
+
+      # Official action does not support profiles
+      - name: Add profile credentials to ~/.aws/credentials
+        run: |
+          aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }}
+          aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }}
+          aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }}
+
+      - name: Install Cloud Nuke
+        run: |
+          wget https://github.com/gruntwork-io/cloud-nuke/releases/download/${{ env.CLOUD_NUKE_VERSION }}/cloud-nuke_linux_amd64
+          chmod +x cloud-nuke_linux_amd64
+
+      # This is likely to fail, therefore we ignore the error
+      # We're ignoring ec2_dhcp_option as they couldn't be deleted
+      # cloudtrail is managed by IT and can't be deleted either
+      - name: Run Cloud Nuke
+        timeout-minutes: 45
+        env:
+          DISABLE_TELEMETRY: "true"
+        run: |
+          ./cloud-nuke_linux_amd64 aws \
+          --region ${{ env.AWS_REGION }} \
+          --force \
+          --newer-than ${{ env.CLEANUP_NEWER_THAN }} \
+          --exclude-resource-type ec2_dhcp_option \
+          --exclude-resource-type cloudtrail || true
+
+      # Following will delete global resources and things that cloud-nuke does not support
+      - name: Delete additional AWS resources
+        timeout-minutes: 15
+        run: .github/workflows/scripts/aws_cleanup.sh "${{ env.AWS_REGION }}"
+
+      # The second run should remove the remaining resources (VPCs) and fail if there's anything left
+      - name: Run Cloud Nuke
+        timeout-minutes: 45
+        env:
+          DISABLE_TELEMETRY: "true"
+        run: |
+          ./cloud-nuke_linux_amd64 aws \
+          --region ${{ env.AWS_REGION }} \
+          --force \
+          --newer-than ${{ env.CLEANUP_NEWER_THAN }} \
+          --exclude-resource-type ec2_dhcp_option \
+          --exclude-resource-type cloudtrail

.github/workflows/scripts/aws_cleanup.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# This script deletes additional AWS resources based on specified criteria.
+
+# Check if the region argument is provided
+if [ -z "$1" ]; then
+    echo "Please provide the AWS region as the first argument."
+    exit 1
+fi
+
+region="$1"
+
+echo "Deleting additional resources in the $region region..."
+
+
+echo "Deleting additional resources..."
+# KMS keys can't be deleted due to resource policies, requires manual intervention
+
+echo "Deleting IAM Roles"
+# Detach permissions and profile instances and delete IAM roles
+role_arns=$(aws iam list-roles --query "Roles[?contains(RoleName, 'nightly')].RoleName" --output text)
+
+read -r -a role_arns_array <<< "$role_arns"
+
+for role_arn in "${role_arns_array[@]}"
+do
+    echo "Removing instance profiles and policies of role: $role_arn"
+    attached_policy_arns=$(aws iam list-attached-role-policies --role-name "$role_arn" --query 'AttachedPolicies[].PolicyArn' --output text)
+    read -r -a attached_policy_arns_array <<< "$attached_policy_arns"
+
+    for policy_arn in "${attached_policy_arns_array[@]}"
+    do
+        echo "Removing attached policy: $policy_arn"
+        aws iam detach-role-policy --role-name "$role_arn" --policy-arn "$policy_arn"
+    done
+
+    policy_arns=$(aws iam list-role-policies --role-name "$role_arn" --query 'PolicyNames' --output text)
+    read -r -a policy_arns_array <<< "$policy_arns"
+
+    for policy_arn in "${policy_arns_array[@]}"
+    do
+        echo "Deleting policy: $policy_arn"
+        aws iam delete-role-policy --role-name "$role_arn" --policy-name "$policy_arn"
+    done
+
+    instance_profile_arns=$(aws iam list-instance-profiles-for-role --role-name "$role_arn" --query 'InstanceProfiles[].InstanceProfileName' --output text)
+    read -r -a instance_profile_arns_array <<< "$instance_profile_arns"
+
+    for instance_profile_arn in "${instance_profile_arns_array[@]}"
+    do
+        echo "Removing instance profile: $instance_profile_arn"
+        aws iam remove-role-from-instance-profile --instance-profile-name "$instance_profile_arn" --role-name "$role_arn"
+    done
+
+    echo "Deleting role: $role_arn"
+    aws iam delete-role --role-name "$role_arn"
+
+done
+
+echo "Deleting IAM Policies"
+# Delete Policies
+iam_policies=$(aws iam list-policies --query "Policies[?contains(PolicyName, 'nightly')].Arn" --output text)
+
+read -r -a iam_policies_array <<< "$iam_policies"
+
+for iam_policy in "${iam_policies_array[@]}"
+do
+    echo "Deleting policy: $iam_policy"
+    aws iam delete-policy --policy-arn "$iam_policy"
+done
+
+echo "Deleting OIDC Providers"
+# Delete OIDC Provider
+oidc_providers=$(aws iam list-open-id-connect-providers --query "OpenIDConnectProviderList[?contains(Arn, 'eu-west-2') || contains(Arn, 'eu-west-3')].Arn" --output text)
+
+read -r -a oidc_providers_array <<< "$oidc_providers"
+
+for oidc_provider in "${oidc_providers_array[@]}"
+do
+    echo "Deleting OIDC Provider: $oidc_provider"
+    aws iam delete-open-id-connect-provider --open-id-connect-provider-arn "$oidc_provider"
+done
+
+echo "Deleting VPC Peering Connections"
+# Delete VPC Peering Connection
+peering_connection_ids=$(aws ec2 describe-vpc-peering-connections --region "$region" --query "VpcPeeringConnections[?Status.Code == 'active' && Tags[?contains(Value, 'nightly')]]".VpcPeeringConnectionId --output text)
+
+read -r -a peering_connection_ids_array <<< "$peering_connection_ids"
+
+for peering_connection_id in "${peering_connection_ids_array[@]}"
+do
+    echo "Deleting VPC Peering Connection: $peering_connection_id"
+    aws ec2 delete-vpc-peering-connection --region "$region" --vpc-peering-connection-id "$peering_connection_id"
+done