feat: support tfvars for inputs and lib for vnext (#251)

# Pull Request ## Description There are three key features implemented here: - Support tfvars as an input format. - Support tfvars pass through for Terraform starter modules in order to provide a good experience for customers with day 2 operations. - Support supplying a lib folder (or any arbitrary files or folders) to support vNext customisation. End to end tests will be run to confirm there are no breaking changes. e2e test run is here: https://github.com/Azure/accelerator-bootstrap-modules/actions/runs/11955283956 ## License By submitting this pull request, I confirm that my contribution is made under the terms of the projects associated license.
Azure · Nov 22, 2024 · b560259 · b560259
1 parent 34a8b3e
commit b560259
Show file tree

Hide file tree

Showing 9 changed files with 216 additions and 87 deletions.
diff --git a/src/ALZ/Private/Config-Helpers/Convert-ParametersToInputConfig.ps1 b/src/ALZ/Private/Config-Helpers/Convert-ParametersToInputConfig.ps1
@@ -14,7 +14,10 @@ function Convert-ParametersToInputConfig {
             if($inputConfig.PsObject.Properties.Name -contains $parameterAlias) {
                 Write-Verbose "Alias $parameterAlias exists in input config, renaming..."
                 $configItem = $inputConfig.PSObject.Properties | Where-Object { $_.Name -eq $parameterAlias }
-                $inputConfig | Add-Member -NotePropertyName $parameterKey -NotePropertyValue $configItem.Value
+                $inputConfig | Add-Member -NotePropertyName $parameterKey -NotePropertyValue @{
+                    Value  = $configItem.Value.Value
+                    Source = $configItem.Value.Source
+                }
                 $inputConfig.PSObject.Properties.Remove($configItem.Name)
                 continue
             }
@@ -34,7 +37,10 @@ function Convert-ParametersToInputConfig {
                 $variableValue = [bool]::Parse($variableValue)
             }
             Write-Verbose "Adding parameter $parameterKey with value $variableValue"
-            $inputConfig | Add-Member -NotePropertyName $parameterKey -NotePropertyValue $variableValue
+            $inputConfig | Add-Member -NotePropertyName $parameterKey -NotePropertyValue @{
+                Value  = $variableValue
+                Source = "parameter"
+            }
         }
     }
 

diff --git a/src/ALZ/Private/Config-Helpers/Get-ALZConfig.ps1 b/src/ALZ/Private/Config-Helpers/Get-ALZConfig.ps1
@@ -3,7 +3,9 @@ function Get-ALZConfig {
         [Parameter(Mandatory = $false)]
         [string] $configFilePath = "",
         [Parameter(Mandatory = $false)]
-        [PSCustomObject] $inputConfig = $null
+        [PSCustomObject] $inputConfig = $null,
+        [Parameter(Mandatory = $false)]
+        [string] $hclParserToolPath = ""
     )
 
     if(!(Test-Path $configFilePath)) {
@@ -39,14 +41,25 @@ function Get-ALZConfig {
             Write-Error $errorMessage
             throw $errorMessage
         }
+    } elseif($extension -eq ".tfvars") {
+        try {
+            $config = [PSCustomObject](& $hclParserToolPath $configFilePath | ConvertFrom-Json)
+        } catch {
+            $errorMessage = "Failed to parse HCL inputs. Please check the HCL file for errors and try again. $_"
+            Write-Error $errorMessage
+            throw $errorMessage
+        }
     } else {
-        throw "The config file must be a json or yaml/yml file"
+        throw "The config file must be a json, yaml/yml or tfvars file"
     }
 
     Write-Verbose "Config file loaded from $configFilePath with $($config.PSObject.Properties.Name.Count) properties."
 
     foreach($property in $config.PSObject.Properties) {
-        $inputConfig | Add-Member -NotePropertyName $property.Name -NotePropertyValue $property.Value
+        $inputConfig | Add-Member -NotePropertyName $property.Name -NotePropertyValue @{
+            Value  = $property.Value
+            Source = $extension
+        }
     }
 
     return $inputConfig

diff --git a/src/ALZ/Private/Config-Helpers/Remove-TerraformMetaFileSet.ps1 b/src/ALZ/Private/Config-Helpers/Remove-TerraformMetaFileSet.ps1
@@ -11,7 +11,8 @@ function Remove-TerraformMetaFileSet {
             "terraform.tfvars",
             ".terraform.lock.hcl",
             "examples",
-            "yaml.tf"
+            "yaml.tf",
+            ".alzlib"
         ),
         [Parameter(Mandatory = $false)]
         [switch]$writeVerboseLogs

diff --git a/src/ALZ/Private/Config-Helpers/Request-SpecialInput.ps1 b/src/ALZ/Private/Config-Helpers/Request-SpecialInput.ps1
@@ -42,7 +42,7 @@ function Request-SpecialInput {
         }
 
         if($type -eq "starter") {
-            foreach($starter in $starterConfig.starter_modules.PsObject.Properties) {
+            foreach($starter in $starterConfig.starter_modules.Value.PsObject.Properties) {
                 if($starter.Name -eq $starterPipelineFolder) {
                     continue
                 }

diff --git a/src/ALZ/Private/Config-Helpers/Set-Config.ps1 b/src/ALZ/Private/Config-Helpers/Set-Config.ps1
@@ -47,20 +47,29 @@ function Set-Config {
                 $indexSplit = $inputConfigName.Split([char[]]@('[', ']'), [System.StringSplitOptions]::RemoveEmptyEntries)
                 $inputConfigItem = $inputConfig.PsObject.Properties | Where-Object { $_.Name -eq $indexSplit[0] }
                 if($null -ne $inputConfigItem) {
-                    if(!$inputConfigItem.Value.GetType().ImplementedInterfaces.Contains([System.Collections.ICollection])) {
+                    $inputConfigItemValue = $inputConfigItem.Value.Value
+                    if(!$inputConfigItemValue.GetType().ImplementedInterfaces.Contains([System.Collections.ICollection])) {
                         Write-Error "Input config item $($inputConfigName) is not an array, but an index was specified."
                         throw "Input config item $($inputConfigName) is not an array, but an index was specified."
                     }
                     $index = [int]$indexSplit[1]
-                    if($inputConfigItem.Value.Length -le $index) {
+                    if($inputConfigItemValue.Length -le $index) {
                         Write-Verbose "Input config item $($inputConfigName) does not have an index of $index."
+                        if($index -eq 0) {
+                            Write-Error "At least one value is required for input config item $($inputConfigName)."
+                            throw "At least one value is required for input config item $($inputConfigName)."
+                        }
                     } else {
-                        $inputConfigItemValue = $inputConfigItem.Value[$index]
-                        if($null -ne $inputConfigItemValue) {
-                            $configurationValue.Value.Value = $inputConfigItemValue
+                        $inputConfigItemIndexValue = $inputConfigItemValue[$index]
+                        if($null -ne $inputConfigItemIndexValue) {
+                            $configurationValue.Value.Value = $inputConfigItemIndexValue
                             continue
                         } else {
                             Write-Verbose "Input config item $($inputConfigName) with index $index is null."
+                            if($index -eq 0) {
+                                Write-Error "At least one value is required for input config item $($inputConfigName)."
+                                throw "At least one value is required for input config item $($inputConfigName)."
+                            }
                         }
                     }
                 } else {
@@ -72,7 +81,7 @@ function Set-Config {
             # Look for input config match
             $inputConfigItem = $inputConfig.PsObject.Properties | Where-Object { $_.Name -eq $inputConfigName }
             if($null -ne $inputConfigItem) {
-                $configurationValue.Value.Value = $inputConfigItem.Value
+                $configurationValue.Value.Value = $inputConfigItem.Value.Value
                 continue
             }
 

diff --git a/src/ALZ/Private/Config-Helpers/Write-TfvarsJsonFile.ps1 b/src/ALZ/Private/Config-Helpers/Write-TfvarsJsonFile.ps1
@@ -5,7 +5,10 @@ function Write-TfvarsJsonFile {
         [string] $tfvarsFilePath,
 
         [Parameter(Mandatory = $false)]
-        [PSObject] $configuration
+        [PSObject] $configuration,
+
+        [Parameter(Mandatory = $false)]
+        [string[]] $skipItems = @()
     )
 
     if ($PSCmdlet.ShouldProcess("Download Terraform Tools", "modify")) {
@@ -17,6 +20,11 @@ function Write-TfvarsJsonFile {
         $jsonObject = [ordered]@{}
 
         foreach($configurationProperty in $configuration.PSObject.Properties | Sort-Object Name) {
+            if($skipItems -contains $configurationProperty.Name) {
+                Write-Verbose "Skipping configuration property: $($configurationProperty.Name)"
+                continue
+            }
+
             $configurationValue = $configurationProperty.Value.Value
 
             if($null -ne $configurationValue -and $configurationValue.ToString() -eq "sourced-from-env") {

diff --git a/src/ALZ/Private/Deploy-Accelerator-Helpers/Get-BootstrapAndStarterConfig.ps1 b/src/ALZ/Private/Deploy-Accelerator-Helpers/Get-BootstrapAndStarterConfig.ps1
@@ -11,8 +11,6 @@ function Get-BootstrapAndStarterConfig {
         [Parameter(Mandatory = $false)]
         [string]$bootstrapConfigPath,
         [Parameter(Mandatory = $false)]
-        [PSCustomObject]$inputConfig,
-        [Parameter(Mandatory = $false)]
         [string]$toolsPath
     )
 
@@ -31,7 +29,7 @@ function Get-BootstrapAndStarterConfig {
         $bootstrapConfigFullPath = Join-Path $bootstrapPath $bootstrapConfigPath
         Write-Verbose "Bootstrap config path $bootstrapConfigFullPath"
         $bootstrapConfig = Get-ALZConfig -configFilePath $bootstrapConfigFullPath
-        $validationConfig = $bootstrapConfig.validators
+        $validationConfig = $bootstrapConfig.validators.Value
 
         # Get the supported regions and availability zones
         Write-Verbose "Getting Supported Regions and Availability Zones with Terraform"
@@ -42,7 +40,7 @@ function Get-BootstrapAndStarterConfig {
         $azureLocationValidator.AllowedValues.Values = $regionsAndZones.supportedRegions
 
         # Get the available bootstrap modules
-        $bootstrapModules = $bootstrapConfig.bootstrap_modules
+        $bootstrapModules = $bootstrapConfig.bootstrap_modules.Value
 
         # Get the bootstrap details and validate it exists (use alias for legacy values)
         $bootstrapDetails = $bootstrapModules.PsObject.Properties | Where-Object { $_.Name -eq $bootstrap -or $bootstrap -in $_.Value.aliases }
@@ -57,9 +55,9 @@ function Get-BootstrapAndStarterConfig {
         if($null -ne $bootstrapStarterModule) {
             # If the bootstrap has starter modules, get the details and url
             $hasStarterModule = $true
-            $starterModules = $bootstrapConfig.PSObject.Properties | Where-Object { $_.Name -eq "starter_modules" }
+            $starterModules = $bootstrapConfig.starter_modules.Value
             $starterModuleType = $bootstrapStarterModule.Value
-            $starterModuleDetails = $starterModules.Value.PSObject.Properties | Where-Object { $_.Name -eq $starterModuleType }
+            $starterModuleDetails = $starterModules.PSObject.Properties | Where-Object { $_.Name -eq $starterModuleType }
             if($null -eq $starterModuleDetails) {
                 Write-InformationColored "The starter modules '$($starterModuleType)' for the bootstrap type '$bootstrap' that you have selected does not exist. This could be an issue with your custom configuration, please check and try again..." -ForegroundColor Red -InformationAction Continue
                 throw

diff --git a/src/ALZ/Private/Deploy-Accelerator-Helpers/New-Bootstrap.ps1 b/src/ALZ/Private/Deploy-Accelerator-Helpers/New-Bootstrap.ps1
@@ -46,7 +46,19 @@ function New-Bootstrap {
 
         [Parameter(Mandatory = $false)]
         [string]
-        $hclParserToolPath
+        $hclParserToolPath,
+
+        [Parameter(Mandatory = $false)]
+        [switch]
+        $convertTfvarsToJson,
+
+        [Parameter(Mandatory = $false)]
+        [string[]]
+        $inputConfigFilePaths = @(),
+
+        [Parameter(Mandatory = $false)]
+        [string[]]
+        $starterAdditionalFiles = @()
     )
 
     if ($PSCmdlet.ShouldProcess("ALZ-Terraform module configuration", "modify")) {
@@ -71,13 +83,16 @@ function New-Bootstrap {
         $starterFoldersToRetain = @()
 
         if($hasStarter) {
-            if($inputConfig.starter_module_name -eq "") {
-                $inputConfig.starter_module_name = Request-SpecialInput -type "starter" -starterConfig $starterConfig
+            if($inputConfig.starter_module_name.Value -eq "") {
+                $inputConfig.starter_module_name = @{
+                    Value  = Request-SpecialInput -type "starter" -starterConfig $starterConfig
+                    Source = "user"
+                }
             }
 
-            $chosenStarterConfig = $starterConfig.starter_modules.$($inputConfig.starter_module_name)
+            $chosenStarterConfig = $starterConfig.starter_modules.Value.$($inputConfig.starter_module_name.Value)
 
-            Write-Verbose "Selected Starter: $($inputConfig.starter_module_name))"
+            Write-Verbose "Selected Starter: $($inputConfig.starter_module_name.Value))"
             $starterModulePath = (Resolve-Path (Join-Path -Path $starterPath -ChildPath $chosenStarterConfig.location)).Path
             $starterRootModuleFolderPath = $starterModulePath
             Write-Verbose "Starter Module Path: $starterModulePath"
@@ -94,7 +109,10 @@ function New-Bootstrap {
                 $starterFoldersToRetain += $starterRootModuleFolder
 
                 # Add the root module folder to bootstrap input config
-                $inputConfig | Add-Member -NotePropertyName "root_module_folder_relative_path" -NotePropertyValue $starterRootModuleFolder
+                $inputConfig | Add-Member -NotePropertyName "root_module_folder_relative_path" -NotePropertyValue @{
+                    Value  = $starterRootModuleFolder
+                    Source = "caluated"
+                }
 
                 # Set the starter root module folder full path
                 $starterRootModuleFolderPath = Join-Path -Path $starterModulePath -ChildPath $starterRootModuleFolder
@@ -126,25 +144,37 @@ function New-Bootstrap {
             }
 
             if($iac -eq "bicep") {
-                $starterParameters = Convert-BicepConfigToInputConfig -bicepConfig $starterConfig.starter_modules.$($inputConfig.starter_module_name) -validators $validationConfig
+                $starterParameters = Convert-BicepConfigToInputConfig -bicepConfig $starterConfig.starter_modules.Value.$($inputConfig.starter_module_name.Value) -validators $validationConfig
             }
         }
 
         # Set computed inputs
-        $inputConfig | Add-Member -NotePropertyName "module_folder_path" -NotePropertyValue $starterModulePath
-        $inputConfig | Add-Member -NotePropertyName "availability_zones_bootstrap" -NotePropertyValue @(Get-AvailabilityZonesSupport -region $inputConfig.bootstrap_location -zonesSupport $zonesSupport)
+        $inputConfig | Add-Member -NotePropertyName "module_folder_path" -NotePropertyValue @{
+            Value  = $starterModulePath
+            Source = "calculated"
+        }
+        $inputConfig | Add-Member -NotePropertyName "availability_zones_bootstrap" -NotePropertyValue @{
+            Value  = @(Get-AvailabilityZonesSupport -region $inputConfig.bootstrap_location.Value -zonesSupport $zonesSupport)
+            Source = "calculated"
+        }
 
         if($inputConfig.PSObject.Properties.Name -contains "starter_location" -and $inputConfig.PSObject.Properties.Name -notcontains "starter_locations") {
-            Write-Verbose "Converting starter_location $($inputConfig.starter_location) to starter_locations..."
-            $inputConfig | Add-Member -NotePropertyName "starter_locations" -NotePropertyValue @($inputConfig.starter_location)
+            Write-Verbose "Converting starter_location $($inputConfig.starter_location.Value) to starter_locations..."
+            $inputConfig | Add-Member -NotePropertyName "starter_locations" -NotePropertyValue @{
+                Value  = @($inputConfig.starter_location.Value)
+                Source = "calculated"
+            }
         }
 
         if($inputConfig.PSObject.Properties.Name -contains "starter_locations") {
             $availabilityZonesStarter = @()
-            foreach($region in $inputConfig.starter_locations) {
+            foreach($region in $inputConfig.starter_locations.Value) {
                 $availabilityZonesStarter += , @(Get-AvailabilityZonesSupport -region $region -zonesSupport $zonesSupport)
             }
-            $inputConfig | Add-Member -NotePropertyName "availability_zones_starter" -NotePropertyValue $availabilityZonesStarter
+            $inputConfig | Add-Member -NotePropertyName "availability_zones_starter" -NotePropertyValue @{
+                Value  = $availabilityZonesStarter
+                Source = "calculated"
+            }
         }
 
         Write-Verbose "Final Input config: $(ConvertTo-Json $inputConfig -Depth 100)"
@@ -188,26 +218,53 @@ function New-Bootstrap {
                 }
             }
             Remove-TerraformMetaFileSet -path $starterModulePath -writeVerboseLogs:$writeVerboseLogs.IsPresent
-            Write-TfvarsJsonFile -tfvarsFilePath $starterTfvarsPath -configuration $starterConfiguration
+            if($convertTfvarsToJson) {
+                Write-TfvarsJsonFile -tfvarsFilePath $starterTfvarsPath -configuration $starterConfiguration
+            } else {
+                $inputsFromTfvars = $inputConfig.PSObject.Properties | Where-Object { $_.Value.Source -eq ".tfvars" } | Select-Object -ExpandProperty Name
+                Write-TfvarsJsonFile -tfvarsFilePath $starterTfvarsPath -configuration $starterConfiguration -skipItems $inputsFromTfvars
+                foreach($inputConfigFilePath in $inputConfigFilePaths | Where-Object { $_ -like "*.tfvars" }) {
+                    $fileName = [System.IO.Path]::GetFileName($inputConfigFilePath)
+                    $fileName = $fileName.Replace(".tfvars", ".auto.tfvars")
+                    $destination = Join-Path -Path $starterRootModuleFolderPath -ChildPath $fileName
+                    Write-Verbose "Copying tfvars file $inputConfigFilePath to $destination"
+                    Copy-Item -Path $inputConfigFilePath -Destination $destination -Force
+                }
+            }
+
+            # Copy additional files
+            foreach($additionalFile in $starterAdditionalFiles) {
+                if(Test-Path $additionalFile -PathType Container) {
+                    $folderName = ([System.IO.DirectoryInfo]::new($additionalFile)).Name
+                    $destination = Join-Path -Path $starterRootModuleFolderPath -ChildPath $folderName
+                    Write-Verbose "Copying folder $additionalFile to $destination"
+                    Copy-Item -Path "$additionalFile/*" -Destination $destination -Recurse -Force
+                } else {
+                    $fileName = [System.IO.Path]::GetFileName($inputConfigFilePath)
+                    $destination = Join-Path -Path $starterRootModuleFolderPath -ChildPath $fileName
+                    Write-Verbose "Copying file $additionalFile to $destination"
+                    Copy-Item -Path $additionalFile -Destination $destination -Force
+                }
+            }
         }
 
         if($iac -eq "bicep") {
-            Copy-ParametersFileCollection -starterPath $starterModulePath -configFiles $starterConfig.starter_modules.$($inputConfig.starter_module_name).deployment_files
+            Copy-ParametersFileCollection -starterPath $starterModulePath -configFiles $starterConfig.starter_modules.Value.$($inputConfig.starter_module_name.Value).deployment_files
             Set-ComputedConfiguration -configuration $starterConfiguration
             Edit-ALZConfigurationFilesInPlace -alzEnvironmentDestination $starterModulePath -configuration $starterConfiguration
             Write-JsonFile -jsonFilePath $starterBicepVarsPath -configuration $starterConfiguration
 
             # Remove unrequired files
-            $foldersOrFilesToRetain = $starterConfig.starter_modules.$($inputConfig.starter_module_name).folders_or_files_to_retain
+            $foldersOrFilesToRetain = $starterConfig.starter_modules.Value.$($inputConfig.starter_module_name.Value).folders_or_files_to_retain
             $foldersOrFilesToRetain += "parameters.json"
             $foldersOrFilesToRetain += "config"
             $foldersOrFilesToRetain += "starter-cache.json"
 
-            foreach($deployment_file in $starterConfig.starter_modules.$($inputConfig.starter_module_name).deployment_files) {
+            foreach($deployment_file in $starterConfig.starter_modules.Value.$($inputConfig.starter_module_name.Value).deployment_files) {
                 $foldersOrFilesToRetain += $deployment_file.templateParametersSourceFilePath
             }
 
-            $subFoldersOrFilesToRemove = $starterConfig.starter_modules.$($inputConfig.starter_module_name).subfolders_or_files_to_remove
+            $subFoldersOrFilesToRemove = $starterConfig.starter_modules.Value.$($inputConfig.starter_module_name.Value).subfolders_or_files_to_remove
 
             Remove-UnrequiredFileSet -path $starterModulePath -foldersOrFilesToRetain $foldersOrFilesToRetain -subFoldersOrFilesToRemove $subFoldersOrFilesToRemove -writeVerboseLogs:$writeVerboseLogs.IsPresent
         }