From 6599b9e88adb5bad24946bf979906b9ff2b4b859 Mon Sep 17 00:00:00 2001 From: Fernando Antivero Date: Tue, 5 Nov 2024 19:16:01 -0300 Subject: [PATCH] feat (deployment): [rgs] follow the region resource recommendations (#99) * follow the region resource recommendations * fix var declaration * modify pre-cluster-stamp for location guidance * fix location recommendation for cluster-stamp modules * Address PR Feedback: fix zones and regions concepts used interchangeable --- cluster-stamp.bicep | 25 ++-------------- docs/deploy/04-subscription.md | 12 ++++---- docs/deploy/05-networking-hub.md | 2 +- docs/deploy/06-aks-jumpboximage.md | 16 +++++----- docs/deploy/08-cluster-networking.md | 14 ++++----- docs/deploy/09-pre-cluster-stamp.md | 12 ++++---- docs/deploy/10-aks-cluster.md | 8 ++--- ...lidate-cluster-access-and-bootstrapping.md | 8 ++--- docs/deploy/15-cleanup.md | 12 ++++---- ...dentityHasRbacToSelfManagedResources.bicep | 22 ++------------ networking/hub-region.v0.bicep | 28 ++++-------------- networking/hub-region.v1.bicep | 29 +++++-------------- networking/hub-region.v2.bicep | 26 ++++------------- networking/spoke-BU0001A0005-00.bicep | 25 ++++------------ networking/spoke-BU0001A0005-01.bicep | 23 ++------------- pre-cluster-stamp.bicep | 23 ++------------- subscription.bicep | 10 +++---- 17 files changed, 81 insertions(+), 214 deletions(-) diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep index 3e5f77ef..4e1bfc6d 100644 --- a/cluster-stamp.bicep +++ b/cluster-stamp.bicep @@ -16,27 +16,6 @@ param k8sControlPlaneAuthorizationTenantId string @secure() param appGatewayListenerCertificate string -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' -]) -@description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') -@minLength(4) -param location string = 'eastus2' - @description('The Azure resource ID of a VM image that will be used for the jump box.') @minLength(70) param jumpBoxImageResourceId string @@ -57,6 +36,9 @@ param gitOpsBootstrappingRepoBranch string = 'main' var kubernetesVersion = '1.30' +@description('AKS Service, Node Pool, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters. This defaults to the resource group\'s location for higher reliability.') +var location = resourceGroup().location + var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' var jumpBoxDefaultAdminUserName = uniqueString(clusterName, resourceGroup().id) @@ -1008,7 +990,6 @@ module ensureClusterIdentityHasRbacToSelfManagedResources 'modules/ensureCluster miClusterControlPlanePrincipalId: miClusterControlPlane.properties.principalId clusterControlPlaneIdentityName: miClusterControlPlane.name vnetSpokeName: vnetSpoke.name - location: location } } diff --git a/docs/deploy/04-subscription.md b/docs/deploy/04-subscription.md index f68771e0..32d14149 100644 --- a/docs/deploy/04-subscription.md +++ b/docs/deploy/04-subscription.md @@ -12,12 +12,12 @@ This reference implementation is split across several resource groups in a singl The following four resource groups will be created in these steps. -| Name | Purpose | -|---------------------------------|-------------------------------------------| -| rg-enterprise-networking-hubs | Contains all of your organization's regional hubs. A regional hub resources in this implementation include the hub Virtual Network, egress firewall, Azure Bastion, and Log Analytics for network logging. They may also contain your VPN Gateways, which are not addressed in this implementation. | -| rg-enterprise-networking-spokes | Contains all of your organization's regional spokes and related networking resources. All spokes will peer with their regional hub and subnets will egress through the regional firewall in the hub. | -| rg-bu0001a0005 | Contains the regulated cluster resources. | -| networkWatcherRG | Contains regional Network Watchers. *(Most subscriptions already have this.)* | +| Name | Purpose | +|-------------------------------------------|-------------------------------------------| +| rg-enterprise-networking-hubs-centralus | Contains all of your organization's regional hubs. A regional hub resources in this implementation include the hub Virtual Network, egress firewall, Azure Bastion, and Log Analytics for network logging. They may also contain your VPN Gateways, which are not addressed in this implementation. | +| rg-enterprise-networking-spokes-centralus | Contains all of your organization's regional spokes and related networking resources. All spokes will peer with their regional hub and subnets will egress through the regional firewall in the hub. | +| rg-bu0001a0005-centralus | Contains the regulated cluster resources. | +| networkWatcherRG | Contains regional Network Watchers. *(Most subscriptions already have this.)* | Both Azure Kubernetes Service and Azure Image Builder Service use a concept of a dynamically-created *infrastructure* resource group. So in addition to the four resource groups mentioned above, as you follow these instructions, you'll end up with six resource groups; two of which are automatically created and their lifecycle tied to their owning service. You will not see these two infrastructure resource groups get created until later in the walkthrough when their owning service is created. diff --git a/docs/deploy/05-networking-hub.md b/docs/deploy/05-networking-hub.md index a48e7fb2..9700785c 100644 --- a/docs/deploy/05-networking-hub.md +++ b/docs/deploy/05-networking-hub.md @@ -29,7 +29,7 @@ Since this walkthrough is expected to be deployed isolated from existing infrast ```bash # [This takes about eight minutes to run.] - az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v0.bicep -p location=eastus2 + az deployment group create -g rg-enterprise-networking-hubs-centralus -f networking/hub-region.v0.bicep ``` The hub deployment will output the following: diff --git a/docs/deploy/06-aks-jumpboximage.md b/docs/deploy/06-aks-jumpboximage.md index 75ea813c..603c26c1 100644 --- a/docs/deploy/06-aks-jumpboximage.md +++ b/docs/deploy/06-aks-jumpboximage.md @@ -37,10 +37,10 @@ You are going to be using Azure Image Builder to generate a Kubernetes-specific 1. Create the AKS jump box image builder network spoke. ```bash - RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs -n hub-region.v0 --query properties.outputs.hubVnetId.value -o tsv) + RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs-centralus -n hub-region.v0 --query properties.outputs.hubVnetId.value -o tsv) # [This takes about one minute to run.] - az deployment group create -g rg-enterprise-networking-spokes -f networking/spoke-BU0001A0005-00.bicep -p location=eastus2 hubVnetResourceId="${RESOURCEID_VNET_HUB}" + az deployment group create -g rg-enterprise-networking-spokes-centralus -f networking/spoke-BU0001A0005-00.bicep -p hubVnetResourceId="${RESOURCEID_VNET_HUB}" ``` 1. Update the regional hub deployment to account for the requirements of the spoke. @@ -50,10 +50,10 @@ You are going to be using Azure Image Builder to generate a Kubernetes-specific > :eyes: If you're curious to see what changed in the regional hub, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v0.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v1.bicep). ```bash - RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv) + RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv) # [This takes about five minutes to run.] - az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v1.bicep -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" + az deployment group create -g rg-enterprise-networking-hubs-centralus -f networking/hub-region.v1.bicep -p aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" ``` ### Build and deploy the jump box image @@ -88,7 +88,7 @@ Now that we have our image building network created, egressing through our hub, ROLEID_IMGDEPLOY=$(az deployment sub show -n DeployAibRbacRoles --query 'properties.outputs.roleResourceIds.value.customImageBuilderImageCreationRole.guid' -o tsv) # [This takes about one minute to run.] - az deployment group create -g rg-bu0001a0005 -f jumpbox/azuredeploy.bicep -p buildInSubnetResourceId=${RESOURCEID_SUBNET_AIB} location=eastus2 imageBuilderNetworkingRoleGuid="${ROLEID_NETWORKING}" imageBuilderImageCreationRoleGuid="${ROLEID_IMGDEPLOY}" -n CreateJumpBoxImageTemplate + az deployment group create -g rg-bu0001a0005-centralus -f jumpbox/azuredeploy.bicep -p buildInSubnetResourceId=${RESOURCEID_SUBNET_AIB} imageBuilderNetworkingRoleGuid="${ROLEID_NETWORKING}" imageBuilderImageCreationRoleGuid="${ROLEID_IMGDEPLOY}" -n CreateJumpBoxImageTemplate ``` 1. Build the general-purpose AKS jump box image. @@ -96,13 +96,13 @@ Now that we have our image building network created, egressing through our hub, Now you'll build the actual VM golden image you will use for your jump box. This uses the image template created in the prior step and is executed by Azure Image Builder under the authority of the managed identity (and its role assignments) also created in the prior step. ```bash - IMAGE_TEMPLATE_NAME=$(az deployment group show -g rg-bu0001a0005 -n CreateJumpBoxImageTemplate --query 'properties.outputs.imageTemplateName.value' -o tsv) + IMAGE_TEMPLATE_NAME=$(az deployment group show -g rg-bu0001a0005-centralus -n CreateJumpBoxImageTemplate --query 'properties.outputs.imageTemplateName.value' -o tsv) # [This takes about >> 30 minutes << to run.] - az image builder run -n $IMAGE_TEMPLATE_NAME -g rg-bu0001a0005 + az image builder run -n $IMAGE_TEMPLATE_NAME -g rg-bu0001a0005-centralus ``` - > A successful run of the command above is typically shown with no output or a success message. An error state will be typically be presented if there was an error. To see whether your image was built successfully, you can go to the **rg-bu0001a0005** resource group in the portal and look for a created VM Image resource. It will have the same name as the Image Template resource created in Step 2. + > A successful run of the command above is typically shown with no output or a success message. An error state will be typically be presented if there was an error. To see whether your image was built successfully, you can go to the **rg-bu0001a0005-centralus** resource group in the portal and look for a created VM Image resource. It will have the same name as the Image Template resource created in Step 2. :coffee: This does take a significant amount of time to run. While the image building is happening, feel free to read ahead, but you should not proceed until this is complete. If you need to perform this reference implementation walk through multiple times, we suggest you create this image in a place that can survive the deleting and re-creating of this reference implementation to save yourself this time in a future execution of this guide. diff --git a/docs/deploy/08-cluster-networking.md b/docs/deploy/08-cluster-networking.md index 262e84a4..ef13b270 100644 --- a/docs/deploy/08-cluster-networking.md +++ b/docs/deploy/08-cluster-networking.md @@ -8,7 +8,7 @@ The regional spoke network in which your cluster is laid into acts as the first ## Expected results -Your `rg-enterprise-networking-spokes` will be populated with the dedicated regional spoke network in which your cluster (and its direct adjacent resources will be connected to). This spoke will have limited Internet exposure and will support Network Security Groups (NSGs) at various levels to further limit network traffic as necessary. +Your `rg-enterprise-networking-spokes-centralus` will be populated with the dedicated regional spoke network in which your cluster (and its direct adjacent resources will be connected to). This spoke will have limited Internet exposure and will support Network Security Groups (NSGs) at various levels to further limit network traffic as necessary. - The network spoke will be called `vnet-spoke-bu0001a0005-01` and have a range of `10.240.0.0/16`. - The spoke is broken into multiple subnets, each with a clearly defined purpose, appropriate IP range, and maximally restrictive NSG. @@ -20,10 +20,10 @@ Your `rg-enterprise-networking-spokes` will be populated with the dedicated regi 1. Deploy the cluster spoke. ```bash - RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs -n hub-region.v0 --query properties.outputs.hubVnetId.value -o tsv) + RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs-centralus -n hub-region.v0 --query properties.outputs.hubVnetId.value -o tsv) # [This takes about five minutes to run.] - az deployment group create -g rg-enterprise-networking-spokes -f networking/spoke-BU0001A0005-01.bicep -p location=eastus2 hubVnetResourceId="${RESOURCEID_VNET_HUB}" + az deployment group create -g rg-enterprise-networking-spokes-centralus -f networking/spoke-BU0001A0005-01.bicep -p hubVnetResourceId="${RESOURCEID_VNET_HUB}" ``` 1. Update the regional hub deployment to account for the runtime requirements of the virtual network. @@ -33,12 +33,12 @@ Your `rg-enterprise-networking-spokes` will be populated with the dedicated regi > :eyes: If you're curious to see what changed in the regional hub, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v1.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v2.bicep). ```bash - RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv) - RESOURCEID_SUBNET_NODEPOOLS="['$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query "properties.outputs.nodepoolSubnetResourceIds.value | join ('\',\'',@)" -o tsv)']" - RESOURCEID_SUBNET_JUMPBOX=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query properties.outputs.jumpboxSubnetResourceId.value -o tsv) + RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv) + RESOURCEID_SUBNET_NODEPOOLS="['$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-01 --query "properties.outputs.nodepoolSubnetResourceIds.value | join ('\',\'',@)" -o tsv)']" + RESOURCEID_SUBNET_JUMPBOX=$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-01 --query properties.outputs.jumpboxSubnetResourceId.value -o tsv) # [This takes about seven minutes to run.] - az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v2.bicep -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" nodepoolSubnetResourceIds="${RESOURCEID_SUBNET_NODEPOOLS}" aksJumpboxSubnetResourceId="${RESOURCEID_SUBNET_JUMPBOX}" + az deployment group create -g rg-enterprise-networking-hubs-centralus -f networking/hub-region.v2.bicep -p aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" nodepoolSubnetResourceIds="${RESOURCEID_SUBNET_NODEPOOLS}" aksJumpboxSubnetResourceId="${RESOURCEID_SUBNET_JUMPBOX}" ``` ### Next step diff --git a/docs/deploy/09-pre-cluster-stamp.md b/docs/deploy/09-pre-cluster-stamp.md index 999a7c98..b4321c67 100644 --- a/docs/deploy/09-pre-cluster-stamp.md +++ b/docs/deploy/09-pre-cluster-stamp.md @@ -31,7 +31,7 @@ An Azure user managed identity is going to be deployed. This identity is the ing > :book: The app team will be deploying to a spoke Virtual Network, that was already provisioned by the network team. ```bash - export RESOURCEID_VNET_CLUSTERSPOKE=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query properties.outputs.clusterVnetResourceId.value -o tsv) + export RESOURCEID_VNET_CLUSTERSPOKE=$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-01 --query properties.outputs.clusterVnetResourceId.value -o tsv) echo RESOURCEID_VNET_CLUSTERSPOKE: $RESOURCEID_VNET_CLUSTERSPOKE ``` @@ -39,7 +39,7 @@ An Azure user managed identity is going to be deployed. This identity is the ing ```bash # [This takes about eight minutes.] - az deployment group create -g rg-bu0001a0005 -f pre-cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} aksIngressControllerCertificate=${INGRESS_CONTROLLER_CERTIFICATE_BASE64} location=eastus2 + az deployment group create -g rg-bu0001a0005-centralus -f pre-cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} aksIngressControllerCertificate=${INGRESS_CONTROLLER_CERTIFICATE_BASE64} ``` ## Quarantine pattern @@ -74,7 +74,7 @@ Using a security agent that is container-aware and can operate from within the c # Get your quarantine Azure Container Registry service name # You only deployed one ACR instance in this walkthrough, but this could be # a separate, dedicated quarantine instance managed by your IT team. - ACR_NAME_QUARANTINE=$(az deployment group show -g rg-bu0001a0005 -n pre-cluster-stamp --query properties.outputs.quarantineContainerRegistryName.value -o tsv) + ACR_NAME_QUARANTINE=$(az deployment group show -g rg-bu0001a0005-centralus -n pre-cluster-stamp --query properties.outputs.quarantineContainerRegistryName.value -o tsv) # [Combined this takes about eight minutes.] az acr import --source docker.io/falcosecurity/falco-no-driver:0.36.0 -t quarantine/falcosecurity/falco-no-driver:0.36.0 -n $ACR_NAME_QUARANTINE && \ @@ -114,7 +114,7 @@ Using a security agent that is container-aware and can operate from within the c ```bash # Get your live Azure Container Registry service name - ACR_NAME=$(az deployment group show -g rg-bu0001a0005 -n pre-cluster-stamp --query properties.outputs.containerRegistryName.value -o tsv) + ACR_NAME=$(az deployment group show -g rg-bu0001a0005-centralus -n pre-cluster-stamp --query properties.outputs.containerRegistryName.value -o tsv) # [Combined this takes about eight minutes.] az acr import --source quarantine/falcosecurity/falco-no-driver:0.36.0 -r $ACR_NAME_QUARANTINE -t live/falcosecurity/falco-no-driver:0.36.0 -n $ACR_NAME && \ @@ -149,7 +149,7 @@ Once web traffic hits Azure Application Gateway (deployed in a future step), pub 1. Obtain the Azure Key Vault details and give the current user permissions and network access to import certificates. ```bash - KEYVAULT_NAME=$(az deployment group show --resource-group rg-bu0001a0005 -n pre-cluster-stamp --query properties.outputs.keyVaultName.value -o tsv) + KEYVAULT_NAME=$(az deployment group show --resource-group rg-bu0001a0005-centralus -n pre-cluster-stamp --query properties.outputs.keyVaultName.value -o tsv) TEMP_ROLEASSIGNMENT_TO_UPLOAD_CERT=$(az role assignment create --role a4417e6f-fecd-4de8-b567-7b0420556985 --assignee-principal-type user --assignee-object-id $(az ad signed-in-user show --query 'id' -o tsv) --scope $(az keyvault show --name $KEYVAULT_NAME --query 'id' -o tsv) --query 'id' -o tsv) echo TEMP_ROLEASSIGNMENT_TO_UPLOAD_CERT: $TEMP_ROLEASSIGNMENT_TO_UPLOAD_CERT @@ -204,7 +204,7 @@ Your cluster will be bootstrapped using the Microsoft-provided GitOps extension, You'll be using the [Secrets Store CSI Driver for Kubernetes](https://learn.microsoft.com/azure/aks/csi-secrets-store-driver) to mount the ingress controller's certificate which you stored in Azure Key Vault. Once mounted, your ingress controller will be able to use it. To make the CSI Provider aware of this certificate, it must be described in a `SecretProviderClass` resource. You'll update the supplied manifest file with this information now. ```bash - INGRESS_CONTROLLER_WORKLOAD_IDENTITY_CLIENT_ID_BU0001A0005_01=$(az deployment group show --resource-group rg-bu0001a0005 -n pre-cluster-stamp --query properties.outputs.ingressClientid.value -o tsv) + INGRESS_CONTROLLER_WORKLOAD_IDENTITY_CLIENT_ID_BU0001A0005_01=$(az deployment group show --resource-group rg-bu0001a0005-centralus -n pre-cluster-stamp --query properties.outputs.ingressClientid.value -o tsv) echo INGRESS_CONTROLLER_WORKLOAD_IDENTITY_CLIENT_ID_BU0001A0005_01: $INGRESS_CONTROLLER_WORKLOAD_IDENTITY_CLIENT_ID_BU0001A0005_01 sed -i -e "s/KEYVAULT_NAME/${KEYVAULT_NAME}/" -e "s/KEYVAULT_TENANT/${TENANTID_AZURERBAC}/" -e "s/INGRESS_CONTROLLER_WORKLOAD_IDENTITY_CLIENT_ID_BU0001A0005_01/${INGRESS_CONTROLLER_WORKLOAD_IDENTITY_CLIENT_ID_BU0001A0005_01}/" ingress-nginx/akv-tls-provider.yaml diff --git a/docs/deploy/10-aks-cluster.md b/docs/deploy/10-aks-cluster.md index 828c0655..6e60dd8d 100644 --- a/docs/deploy/10-aks-cluster.md +++ b/docs/deploy/10-aks-cluster.md @@ -14,7 +14,7 @@ Now that the all the [necessary bootstrapping requirements are deployed](./09-pr 1. Get the already-deployed, Virtual Network resource ID that this cluster will be attached to. ```bash - RESOURCEID_VNET_CLUSTERSPOKE=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query properties.outputs.clusterVnetResourceId.value -o tsv) + RESOURCEID_VNET_CLUSTERSPOKE=$(az deployment group show -g rg-enterprise-networking-spokes-centralus -n spoke-BU0001A0005-01 --query properties.outputs.clusterVnetResourceId.value -o tsv) echo RESOURCEID_VNET_CLUSTERSPOKE: $RESOURCEID_VNET_CLUSTERSPOKE ``` @@ -22,7 +22,7 @@ Now that the all the [necessary bootstrapping requirements are deployed](./09-pr ```bash # If you used a pre-existing image and not the one built by this walk through, replace the command below with the resource id of that image. - RESOURCEID_IMAGE_JUMPBOX=$(az deployment group show -g rg-bu0001a0005 -n CreateJumpBoxImageTemplate --query 'properties.outputs.distributedImageResourceId.value' -o tsv) + RESOURCEID_IMAGE_JUMPBOX=$(az deployment group show -g rg-bu0001a0005-centralus -n CreateJumpBoxImageTemplate --query 'properties.outputs.distributedImageResourceId.value' -o tsv) echo RESOURCEID_IMAGE_JUMPBOX: $RESOURCEID_IMAGE_JUMPBOX ``` @@ -50,10 +50,10 @@ Now that the all the [necessary bootstrapping requirements are deployed](./09-pr echo GITOPS_CURRENT_BRANCH_NAME: $GITOPS_CURRENT_BRANCH_NAME # [This takes about 20 minutes to run.] - az deployment group create -g rg-bu0001a0005 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminEntraGroupObjectId=${OBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} gitOpsBootstrappingRepoBranch=${GITOPS_CURRENT_BRANCH_NAME} + az deployment group create -g rg-bu0001a0005-centralus -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminEntraGroupObjectId=${OBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} gitOpsBootstrappingRepoBranch=${GITOPS_CURRENT_BRANCH_NAME} # Or if you updated and wish to use the parameters file … - #az deployment group create -g rg-bu0001a0005 -f cluster-stamp.bicep -p "@azuredeploy.parameters.prod.json" + #az deployment group create -g rg-bu0001a0005-centralus -f cluster-stamp.bicep -p "@azuredeploy.parameters.prod.json" ``` ### Next step diff --git a/docs/deploy/11-validate-cluster-access-and-bootstrapping.md b/docs/deploy/11-validate-cluster-access-and-bootstrapping.md index 574929c0..8e0317e0 100644 --- a/docs/deploy/11-validate-cluster-access-and-bootstrapping.md +++ b/docs/deploy/11-validate-cluster-access-and-bootstrapping.md @@ -19,7 +19,7 @@ Your cluster was deployed with Azure Policy and the Flux GitOps extension. You'l If this is the first time you've used Azure Bastion, here is a detailed walk through of this process. 1. Open the [Azure Portal](https://portal.azure.com). - 1. Navigate to the **rg-bu0001a0005** resource group. + 1. Navigate to the **rg-bu0001a0005-centralus** resource group. 1. Click on the virtual machine Scale Set resource named **vmss-jumpboxes**. 1. Click **Instances**. 1. Click the name of any of the two listed instances. Such as **vmss-jumpboxes_0** @@ -39,7 +39,7 @@ Your cluster was deployed with Azure Policy and the Flux GitOps extension. You'l ```bash az login - # This will give you a link to https://microsoft.com/devicelogin where you can enter + # This will give you a link to https://microsoft.com/devicelogin where you can enter # the provided code and perform authentication. # Ensure you're on the correct subscription @@ -54,9 +54,9 @@ Your cluster was deployed with Azure Policy and the Flux GitOps extension. You'l 1. *From your Azure Bastion connection*, get your AKS credentials and set your `kubectl` context to your cluster. ```bash - AKS_CLUSTER_NAME=$(az deployment group show -g rg-bu0001a0005 -n cluster-stamp --query properties.outputs.aksClusterName.value -o tsv) + AKS_CLUSTER_NAME=$(az deployment group show -g rg-bu0001a0005-centralus -n cluster-stamp --query properties.outputs.aksClusterName.value -o tsv) - az aks get-credentials -g rg-bu0001a0005 -n $AKS_CLUSTER_NAME + az aks get-credentials -g rg-bu0001a0005-centralus -n $AKS_CLUSTER_NAME ``` 1. *From your Azure Bastion connection*, test cluster access and authenticate as a cluster admin user. diff --git a/docs/deploy/15-cleanup.md b/docs/deploy/15-cleanup.md index 70997c27..9bb283ed 100644 --- a/docs/deploy/15-cleanup.md +++ b/docs/deploy/15-cleanup.md @@ -15,9 +15,9 @@ After you are done exploring your deployed [AKS Baseline Cluster for Regulated W :warning: Ensure you are using the correct subscription, and validate that the only resources that exist in these groups are ones you're okay deleting. ```bash - az group delete -n rg-bu0001a0005 - az group delete -n rg-enterprise-networking-spokes - az group delete -n rg-enterprise-networking-hubs + az group delete -n rg-bu0001a0005-centralus + az group delete -n rg-enterprise-networking-spokes-centralus + az group delete -n rg-enterprise-networking-hubs-centralus ``` Depending on your subscription's starting point, this walkthrough might have also deployed a resource group by the name of `networkWatcherRG`. If you know this to be the case, and wish to remove it as well, you can execute `az group delete -n networkWatcherRG`. If you are not sure, you can leave that resource group in place, the resources deployed as part of this walkthrough to that resource group are not cost or security impacting. @@ -37,9 +37,9 @@ After you are done exploring your deployed [AKS Baseline Cluster for Regulated W Execute the following commands will handle all Resource Group-scoped policies: ```bash - for p in $(az policy assignment list --disable-scope-strict-match --query "[?resourceGroup=='rg-bu0001a0005'].name" -o tsv); do az policy assignment delete -n ${p} -g rg-bu0001a0005; done - for p in $(az policy assignment list --disable-scope-strict-match --query "[?resourceGroup=='rg-enterprise-networking-spokes'].name" -o tsv); do az policy assignment delete -n ${p} -g rg-enterprise-networking-spokes; done - for p in $(az policy assignment list --disable-scope-strict-match --query "[?resourceGroup=='rg-enterprise-networking-hubs'].name" -o tsv); do az policy assignment delete -n ${p} -g rg-enterprise-networking-hubs; done + for p in $(az policy assignment list --disable-scope-strict-match --query "[?resourceGroup=='rg-bu0001a0005-centralus'].name" -o tsv); do az policy assignment delete -n ${p} -g rg-bu0001a0005; done + for p in $(az policy assignment list --disable-scope-strict-match --query "[?resourceGroup=='rg-enterprise-networking-spokes-centralus'].name" -o tsv); do az policy assignment delete -n ${p} -g rg-enterprise-networking-spokes; done + for p in $(az policy assignment list --disable-scope-strict-match --query "[?resourceGroup=='rg-enterprise-networking-hubs-centralus'].name" -o tsv); do az policy assignment delete -n ${p} -g rg-enterprise-networking-hubs; done ``` 1. Remove *custom* Azure Policy definitions. diff --git a/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep index 0b0c6307..4c7faf49 100644 --- a/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep +++ b/modules/ensureClusterIdentityHasRbacToSelfManagedResources.bicep @@ -16,26 +16,10 @@ param clusterControlPlaneIdentityName string @minLength(1) param vnetSpokeName string -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' -]) +/*** VARIABLES ***/ + @description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') -@minLength(4) -param location string +var location = resourceGroup().location /*** EXISTING SUBSCRIPTION RESOURCES ***/ diff --git a/networking/hub-region.v0.bicep b/networking/hub-region.v0.bicep index 34d97073..426b1875 100644 --- a/networking/hub-region.v0.bicep +++ b/networking/hub-region.v0.bicep @@ -2,27 +2,6 @@ targetScope = 'resourceGroup' /*** PARAMETERS ***/ -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' -]) -@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') -@minLength(4) -param location string - @description('A /24 to contain the regional firewall, management, and gateway subnet') @minLength(10) @maxLength(18) @@ -46,6 +25,11 @@ param azureBastionSubnetAddressSpace string = '10.200.0.96/27' @description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') param deployFlowLogResources bool = true +/*** VARIABLES ***/ + +@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains an approved regional list which is a subset of zones with Availability Zone support. Defaults to the resource group\'s location for higher availability.') +var location = resourceGroup().location + /*** EXISTING RESOURCES ***/ resource networkWatcherResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (deployFlowLogResources) { @@ -361,7 +345,7 @@ resource pipAzureBastion 'Microsoft.Network/publicIPAddresses@2021-05-01' = { zones: [ '1' '2' - '3' + '3' ] properties: { publicIPAllocationMethod: 'Static' diff --git a/networking/hub-region.v1.bicep b/networking/hub-region.v1.bicep index 79084a04..5eda2bf1 100644 --- a/networking/hub-region.v1.bicep +++ b/networking/hub-region.v1.bicep @@ -6,27 +6,6 @@ targetScope = 'resourceGroup' @minLength(79) param aksImageBuilderSubnetResourceId string -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' -]) -@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') -@minLength(4) -param location string = 'eastus2' - @description('A /24 to contain the regional firewall, management, and gateway subnet') @minLength(10) @maxLength(18) @@ -50,6 +29,12 @@ param azureBastionSubnetAddressSpace string = '10.200.0.96/27' @description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') param deployFlowLogResources bool = true +/*** VARIABLES ***/ + +@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains an approved regional list which is a subset of regions with Availability Zone support. Defaults to the resource group\'s location for higher availability.') +var location = resourceGroup().location + + /*** EXISTING RESOURCES ***/ @description('The resource group name containing virtual network in which Azure Image Builder will drop the compute into to perform the image build.') @@ -383,7 +368,7 @@ resource pipAzureBastion 'Microsoft.Network/publicIPAddresses@2021-05-01' = { zones: [ '1' '2' - '3' + '3' ] properties: { publicIPAllocationMethod: 'Static' diff --git a/networking/hub-region.v2.bicep b/networking/hub-region.v2.bicep index 2957bda4..8d36c19e 100644 --- a/networking/hub-region.v2.bicep +++ b/networking/hub-region.v2.bicep @@ -13,27 +13,6 @@ param aksImageBuilderSubnetResourceId string @minLength(79) param aksJumpboxSubnetResourceId string -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' -]) -@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') -@minLength(4) -param location string = 'eastus2' - @description('A /24 to contain the regional firewall, management, and gateway subnet') @minLength(10) @maxLength(18) @@ -57,6 +36,11 @@ param azureBastionSubnetAddressSpace string = '10.200.0.96/27' @description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') param deployFlowLogResources bool = true +/*** VARIABLES ***/ + +@description('The hub\'s regional affinity. All resources tied to this hub will also be homed in this region. The network team maintains an approved regional list which is a subset of regions with Availability Zone support. Defaults to the resource group\'s location for higher availability.') +var location = resourceGroup().location + /*** EXISTING RESOURCES ***/ @description('The resource group name containing virtual network in which Azure Image Builder will drop the compute into to perform the image build.') diff --git a/networking/spoke-BU0001A0005-00.bicep b/networking/spoke-BU0001A0005-00.bicep index bf75cf37..4eba529e 100644 --- a/networking/spoke-BU0001A0005-00.bicep +++ b/networking/spoke-BU0001A0005-00.bicep @@ -6,29 +6,14 @@ targetScope = 'resourceGroup' @minLength(79) param hubVnetResourceId string -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' - ]) -@description('The spokes\'s regional affinity, must be the same as the hub\'s location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') -param location string - @description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') param deployFlowLogResources bool = true +/*** VARIABLES ***/ + +@description('The spokes\'s regional affinity, must be the same as the hub\'s location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of regions with Availability Zone support.') +var location = resourceGroup().location + /*** EXISTING RESOURCES ***/ @description('The resource group name containing virtual network in which the regional Azure Firewall is deployed.') diff --git a/networking/spoke-BU0001A0005-01.bicep b/networking/spoke-BU0001A0005-01.bicep index 9f2fcfac..db5424bf 100644 --- a/networking/spoke-BU0001A0005-01.bicep +++ b/networking/spoke-BU0001A0005-01.bicep @@ -6,26 +6,6 @@ targetScope = 'resourceGroup' @minLength(79) param hubVnetResourceId string -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' - ]) -@description('The spokes\'s regional affinity, must be the same as the hub\'s location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of zones with Availability Zone support.') -param location string - @description('Flow Logs are enabled by default, if for some reason they cause conflicts with flow log policies already in place in your subscription, you can disable them by passing "false" to this parameter.') param deployFlowLogResources bool = true @@ -34,6 +14,9 @@ param deployFlowLogResources bool = true @description('The organization\'s application ID') var orgAppId = 'BU0001A0005' +@description('The spokes\'s regional affinity, must be the same as the hub\'s location. All resources tied to this spoke will also be homed in this region. The network team maintains this approved regional list which is a subset of regions with Availability Zone support.') +var location = resourceGroup().location + /*** EXISTING RESOURCES ***/ @description('The resource group name containing virtual network in which the regional Azure Firewall is deployed.') diff --git a/pre-cluster-stamp.bicep b/pre-cluster-stamp.bicep index 97a1997f..f3bc9c3b 100644 --- a/pre-cluster-stamp.bicep +++ b/pre-cluster-stamp.bicep @@ -6,27 +6,6 @@ targetScope = 'resourceGroup' @minLength(79) param targetVnetResourceId string -@allowed([ - 'australiaeast' - 'canadacentral' - 'centralus' - 'eastus' - 'eastus2' - 'westus2' - 'francecentral' - 'germanywestcentral' - 'northeurope' - 'southafricanorth' - 'southcentralus' - 'uksouth' - 'westeurope' - 'japaneast' - 'southeastasia' -]) -@description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') -@minLength(4) -param location string = 'eastus2' - @allowed([ 'australiasoutheast' 'canadaeast' @@ -59,6 +38,8 @@ param geoRedundancyLocation string = 'centralus' param aksIngressControllerCertificate string /*** VARIABLES ***/ +@description('AKS Service, Node Pools, and supporting services (KeyVault, App Gateway, etc) region. This needs to be the same region as the vnet provided in these parameters.') +var location = resourceGroup().location var subRgUniqueString = uniqueString('aks', subscription().subscriptionId, resourceGroup().id) var clusterName = 'aks-${subRgUniqueString}' diff --git a/subscription.bicep b/subscription.bicep index 40efb962..29f5fe63 100644 --- a/subscription.bicep +++ b/subscription.bicep @@ -1,6 +1,6 @@ targetScope = 'subscription' -/* Required Permissions +/* Required Permissions - Scope: Subscription Role: Contributor Reason: Creating resource groups, azure policy definations and assignments @@ -26,7 +26,7 @@ param networkWatcherRGRegion string = '' /*** VARIABLES ***/ @description('This region is used as the default for all generic resource groups and for any additional deployment resources. No resources are actually deployed to this resource group.') -var deploymentResourceRegion = 'centralus' +var deploymentResourceRegion = 'centralus' /*** EXISTING RESOURCES ***/ @@ -39,19 +39,19 @@ resource securityAdminRole 'Microsoft.Authorization/roleDefinitions@2018-01-01-p @description('This contains all of our regional hubs. Typically this would be found in your enterprise\'s Connectivity subscription.') resource rgHubs 'Microsoft.Resources/resourceGroups@2021-04-01' = { - name: 'rg-enterprise-networking-hubs' + name: 'rg-enterprise-networking-hubs-${deploymentResourceRegion}' location: deploymentResourceRegion } @description('This contains all of our regional spokes. Typically this would be found in your enterprise\'s Connectivity subscription or in the workload\'s subscription.') resource rgSpokes 'Microsoft.Resources/resourceGroups@2021-04-01' = { - name: 'rg-enterprise-networking-spokes' + name: 'rg-enterprise-networking-spokes-${deploymentResourceRegion}' location: deploymentResourceRegion } @description('This is the resource group for BU001A0005. Typically this would be found in your workload\'s subscription.') resource rgbu0001a0005 'Microsoft.Resources/resourceGroups@2021-04-01' = { - name: 'rg-bu0001a0005' + name: 'rg-bu0001a0005-${deploymentResourceRegion}' location: deploymentResourceRegion }