CAPT-1632 Implement review apps (#2737)

Migration to AKS

.dockerignore

@@ -16,3 +16,7 @@ localhost.key
 
 **/.terraform
 .vscode
+
+.github
+azure
+terraform

.github/actions/deploy-environment/action.yml

@@ -0,0 +1,44 @@
+name: Deploy environment
+description: Deploys an application environment
+
+inputs:
+  environment:
+    description: The name of the environment
+    required: true
+  docker-image:
+    description: The Docker image to deploy
+    required: true
+  azure-credentials:
+    description: JSON object containing a service principal that can read from Azure Key Vault
+    required: true
+  pull-request-number:
+    description: The pull request number which triggered this deploy.
+    required: false
+
+runs:
+  using: composite
+
+  steps:
+    - uses: hashicorp/setup-terraform@v3
+      with:
+        terraform_version: 1.8.4
+        terraform_wrapper: false
+
+    - uses: DFE-Digital/github-actions/set-kubelogin-environment@master
+      with:
+        azure-credentials: ${{ inputs.azure-credentials }}
+
+    - name: Terraform Apply
+      shell: bash
+      run: |
+        make ci ${{ inputs.environment }} terraform-apply-aks
+      env:
+        DOCKER_IMAGE_TAG: ${{ inputs.docker-image }}
+        PR_NUMBER: ${{ inputs.pull-request-number }}
+
+    - name: Extract Terraform outputs
+      shell: bash
+      id: set_outputs
+      run: |
+        output=$(terraform -chdir=terraform/application output -raw url)
+        echo "APP_URL=$output" >> $GITHUB_ENV

.github/workflows/build_and_deploy.yml

@@ -0,0 +1,80 @@
+name: Build and deploy to AKS cluster
+
+on:
+  pull_request:
+    types: [labeled, opened, reopened, synchronize]
+
+concurrency: deploy-${{ github.ref }}
+
+permissions:
+  packages: write
+  pull-requests: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'deploy') }}
+    outputs:
+      docker-image-tag: ${{ steps.build-image.outputs.tag }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build and push docker image
+        id: build-image
+        uses: DFE-Digital/github-actions/build-docker-image@master
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          target: web
+          context: .
+
+  deploy_review:
+    name: Deploy to review environment
+    concurrency: deploy_review_${{ github.event.pull_request.number }}
+    runs-on: ubuntu-latest
+    needs: [build]
+    environment:
+      name: review-aks
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - uses: ./.github/actions/deploy-environment
+        id: deploy
+        with:
+          environment: review-aks
+          docker-image: ${{ needs.build.outputs.docker-image-tag }}
+          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+          pull-request-number: ${{ github.event.pull_request.number }}
+
+      - uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Set kubectl
+        uses: DFE-Digital/github-actions/set-kubectl@master
+
+      - name: Seed review app
+        shell: bash
+        if: github.event.number != ''
+        run: |
+          make ci review-aks get-cluster-credentials
+          kubectl exec -n srtl-development deployment/claim-additional-payments-for-teaching-review-${{ github.event.pull_request.number }}-worker -- sh -c "DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/prepare-database"
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+
+      - name: Post comment to Pull Request ${{ github.event.number }}
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: aks
+          message: |
+            ### Deployments
+
+            | Journey             | URL                                  |
+            | ------------------- | ------------------------------------ |
+            | Additional Payments | <${{ env.APP_URL }}/additional-payments/claim> |
+            | Student Loans       | <${{ env.APP_URL }}/student-loans/claim>       |
+            | Admin               | <${{ env.APP_URL }}/admin>                     |

.github/workflows/ci.yaml

@@ -24,6 +24,52 @@ jobs:
       - name: Linting - Standardrb
         run: bin/rails standard
 
+  lint-dfe-analytics:
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16-alpine
+        ports:
+          - '5432:5432'
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+    env:
+      RAILS_ENV: test
+      DFE_TEACHERS_PAYMENT_SERVICE_DATABASE_USERNAME: postgres
+      DFE_TEACHERS_PAYMENT_SERVICE_DATABASE_PASSWORD: password
+      DFE_TEACHERS_PAYMENT_SERVICE_DATABASE_HOST: localhost
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Install Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - name: Prepare DB
+        run: bin/rails db:prepare
+      - name: Linting - DfE::Analytics
+        run: bin/rails dfe:analytics:check
+
+  lint-terraform:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.8.4
+      - run: terraform fmt -check -diff
+      - uses: actions/cache@v4
+        name: Cache tflint plugin dir
+        with:
+          path: ~/.tflint.d/plugins
+          key: tflint-${{ hashFiles('.tflint.hcl') }}
+      - uses: terraform-linters/setup-tflint@v4
+        name: Setup TFLint
+      - run: tflint --init
+      - run: tflint -f compact
+
   test:
     runs-on: ubuntu-latest
     strategy:
@@ -33,7 +79,7 @@ jobs:
         ci_node_index: [0, 1, 2, 3, 4, 5, 6, 7]
     services:
       postgres:
-        image: postgres:11-alpine
+        image: postgres:16-alpine
         ports:
           - '5432:5432'
         env:

.github/workflows/delete_review_app.yml

@@ -0,0 +1,32 @@
+name: Delete review app on AKS
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  delete-review-app:
+    name: Delete review app ${{ github.event.pull_request.number }}
+    concurrency: deploy_review_${{ github.event.pull_request.number }}
+    runs-on: ubuntu-latest
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'deploy') }}
+    environment: review-aks
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: 1.8.4
+          terraform_wrapper: false
+
+      - uses: DFE-Digital/github-actions/set-kubelogin-environment@master
+        with:
+          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Terraform destroy
+        run: |
+          make ci review-aks terraform-destroy-aks
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}

.gitignore

@@ -55,3 +55,11 @@ coverage/
 
 # Assets
 public/assets
+
+# AKS deployment
+.terraform
+terraform/application/vendor
+terraform/domains/environment_domains/vendor
+terraform.tfstate*
+bin/terrafile
+bin/konduit.sh

.tflint.hcl

@@ -0,0 +1,10 @@
+plugin "terraform" {
+  enabled = true
+  preset  = "recommended"
+}
+
+plugin "azurerm" {
+    enabled = true
+    version = "0.26.0"
+    source  = "github.com/terraform-linters/tflint-ruleset-azurerm"
+}

.tool-versions

@@ -1,3 +1,3 @@
-terraform 1.2.4
+terraform 1.8.4
 ruby 3.2.4
 nodejs 16.17.0

Makefile

@@ -34,12 +34,13 @@ production:
 	$(eval CONTAINER_NAME=s118p01conttfstate)
 	$(eval DEPLOY_ENV=production)
 
-.PHONY: review_aks
-review_aks: test-cluster
+.PHONY: review-aks
+review-aks: test-cluster
 	$(if ${PR_NUMBER},,$(error Missing PR_NUMBER))
 	$(eval ENVIRONMENT=review-${PR_NUMBER})
 	$(eval export TF_VAR_environment=${ENVIRONMENT})
 	$(eval include global_config/review.sh)
+	echo https://claim-additional-payments-for-teaching-review-$(PR_NUMBER).test.teacherservices.cloud will be created in aks
 
 set-azure-account:
 	az account set -s ${AZ_SUBSCRIPTION}
@@ -55,27 +56,63 @@ terraform-init: set-azure-account
 		-backend-config=container_name=${CONTAINER_NAME} \
 		${BACKEND_KEY}
 
+terraform-init-aks: composed-variables bin/terrafile set-azure-account-aks
+	$(if ${DOCKER_IMAGE_TAG}, , $(eval DOCKER_IMAGE_TAG=master))
+
+	./bin/terrafile -p terraform/application/vendor/modules -f terraform/application/config/$(CONFIG)_Terrafile
+	terraform -chdir=terraform/application init -upgrade -reconfigure \
+		-backend-config=resource_group_name=${RESOURCE_GROUP_NAME} \
+		-backend-config=storage_account_name=${STORAGE_ACCOUNT_NAME} \
+		-backend-config=key=${ENVIRONMENT}_kubernetes.tfstate
+
+	$(eval export TF_VAR_azure_resource_prefix=${AZURE_RESOURCE_PREFIX})
+	$(eval export TF_VAR_config_short=${CONFIG_SHORT})
+	$(eval export TF_VAR_service_name=${SERVICE_NAME})
+	$(eval export TF_VAR_service_short=${SERVICE_SHORT})
+	$(eval export TF_VAR_docker_image=${DOCKER_REPOSITORY}:${DOCKER_IMAGE_TAG})
+
 terraform-plan: terraform-init
 	terraform -chdir=azure/terraform plan \
 		-var="input_container_version=${IMAGE_TAG}" \
 		-var-file workspace_variables/${DEPLOY_ENV}.tfvars.json
 
+terraform-plan-aks: terraform-init-aks
+	terraform -chdir=terraform/application plan -var-file "config/${CONFIG}.tfvars.json"
+
 terraform-apply: terraform-init
 	terraform -chdir=azure/terraform apply \
 		-var="input_container_version=${IMAGE_TAG}" \
 		-var-file workspace_variables/${DEPLOY_ENV}.tfvars.json
 
+terraform-apply-aks: terraform-init-aks
+	terraform -chdir=terraform/application apply -var-file "config/${CONFIG}.tfvars.json" ${AUTO_APPROVE}
+
 terraform-destroy: terraform-init
 	terraform -chdir=azure/terraform destroy \
 		-var="input_container_version=${IMAGE_TAG}" \
 		-var-file workspace_variables/${DEPLOY_ENV}.tfvars.json
 
+terraform-destroy-aks: terraform-init-aks
+	terraform -chdir=terraform/application destroy -var-file "config/${CONFIG}.tfvars.json" ${AUTO_APPROVE}
+
+domains:
+	$(eval include global_config/domains.sh)
+
 composed-variables:
 	$(eval RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg)
 	$(eval KEYVAULT_NAMES='("${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-app-kv", "${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-inf-kv")')
 	$(eval STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}${CONFIG_SHORT}tfsa)
 	$(eval LOG_ANALYTICS_WORKSPACE_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-log)
 
+ci:
+	$(eval AUTO_APPROVE=-auto-approve)
+	$(eval SKIP_AZURE_LOGIN=true)
+	$(eval SKIP_CONFIRM=true)
+
+bin/terrafile: ## Install terrafile to manage terraform modules
+	curl -sL https://github.com/coretech/terrafile/releases/download/v${TERRAFILE_VERSION}/terrafile_${TERRAFILE_VERSION}_$$(uname)_x86_64.tar.gz \
+		| tar xz -C ./bin terrafile
+
 set-what-if:
 	$(eval WHAT_IF=--what-if)
 
@@ -99,3 +136,15 @@ validate-arm-resources: set-what-if arm-deployment ## Validate ARM resource depl
 test-cluster:
 	$(eval CLUSTER_RESOURCE_GROUP_NAME=s189t01-tsc-ts-rg)
 	$(eval CLUSTER_NAME=s189t01-tsc-test-aks)
+
+production-cluster:
+	$(eval CLUSTER_RESOURCE_GROUP_NAME=s189p01-tsc-pd-rg)
+	$(eval CLUSTER_NAME=s189p01-tsc-production-aks)
+
+get-cluster-credentials: set-azure-account-aks
+	az aks get-credentials --overwrite-existing -g ${CLUSTER_RESOURCE_GROUP_NAME} -n ${CLUSTER_NAME}
+	kubelogin convert-kubeconfig -l $(if ${GITHUB_ACTIONS},spn,azurecli)
+
+bin/konduit.sh:
+	curl -s https://raw.githubusercontent.com/DFE-Digital/teacher-services-cloud/main/scripts/konduit.sh -o bin/konduit.sh \
+		&& chmod +x bin/konduit.sh

README.md

@@ -280,21 +280,12 @@ https://test.additional-teaching-payment.education.gov.uk.
 The release process for Production is documented in
 [`docs/release-process.md`](docs/release-process.md)
 
-### Heroku Review Apps
-
-Pull requests in the
-[GitHub Repository](https://github.com/DFE-Digital/dfe-teachers-payment-service)
-will automatically have a
-[review app](https://devcenter.heroku.com/articles/github-integration-review-apps)
-created in Heroku once CI has passed.
-
-For more information, see the [app's Heroku docs](docs/heroku.md)
-
 ### Azure review apps
 
-Pull requests automatically build a review app in Azure using the same
-technologies as the production environment. Each one can be accessed via this
-URL: `https://s118d02-app-pr-<PR NUMBER>-as.azurewebsites.net`
+Adding the 'deploy' label to pull requests builds a review app in Azure using
+the same technologies as the production environment. Each one can be accessed
+via this URL:
+`https://claim-additional-payments-for-teaching-review-<PR_NUMBER>.test.teacherservices.cloud`
 
 The review app is then destroyed when the PR is closed or merged.
 

config/environments/production.rb

@@ -41,6 +41,7 @@
 
   # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies.
   config.force_ssl = true
+  config.ssl_options = {redirect: {exclude: ->(request) { /healthcheck/.match?(request.path) }}}
 
   # Include generic and useful information about system operation, but avoid logging too much
   # information to avoid inadvertent exposure of personally identifiable information (PII).
@@ -92,6 +93,10 @@
     logger = ActiveSupport::Logger.new($stdout)
     logger.formatter = config.log_formatter
     config.logger = ActiveSupport::TaggedLogging.new(logger)
+
+    $stdout.sync = true
+    config.rails_semantic_logger.add_file_appender = false
+    config.semantic_logger.add_appender(io: $stdout, formatter: config.rails_semantic_logger.format)
   end
 
   # Do not dump schema after migrations.