restructure repo

.github/workflows/_deployment.yaml

@@ -147,16 +147,13 @@ jobs:
           export HCA_PROJECT_ID="${{ vars.hca_project_id }}"
           export TCO_ID="${{ vars.tco_id }}"
           export ENVIRONMENT="${{ inputs.environment }}"
-          envsubst < ./sample-deploy-code/service-yaml/container-${{ inputs.environment }}.yaml > container-${{ inputs.environment }}.yaml
+          envsubst < ./service-yaml/container-${{ inputs.environment }}.yaml > container-${{ inputs.environment }}.yaml
 
       - name: Deploy to ${{ inputs.environment }} Cloud Run
         id: deploy-dev-qa
         uses: google-github-actions/deploy-cloudrun@v2
-        continue-on-error: true
         with:
           project_id: ${{ vars.gcp_project_id }}
           service: ${{ vars.service_name }}
           region: ${{ vars.region }}
-          metadata: container-${{ inputs.environment }}.yaml
-      - run: echo "OK"
-        if: job.steps.bad.status == failure()
+          metadata: container-${{ inputs.environment }}.yaml

.github/workflows/ci.yml

@@ -1,4 +1,4 @@
-name: Continuous Integration (CI) Build
+name: Continuous Integration (CI) Build for QA and Dev Environments
 
 on:
   pull_request:

.github/workflows/liquibase-ci.yml

@@ -1,4 +1,4 @@
-name: Reusable Continuous Integration (CI) Build Workflow
+name: Reusable Continuous Integration (CI) Build Workflow for Dev, QA and prod-plan
 
 on:
   workflow_call:
@@ -31,6 +31,9 @@ jobs:
       pull-requests: write
       actions: read
       security-events: write
+    defaults:
+      run:
+        working-directory: ./db/
 
     steps:
     - id: 'auth'
@@ -70,15 +73,15 @@ jobs:
 
     - name: Liquibase Status
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} status --verbose
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} status --verbose
 
     - name: Liquibase Validate
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} validate   
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} validate   
 
     - name: Liquibase Print SQL
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} --output-file=./artifacts/update-sql.sql update-sql
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} --output-file=./artifacts/update-sql.sql update-sql
 
     - name: Set Timestamp
       run: echo "TIMESTAMP=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_ENV
@@ -92,4 +95,4 @@ jobs:
     - name: Liquibase Update
       if: github.event_name == 'push'
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} update
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} update

.github/workflows/liquibase-release.yml

@@ -1,4 +1,4 @@
-name: Reusable Liquibase Publishing Workflow
+name: Reusable Liquibase & Cloud Run Publishing Workflow for Production Environment
 
 on:
   workflow_call:
@@ -30,6 +30,9 @@ jobs:
       pull-requests: write
       actions: read
       security-events: write
+    defaults:
+      run:
+        working-directory: ./db/
 
     steps:
     - id: 'auth'
@@ -69,11 +72,11 @@ jobs:
 
     - name: Liquibase Status
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} status --verbose
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} status --verbose
 
     - name: Liquibase Print SQL
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} --output-file=./artifacts/update-sql.sql update-sql
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} --output-file=./artifacts/update-sql.sql update-sql
 
     - name: Set Timestamp
       run: echo "TIMESTAMP=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_ENV
@@ -86,4 +89,4 @@ jobs:
 
     - name: Liquibase Update
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} update
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{vars.LOG_LEVEL}} --sql-log-level ${{vars.SQL_LOG_LEVEL}} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} update

.github/workflows/liquibase-rollback.yml

@@ -1,4 +1,4 @@
-name: Liquibase Rollback Reusable Workflow
+name: Reusable Liquibase Rollback Reusable Workflow
 
 on:
   workflow_call:
@@ -31,6 +31,9 @@ jobs:
       pull-requests: write
       actions: read
       security-events: write
+    defaults:
+      run:
+        working-directory: ./db/
 
     steps:
     - id: 'auth'
@@ -70,7 +73,7 @@ jobs:
 
     - name: Liquibase Print Rollback SQL
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} --output-file=./artifacts/rollback-sql.sql rollback-sql ${{ vars.LAST_STABLE_VERSION_TAG }}
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} --output-file=./artifacts/rollback-sql.sql rollback-sql ${{ vars.LAST_STABLE_VERSION_TAG }}
 
     - name: Set Timestamp
       run: echo "TIMESTAMP=$(date +"%Y%m%d%H%M%S")" >> $GITHUB_ENV
@@ -83,4 +86,4 @@ jobs:
 
     - name: Liquibase Rollback
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} rollback ${{ vars.LAST_STABLE_VERSION_TAG }}
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} rollback ${{ vars.LAST_STABLE_VERSION_TAG }}

.github/workflows/liquibase-tag.yml

@@ -1,4 +1,4 @@
-name: Reusable Continuous Integration (CI) Build Workflow
+name: Reusable Workflow for Liquibase Tagging
 
 on:
   workflow_call:
@@ -37,6 +37,9 @@ jobs:
       pull-requests: write
       actions: read
       security-events: write
+    defaults:
+      run:
+        working-directory: ./db/
 
     steps:
     - id: 'auth'
@@ -76,7 +79,7 @@ jobs:
 
     - name: Liquibase Tag
       run: |
-        ./liquibase/liquibase --defaultsFile=./liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} tag --tag=${{ inputs.liquibase-tag }}
+        ./liquibase/liquibase --defaultsFile=./db/liquibase/liquibase.properties --log-level ${{ vars.LOG_LEVEL }} --sql-log-level ${{ vars.SQL_LOG_LEVEL }} --url ${{ vars.URL }} --username ${{ secrets.USERNAME }} tag --tag=${{ inputs.liquibase-tag }}
 
     - name: Update Repository Env Variable
       run: |

.github/workflows/release.yml

@@ -1,4 +1,5 @@
-name: Release
+
+name: Release Workflow for PROD Environments
 on:
   release:
     types: [published]
@@ -40,7 +41,7 @@ jobs:
       environment: 'prod'
     secrets: inherit
 
-    prod-liquibase-rollback:
+  prod-liquibase-rollback:
       name: PROD liquibase rollback
       uses: ./.github/workflows/liquibase-rollback.yml
       if: failure() || cancelled()
@@ -52,7 +53,7 @@ jobs:
       with:
         environment: 'prod'
         runner: 'ubuntu-latest'
-    prod-liquibase-update-tag:
+  prod-liquibase-update-tag:
       name: PROD Liquibase Update tag and repository variable
       needs: [ prod-liquibase-deploy, prod-docker ]
       uses: ./.github/workflows/liquibase-tag.yml

README.md

@@ -1,23 +1,51 @@
-##DB Change Management With Liquibase
-
-This serves as an example liquibase set up following the liquibase's best practices as outlined on the [liquibase website](http://www.liquibase.org/bestpractices.html).
-
-###SqlFire Example
-You will find the required jar files in the lib folder of the packaged version of liquibase. 
-
-* sqlfireclient.jar
-* liquibase-sqlfire-3.0.0.jar (sqlfire extensions for liquibase)
-
-The example creates the example schema that was shipped with verion 1.0 of SqlFire and was used in thier documentation. (Airline, Cities, Flights, etc).
-
-The schema generation uses the functionality added in liquibase 3.0 to use annotated sql scripts. This allows for the use of sqlfire's extended keyword set required for sqlfire schemas (collocate, replicate, etc).
-
-It also loads reference data that was provided with the example sqlfire download.
-
-It is important to note that due to the extended keyword set and changes to the information schema from Apache Derby to support sqlfire's distribution, liquibase funcitionality such as database diff/generateChangelog is not supported.
-
-
-###SqlServer Example
-
-
-###PostgreSql Example
+# Continuous Integration & Continuous Deployment for Cloud Run
+This repository is designed to lint code, scan code and deploy packaged code to Cloud Run. It manages the promotion process from development to production through pull requests and releases while also allowing for canary deployments through workflow dispatch. 
+
+## Prerequisites
+ * Develop, QA and Production Google Cloud projects are created.
+ * Workload identity [pools and providers](https://cloud.google.com/iam/docs/manage-workload-identity-pools-providers) have been created and added to [GitHub Secrets](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions).
+ * The roles/iam.workloadIdentityUser role has been granted to the service account that will authenticate with workload identity federation. See [documentation](https://cloud.google.com/blog/products/identity-security/secure-your-use-of-third-party-tools-with-identity-federation) for guidance on provisioning workload identity with GitHub Actions.
+ * The service account has been added to GitHub Secrets and has the following roles. 
+    * roles/artifactregistry.writer
+    * roles/run.admin
+    * roles/secretmanager.secretAccessor
+    * roles/cloudsql.client
+ * All required APIs for Google Cloud services have been enabled.
+ * [Artifact Registry](https://cloud.google.com/artifact-registry/docs/docker/store-docker-container-images) repository must be created. 
+ * Branches are created for develop and main. 
+ * Environments for dev, qa and prod are created within GitHub Environments. 
+ * Following environment variables are created in each environment. 
+    * ARTIFACT_REGISTRY_PROJECT
+    * ARTIFACT_REGISTRY_REPO
+    * CLOUD_RUN_SA
+    * GCP_PROJECT_ID
+    * SERVICE_NAME
+ * Following repsository variables are created in each environment. 
+    * CODE_DIRECTORY
+    * LANGUAGE
+    * REGION
+
+
+## Deploying to DEV 
+1. Create a new feature branch from main, make necessary changes to your code. 
+2. Raise a pull request from your new feature branch to develop. 
+3. When the pull request is raised, the workflow will lint the code to ensure quality and CodeQL will scan the code to ensure there are no vulnerabilities in the code. 
+4. If there are no linting issues or CodeQL vulnerabilities, the pull request can be merged after the workflow completes and approvals are received. 
+5. Once merged, the image would be built and pushed to Artifact Registry in the Google Cloud project used for development.
+6. In develop, once the image is built, it will immediately be deployed to Cloud Run as a new revision in the development project. 
+
+## Deploying to QA 
+1. Raise a pull request from develop to main. This will not trigger a workflow. 
+2. Once develop is merged to main, the image is built and pushed to the **production** Artifact Registry repository. The reason this is done is to test the image in QA, then re-tag the image for use in production if QA testing is sucessful. 
+3. Once the image is pushed to the production Artifact Registry, Cloud Run will pull the image and deploy it to the QA Google Cloud project. 
+
+## Canary Deployments to Production 
+1. Go to the Google Cloud console to retrieve the existing revision name. 
+2. Go to the workflow named *Canary Deployment to Cloud Run* to trigger the workflow from workflow dispatch. 
+3. Insert the existing revision name into the field named *Old Revision Name* and set the traffic split so it adds up to 100%. Feel free to do this a few times to gradually rollout the new revision, increasing the traffic to the new revision each time. 
+4. In the console, you can see the new revision will have the URL tag *blue* and the old revision will have the URL tag *green*. This can be used to see which users hit each revision or to have users test the new revision by using the revision URL. 
+
+## Deploying to Production
+1. Create a [GitHub Release](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository) to trigger a new production deployment. Once the release is published, the workflow will be triggered. 
+2. This environment should have approvals on the workflow, so approvers will need to approve before the image build and before the Cloud Run deployment. 
+3. This workflow will re-tag the image with the release tag and will deploy 100% of traffic to the new revision. 

liquibase-postgres-master.xml → db/liquibase-postgres-master.xml

@@ -7,7 +7,7 @@
         http://www.liquibase.org/xml/ns/dbchangelog-ext http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-ext.xsd">
 	<!-- Include postgresql files -->
 <!--	<include file="changesets/postgresql/initial-schema-postgresql.sql"/>-->
-	    <includeAll path="changesets/postgresql/"/>
+	    <includeAll path="/changesets/postgresql/" relativeToChangelogFile="true" />
 
 
 </databaseChangeLog>

docs/Deployment.md

@@ -9,26 +9,36 @@ There are two Github workflows under the .github/workflows/ folder used to facil
     * Establish cloudSQL proxy connection to the SQL database
     * Setup liquibase and run liquibase status. Print SQL to be executed as an artifact to the workflow
     * Additionally, if triggered due to a push request, the liquibase update is run to execute the changes on the respective environment based on which the push is triggered (dev for develop branch and QA for main branch)
+    * Build the docker application and deploy it to cloud run
+    * If the Docker deployment is successful, liquibase changelog is tagged with the commit ID and the Repository environment Variable for the latest stable commit will be updated to this commit ID.
+    * If the Docker deployment fails, liquibase rollback will be triggered to revert the database to the last stable version tag based on the repository environment variable `LAST_STABLE_VERSION_TAG`
 - release.yaml
     * Executes database changes in the production environment on creation of a release from the main branch.
+    * Similar to the `ci.yaml` workflow, app is deployed to cloud run. Based on the success or failure of app deployment, the liquibase changelog will be tagged or the changes will be reverted to the last stable commit.
 
 ## Deployment into dev, QA, and prod
 ### Dev environment
 - Make a new branch, `your-branch`, off of the `develop` branch and make your changes.
 - Create a PR merging `your-branch` into `develop`.
     - This will run checks on your changes, attaches the SQL script to be executed as an artifact to the workflow run. This should be reviewed and approved before merging.
 - Once merged, liquibase update command will be run and changes will be deployed to the `dev` environment.
+- Docker image is built and deployed to Cloud Run in the `dev` environment
+- If the Cloud Run deployment is successful, then Liquibase version is tagged and updated in the `LAST_STABLE_VERSION_TAG` Github Environment variable for `dev`
+- If the Cloud Run deployment is failed, then Liquibase Rollback is run to rollback the database changes to the current `LAST_STABLE_VERSION_TAG` in the Github Environment variables for `dev`
 - Test on `dev`
 ### QA environment
 - When testing is completed successfully in dev environment create a PR merging `develop` into `main`.
     - This will again run checks on your changes, adds a SQL script to be reviewed before merging to the workflow.
-- Then merge `develop` into `main` and the database changes will be deployed to the `qa` environment.
+- Then merge `develop` into `main` and the changes will be deployed to the `qa` environment.
+- Docker image is built and deployed to Cloud Run in the `qa` environment
+- If the Cloud Run deployment is successful, then Liquibase version is tagged and updated in the `LAST_STABLE_VERSION_TAG` Github Environment variable for `qa`
+- If the Cloud Run deployment is failed, then Liquibase Rollback is run to rollback the database changes to the current `LAST_STABLE_VERSION_TAG` in the Github Environment variables for `qa`
 - Test again on `qa ` for quality assurance.
 ### Prod environment
 - When QA is completed successfully, you can now promote to `prod`.
 - To do this, you will have to create release from the `main` branch with the appropriate tag and release notes.
-- Once this is done, a plan will be run on the github workflow and an artifact of the SQL script to be run will be attached to the workflow.
-- If approved and published, this will execute the same database changes on `prod`.
+- Once this is done, a plan will be run on the github workflow using an idential `prod-plan` github environment and an artifact of the SQL script to be run will be attached to the workflow.
+- Once the artifact is reviewed and the `prod` job workflow is approved, this will execute the same database changes and app deployment on `prod`.
 
 
 ## References