Run unit tests using multiple configurations (#1903)

* set FF_TEMPLATE_CATEGORY on for testing

* feat: add new workflow to run tests with ff OFF

* task: update config on the fly to turn FF on; echo config.py contents

* task: echo config.py contents before tests start

* task: turn the FF back on in the config

* fix: fix unrelated, failing test

* fix: load production set of feature flags when running unit tests the second time

- updates `pytest.ini` to set `NOTIFY_ENVIRONMENT` = 'production' when running the tests
- update the 'production' config to use the same feature flags as prod does right now
- update the 'test' config to use the same feature flags as staging does right now

* chore: rename action

* chore: update prod config

* chore: formatting

* fix: add more things to the prod config (taken from the Test config)

* fix(debug): cat the `pytest.ini` instead of `config.py`

* fix: try changing some configs

* fix: tweak prod config again

* chore: update workflow names, filename

* debug: add a few more debug things

* fix: put back configs im unsure of

* fix: try doing these overrides in pytest.ini so they dont affect prod

* fix: try unsetting these values

* fix: try another thing

* fix: add new config for running prod feature flags; revert production config

* fix: use new config for 2nd unit test run

* chore: formatting

* test: force a test to fail when FF is OFF

* chore: formatting

* chore: remove debugging code

* Update .github/workflows/test_prod_config.yaml

.github/workflows/test.yaml

@@ -3,7 +3,7 @@ on:
   schedule:
     - cron: '1 0-12,20-23 * * 1-5'
     - cron: '1 * * * 0,6'
-name: Continuous Integration Testing
+name: Continuous Integration Testing (staging feature flags)
 jobs:
   test:
     runs-on: ubuntu-latest

.github/workflows/test_prod_config.yaml

@@ -0,0 +1,49 @@
+on:
+  push:
+  schedule:
+    - cron: '1 0-12,20-23 * * 1-5'
+    - cron: '1 * * * 0,6'
+name: Continuous Integration Testing (prod feature flags)
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
+      with:
+        python-version: '3.10'
+    - uses: actions/setup-node@1a4442cacd436585916779262731d5b162bc6ec7 # v3.8.2
+      with:
+        node-version: '16.x'
+    - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Install poetry
+      env:
+        POETRY_VERSION: "1.7.1"
+      run: pip install poetry==${POETRY_VERSION} && poetry --version
+    - name: Install requirements
+      run: poetry install --with test
+    - uses: actions/cache@e12d46a63a90f2fae62d114769bbf2a179198b5c # v3.3.3
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    
+    - name: Check for missing translations
+      run: make test-translations
+    - name: Update NOTIFY_ENVIRONMENT for pytest
+      run: sed -i 's/NOTIFY_ENVIRONMENT=test/NOTIFY_ENVIRONMENT=production_FF/' pytest.ini
+    - run: /bin/bash -c "scripts/bootstrap.sh && poetry run make test"
+      env:
+        A11Y_TRACKER_KEY: ${{ secrets.A11Y_TRACKER_KEY }}
+        MIXPANEL_PROJECT_TOKEN: <project_token>
+    - name: Notify Slack channel if this job failed in default branch
+      if: ${{ failure() && github.ref == 'refs/heads/main' }}
+      run: |
+        json="{'text':'<!here> CI is failing in <https://github.com/cds-snc/notification-admin/|notification-admin> !'}"
+        curl -X POST -H 'Content-type: application/json' --data "$json"  ${{ secrets.SLACK_WEBHOOK }}

app/config.py

@@ -190,7 +190,7 @@ class Development(Config):
     SESSION_PROTECTION = None
     SYSTEM_STATUS_URL = "https://localhost:3000"
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
-    FF_TEMPLATE_CATEGORY = True
+    FF_TEMPLATE_CATEGORY = False
 
 
 class Test(Development):
@@ -215,9 +215,52 @@ class Test(Development):
     FF_SALESFORCE_CONTACT = False
     SYSTEM_STATUS_URL = "https://localhost:3000"
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
+
+    FF_SPIKE_SMS_DAILY_LIMIT = True
+    FF_SMS_PARTS_UI = False
+    FF_BOUNCE_RATE_V1 = True
+    FF_BOUNCE_RATE_V15 = True
+    FF_BOUNCE_RATE_BACKEND = True
+    FF_ABTEST_SERVICE_ID = ""
+    FF_NEW_BRANDING = True
+    FF_TOU = True
     FF_TEMPLATE_CATEGORY = True
 
 
+class ProductionFF(Config):
+    ADMIN_CLIENT_SECRET = os.environ.get("ADMIN_CLIENT_SECRET", "dev-notify-secret-key")
+    ANTIVIRUS_API_HOST = "https://test-antivirus"
+    ANTIVIRUS_API_KEY = "test-antivirus-secret"
+    API_HOST_NAME = os.environ.get("API_HOST_NAME", "http://localhost:6011")
+    ASSET_DOMAIN = "static.example.com"
+    CRM_ORG_LIST_URL = "test-domain-dot-com"
+    CRM_GITHUB_PERSONAL_ACCESS_TOKEN = "not-a-real-token"
+    DANGEROUS_SALT = os.environ.get("DANGEROUS_SALT", "dev-notify-salt")
+    DEBUG = True
+    DEBUG_KEY = "debug"
+    MOU_BUCKET_NAME = "test-mou"
+    NOTIFY_ENVIRONMENT = "test"
+    SECRET_KEY = ["dev-notify-secret-key"]
+    TEMPLATE_PREVIEW_API_HOST = "http://localhost:9999"
+    TEMPLATE_PREVIEW_API_KEY = "dev-notify-secret-key"
+    TESTING = True
+    WTF_CSRF_ENABLED = False
+    GC_ARTICLES_API = "articles.alpha.canada.ca/notification-gc-notify"
+    FF_SALESFORCE_CONTACT = False
+    SYSTEM_STATUS_URL = "https://localhost:3000"
+    NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
+
+    FF_SPIKE_SMS_DAILY_LIMIT = True
+    FF_SMS_PARTS_UI = False
+    FF_BOUNCE_RATE_V1 = True
+    FF_BOUNCE_RATE_V15 = True
+    FF_BOUNCE_RATE_BACKEND = True
+    FF_ABTEST_SERVICE_ID = ""
+    FF_NEW_BRANDING = True
+    FF_TOU = False
+    FF_TEMPLATE_CATEGORY = False
+
+
 class Production(Config):
     CHECK_PROXY_HEADER = False
     HTTP_PROTOCOL = "https"
@@ -250,6 +293,7 @@ class Dev(Production):
     "test": Test,
     "staging": Staging,
     "production": Production,
+    "production_FF": ProductionFF,
     "scratch": Scratch,
     "dev": Dev,
 }