Merge pull request #3192 from raft-tech/3046-plg-cloud

PLG Cloud.gov

.circleci/build-and-test/commands.yml

@@ -25,7 +25,7 @@
             fi
             echo "export CURRENT_FLAG=$CURRENT_FLAG" >> $BASH_ENV
       - run:
-          name: Upload code coverage report if target branch
+          name: Upload code coverage report of target branch
           command: codecov -t "$CODECOV_TOKEN" -f <<parameters.coverage-report>> -F "$CURRENT_FLAG"
 
   install-nodejs-machine:

.gitconfig

@@ -14,3 +14,4 @@
 	allowed = .git/config:.*
 	allowed = .gitconfig:.*
 	allowed = .*DJANGO_SECRET_KEY=.*
+	allowed = ./tdrs-backend/plg/loki/manifest.yml:*

.gitignore

@@ -109,4 +109,9 @@ cypress.env.json
 
 # Patches
 *.patch
+
+# Logs
+*.log
+
+# DB seeds
 tdrs-backend/*.pg

Taskfile.yml

@@ -2,11 +2,6 @@ version: '3'
 
 tasks:
 
-  upload-kibana-objs:
-    desc: Upload dashboards to Kibana server
-    cmds:
-      - 'curl -X POST localhost:5601/api/saved_objects/_import -H "kbn-xsrf: true" --form file=@tdrs-backend/tdpservice/search_indexes/kibana_saved_objs.ndjson'
-
   create-network:
     desc: Create the external network
     cmds:
@@ -251,7 +246,7 @@ tasks:
     desc: Open a shell in the frontend container
     dir: tdrs-frontend
     cmds:
-      - docker-compose -f docker-compose.yml exec tdp-frontend sh
+      - docker-compose -f docker-compose.yml exec tdp-frontend bash
 
   up:
     desc: Start both frontend and backend web servers
@@ -268,4 +263,4 @@ tasks:
   help:
     desc: Show this help message
     cmds:
-      - task --list
+      - task --list

codecov.yml

@@ -42,4 +42,7 @@ flags:
     carryforward: true
 
 ignore:
-  - "tdrs-backend/tdpservice/scheduling/db_backup.py"
+  - "tdrs-backend/tdpservice/scheduling/db_backup.py"
+  - "tdrs-backend/tdpservice/search_indexes/admin/mulitselect_filter.py"
+  - "tdrs-backend/tdpservice/email/helpers/account_access_requests.py"
+  - "tdrs-backend/tdpservice/search_indexes/admin/filters.py"

scripts/cf-check.sh

@@ -3,15 +3,15 @@ set -e
 if command -v cf /dev/null 2>&1; then
     echo The command cf is available
 else
-
     apt-get update
     apt-get install wget gnupg2 apt-transport-https
 
-    wget -q -O - https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | sudo apt-key add -
-
-    echo "deb https://packages.cloudfoundry.org/debian stable main" | sudo tee /etc/apt/sources.list.d/cloudfoundry-cli.list
-
-    apt-get update
-    apt-get install cf7-cli
+    NEXUS_ARCHIVE="cf7-cli_7.7.13_linux_x86-64.tgz"
+    NEXUS_URL="https://tdp-nexus.dev.raftlabs.tech/repository/tdp-bin/cloudfoundry-cli/$NEXUS_ARCHIVE"
+    curl $NEXUS_URL -o $NEXUS_ARCHIVE  # prefers anonymous, use of -u failed.
+    tar xzf $NEXUS_ARCHIVE
+    mv ./cf /usr/local/bin/
+    chmod +x /usr/local/bin/cf
+    cf --version
 
 fi

scripts/deploy-backend.sh

@@ -105,6 +105,27 @@ update_kibana()
   cf run-task $CGAPPNAME_BACKEND --command "$CMD" --name kibana-obj-upload
 }
 
+prepare_promtail() {
+  pushd tdrs-backend/plg/promtail
+  CONFIG=config.yml
+  yq eval -i ".scrape_configs[0].job_name = \"system-$backend_app_name\""  $CONFIG
+  yq eval -i ".scrape_configs[0].static_configs[0].labels.job = \"system-$backend_app_name\""  $CONFIG
+  yq eval -i ".scrape_configs[1].job_name = \"backend-$backend_app_name\""  $CONFIG
+  yq eval -i ".scrape_configs[1].static_configs[0].labels.job = \"backend-$backend_app_name\""  $CONFIG
+  popd
+}
+
+update_plg_networking() {
+  # Need to switch the space after deploy since we're not always in dev space to handle specific networking from dev
+  # PLG apps to the correct backend app.
+  cf target -o hhs-acf-ofa -s tanf-dev
+  cf add-network-policy prometheus "$CGAPPNAME_BACKEND" -s "$CF_SPACE" --protocol tcp --port 8080
+  cf target -o hhs-acf-ofa -s "$CF_SPACE"
+
+  # Promtial needs to send logs to Loki
+  cf add-network-policy "$CGAPPNAME_BACKEND" loki -s "tanf-dev" --protocol tcp --port 8080
+}
+
 update_backend()
 {
     cd tdrs-backend || exit
@@ -143,6 +164,9 @@ update_backend()
     # Add network policy to allow frontend to access backend
     cf add-network-policy "$CGAPPNAME_FRONTEND" "$CGAPPNAME_BACKEND" --protocol tcp --port 8080
 
+    # Add PLG routing
+    update_plg_networking
+
     if [ "$CF_SPACE" = "tanf-prod" ]; then
       # Add network policy to allow backend to access tanf-prod services
       cf add-network-policy "$CGAPPNAME_BACKEND" clamav-rest --protocol tcp --port 9000
@@ -229,6 +253,7 @@ else
   CYPRESS_TOKEN=$CYPRESS_TOKEN
 fi
 
+prepare_promtail
 if [ "$DEPLOY_STRATEGY" = "rolling" ] ; then
     # Perform a rolling update for the backend and frontend deployments if
     # specified, otherwise perform a normal deployment

scripts/deploy-frontend.sh

@@ -13,6 +13,7 @@ CF_SPACE=${5}
 ENVIRONMENT=${6}
 
 env=${CF_SPACE#"tanf-"}
+frontend_app_name=$(echo $CGHOSTNAME_FRONTEND | cut -d"-" -f3)
 
 # Update the Kibana name to include the environment
 KIBANA_BASE_URL="${CGAPPNAME_KIBANA}-${env}.apps.internal"
@@ -52,7 +53,7 @@ update_frontend()
 
     cf set-env "$CGHOSTNAME_FRONTEND" BACKEND_HOST "$CGHOSTNAME_BACKEND"
     cf set-env "$CGHOSTNAME_FRONTEND" KIBANA_BASE_URL "$KIBANA_BASE_URL"
-    
+
     npm run build:$ENVIRONMENT
     unlink .env.production
     mkdir deployment
@@ -86,7 +87,7 @@ update_frontend()
     else
         cf map-route "$CGHOSTNAME_FRONTEND" app.cloud.gov --hostname "${CGHOSTNAME_FRONTEND}"
     fi
-    
+
     cd ../..
     rm -r tdrs-frontend/deployment
 }

scripts/localstack-setup.sh

@@ -5,3 +5,7 @@ awslocal s3api create-bucket --bucket $AWS_BUCKET --region $AWS_REGION_NAME
 
 # Enable object versioning on the bucket
 awslocal s3api put-bucket-versioning --bucket $AWS_BUCKET --versioning-configuration Status=Enabled
+
+# Add bucket for Loki to store logs
+awslocal s3api create-bucket --bucket loki-logs --region $AWS_REGION_NAME
+awslocal s3api put-bucket-versioning --bucket loki-logs --versioning-configuration Status=Enabled

tdrs-backend/Dockerfile

@@ -9,18 +9,23 @@ ENV DJANGO_SETTINGS_MODULE=tdpservice.settings.local
 ENV DJANGO_CONFIGURATION=Local
 # Allows docker to cache installed dependencies between builds
 COPY Pipfile Pipfile.lock /tdpapp/
+COPY sources.list /etc/apt/sources.list
 WORKDIR /tdpapp/
 # Download latest listing of available packages:
 RUN apt-get -y update
 # Upgrade already installed packages:
 RUN apt-get -y upgrade
+# Install packages:
+RUN apt-get install -y gcc graphviz graphviz-dev libpq-dev python3-dev vim curl ca-certificates
+
 # Postgres client setup
-RUN apt --purge remove postgresql postgresql-* && apt install -y postgresql-common curl ca-certificates && install -d /usr/share/postgresql-common/pgdg && \
+#RUN bash -c 'echo "deb [trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pdpg main" >> /etc/apt/sources.list'
+RUN apt-get update -y && apt-get upgrade -y
+RUN apt install -y postgresql-common && install -d /usr/share/postgresql-common/pgdg && \
+sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc trusted=yes] https://tdp-nexus.dev.raftlabs.tech/repository/apt-proxy-postgres/ bullseye-pgdg main" >> /etc/apt/sources.list' && \
 curl -o /usr/share/postgresql-common/pgdg/apt.postgresql.org.asc --fail https://www.postgresql.org/media/keys/ACCC4CF8.asc && \
-sh -c 'echo "deb [signed-by=/usr/share/postgresql-common/pgdg/apt.postgresql.org.asc] https://apt.postgresql.org/pub/repos/apt bullseye-pgdg main" > /etc/apt/sources.list.d/pgdg.list' && \
-apt -y update && apt install postgresql-client-15 -y
-# Install packages:
-RUN apt install -y gcc graphviz graphviz-dev libpq-dev python3-dev vim
+apt -y update && apt -y upgrade && apt install postgresql-client-15 -y
+
 # Install pipenv
 RUN pip install --upgrade pip pipenv
 RUN pipenv install --dev --system --deploy

tdrs-backend/Pipfile

@@ -1,6 +1,7 @@
 [[source]]
 name = "pypi"
-url = "https://pypi.org/simple"
+trusted-host = "https://tdp-nexus.dev.raftlabs.tech/"
+url = "https://tdp-nexus.dev.raftlabs.tech/repository/pypi-proxy/simple"
 verify_ssl = true
 
 [dev-packages]
@@ -62,4 +63,4 @@ django_prometheus = "==2.3.1"
 sentry-sdk = "==2.11.0"
 
 [requires]
-python_version = "3.10.8"
+python_version = "3.10.8"