Merge pull request #807 from 18F/staging

Production deploy to fix sampling in 30 day hostname report

.github/workflows/ci.yml

@@ -0,0 +1,112 @@
+on:
+  push:
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "lts/*"
+          cache: 'npm'
+      - name: Install node dependencies
+        run: npm ci
+      - name: Lint javascript
+        run: npm run lint
+  test:
+    needs: lint
+    runs-on: ubuntu-latest
+    # Start Postgres as a service, wait until healthy. Uses latest Postgres version.
+    services:
+      postgres:
+        image: postgres:latest
+        env:
+          POSTGRES_DB: analytics_reporter_test
+          POSTGRES_USER: analytics
+          POSTGRES_PASSWORD: 123abc
+        ports:
+          - 5432:5432
+        options:
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "lts/*"
+          cache: 'npm'
+      - name: Install node dependencies
+        run: npm ci
+      - name: Run tests
+        run: npm test
+  deploy_dev:
+    needs:
+      - lint
+      - test
+    if: github.ref == 'refs/heads/develop'
+    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
+    with:
+      ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}
+      ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }}
+      APP_NAME: ${{ vars.APP_NAME_DEV }}
+      CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }}
+      CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_DEV }}
+      DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_DEV }}
+      NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_DEV }}
+      S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_DEV }}
+    secrets:
+      ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
+      CF_USERNAME: ${{ secrets.CF_USERNAME_DEV }}
+      CF_PASSWORD: ${{ secrets.CF_PASSWORD_DEV }}
+      GA4_CREDS: ${{ secrets.GA4_CREDS }}
+      NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_DEV }}
+  deploy_stg:
+    needs:
+      - lint
+      - test
+    if: github.ref == 'refs/heads/staging'
+    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
+    with:
+      ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}
+      ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }}
+      APP_NAME: ${{ vars.APP_NAME_STG }}
+      CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }}
+      CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_STG }}
+      DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_STG }}
+      NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_STG }}
+      S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_STG }}
+    secrets:
+      ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
+      CF_USERNAME: ${{ secrets.CF_USERNAME_STG }}
+      CF_PASSWORD: ${{ secrets.CF_PASSWORD_STG }}
+      GA4_CREDS: ${{ secrets.GA4_CREDS }}
+      NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_STG }}
+  deploy_prd:
+    needs:
+      - lint
+      - test
+    if: github.ref == 'refs/heads/master'
+    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
+    with:
+      ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}
+      ANALYTICS_REPORT_EMAIL: ${{ vars.ANALYTICS_REPORT_EMAIL }}
+      APP_NAME: ${{ vars.APP_NAME_PRD }}
+      CF_ORGANIZATION_NAME: ${{ vars.CF_ORGANIZATION_NAME }}
+      CF_SPACE_NAME: ${{ vars.CF_SPACE_NAME_PRD }}
+      DB_SERVICE_NAME: ${{ vars.DB_SERVICE_NAME_PRD }}
+      NEW_RELIC_APP_NAME: ${{ vars.NEW_RELIC_APP_NAME_PRD }}
+      S3_SERVICE_NAME: ${{ vars.S3_SERVICE_NAME_PRD }}
+    secrets:
+      ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
+      CF_USERNAME: ${{ secrets.CF_USERNAME_PRD }}
+      CF_PASSWORD: ${{ secrets.CF_PASSWORD_PRD }}
+      GA4_CREDS: ${{ secrets.GA4_CREDS }}
+      NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY_PRD }}

.github/workflows/deploy.yml

@@ -0,0 +1,91 @@
+on:
+  workflow_call:
+    inputs:
+      ANALYTICS_KEY_FILE_NAME:
+        required: true
+        type: string
+      ANALYTICS_REPORT_EMAIL:
+        required: true
+        type: string
+      APP_NAME:
+        required: true
+        type: string
+      CF_ORGANIZATION_NAME:
+        required: true
+        type: string
+      CF_SPACE_NAME:
+        required: true
+        type: string
+      DB_SERVICE_NAME:
+        required: true
+        type: string
+      NEW_RELIC_APP_NAME:
+        type: string
+      S3_SERVICE_NAME:
+        required: true
+        type: string
+    secrets:
+      ANALYTICS_CREDENTIALS:
+        required: true
+      CF_USERNAME:
+        required: true
+      CF_PASSWORD:
+        required: true
+      GA4_CREDS:
+        required: true
+      NEW_RELIC_LICENSE_KEY:
+
+env:
+  ANALYTICS_CREDENTIALS: ${{ secrets.ANALYTICS_CREDENTIALS }}
+  ANALYTICS_KEY_FILE_NAME: ${{ inputs.ANALYTICS_KEY_FILE_NAME }}
+  ANALYTICS_REPORT_EMAIL: ${{ inputs.ANALYTICS_REPORT_EMAIL }}
+  APP_NAME: ${{ inputs.APP_NAME }}
+  CF_USERNAME: ${{ secrets.CF_USERNAME }}
+  CF_PASSWORD: ${{ secrets.CF_PASSWORD }}
+  CF_ORGANIZATION_NAME: ${{ inputs.CF_ORGANIZATION_NAME }}
+  CF_SPACE_NAME: ${{ inputs.CF_SPACE_NAME }}
+  DB_SERVICE_NAME: ${{ inputs.DB_SERVICE_NAME }}
+  GA4_CREDS: ${{ secrets.GA4_CREDS }}
+  NEW_RELIC_APP_NAME: ${{ inputs.NEW_RELIC_APP_NAME }}
+  NEW_RELIC_LICENSE_KEY: ${{ secrets.NEW_RELIC_LICENSE_KEY }}
+  S3_SERVICE_NAME: ${{ inputs.S3_SERVICE_NAME }}
+
+jobs:
+  deploy_reporter:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Code checkout
+        uses: actions/checkout@v4
+      - name: Install node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "lts/*"
+          cache: 'npm'
+      - name: Install node dependencies
+        run: npm ci
+      - name: Install cloud foundry CLI for interacting with cloud.gov
+        run: |
+          sudo curl -v -L -o cf8-cli-installer_8.7.4_x86-64.deb 'https://packages.cloudfoundry.org/stable?release=debian64&version=8.7.4'
+          sudo dpkg -i cf8-cli-installer_8.7.4_x86-64.deb
+      - name: Write Google GA4 Credentials file from the value in GA4_CREDS env var.
+        run: |
+          echo $GA4_CREDS > ./my-analytics-ga4-65057af58daa.json
+      - name: Run envsubst on manifest.yml to set environment specific values
+        run: |
+          mv manifest.yml manifest.yml.src
+          envsubst < manifest.yml.src > manifest.yml
+          cat manifest.yml
+      - name: Replace config.js and knexfile.js with .cloudgov versions of those files
+        run: |
+          rm ./src/config.js
+          mv ./src/config.cloudgov.js ./src/config.js
+          rm knexfile.js
+          mv knexfile.cloudgov.js knexfile.js
+      - name: Login to cloud.gov and deploy
+        run: |
+          set -e
+          # Log into cloud.gov
+          cf api api.fr.cloud.gov
+          cf login -u $CF_USERNAME -p $CF_PASSWORD -o $CF_ORGANIZATION_NAME -s $CF_SPACE_NAME
+          cf push -f "./manifest.yml" --strategy rolling
+          cf logout

.gitignore

@@ -11,6 +11,7 @@
 npm-debug.log
 *.swp
 .git
+/.nyc_output
 
 # Track selected JSON files
 !package.json

README.md

@@ -418,6 +418,21 @@ Compose:
 docker-compose up
 ```
 
+## Linting
+
+This repo uses Eslint and Prettier for code static analysis and formatting. Run
+the linter with:
+
+```shell
+npm run lint
+```
+
+Automatically fix lint issues with:
+
+```shell
+npm run lint:fix
+```
+
 ## Running the unit tests
 
 The unit tests for this repo require a local PostgreSQL database. You can run a
@@ -441,6 +456,16 @@ Run the tests (pre-test hook runs DB migrations):
 npm test
 ```
 
+### Running the unit tests with code coverage reporting
+
+If you wish to see a code coverage report after running the tests, use the
+following command. This runs the DB migrations, tests, and the NYC code coverage
+tool:
+
+```shell
+npm run coverage
+```
+
 ## Public domain
 
 This project is in the worldwide [public domain](LICENSE.md). As stated in [CONTRIBUTING](CONTRIBUTING.md):

deploy/cron.js

@@ -7,7 +7,7 @@ if (process.env.NEW_RELIC_APP_NAME) {
 }
 
 const spawn = require("child_process").spawn;
-const logger = require('../src/logger').initialize();
+const logger = require("../src/logger").initialize();
 
 logger.info("===================================");
 logger.info("=== STARTING ANALYTICS-REPORTER ===");
@@ -25,41 +25,41 @@ const runScriptWithLogName = (scriptPath, scriptLoggingName) => {
   childProcess.stdout.on("data", (data) => {
     logger.info(`[${scriptLoggingName}]`);
     // Writes logging output from child processes to console.
-    console.log(data.toString().trim())
+    console.log(data.toString().trim());
   });
 
   childProcess.stderr.on("data", (data) => {
     logger.error(`[${scriptLoggingName}]`);
     // Writes error logging output from child processes to console.
-    console.log(data.toString().trim())
+    console.log(data.toString().trim());
   });
 
-  childProcess.on("exit", (code, signal) => {
+  childProcess.on("close", (code, signal) => {
     logger.info(`${scriptLoggingName} exitted with code: ${code}`);
     if (signal) {
       logger.info(`${scriptLoggingName} received signal: ${signal}`);
     }
   });
-}
+};
 
 const api_ua_run = () => {
-  runScriptWithLogName(`${scriptUARootPath}/api.sh`, 'ua - api.sh')
+  runScriptWithLogName(`${scriptUARootPath}/api.sh`, "ua - api.sh");
 };
 
 const api_run = () => {
-  runScriptWithLogName(`${scriptRootPath}/api.sh`, 'api.sh')
+  runScriptWithLogName(`${scriptRootPath}/api.sh`, "api.sh");
 };
 
 const daily_run = () => {
-  runScriptWithLogName(`${scriptRootPath}/daily.sh`, 'daily.sh')
+  runScriptWithLogName(`${scriptRootPath}/daily.sh`, "daily.sh");
 };
 
 const hourly_run = () => {
-  runScriptWithLogName(`${scriptRootPath}/hourly.sh`, 'hourly.sh')
+  runScriptWithLogName(`${scriptRootPath}/hourly.sh`, "hourly.sh");
 };
 
 const realtime_run = () => {
-  runScriptWithLogName(`${scriptRootPath}/realtime.sh`, 'realtime.sh')
+  runScriptWithLogName(`${scriptRootPath}/realtime.sh`, "realtime.sh");
 };
 
 /**
@@ -72,7 +72,7 @@ const calculateNextDailyRunTimeOffset = () => {
     currentTime.getFullYear(),
     currentTime.getMonth(),
     currentTime.getDate() + 1,
-    10 - currentTime.getTimezoneOffset() / 60
+    10 - currentTime.getTimezoneOffset() / 60,
   );
   return (nextRunTime - currentTime) % (1000 * 60 * 60 * 24);
 };
@@ -92,14 +92,15 @@ setTimeout(() => {
   // Run at 10 AM UTC, then every 24 hours afterwards
   daily_run();
   setInterval(daily_run, 1000 * 60 * 60 * 24);
-  //api
+  // API
   api_run();
   setInterval(api_run, 1000 * 60 * 60 * 24);
-  //ua api
+  // UA API
   api_ua_run();
   setInterval(api_ua_run, 1000 * 60 * 60 * 24);
 }, calculateNextDailyRunTimeOffset());
-//hourly
+// hourly
 setInterval(hourly_run, 1000 * 60 * 60);
-//realtime
-setInterval(realtime_run, 1000 * 60 * 5);
+// realtime. Runs every 15 minutes.
+// Google updates realtime reports every 30 minutes, so there is some overlap.
+setInterval(realtime_run, 1000 * 60 * 15);

eslint.config.js

@@ -0,0 +1,20 @@
+const { configs: eslintConfigs } = require("@eslint/js");
+const eslintPluginPrettierRecommended = require("eslint-plugin-prettier/recommended");
+const globals = require("globals");
+
+module.exports = [
+  {
+    languageOptions: {
+      globals: {
+        ...globals.node,
+        ...globals.mocha,
+      },
+    },
+  },
+  {
+    // UA code is deprecated, so don't bother with static analysis rules.
+    ignores: ["ua/**/*.js"],
+    ...eslintConfigs.recommended,
+  },
+  eslintPluginPrettierRecommended,
+];