Merge pull request #408 from MuckRock/370-envs

Use JS config files instead of environment variables

.gitignore

@@ -27,9 +27,8 @@ public/assets/
 .netlify
 
 # Local test env file
-.env.secret
-.env.test
-.env.sentry-build-plugin
+.env
+.env.*
 playwright-report
 test-results
 

Makefile

@@ -34,12 +34,6 @@ build-serve:
 build-analyze:
 	docker compose -f local.yml up documentcloud_frontend_analyze
 
-test:
-	docker compose -f local.builder.yml run --rm test
-
-test-watch:
-	docker compose -f local.builder.yml run --rm test-watch
-
 prettier-check:
 	prettier --check --plugin-search-dir=. src
 

README.md

@@ -6,16 +6,16 @@ The main frontend for DocumentCloud, written in [Svelte](https://svelte.dev/).
 
 ## Usage
 
-This project is a standard Node project but wrapped to run in Docker compose.
+This project depends on both [Squarelet](https://github.com/muckrock/squarelet) and the [DocumentCloud (Django)](https://github.com/muckrock/documentcloud). Follow the steps in their READMEs before setting up this project.
 
-In order to install, run:
+This project is a Svelte + Webpack project wrapped in Docker compose.
+
+In order to install dependencies inside the Docker container, run:
 
 ```bash
 make install
 ```
 
-If this process fails because it can't find `.env.test`, run `cp .env .env.test` to create the test environment file.
-
 Once the node modules have been installed, start the app with:
 
 ```bash
@@ -32,8 +32,6 @@ echo "127.0.0.1 www.dev.documentcloud.org" | sudo tee -a /etc/hosts
 
 Once everything is up and running, you should be able to see the website live at [www.dev.documentcloud.org](http://www.dev.documentcloud.org/). Note the frontend is not yet functional.
 
-To utilize the frontend with its necessary authentication system and backend, [Squarelet](https://github.com/muckrock/squarelet) and [DocumentCloud (Django)](https://github.com/muckrock/documentcloud) are a requirement. Follow the steps in their READMEs.
-
 ## Building for production
 
 Run `make build` to build the production version of the app. The project will be output in the `public` directory.
@@ -58,56 +56,49 @@ Run the relevant `npm install ...` command and then get the change mirrored on t
 
 ## Unit tests
 
-Run unit tests with `make test`. This will run the tests via the builder Docker image.
+Run unit tests with `npm test`.
 
-## Functional tests
+## Browser tests
 
-All of the functional test commands depend on the front end running, so start the app with `make dev-app` and start the backend and Squarelet as well.
+All of the browser test commands depend on the front end running, so start the app with `make dev` and start the backend and Squarelet as well.
 
 ### Running tests locally
 
-Run `make browser-test` in another terminal. This will run the `browser-test` Docker image against the running front-end app using all of the browsers, except Chromium for now.
-
-You will need to create a user that is verified for uploading as described in the [backend documentation](muckrock/documentcloud). Then, you need to put that users credentials in a `.env.test` file in the project root that looks like this:
-
-TEST_USER=<the test user>
-TEST_PASS=<the password>
-APP_URL=https://www.dev.documentcloud.org/
-
-To run the functional tests without the Docker image, run `make browser-test-direct`. This will run the test suite files via your computer's Node. It will use the webkit browser only (but you can change this in the Makefile if you like).
-
-`make browser-test-direct` will do the same thing, except it will use all of the browsers.
-
-The above commands run the browsers headlessly. If you want to see what's going on, you can use `make browser-test-headful`.
-
-If you want to step through the tests with the debugger, use `make browser-test-debug`.
-
-### Running against staging
-
-Add the following for a verified-to-upload user an staging to `.env.staging`:
-
-TEST_USER=<the test user>
-TEST_PASS=<the password>
-
-Then, run `make browser-test-staging`. To run it headfully, use `make browser-test-headful-staging`.
+Run `npm run test:browser` in another terminal. This will run Playwright using Chromium and Firefox.
 
 ### Development
 
 The functional tests are organized like this:
 
-`tests/functional`: Common utilities for the test go here.
-`tests/functional/cases`: The bodies of individual test cases.
-`tests/functional/suites`: These files are the test runner entry points ([`tape`](https://github.com/ljharb/tape/) is the test runner). They use the utilties to start up and shut down the browsers via Playwright and load and run individual test cases. It may make sense to repeat some test cases across suites, like signing in, uploading a document, and deleting an uploaded document, for example.
-`tests/functional/fixtures`: Artifacts and data needed by the tests go here.
+```
+tests
+├── README.md
+├── anonymous
+│   ├── manager
+│   │   └── app.spec.js
+│   ├── pages
+│   │   └── home.spec.js
+│   └── viewer
+│       └── document.spec.js
+└── fixtures
+    ├── Small pdf.pdf
+    ├── development.json
+    ├── production.json
+    ├── staging.json
+    └── the-nature-of-the-firm-CPEC11.pdf
+```
+
+Tests are organized around major parts of the codebase -- `manager`, `pages` and `viewer`. Tests under `anonymous` don't use an authenticated user.
+
+Tests rely on specific docouments available in each environment, which will have different URLs, so lists of known documents are provided in `development.json`, `staging.json` and `production.json`. Those correspond to the `NODE_ENV` environment variable.
 
 ## Storybooks
 
-Storybooks are used to create isolated environments for developing, testing, and demonstrating the Svelte components that compose the user interface.
+Storybooks are used to create isolated environments for developing, testing and demonstrating the Svelte components that compose the user interface.
 
-For now, Storybooks run locally to your machine, not in the Docker container.
-They also require Node v16 or later to run (the frontend containers currently run on v12).
+Storybooks run locally to your machine, not in the Docker container.
 
-To run the Storybook dev server with Node 16 enabled:
+To run the Storybook dev server:
 
 ```sh
 npm run storybook

local.builder.yml

@@ -77,8 +77,8 @@ services:
       squarelet_default:
         aliases:
           - www.dev.documentcloud.org
-    env_file:
-      - .env.test
+    environment:
+      - NODE_ENV=test
     command: ./browser-test-all.sh
 
   browser-test-staging:
@@ -91,8 +91,8 @@ services:
       squarelet_default:
         aliases:
           - www.dev.documentcloud.org
-    env_file:
-      - .env.staging
+    environment:
+      - NODE_ENV=staging
     command: ./browser-test-all.sh
 
 volumes:

src/addons/progress/AddonStatus.svelte

@@ -4,8 +4,7 @@
 
   import AddonRun, { runs } from "./AddonRun.svelte";
   import { baseApiUrl } from "../../api/base.js";
-
-  const POLL_INTERVAL = parseInt(process.env.POLL_INTERVAL, 10);
+  import { POLL_INTERVAL } from "../../config/config.js";
 
   const endpoint = new URL(
     "/api/addon_runs/?expand=addon&dismissed=false&per_page=20",

src/api/auth.js

@@ -1,14 +1,15 @@
 import { Svue } from "svue";
+import * as config from "../config/config.js";
 
 const DOCUMENTCLOUD_TOKEN_STORAGE_KEY = "documentcloud_token";
 
-export const SQUARELET_URL = process.env.SQUARELET_BASE;
-export const SIGN_IN_URL = process.env.DC_BASE + process.env.DC_LOGIN;
+export const SQUARELET_URL = config.SQUARELET_BASE;
+export const SIGN_IN_URL = config.DC_BASE + config.DC_LOGIN;
 export const SIGN_UP_URL =
-  process.env.SQUARELET_BASE +
-  process.env.SQUARELET_SIGNUP +
+  config.SQUARELET_BASE +
+  config.SQUARELET_SIGNUP +
   encodeURIComponent(window.location.href);
-export const SIGN_OUT_URL = process.env.DC_BASE + process.env.DC_LOGOUT;
+export const SIGN_OUT_URL = config.DC_BASE + config.DC_LOGOUT;
 
 export const auth = new Svue({
   data() {

src/api/base.js

@@ -1,4 +1,6 @@
-export const baseApiUrl = process.env.DC_BASE + process.env.API;
+import { DC_BASE, API } from "../config/config.js";
+
+export const baseApiUrl = DC_BASE + API;
 
 export function apiUrl(url) {
   return `${baseApiUrl}${url}`;

src/api/document.js

@@ -15,15 +15,15 @@ import axios from "axios";
 
 import { Document, transformHighlights } from "@/structure/document.js";
 
-const POLL_TIMEOUT = process.env.POLL_TIMEOUT;
-
-const GET_BATCH = parseInt(process.env.GET_BATCH);
-const GET_BATCH_DELAY = parseInt(process.env.GET_BATCH_DELAY);
-const UPLOAD_BATCH = parseInt(process.env.UPLOAD_BATCH);
-const UPLOAD_BATCH_DELAY = parseInt(process.env.UPLOAD_BATCH_DELAY);
-
-const HIGHLIGHT_START = process.env.HIGHLIGHT_START;
-const HIGHLIGHT_END = process.env.HIGHLIGHT_END;
+import {
+  POLL_INTERVAL,
+  GET_BATCH,
+  GET_BATCH_DELAY,
+  UPLOAD_BATCH,
+  UPLOAD_BATCH_DELAY,
+  HIGHLIGHT_START,
+  HIGHLIGHT_END,
+} from "../config/config.js";
 
 // Statuses
 export const PENDING = "pending";
@@ -169,7 +169,7 @@ export async function changeRevisionControl(id, revision_control) {
         expand: [DEFAULT_EXPAND, "revisions"].join(","),
       }),
     ),
-    {revision_control}
+    { revision_control },
   );
   return data;
 }
@@ -254,7 +254,7 @@ export async function pollDocument(
   }
 
   // Retrigger after timeout
-  await timeout(POLL_TIMEOUT);
+  await timeout(POLL_INTERVAL);
   pollDocument(id, docFn, doneFn, conditionFn);
 }
 

src/api/languages.js

@@ -1,6 +1,10 @@
-const languageCodes = process.env.LANGUAGE_CODES.split("|");
-const languageNames = process.env.LANGUAGE_NAMES.split("|");
-export const defaultLanguage = process.env.DEFAULT_LANGUAGE;
+import {
+  LANGUAGE_CODES,
+  LANGUAGE_NAMES,
+  DEFAULT_LANGUAGE,
+} from "../config/config.js";
+
+export const defaultLanguage = DEFAULT_LANGUAGE;
 
 function makeLanguagePairs(codes, names) {
   const results = [];
@@ -11,7 +15,7 @@ function makeLanguagePairs(codes, names) {
   return results;
 }
 
-export const languages = makeLanguagePairs(languageCodes, languageNames);
+export const languages = makeLanguagePairs(LANGUAGE_CODES, LANGUAGE_NAMES);
 
 const textractLanguageCodes = ["eng", "spa", "ita", "por", "fra", "deu"];
 export const textractLanguages = languages.filter(

src/api/session.js

@@ -1,6 +1,8 @@
 import axios from "axios";
 import axiosRetry from "axios-retry";
 
+import { DC_BASE } from "../config/config.js";
+
 // Hook in failed request interceptor
 axiosRetry(axios, {
   retries: 3,
@@ -97,7 +99,7 @@ session.getStatic = async function getStatic(url) {
   // On the second request, we do not sent the session cookie
   // If we are fetching a public asset, the first request is directly to S3.  In
   // that case we must not send the session cookie on the first request.
-  if (url.startsWith(process.env.DC_BASE)) {
+  if (url.startsWith(DC_BASE)) {
     result = await session.get(url).then((r) => r.data);
     let redirect = result.location;
 

src/api/viewer.js

@@ -1,13 +1,4 @@
-function processImageWidths(widthSpec) {
-  let result = widthSpec.split(",").map((width) => {
-    const parts = width.split(":");
-    return [parseFloat(parts[1]), parts[0]];
-  });
-  result = result.sort((a, b) => a[0] - b[0]);
-  return result;
-}
-
-export const imageWidths = processImageWidths(process.env.IMAGE_WIDTHS);
+import { IMAGE_WIDTHS } from "../config/config.js";
 
 export function documentDimensionUrl(document) {
   return `${document.assetUrl}documents/${document.id}/${document.slug}.pagesize`;
@@ -19,11 +10,11 @@ export function documentDimensionUrl(document) {
  * @param {number} desiredWidth The desired width of the page image.
  */
 function getDesiredSize(desiredWidth) {
-  for (let i = 0; i < imageWidths.length; i++) {
-    const [width, name] = imageWidths[i];
+  for (let i = 0; i < IMAGE_WIDTHS.length; i++) {
+    const [width, name] = IMAGE_WIDTHS[i];
     if (desiredWidth <= width) return name;
   }
-  return imageWidths[imageWidths.length - 1][1];
+  return IMAGE_WIDTHS[IMAGE_WIDTHS.length - 1][1];
 }
 
 export function pageImageUrl(document, pageNumber, desiredWidth) {