diff --git a/.env b/.env
deleted file mode 100644
index 272f4e2b3..000000000
--- a/.env
+++ /dev/null
@@ -1,67 +0,0 @@
-DC_BASE=https://api.dev.documentcloud.org
-APP_URL=https://www.dev.documentcloud.org/
-EMBED_URL=https://www.dev.documentcloud.org/
-SQUARELET_BASE=https://dev.squarelet.com
-
-DC_LOGIN=/accounts/login/squarelet
-DC_LOGOUT=/accounts/logout/
-SQUARELET_SIGNUP=/accounts/signup/?intent=documentcloud&next=
-
-POLL_INTERVAL=5000
-IMAGE_WIDTHS=xlarge:2000,large:1000,normal:700,small:180,thumbnail:60
-
-MAX_PER_PAGE=100
-
-PDF_SIZE_LIMIT=525336576
-PDF_SIZE_LIMIT_READABLE=500 MB
-
-DOCUMENT_SIZE_LIMIT=27262976
-DOCUMENT_SIZE_LIMIT_READABLE=25 MB
-
-TOAST_LENGTH=5000
-TOAST_FADE=800
-
-LEGACY_CUT_OFF=20000000
-
-HIGHLIGHT_START=
-HIGHLIGHT_END=
-
-API=/api/
-
-UPLOAD_LIMIT=1000
-UPLOAD_BATCH=25
-UPLOAD_BATCH_DELAY=1000
-GET_BATCH=25
-GET_BATCH_DELAY=1000
-
-TAG_KEY=_tag
-
-SPECIAL_VERSION=Thanks for using DocumentCloud!
-SPECIAL_CONTACT=mailto:info@documentcloud.org?subject=DocumentCloud Feedback
-TIP_OF_THE_DAY=tipofday/
-LOGIN_ERROR=DocumentCloud is only available to users we have explicitly granted permission.
-
-LANGUAGE_CODES=afr|amh|ara|asm|aze|aze_cyrl|bel|ben|bod|bos|bul|cat|ceb|ces|zho|tra|chr|cym|dan|deu|dzo|ell|eng|enm|epo|est|eus|fas|fin|fra|frk|frm|gle|glg|grc|guj|hat|heb|hin|hrv|hun|iku|ind|isl|ita|ita_old|jav|jpn|kan|kat|kat_old|kaz|khm|kir|kor|kur|lao|lat|lav|lit|mal|mar|mkd|mlt|msa|mya|nep|nld|nor|ori|pan|pol|por|pus|ron|rus|san|sin|slk|slv|spa|spa_old|sqi|srp|srp_latn|swa|swe|syr|tam|tel|tgk|tgl|tha|tir|tur|uig|ukr|urd|uzb|uzb_cyrl|vie|yid
-LANGUAGE_NAMES=Afrikaans|Amharic|Arabic|Assamese|Azerbaijani|Azerbaijani - Cyrillic|Belarusian|Bengali|Tibetan|Bosnian|Bulgarian|Catalan|Cebuano|Czech|Chinese - Simplified|Chinese - Traditional|Cherokee|Welsh|Danish|German|Dzongkha|Greek|English|Middle English|Esperanto|Estonian|Basque|Persian|Finnish|French|German Fraktur|Middle French|Irish|Galician|Ancient Greek|Gujarati|Haitian; Haitian Creole|Hebrew|Hindi|Croatian|Hungarian|Inuktitut|Indonesian|Icelandic|Italian|Italian - Old|Javanese|Japanese|Kannada|Georgian|Georgian - Old|Kazakh|Central Khmer|Kirghiz; Kyrgyz|Korean|Kurdish|Lao|Latin|Latvian|Lithuanian|Malayalam|Marathi|Macedonian|Maltese|Malay|Burmese|Nepali|Dutch; Flemish|Norwegian|Oriya|Panjabi; Punjabi|Polish|Portuguese|Pushto; Pashto|Romanian; Moldavian; Moldovan|Russian|Sanskrit|Sinhala; Sinhalese|Slovak|Slovenian|Spanish; Castilian|Spanish; Castilian - Old|Albanian|Serbian|Serbian - Latin|Swahili|Swedish|Syriac|Tamil|Telugu|Tajik|Tagalog|Thai|Tigrinya|Turkish|Uighur; Uyghur|Ukrainian|Urdu|Uzbek|Uzbek - Cyrillic|Vietnamese|Yiddish
-DEFAULT_LANGUAGE=eng
-
-DOCUMENT_TYPES=123,602,abw,agd,bmp,cdr,cgm,cmx,csv,cwk,dbf,dif,doc,docx,dot,emf,eps,fb2,fhd,fodg,fodp,fods,fodt,gif,gnm,gnumeric,htm,html,hwp,jpeg,jpg,jtd,jtt,key,kth,mml,numbers,odb,odf,odg,odp,ods,odt,p65,pages,pbm,pcd,pct,pcx,pdf,pgm,plt,pm3,pm4,pm5,pm6,pmd,png,pot,ppm,pps,ppt,pptx,psd,pub,qxp,ras,rlf,rtf,sda,sdc,sdd,sdp,sdw,sgf,sgl,sgv,slk,stc,std,sti,stw,svg,svm,sxc,sxd,sxi,sxm,sxw,tga,tif,tiff,txt,uof,uop,uos,uot,vor,vsd,wb2,wdb,wk1,wk3,wk4,wks,wpd,wps,wq1,wq2,wri,xbm,xls,xlsx,xlt,xlw,xml,xpm,zabw,zmf
-
-STAFF_ONLY_S3_URL=http://minio.documentcloud.org:9000/minio/documents/documents/$$ID$$/
-
-
-DOCUMENT_TITLE_CHAR_LIMIT=1000
-DOCUMENT_DESCRIPTION_CHAR_LIMIT=4000
-DOCUMENT_SOURCE_CHAR_LIMIT=1000
-RELATED_ARTICLE_URL_CHAR_LIMIT=1024
-PUBLISHED_URL_CHAR_LIMIT=1024
-
-PROJECT_DESCRIPTION_CHAR_LIMIT=1000
-PROJECT_TITLE_CHAR_LIMIT=255
-
-NOTE_TITLE_CHAR_LIMIT=500
-NOTE_CONTENT_CHAR_LIMIT=2000
-SECTION_TITLE_CHAR_LIMIT=200
-
-PROJECT_REDIRECT_HASH_URL=https://s3.amazonaws.com/s3.documentcloud.org/legacy/project_redirects.bin
-ORG_REDIRECT_HASH_URL=https://s3.amazonaws.com/s3.documentcloud.org/legacy/org_redirects.bin
diff --git a/.env.production b/.env.production
deleted file mode 100644
index e0f709935..000000000
--- a/.env.production
+++ /dev/null
@@ -1,5 +0,0 @@
-DC_BASE=https://api.www.documentcloud.org
-APP_URL=https://www.documentcloud.org/
-EMBED_URL=https://embed.documentcloud.org/
-SQUARELET_BASE=https://accounts.muckrock.com
-STAFF_ONLY_S3_URL=https://s3.console.aws.amazon.com/s3/buckets/s3.documentcloud.org?region=us-east-1&prefix=documents/$$ID$$/&showversions=false
diff --git a/.env.staging b/.env.staging
deleted file mode 100644
index 47bb2951d..000000000
--- a/.env.staging
+++ /dev/null
@@ -1,5 +0,0 @@
-DC_BASE=https://api.muckcloud.com
-APP_URL=https://muckcloud.com
-EMBED_URL=https://muckcloud.com
-SQUARELET_BASE=https://squarelet-staging.herokuapp.com
-STAFF_ONLY_S3_URL=https://s3.console.aws.amazon.com/s3/buckets/documentcloud-staging-files?region=us-east-1&prefix=documents/$$ID$$/&showversions=false
diff --git a/.gitignore b/.gitignore
index db643eacd..2b5317021 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,9 +27,8 @@ public/assets/
.netlify
# Local test env file
-.env.secret
-.env.test
-.env.sentry-build-plugin
+.env
+.env.*
playwright-report
test-results
diff --git a/Makefile b/Makefile
index 32f7e47fc..b92277f8f 100644
--- a/Makefile
+++ b/Makefile
@@ -34,12 +34,6 @@ build-serve:
build-analyze:
docker compose -f local.yml up documentcloud_frontend_analyze
-test:
- docker compose -f local.builder.yml run --rm test
-
-test-watch:
- docker compose -f local.builder.yml run --rm test-watch
-
prettier-check:
prettier --check --plugin-search-dir=. src
diff --git a/README.md b/README.md
index 81bbca395..03d2c805c 100644
--- a/README.md
+++ b/README.md
@@ -6,16 +6,16 @@ The main frontend for DocumentCloud, written in [Svelte](https://svelte.dev/).
## Usage
-This project is a standard Node project but wrapped to run in Docker compose.
+This project depends on both [Squarelet](https://github.com/muckrock/squarelet) and the [DocumentCloud (Django)](https://github.com/muckrock/documentcloud). Follow the steps in their READMEs before setting up this project.
-In order to install, run:
+This project is a Svelte + Webpack project wrapped in Docker compose.
+
+In order to install dependencies inside the Docker container, run:
```bash
make install
```
-If this process fails because it can't find `.env.test`, run `cp .env .env.test` to create the test environment file.
-
Once the node modules have been installed, start the app with:
```bash
@@ -32,8 +32,6 @@ echo "127.0.0.1 www.dev.documentcloud.org" | sudo tee -a /etc/hosts
Once everything is up and running, you should be able to see the website live at [www.dev.documentcloud.org](http://www.dev.documentcloud.org/). Note the frontend is not yet functional.
-To utilize the frontend with its necessary authentication system and backend, [Squarelet](https://github.com/muckrock/squarelet) and [DocumentCloud (Django)](https://github.com/muckrock/documentcloud) are a requirement. Follow the steps in their READMEs.
-
## Building for production
Run `make build` to build the production version of the app. The project will be output in the `public` directory.
@@ -58,56 +56,49 @@ Run the relevant `npm install ...` command and then get the change mirrored on t
## Unit tests
-Run unit tests with `make test`. This will run the tests via the builder Docker image.
+Run unit tests with `npm test`.
-## Functional tests
+## Browser tests
-All of the functional test commands depend on the front end running, so start the app with `make dev-app` and start the backend and Squarelet as well.
+All of the browser test commands depend on the front end running, so start the app with `make dev` and start the backend and Squarelet as well.
### Running tests locally
-Run `make browser-test` in another terminal. This will run the `browser-test` Docker image against the running front-end app using all of the browsers, except Chromium for now.
-
-You will need to create a user that is verified for uploading as described in the [backend documentation](muckrock/documentcloud). Then, you need to put that users credentials in a `.env.test` file in the project root that looks like this:
-
-TEST_USER=
-TEST_PASS=
-APP_URL=https://www.dev.documentcloud.org/
-
-To run the functional tests without the Docker image, run `make browser-test-direct`. This will run the test suite files via your computer's Node. It will use the webkit browser only (but you can change this in the Makefile if you like).
-
-`make browser-test-direct` will do the same thing, except it will use all of the browsers.
-
-The above commands run the browsers headlessly. If you want to see what's going on, you can use `make browser-test-headful`.
-
-If you want to step through the tests with the debugger, use `make browser-test-debug`.
-
-### Running against staging
-
-Add the following for a verified-to-upload user an staging to `.env.staging`:
-
-TEST_USER=
-TEST_PASS=
-
-Then, run `make browser-test-staging`. To run it headfully, use `make browser-test-headful-staging`.
+Run `npm run test:browser` in another terminal. This will run Playwright using Chromium and Firefox.
### Development
The functional tests are organized like this:
-`tests/functional`: Common utilities for the test go here.
-`tests/functional/cases`: The bodies of individual test cases.
-`tests/functional/suites`: These files are the test runner entry points ([`tape`](https://github.com/ljharb/tape/) is the test runner). They use the utilties to start up and shut down the browsers via Playwright and load and run individual test cases. It may make sense to repeat some test cases across suites, like signing in, uploading a document, and deleting an uploaded document, for example.
-`tests/functional/fixtures`: Artifacts and data needed by the tests go here.
+```
+tests
+├── README.md
+├── anonymous
+│ ├── manager
+│ │ └── app.spec.js
+│ ├── pages
+│ │ └── home.spec.js
+│ └── viewer
+│ └── document.spec.js
+└── fixtures
+ ├── Small pdf.pdf
+ ├── development.json
+ ├── production.json
+ ├── staging.json
+ └── the-nature-of-the-firm-CPEC11.pdf
+```
+
+Tests are organized around major parts of the codebase -- `manager`, `pages` and `viewer`. Tests under `anonymous` don't use an authenticated user.
+
+Tests rely on specific docouments available in each environment, which will have different URLs, so lists of known documents are provided in `development.json`, `staging.json` and `production.json`. Those correspond to the `NODE_ENV` environment variable.
## Storybooks
-Storybooks are used to create isolated environments for developing, testing, and demonstrating the Svelte components that compose the user interface.
+Storybooks are used to create isolated environments for developing, testing and demonstrating the Svelte components that compose the user interface.
-For now, Storybooks run locally to your machine, not in the Docker container.
-They also require Node v16 or later to run (the frontend containers currently run on v12).
+Storybooks run locally to your machine, not in the Docker container.
-To run the Storybook dev server with Node 16 enabled:
+To run the Storybook dev server:
```sh
npm run storybook
diff --git a/local.builder.yml b/local.builder.yml
index acc920f98..d4668db7e 100644
--- a/local.builder.yml
+++ b/local.builder.yml
@@ -77,8 +77,8 @@ services:
squarelet_default:
aliases:
- www.dev.documentcloud.org
- env_file:
- - .env.test
+ environment:
+ - NODE_ENV=test
command: ./browser-test-all.sh
browser-test-staging:
@@ -91,8 +91,8 @@ services:
squarelet_default:
aliases:
- www.dev.documentcloud.org
- env_file:
- - .env.staging
+ environment:
+ - NODE_ENV=staging
command: ./browser-test-all.sh
volumes:
diff --git a/src/addons/progress/AddonStatus.svelte b/src/addons/progress/AddonStatus.svelte
index 0c1160316..9ad7b647f 100644
--- a/src/addons/progress/AddonStatus.svelte
+++ b/src/addons/progress/AddonStatus.svelte
@@ -4,8 +4,7 @@
import AddonRun, { runs } from "./AddonRun.svelte";
import { baseApiUrl } from "../../api/base.js";
-
- const POLL_INTERVAL = parseInt(process.env.POLL_INTERVAL, 10);
+ import { POLL_INTERVAL } from "../../config/config.js";
const endpoint = new URL(
"/api/addon_runs/?expand=addon&dismissed=false&per_page=20",
diff --git a/src/api/auth.js b/src/api/auth.js
index c76bb191f..459d5d4cb 100644
--- a/src/api/auth.js
+++ b/src/api/auth.js
@@ -1,14 +1,15 @@
import { Svue } from "svue";
+import * as config from "../config/config.js";
const DOCUMENTCLOUD_TOKEN_STORAGE_KEY = "documentcloud_token";
-export const SQUARELET_URL = process.env.SQUARELET_BASE;
-export const SIGN_IN_URL = process.env.DC_BASE + process.env.DC_LOGIN;
+export const SQUARELET_URL = config.SQUARELET_BASE;
+export const SIGN_IN_URL = config.DC_BASE + config.DC_LOGIN;
export const SIGN_UP_URL =
- process.env.SQUARELET_BASE +
- process.env.SQUARELET_SIGNUP +
+ config.SQUARELET_BASE +
+ config.SQUARELET_SIGNUP +
encodeURIComponent(window.location.href);
-export const SIGN_OUT_URL = process.env.DC_BASE + process.env.DC_LOGOUT;
+export const SIGN_OUT_URL = config.DC_BASE + config.DC_LOGOUT;
export const auth = new Svue({
data() {
diff --git a/src/api/base.js b/src/api/base.js
index 794350c3c..e4c01dd96 100644
--- a/src/api/base.js
+++ b/src/api/base.js
@@ -1,4 +1,6 @@
-export const baseApiUrl = process.env.DC_BASE + process.env.API;
+import { DC_BASE, API } from "../config/config.js";
+
+export const baseApiUrl = DC_BASE + API;
export function apiUrl(url) {
return `${baseApiUrl}${url}`;
diff --git a/src/api/document.js b/src/api/document.js
index 66bc9907c..5539f52a9 100644
--- a/src/api/document.js
+++ b/src/api/document.js
@@ -15,15 +15,15 @@ import axios from "axios";
import { Document, transformHighlights } from "@/structure/document.js";
-const POLL_TIMEOUT = process.env.POLL_TIMEOUT;
-
-const GET_BATCH = parseInt(process.env.GET_BATCH);
-const GET_BATCH_DELAY = parseInt(process.env.GET_BATCH_DELAY);
-const UPLOAD_BATCH = parseInt(process.env.UPLOAD_BATCH);
-const UPLOAD_BATCH_DELAY = parseInt(process.env.UPLOAD_BATCH_DELAY);
-
-const HIGHLIGHT_START = process.env.HIGHLIGHT_START;
-const HIGHLIGHT_END = process.env.HIGHLIGHT_END;
+import {
+ POLL_INTERVAL,
+ GET_BATCH,
+ GET_BATCH_DELAY,
+ UPLOAD_BATCH,
+ UPLOAD_BATCH_DELAY,
+ HIGHLIGHT_START,
+ HIGHLIGHT_END,
+} from "../config/config.js";
// Statuses
export const PENDING = "pending";
@@ -169,7 +169,7 @@ export async function changeRevisionControl(id, revision_control) {
expand: [DEFAULT_EXPAND, "revisions"].join(","),
}),
),
- {revision_control}
+ { revision_control },
);
return data;
}
@@ -254,7 +254,7 @@ export async function pollDocument(
}
// Retrigger after timeout
- await timeout(POLL_TIMEOUT);
+ await timeout(POLL_INTERVAL);
pollDocument(id, docFn, doneFn, conditionFn);
}
diff --git a/src/api/languages.js b/src/api/languages.js
index d9ea8b438..9fc6e4747 100644
--- a/src/api/languages.js
+++ b/src/api/languages.js
@@ -1,6 +1,10 @@
-const languageCodes = process.env.LANGUAGE_CODES.split("|");
-const languageNames = process.env.LANGUAGE_NAMES.split("|");
-export const defaultLanguage = process.env.DEFAULT_LANGUAGE;
+import {
+ LANGUAGE_CODES,
+ LANGUAGE_NAMES,
+ DEFAULT_LANGUAGE,
+} from "../config/config.js";
+
+export const defaultLanguage = DEFAULT_LANGUAGE;
function makeLanguagePairs(codes, names) {
const results = [];
@@ -11,7 +15,7 @@ function makeLanguagePairs(codes, names) {
return results;
}
-export const languages = makeLanguagePairs(languageCodes, languageNames);
+export const languages = makeLanguagePairs(LANGUAGE_CODES, LANGUAGE_NAMES);
const textractLanguageCodes = ["eng", "spa", "ita", "por", "fra", "deu"];
export const textractLanguages = languages.filter(
diff --git a/src/api/session.js b/src/api/session.js
index 2b4b53e47..fc6267ebd 100644
--- a/src/api/session.js
+++ b/src/api/session.js
@@ -1,6 +1,8 @@
import axios from "axios";
import axiosRetry from "axios-retry";
+import { DC_BASE } from "../config/config.js";
+
// Hook in failed request interceptor
axiosRetry(axios, {
retries: 3,
@@ -97,7 +99,7 @@ session.getStatic = async function getStatic(url) {
// On the second request, we do not sent the session cookie
// If we are fetching a public asset, the first request is directly to S3. In
// that case we must not send the session cookie on the first request.
- if (url.startsWith(process.env.DC_BASE)) {
+ if (url.startsWith(DC_BASE)) {
result = await session.get(url).then((r) => r.data);
let redirect = result.location;
diff --git a/src/api/viewer.js b/src/api/viewer.js
index f16c43ed0..d83c2c1be 100644
--- a/src/api/viewer.js
+++ b/src/api/viewer.js
@@ -1,13 +1,4 @@
-function processImageWidths(widthSpec) {
- let result = widthSpec.split(",").map((width) => {
- const parts = width.split(":");
- return [parseFloat(parts[1]), parts[0]];
- });
- result = result.sort((a, b) => a[0] - b[0]);
- return result;
-}
-
-export const imageWidths = processImageWidths(process.env.IMAGE_WIDTHS);
+import { IMAGE_WIDTHS } from "../config/config.js";
export function documentDimensionUrl(document) {
return `${document.assetUrl}documents/${document.id}/${document.slug}.pagesize`;
@@ -19,11 +10,11 @@ export function documentDimensionUrl(document) {
* @param {number} desiredWidth The desired width of the page image.
*/
function getDesiredSize(desiredWidth) {
- for (let i = 0; i < imageWidths.length; i++) {
- const [width, name] = imageWidths[i];
+ for (let i = 0; i < IMAGE_WIDTHS.length; i++) {
+ const [width, name] = IMAGE_WIDTHS[i];
if (desiredWidth <= width) return name;
}
- return imageWidths[imageWidths.length - 1][1];
+ return IMAGE_WIDTHS[IMAGE_WIDTHS.length - 1][1];
}
export function pageImageUrl(document, pageNumber, desiredWidth) {
diff --git a/src/common/FilePicker.svelte b/src/common/FilePicker.svelte
index 11231b2ae..62c8b701e 100644
--- a/src/common/FilePicker.svelte
+++ b/src/common/FilePicker.svelte
@@ -1,4 +1,5 @@
-
+
+
-