Merge branch 'main' into WP-699--rt-attachment-fix

.gitignore

@@ -20,6 +20,7 @@ designsafe/templates/react-assets.html
 designsafe.env
 mysql.env
 rabbitmq.env
+ngrok.env
 rabbitmq.conf
 mysql.cnf
 

.pylintrc

@@ -52,7 +52,7 @@ ignore=CVS,tests.py
 # ignore-list. The regex matches against paths and can be in Posix or Windows
 # format. Because '\\' represents the directory delimiter on Windows systems,
 # it can't be used as an escape character.
-ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$
+ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$,^.*test_.*$
 
 # Files or directories matching the regular expression patterns are skipped.
 # The regex matches against base names, not paths. The default value ignores

Makefile

@@ -1,3 +1,9 @@
+NGROK_ENV_FILE = ./conf/env_files/ngrok.env
+ifeq ("$(wildcard $(NGROK_ENV_FILE))","")
+ NGROK_ENV_FILE = ./conf/env_files/ngrok.sample.env
+endif
+
+
 .PHONY: build
 build:
  docker compose -f ./conf/docker/docker-compose.yml build
@@ -8,16 +14,16 @@ build-dev:
 
 .PHONY: start
 start:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml up
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml up
 
 .PHONY: stop
 stop:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml down
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml down
 
 .PHONY: start-m1
 start-m1:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up
 
 .PHONY: stop-m1
 stop-m1:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down

README.md

@@ -1,6 +1,3 @@
-[![Build Status](https://travis-ci.org/DesignSafe-CI/portal.svg?branch=master)](https://travis-ci.org/DesignSafe-CI/portal)
-[![codecov](https://codecov.io/gh/DesignSafe-CI/portal/branch/master/graph/badge.svg)](https://codecov.io/gh/DesignSafe-CI/portal)
-
 # DesignSafe-CI Portal
 
 ## Prequisites for running the portal application
@@ -11,8 +8,8 @@ on.
 
 - [Install Docker][3]
 - [Install Docker Compose][4]
-- [Install Make][12]
-- [Node.js][13] 16.x
+- [Install Make][11]
+- [Node.js][12] 20.x
 
 If you are on a Mac or a Windows machine, the recommended method is to install
 [Docker Desktop][5], which will install both Docker and Docker Compose, which is required to run Docker on Mac/Windows hosts.
@@ -37,7 +34,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  - `DJANGO_DEBUG`: should be set to `True` for development
  - `DJANGO_SECRET`: should be changed for production
  - `TAS_*`: should be set to enable direct access to `django.contrib.admin`
- - `AGAVE_*`: should be set to enable Agave API integration (authentication, etc.)
+ - `TAPIS_*`: should be set to enable Tapis API integration (authentication, etc.)
  - `RT_*`: should be set to enable ticketing
 
  Make a copy of [rabbitmq.sample.env](conf/env_files/rabbitmq.sample.env)
@@ -46,7 +43,15 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  Make a copy of [external_resource_secrets.sample.py](designsafe/settings/external_resource_secrets.sample.py)
  and rename it to `external_resource_secrets.py`.
 
-3. Build the containers and frontend packages
+3. Configure ngrok
+
+ a. Install [ngrok](https://ngrok.com/docs/getting-started/), and create an ngrok account.
+
+ b. Copy [conf/env_files/ngrok.sample.env](conf/env_files/ngrok.sample.env) to `conf/env_files/ngrok.env`.
+
+ c. In `conf/env_files/ngrok.env`, set the `NGROK_AUTHTOKEN` and `NGROK_DOMAIN` variables using your authtoken and static ngrok domain found in your [ngrok dashboard](https://dashboard.ngrok.com/).
+
+4. Build the containers and frontend packages
 
  1. Containers:
  ```sh
@@ -72,7 +77,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  npm run start
  ```
 
-4. Start local containers
+5. Start local containers
 
  ```
  $ make start
@@ -89,11 +94,11 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  $ ./manage.py createsuperuser
  ```
 
-5. Setup local access to the portal:
+6. Setup local access to the portal:
 
  Add a record to your local hosts file for `127.0.0.1 designsafe.dev`
  ```
- sudo vim /etc/hosts
+ $ sudo vim /etc/hosts
  ```
 
  Now you can navigate to [designsafe.dev](designsafe.dev) in your browser.
@@ -178,11 +183,10 @@ See the [DesignSafe Styles Reference][7] for style reference and custom CSS docu
 
 ### Updating Python dependencies
 
-For simplicity the Dockerfile uses a `requirements.txt` exported from Poetry. To add a new dependency:
+This project uses [Python Poetry](https://python-poetry.org/docs/) to manage dependencies. To add a new dependency:
 
 1. Run `poetry add $NEW_DEPENDENCY`.
-2. Run `poetry export > requirements.txt --dev --without-hashes` in the repository root.
-3. Rebuild the dev image with `docker-compose -f conf/docker/docker-compose.yml build`
+2. Rebuild the dev image with `make build-dev`
 
 ## Testing
 
@@ -200,64 +204,46 @@ Django tests should be written according to standard [Django testing procedures]
 You can run Django tests with the following command:
 
 ```shell
-$ docker exec -it des_django pytest designsafe
+$ docker exec -it des_django pytest -ra designsafe
 ```
 
 ### Frontend tests
 
-Frontend tests are [Jasmine][9] tests executed using the [Karma engine][10]. Testing
-guidelines can be found in the [AngularJS Developer Guide on Unit Testing][11].
+Frontend tests are [Vitest][9] tests executed using [Nx][10].
 
-To run frontend tests, ensure that all scripts and test scripts are configured in
-[`karma-conf.js`](karma-conf.js) and then run the command:
+To run frontend tests, run the command:
 
 ```shell
 $ npm run test
 ```
 
 ## Development setup
 
-Use `docker-compose` to run the portal in development. The default compose file,
-[`docker-compose.yml`](docker-compose.yml) runs the main django server in development
+Use `docker compose` to run the portal in development. The default compose file,
+[`docker-compose.yml`](conf/docker/docker-compose.yml) runs the main django server in development
 mode with a redis service for websockets support. You can optionally enable the EF sites
 for testing.
 
 ```shell
-$ docker-compose -f conf/docker/docker-compose.yml build
-$ docker-compose -f conf/docker/docker-compose-dev.all.debug.yml up
-$ npm run dev
+$ make build-dev
+$ make start
+$ npm run start
+$ docker run -v `pwd`:`pwd` -w `pwd` -it node:16 /bin/bash -c "npm run dev"
 ```
 
 When using this compose file, your Tapis Client should be configured with a `callback_url`
 of `http://$DOCKER_HOST_IP:8000/auth/tapis/callback/`.
 
-For developing some services, e.g. Box.com integration, https support is required. To
-enable an Nginx http proxy run using the [`docker-compose-http.yml`](docker-compose-http.yml)
-file. This file configures the same services as the default compose file, but it also sets
-up an Nginx proxy secured by a self-signed certificate.
 
 ```shell
 $ docker-compose -f docker-compose-http.yml build
 $ docker-compose -f docker-compose-http.yml up
 ```
 
-### Agave filesystem setup
-1. Delete all of the old metadata objects using this command:
-
- `metadata-list Q '{"name": "designsafe metadata"}' | while read x; do metadata-delete $x; done;`
-2. Run `dsapi/agave/tools/bin/walker.py` to create the metadata objects for the existing files in your FS.
-
- `python portal/dsapi/agave/tools/bin/walker.py <command> <api_server> <token> <systemId> <base_folder>`
- - `base_folder` is your username, if you want to fix everything under your home dir.
- - `command`:
- - `files`: Walk through the files and print their path.
- - `meta`: Walk through the metadata objs in a filesystem-like manner and print their path.
- - `files-fix`: Check if there's a meta obj for every file, if not create the meta obj.
- - `meta-fix`: Check if there's a file for every meta obj, if not delete the meta obj.
 
 ## Production setup
 
-Production deployment is managed by ansible. See https://github.com/designsafe-ci/ansible.
+Production deployment is managed by Camino. See https://github.com/TACC/Camino.
 
 
 [1]: https://docs.docker.com/
@@ -267,8 +253,7 @@ Production deployment is managed by ansible. See https://github.com/designsafe-c
 [5]: https://docs.docker.com/desktop/
 [7]: https://github.com/DesignSafe-CI/portal/wiki/CSS-Styles-Reference
 [8]: https://docs.djangoproject.com/en/dev/topics/testing/
-[9]: http://jasmine.github.io/1.3/introduction.html
-[10]: http://karma-runner.github.io/0.12/intro/installation.html
-[11]: https://docs.angularjs.org/guide/unit-testing
-[12]: https://www.gnu.org/software/make/
-[13]: https://nodejs.org/
+[9]: https://vitest.dev/
+[10]: https://nx.dev/getting-started/intro
+[11]: https://www.gnu.org/software/make/
+[12]: https://nodejs.org/

client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx

@@ -14,7 +14,9 @@ export const ProjectCitation: React.FC<{
  const { data } = useProjectDetail(projectId);
  const entityDetails = data?.entities.find((e) => e.uuid === entityUuid);
  const authors =
- entityDetails?.value.authors?.filter((a) => a.fname && a.lname) ?? [];
+ entityDetails?.value.authors?.filter(
+ (a) => a.fname && a.lname && a.authorship !== false
+ ) ?? [];
  if (!data || !entityDetails) return null;
  return (
  <div>
@@ -44,7 +46,8 @@ export const PublishedCitation: React.FC<{
  (child) => child.uuid === entityUuid && child.version === version
  );
 
- const authors = entityDetails?.value.authors ?? [];
+ const authors =
+ entityDetails?.value.authors?.filter((a) => a.authorship !== false) ?? [];
  if (!data || !entityDetails) return null;
 
  const doi =

conf/docker/docker-compose-dev.all.debug.m1.yml

@@ -77,7 +77,10 @@ services:
 
  django:
  image: designsafeci/portal:tapis-v3
- env_file: ../env_files/designsafe.env
+ env_file:
+ - path: ../env_files/designsafe.env
+ - path: ../env_files/ngrok.env
+ required: false
  links:
  - memcached:memcached
  - rabbitmq:rabbitmq
@@ -98,7 +101,10 @@ services:
 
  websockets:
  image: designsafeci/portal:tapis-v3
- env_file: ../env_files/designsafe.env
+ env_file:
+ - path: ../env_files/designsafe.env
+ - path: ../env_files/ngrok.env
+ required: false
  volumes:
  - ../../.:/srv/www/designsafe
  - ../../data/media:/srv/www/designsafe/media
@@ -108,7 +114,10 @@ services:
 
  workers:
  image: designsafeci/portal:tapis-v3
- env_file: ../env_files/designsafe.env
+ env_file:
+ - path: ../env_files/designsafe.env
+ - path: ../env_files/ngrok.env
+ required: false
  links:
  - memcached:memcached
  - rabbitmq:rabbitmq
@@ -125,6 +134,20 @@ services:
  container_name: des_workers
  hostname: des_workers
 
+ ngrok:
+ image: ngrok/ngrok:latest
+ platform: "linux/amd64"
+ environment:
+ NGROK_AUTHTOKEN: ${NGROK_AUTHTOKEN}
+ command:
+ - "http"
+ - --url=${NGROK_DOMAIN}
+ - "https://host.docker.internal:443"
+ ports:
+ - 4040:4040
+ container_name: des_ngrok
+ hostname: des_ngrok
+
 volumes:
  redis_data_v3:
  des_postgres_data_v3:

conf/docker/docker-compose-dev.all.debug.yml

@@ -73,7 +73,10 @@ services:
 
  django:
  image: designsafeci/portal:tapis-v3
- env_file: ../env_files/designsafe.env
+ env_file:
+ - path: ../env_files/designsafe.env
+ - path: ../env_files/ngrok.env
+ required: false
  links:
  - memcached:memcached
  - rabbitmq:rabbitmq
@@ -94,7 +97,10 @@ services:
 
  websockets:
  image: designsafeci/portal:tapis-v3
- env_file: ../env_files/designsafe.env
+ env_file:
+ - path: ../env_files/designsafe.env
+ - path: ../env_files/ngrok.env
+ required: false
  volumes:
  - ../../.:/srv/www/designsafe
  - ../../data/media:/srv/www/designsafe/media
@@ -104,7 +110,10 @@ services:
 
  workers:
  image: designsafeci/portal:tapis-v3
- env_file: ../env_files/designsafe.env
+ env_file:
+ - path: ../env_files/designsafe.env
+ - path: ../env_files/ngrok.env
+ required: false
  links:
  - memcached:memcached
  - rabbitmq:rabbitmq
@@ -121,6 +130,19 @@ services:
  container_name: des_workers
  hostname: des_workers
 
+ ngrok:
+ image: ngrok/ngrok:latest
+ environment:
+ NGROK_AUTHTOKEN: ${NGROK_AUTHTOKEN}
+ command:
+ - "http"
+ - --url=${NGROK_DOMAIN}
+ - "https://host.docker.internal:443"
+ ports:
+ - 4040:4040
+ container_name: des_ngrok
+ hostname: des_ngrok
+
 volumes:
  redis_data_v3:
  des_postgres_data_v3:

conf/env_files/designsafe.sample.env

@@ -80,7 +80,6 @@ AGAVE_CLIENT_SECRET=
 AGAVE_SUPER_TOKEN=
 
 AGAVE_STORAGE_SYSTEM=
-AGAVE_WORKING_SYSTEM=
 AGAVE_JWT_PUBKEY=
 AGAVE_JWT_ISSUER=
 AGAVE_JWT_HEADER=

conf/env_files/ngrok.sample.env

@@ -0,0 +1,5 @@
+# Get authtoken from https://dashboard.ngrok.com/get-started/your-authtoken
+NGROK_AUTHTOKEN=your_ngrok_authtoken
+
+# Get static domain from https://dashboard.ngrok.com/domains
+NGROK_DOMAIN=your-ngrok-subdomain.ngrok-free.app