Merge branch 'main' into task/WC-14--citation-version-format

.gitignore

@@ -20,6 +20,7 @@ designsafe/templates/react-assets.html
 designsafe.env
 mysql.env
 rabbitmq.env
+ngrok.env
 rabbitmq.conf
 mysql.cnf
 

.pylintrc

@@ -52,7 +52,7 @@ ignore=CVS,tests.py
 # ignore-list. The regex matches against paths and can be in Posix or Windows
 # format. Because '\\' represents the directory delimiter on Windows systems,
 # it can't be used as an escape character.
-ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$
+ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$,^.*test_.*$
 
 # Files or directories matching the regular expression patterns are skipped.
 # The regex matches against base names, not paths. The default value ignores

Makefile

@@ -1,3 +1,9 @@
+NGROK_ENV_FILE = ./conf/env_files/ngrok.env
+ifeq ("$(wildcard $(NGROK_ENV_FILE))","")
+ NGROK_ENV_FILE = ./conf/env_files/ngrok.sample.env
+endif
+
+
 .PHONY: build
 build:
  docker compose -f ./conf/docker/docker-compose.yml build
@@ -8,16 +14,16 @@ build-dev:
 
 .PHONY: start
 start:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml up
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml up
 
 .PHONY: stop
 stop:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml down
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml down
 
 .PHONY: start-m1
 start-m1:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up
 
 .PHONY: stop-m1
 stop-m1:
- docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down
+ docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down

README.md

@@ -1,6 +1,3 @@
-[![Build Status](https://travis-ci.org/DesignSafe-CI/portal.svg?branch=master)](https://travis-ci.org/DesignSafe-CI/portal)
-[![codecov](https://codecov.io/gh/DesignSafe-CI/portal/branch/master/graph/badge.svg)](https://codecov.io/gh/DesignSafe-CI/portal)
-
 # DesignSafe-CI Portal
 
 ## Prequisites for running the portal application
@@ -11,8 +8,8 @@ on.
 
 - [Install Docker][3]
 - [Install Docker Compose][4]
-- [Install Make][12]
-- [Node.js][13] 16.x
+- [Install Make][11]
+- [Node.js][12] 20.x
 
 If you are on a Mac or a Windows machine, the recommended method is to install
 [Docker Desktop][5], which will install both Docker and Docker Compose, which is required to run Docker on Mac/Windows hosts.
@@ -37,7 +34,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  - `DJANGO_DEBUG`: should be set to `True` for development
  - `DJANGO_SECRET`: should be changed for production
  - `TAS_*`: should be set to enable direct access to `django.contrib.admin`
- - `AGAVE_*`: should be set to enable Agave API integration (authentication, etc.)
+ - `TAPIS_*`: should be set to enable Tapis API integration (authentication, etc.)
  - `RT_*`: should be set to enable ticketing
 
  Make a copy of [rabbitmq.sample.env](conf/env_files/rabbitmq.sample.env)
@@ -46,7 +43,15 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  Make a copy of [external_resource_secrets.sample.py](designsafe/settings/external_resource_secrets.sample.py)
  and rename it to `external_resource_secrets.py`.
 
-3. Build the containers and frontend packages
+3. Configure ngrok
+
+ a. Install [ngrok](https://ngrok.com/docs/getting-started/), and create an ngrok account.
+
+ b. Copy [conf/env_files/ngrok.sample.env](conf/env_files/ngrok.sample.env) to `conf/env_files/ngrok.env`.
+
+ c. In `conf/env_files/ngrok.env`, set the `NGROK_AUTHTOKEN` and `NGROK_DOMAIN` variables using your authtoken and static ngrok domain found in your [ngrok dashboard](https://dashboard.ngrok.com/).
+
+4. Build the containers and frontend packages
 
  1. Containers:
  ```sh
@@ -72,7 +77,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  npm run start
  ```
 
-4. Start local containers
+5. Start local containers
 
  ```
  $ make start
@@ -89,11 +94,11 @@ If you are on a Mac or a Windows machine, the recommended method is to install
  $ ./manage.py createsuperuser
  ```
 
-5. Setup local access to the portal:
+6. Setup local access to the portal:
 
  Add a record to your local hosts file for `127.0.0.1 designsafe.dev`
  ```
- sudo vim /etc/hosts
+ $ sudo vim /etc/hosts
  ```
 
  Now you can navigate to [designsafe.dev](designsafe.dev) in your browser.
@@ -178,11 +183,10 @@ See the [DesignSafe Styles Reference][7] for style reference and custom CSS docu
 
 ### Updating Python dependencies
 
-For simplicity the Dockerfile uses a `requirements.txt` exported from Poetry. To add a new dependency:
+This project uses [Python Poetry](https://python-poetry.org/docs/) to manage dependencies. To add a new dependency:
 
 1. Run `poetry add $NEW_DEPENDENCY`.
-2. Run `poetry export > requirements.txt --dev --without-hashes` in the repository root.
-3. Rebuild the dev image with `docker-compose -f conf/docker/docker-compose.yml build`
+2. Rebuild the dev image with `make build-dev`
 
 ## Testing
 
@@ -200,64 +204,46 @@ Django tests should be written according to standard [Django testing procedures]
 You can run Django tests with the following command:
 
 ```shell
-$ docker exec -it des_django pytest designsafe
+$ docker exec -it des_django pytest -ra designsafe
 ```
 
 ### Frontend tests
 
-Frontend tests are [Jasmine][9] tests executed using the [Karma engine][10]. Testing
-guidelines can be found in the [AngularJS Developer Guide on Unit Testing][11].
+Frontend tests are [Vitest][9] tests executed using [Nx][10].
 
-To run frontend tests, ensure that all scripts and test scripts are configured in
-[`karma-conf.js`](karma-conf.js) and then run the command:
+To run frontend tests, run the command:
 
 ```shell
 $ npm run test
 ```
 
 ## Development setup
 
-Use `docker-compose` to run the portal in development. The default compose file,
-[`docker-compose.yml`](docker-compose.yml) runs the main django server in development
+Use `docker compose` to run the portal in development. The default compose file,
+[`docker-compose.yml`](conf/docker/docker-compose.yml) runs the main django server in development
 mode with a redis service for websockets support. You can optionally enable the EF sites
 for testing.
 
 ```shell
-$ docker-compose -f conf/docker/docker-compose.yml build
-$ docker-compose -f conf/docker/docker-compose-dev.all.debug.yml up
-$ npm run dev
+$ make build-dev
+$ make start
+$ npm run start
+$ docker run -v `pwd`:`pwd` -w `pwd` -it node:16 /bin/bash -c "npm run dev"
 ```
 
 When using this compose file, your Tapis Client should be configured with a `callback_url`
 of `http://$DOCKER_HOST_IP:8000/auth/tapis/callback/`.
 
-For developing some services, e.g. Box.com integration, https support is required. To
-enable an Nginx http proxy run using the [`docker-compose-http.yml`](docker-compose-http.yml)
-file. This file configures the same services as the default compose file, but it also sets
-up an Nginx proxy secured by a self-signed certificate.
 
 ```shell
 $ docker-compose -f docker-compose-http.yml build
 $ docker-compose -f docker-compose-http.yml up
 ```
 
-### Agave filesystem setup
-1. Delete all of the old metadata objects using this command:
-
- `metadata-list Q '{"name": "designsafe metadata"}' | while read x; do metadata-delete $x; done;`
-2. Run `dsapi/agave/tools/bin/walker.py` to create the metadata objects for the existing files in your FS.
-
- `python portal/dsapi/agave/tools/bin/walker.py <command> <api_server> <token> <systemId> <base_folder>`
- - `base_folder` is your username, if you want to fix everything under your home dir.
- - `command`:
- - `files`: Walk through the files and print their path.
- - `meta`: Walk through the metadata objs in a filesystem-like manner and print their path.
- - `files-fix`: Check if there's a meta obj for every file, if not create the meta obj.
- - `meta-fix`: Check if there's a file for every meta obj, if not delete the meta obj.
 
 ## Production setup
 
-Production deployment is managed by ansible. See https://github.com/designsafe-ci/ansible.
+Production deployment is managed by Camino. See https://github.com/TACC/Camino.
 
 
 [1]: https://docs.docker.com/
@@ -267,8 +253,7 @@ Production deployment is managed by ansible. See https://github.com/designsafe-c
 [5]: https://docs.docker.com/desktop/
 [7]: https://github.com/DesignSafe-CI/portal/wiki/CSS-Styles-Reference
 [8]: https://docs.djangoproject.com/en/dev/topics/testing/
-[9]: http://jasmine.github.io/1.3/introduction.html
-[10]: http://karma-runner.github.io/0.12/intro/installation.html
-[11]: https://docs.angularjs.org/guide/unit-testing
-[12]: https://www.gnu.org/software/make/
-[13]: https://nodejs.org/
+[9]: https://vitest.dev/
+[10]: https://nx.dev/getting-started/intro
+[11]: https://www.gnu.org/software/make/
+[12]: https://nodejs.org/

client/modules/_hooks/src/notifications/useNotifications.ts

@@ -6,7 +6,11 @@ import {
 } from '@tanstack/react-query';
 import apiClient from '../apiClient';
 
-type TPortalEventType = 'data_depot' | 'job' | 'interactive_session_ready';
+type TPortalEventType =
+ | 'data_depot'
+ | 'job'
+ | 'interactive_session_ready'
+ | 'markAllNotificationsAsRead';
 
 export type TJobStatusNotification = {
  action_link: string;

client/modules/_hooks/src/workspace/types.ts

@@ -1,6 +1,7 @@
 export type TParameterSetNotes = {
  isHidden?: boolean;
  fieldType?: string;
+ inputType?: string;
  validator?: {
  regex: string;
  message: string;
@@ -191,7 +192,7 @@ export type TTapisJob = {
  stageAppTransactionId?: string;
  status: string;
  subscriptions: string;
- tags: string[];
+ tags: string[] | null;
  tapisQueue: string;
  tenant: string;
  uuid: string;

client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx

@@ -14,7 +14,9 @@ export const ProjectCitation: React.FC<{
  const { data } = useProjectDetail(projectId);
  const entityDetails = data?.entities.find((e) => e.uuid === entityUuid);
  const authors =
- entityDetails?.value.authors?.filter((a) => a.fname && a.lname) ?? [];
+ entityDetails?.value.authors?.filter(
+ (a) => a.fname && a.lname && a.authorship !== false
+ ) ?? [];
  if (!data || !entityDetails) return null;
  return (
  <div>
@@ -44,7 +46,8 @@ export const PublishedCitation: React.FC<{
  (child) => child.uuid === entityUuid && child.version === version
  );
 
- const authors = entityDetails?.value.authors ?? [];
+ const authors =
+ entityDetails?.value.authors?.filter((a) => a.authorship !== false) ?? [];
  if (!data || !entityDetails) return null;
 
  const doi =

client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx

@@ -17,8 +17,8 @@ export const ProjectCollapse: React.FC<
  if (header.length > 0) {
  const headerElement = header[0] as HTMLElement;
  headerElement.style.backgroundColor =
- PROJECT_COLORS[entityName]['fill'];
- headerElement.style.border = `1px solid ${PROJECT_COLORS[entityName]['outline']}`;
+ PROJECT_COLORS[entityName]?.['fill'];
+ headerElement.style.border = `1px solid ${PROJECT_COLORS[entityName]?.['outline']}`;
  }
  }
  },
@@ -36,7 +36,8 @@ export const ProjectCollapse: React.FC<
  forceRender: true,
  label: (
  <span>
- {DISPLAY_NAMES[entityName]} | <strong>{title}</strong>
+ {DISPLAY_NAMES[entityName] ?? 'Collection'} |{' '}
+ <strong>{title}</strong>
  </span>
  ),
  children,

client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx

@@ -137,8 +137,8 @@ const RecursiveTree: React.FC<{
  <div
  className={styles['tree-list-badge']}
  style={{
- backgroundColor: PROJECT_COLORS[treeData.name].fill,
- outline: `1px solid ${PROJECT_COLORS[treeData.name].outline}`,
+ backgroundColor: PROJECT_COLORS[treeData.name]?.fill,
+ outline: `1px solid ${PROJECT_COLORS[treeData.name]?.outline}`,
  }}
  >
  {DISPLAY_NAMES[treeData.name]}

client/modules/datafiles/src/projects/constants.ts

@@ -17,6 +17,8 @@ export const SIMULATION_MODEL = 'designsafe.project.simulation.model';
 export const SIMULATION_INPUT = 'designsafe.project.simulation.input';
 export const SIMULATION_OUTPUT = 'designsafe.project.simulation.output';
 // Field Research
+export const FIELD_RECON_COLLECTION =
+ 'designsafe.project.field_recon.collection';
 export const FIELD_RECON_MISSION = 'designsafe.project.field_recon.mission';
 export const FIELD_RECON_REPORT = 'designsafe.project.field_recon.report';
 export const FIELD_RECON_SOCIAL_SCIENCE =
@@ -72,6 +74,7 @@ export const PROJECT_COLORS: Record<string, { outline: string; fill: string }> =
  [HYBRID_SIM_SIM_SUBSTRUCTURE]: { outline: '#BD5717', fill: '#EBCCB9' },
  [HYBRID_SIM_SIM_OUTPUT]: { outline: '#B59300', fill: '#ECE4BF' },
 
+ [FIELD_RECON_COLLECTION]: { outline: '#43A59D', fill: '#CAE9E6' },
  [FIELD_RECON_REPORT]: { outline: '#cccccc', fill: '#f5f5f5' },
  [FIELD_RECON_MISSION]: { outline: '#000000', fill: '#ffffff' },
  [FIELD_RECON_GEOSCIENCE]: { outline: '#43A59D', fill: '#CAE9E6' },
@@ -191,6 +194,7 @@ export const DISPLAY_NAMES: Record<string, string> = {
  [HYBRID_SIM_SIM_OUTPUT]: 'Simulation Output',
  // Field Recon
  [FIELD_RECON_MISSION]: 'Mission',
+ [FIELD_RECON_COLLECTION]: 'Collection',
  [FIELD_RECON_GEOSCIENCE]: 'Geoscience Collection',
  [FIELD_RECON_SOCIAL_SCIENCE]: 'Social Science Collection',
  [FIELD_RECON_REPORT]: 'Document Collection',

client/modules/workspace/src/AppsWizard/AppsFormSchema.ts

@@ -56,8 +56,7 @@ export type TField = {
  parameterSet?: string;
  description?: string;
  options?: TFieldOptions[];
- tapisFile?: boolean;
- tapisFileSelectionMode?: string;
+ fileSettings?: TAppFileSettings;
  placeholder?: string;
  readOnly?: boolean;
 };
@@ -96,7 +95,7 @@ export type TAppFormSchema = {
  };
 };
 
-export const inputFileRegex = /^tapis:\/\/(?<storageSystem>[^/]+)/;
+export const tapisInputFileRegex = /^tapis:\/\/(?<storageSystem>[^/]+)/;
 
 export const fieldDisplayOrder: Record<string, string[]> = {
  configuration: [
@@ -109,6 +108,14 @@ export const fieldDisplayOrder: Record<string, string[]> = {
  outputs: ['name', 'archiveSystemId', 'archiveSystemDir'],
 };
 
+export type TAppFilePathRepresentation = 'FullTapisPath' | 'NameOnly';
+export type TAppFileSelectionMode = 'both' | 'file' | 'directory';
+
+export type TAppFileSettings = {
+ fileNameRepresentation: TAppFilePathRepresentation;
+ fileSelectionMode: TAppFileSelectionMode;
+};
+
 // See https://github.com/colinhacks/zod/issues/310 for Zod issue
 const emptyStringToUndefined = z.literal('').transform(() => undefined);
 function asOptionalField<T extends z.ZodTypeAny>(schema: T) {
@@ -316,6 +323,12 @@ const FormSchema = (
  name: `parameters.${parameterSet}.${label}`,
  key: paramId,
  type: 'text',
+ ...(param.notes?.inputType === 'fileInput' && {
+ fileSettings: {
+ fileNameRepresentation: 'NameOnly',
+ fileSelectionMode: 'file',
+ },
+ }),
  };
 
  if (param.notes?.enum_values) {
@@ -399,11 +412,14 @@ const FormSchema = (
  required: input.inputMode === 'REQUIRED',
  name: `inputs.${input.name}`,
  key: `inputs.${input.name}`,
- tapisFile: true,
  type: 'text',
  placeholder: 'Browse Data Files',
  readOnly: input.inputMode === 'FIXED',
- tapisFileSelectionMode: input.notes?.selectionMode ?? 'both',
+ fileSettings: {
+ fileNameRepresentation: 'FullTapisPath',
+ fileSelectionMode:
+ (input.notes?.selectionMode as TAppFileSelectionMode) ?? 'both',
+ },
  };
 
  appFields.fileInputs.schema[input.name] = z.string();