diff --git a/.gitignore b/.gitignore index dbcc6364ed..bcd4a23c88 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ designsafe/templates/react-assets.html designsafe.env mysql.env rabbitmq.env +ngrok.env rabbitmq.conf mysql.cnf diff --git a/.pylintrc b/.pylintrc index d8f0f80226..3458e31246 100644 --- a/.pylintrc +++ b/.pylintrc @@ -52,7 +52,7 @@ ignore=CVS,tests.py # ignore-list. The regex matches against paths and can be in Posix or Windows # format. Because '\\' represents the directory delimiter on Windows systems, # it can't be used as an escape character. -ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$ +ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$,^.*test_.*$ # Files or directories matching the regular expression patterns are skipped. # The regex matches against base names, not paths. The default value ignores diff --git a/Makefile b/Makefile index 0104e5c8b2..dc85626ec1 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,9 @@ +NGROK_ENV_FILE = ./conf/env_files/ngrok.env +ifeq ("$(wildcard $(NGROK_ENV_FILE))","") + NGROK_ENV_FILE = ./conf/env_files/ngrok.sample.env +endif + + .PHONY: build build: docker compose -f ./conf/docker/docker-compose.yml build @@ -8,16 +14,16 @@ build-dev: .PHONY: start start: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml up + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml up .PHONY: stop stop: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml down + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml down .PHONY: start-m1 start-m1: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up .PHONY: stop-m1 stop-m1: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down diff --git a/README.md b/README.md index b2e91f8b7a..b5de1b005a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ -[![Build Status](https://travis-ci.org/DesignSafe-CI/portal.svg?branch=master)](https://travis-ci.org/DesignSafe-CI/portal) -[![codecov](https://codecov.io/gh/DesignSafe-CI/portal/branch/master/graph/badge.svg)](https://codecov.io/gh/DesignSafe-CI/portal) - # DesignSafe-CI Portal ## Prequisites for running the portal application @@ -11,8 +8,8 @@ on. - [Install Docker][3] - [Install Docker Compose][4] -- [Install Make][12] -- [Node.js][13] 16.x +- [Install Make][11] +- [Node.js][12] 20.x If you are on a Mac or a Windows machine, the recommended method is to install [Docker Desktop][5], which will install both Docker and Docker Compose, which is required to run Docker on Mac/Windows hosts. @@ -37,7 +34,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install - `DJANGO_DEBUG`: should be set to `True` for development - `DJANGO_SECRET`: should be changed for production - `TAS_*`: should be set to enable direct access to `django.contrib.admin` - - `AGAVE_*`: should be set to enable Agave API integration (authentication, etc.) + - `TAPIS_*`: should be set to enable Tapis API integration (authentication, etc.) - `RT_*`: should be set to enable ticketing Make a copy of [rabbitmq.sample.env](conf/env_files/rabbitmq.sample.env) @@ -46,7 +43,15 @@ If you are on a Mac or a Windows machine, the recommended method is to install Make a copy of [external_resource_secrets.sample.py](designsafe/settings/external_resource_secrets.sample.py) and rename it to `external_resource_secrets.py`. -3. Build the containers and frontend packages +3. Configure ngrok + + a. Install [ngrok](https://ngrok.com/docs/getting-started/), and create an ngrok account. + + b. Copy [conf/env_files/ngrok.sample.env](conf/env_files/ngrok.sample.env) to `conf/env_files/ngrok.env`. + + c. In `conf/env_files/ngrok.env`, set the `NGROK_AUTHTOKEN` and `NGROK_DOMAIN` variables using your authtoken and static ngrok domain found in your [ngrok dashboard](https://dashboard.ngrok.com/). + +4. Build the containers and frontend packages 1. Containers: ```sh @@ -72,7 +77,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install npm run start ``` -4. Start local containers +5. Start local containers ``` $ make start @@ -89,11 +94,11 @@ If you are on a Mac or a Windows machine, the recommended method is to install $ ./manage.py createsuperuser ``` -5. Setup local access to the portal: +6. Setup local access to the portal: Add a record to your local hosts file for `127.0.0.1 designsafe.dev` ``` - sudo vim /etc/hosts + $ sudo vim /etc/hosts ``` Now you can navigate to [designsafe.dev](designsafe.dev) in your browser. @@ -178,11 +183,10 @@ See the [DesignSafe Styles Reference][7] for style reference and custom CSS docu ### Updating Python dependencies -For simplicity the Dockerfile uses a `requirements.txt` exported from Poetry. To add a new dependency: +This project uses [Python Poetry](https://python-poetry.org/docs/) to manage dependencies. To add a new dependency: 1. Run `poetry add $NEW_DEPENDENCY`. -2. Run `poetry export > requirements.txt --dev --without-hashes` in the repository root. -3. Rebuild the dev image with `docker-compose -f conf/docker/docker-compose.yml build` +2. Rebuild the dev image with `make build-dev` ## Testing @@ -200,16 +204,14 @@ Django tests should be written according to standard [Django testing procedures] You can run Django tests with the following command: ```shell -$ docker exec -it des_django pytest designsafe +$ docker exec -it des_django pytest -ra designsafe ``` ### Frontend tests -Frontend tests are [Jasmine][9] tests executed using the [Karma engine][10]. Testing -guidelines can be found in the [AngularJS Developer Guide on Unit Testing][11]. +Frontend tests are [Vitest][9] tests executed using [Nx][10]. -To run frontend tests, ensure that all scripts and test scripts are configured in -[`karma-conf.js`](karma-conf.js) and then run the command: +To run frontend tests, run the command: ```shell $ npm run test @@ -217,47 +219,31 @@ $ npm run test ## Development setup -Use `docker-compose` to run the portal in development. The default compose file, -[`docker-compose.yml`](docker-compose.yml) runs the main django server in development +Use `docker compose` to run the portal in development. The default compose file, +[`docker-compose.yml`](conf/docker/docker-compose.yml) runs the main django server in development mode with a redis service for websockets support. You can optionally enable the EF sites for testing. ```shell -$ docker-compose -f conf/docker/docker-compose.yml build -$ docker-compose -f conf/docker/docker-compose-dev.all.debug.yml up -$ npm run dev +$ make build-dev +$ make start +$ npm run start +$ docker run -v `pwd`:`pwd` -w `pwd` -it node:16 /bin/bash -c "npm run dev" ``` When using this compose file, your Tapis Client should be configured with a `callback_url` of `http://$DOCKER_HOST_IP:8000/auth/tapis/callback/`. -For developing some services, e.g. Box.com integration, https support is required. To -enable an Nginx http proxy run using the [`docker-compose-http.yml`](docker-compose-http.yml) -file. This file configures the same services as the default compose file, but it also sets -up an Nginx proxy secured by a self-signed certificate. ```shell $ docker-compose -f docker-compose-http.yml build $ docker-compose -f docker-compose-http.yml up ``` -### Agave filesystem setup -1. Delete all of the old metadata objects using this command: - - `metadata-list Q '{"name": "designsafe metadata"}' | while read x; do metadata-delete $x; done;` -2. Run `dsapi/agave/tools/bin/walker.py` to create the metadata objects for the existing files in your FS. - - `python portal/dsapi/agave/tools/bin/walker.py ` - - `base_folder` is your username, if you want to fix everything under your home dir. - - `command`: - - `files`: Walk through the files and print their path. - - `meta`: Walk through the metadata objs in a filesystem-like manner and print their path. - - `files-fix`: Check if there's a meta obj for every file, if not create the meta obj. - - `meta-fix`: Check if there's a file for every meta obj, if not delete the meta obj. ## Production setup -Production deployment is managed by ansible. See https://github.com/designsafe-ci/ansible. +Production deployment is managed by Camino. See https://github.com/TACC/Camino. [1]: https://docs.docker.com/ @@ -267,8 +253,7 @@ Production deployment is managed by ansible. See https://github.com/designsafe-c [5]: https://docs.docker.com/desktop/ [7]: https://github.com/DesignSafe-CI/portal/wiki/CSS-Styles-Reference [8]: https://docs.djangoproject.com/en/dev/topics/testing/ -[9]: http://jasmine.github.io/1.3/introduction.html -[10]: http://karma-runner.github.io/0.12/intro/installation.html -[11]: https://docs.angularjs.org/guide/unit-testing -[12]: https://www.gnu.org/software/make/ -[13]: https://nodejs.org/ +[9]: https://vitest.dev/ +[10]: https://nx.dev/getting-started/intro +[11]: https://www.gnu.org/software/make/ +[12]: https://nodejs.org/ diff --git a/client/modules/_hooks/src/notifications/useNotifications.ts b/client/modules/_hooks/src/notifications/useNotifications.ts index 566a0633ed..cfbcebe900 100644 --- a/client/modules/_hooks/src/notifications/useNotifications.ts +++ b/client/modules/_hooks/src/notifications/useNotifications.ts @@ -6,7 +6,11 @@ import { } from '@tanstack/react-query'; import apiClient from '../apiClient'; -type TPortalEventType = 'data_depot' | 'job' | 'interactive_session_ready'; +type TPortalEventType = + | 'data_depot' + | 'job' + | 'interactive_session_ready' + | 'markAllNotificationsAsRead'; export type TJobStatusNotification = { action_link: string; diff --git a/client/modules/_hooks/src/workspace/types.ts b/client/modules/_hooks/src/workspace/types.ts index 9d73e2ef6a..035d1ec608 100644 --- a/client/modules/_hooks/src/workspace/types.ts +++ b/client/modules/_hooks/src/workspace/types.ts @@ -1,6 +1,7 @@ export type TParameterSetNotes = { isHidden?: boolean; fieldType?: string; + inputType?: string; validator?: { regex: string; message: string; @@ -191,7 +192,7 @@ export type TTapisJob = { stageAppTransactionId?: string; status: string; subscriptions: string; - tags: string[]; + tags: string[] | null; tapisQueue: string; tenant: string; uuid: string; diff --git a/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx b/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx index 28c9fe72e7..6fd4f87488 100644 --- a/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx +++ b/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx @@ -14,7 +14,9 @@ export const ProjectCitation: React.FC<{ const { data } = useProjectDetail(projectId); const entityDetails = data?.entities.find((e) => e.uuid === entityUuid); const authors = - entityDetails?.value.authors?.filter((a) => a.fname && a.lname) ?? []; + entityDetails?.value.authors?.filter( + (a) => a.fname && a.lname && a.authorship !== false + ) ?? []; if (!data || !entityDetails) return null; return (
@@ -44,7 +46,8 @@ export const PublishedCitation: React.FC<{ (child) => child.uuid === entityUuid && child.version === version ); - const authors = entityDetails?.value.authors ?? []; + const authors = + entityDetails?.value.authors?.filter((a) => a.authorship !== false) ?? []; if (!data || !entityDetails) return null; const doi = diff --git a/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx b/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx index aacdf9250f..aac0067b4c 100644 --- a/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx +++ b/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx @@ -17,8 +17,8 @@ export const ProjectCollapse: React.FC< if (header.length > 0) { const headerElement = header[0] as HTMLElement; headerElement.style.backgroundColor = - PROJECT_COLORS[entityName]['fill']; - headerElement.style.border = `1px solid ${PROJECT_COLORS[entityName]['outline']}`; + PROJECT_COLORS[entityName]?.['fill']; + headerElement.style.border = `1px solid ${PROJECT_COLORS[entityName]?.['outline']}`; } } }, @@ -36,7 +36,8 @@ export const ProjectCollapse: React.FC< forceRender: true, label: ( - {DISPLAY_NAMES[entityName]} | {title} + {DISPLAY_NAMES[entityName] ?? 'Collection'} |{' '} + {title} ), children, diff --git a/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx b/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx index 76208dfeee..b05ab19159 100644 --- a/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx +++ b/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx @@ -137,8 +137,8 @@ const RecursiveTree: React.FC<{
{DISPLAY_NAMES[treeData.name]} diff --git a/client/modules/datafiles/src/projects/constants.ts b/client/modules/datafiles/src/projects/constants.ts index bda51640bd..51eb5d2cd0 100644 --- a/client/modules/datafiles/src/projects/constants.ts +++ b/client/modules/datafiles/src/projects/constants.ts @@ -17,6 +17,8 @@ export const SIMULATION_MODEL = 'designsafe.project.simulation.model'; export const SIMULATION_INPUT = 'designsafe.project.simulation.input'; export const SIMULATION_OUTPUT = 'designsafe.project.simulation.output'; // Field Research +export const FIELD_RECON_COLLECTION = + 'designsafe.project.field_recon.collection'; export const FIELD_RECON_MISSION = 'designsafe.project.field_recon.mission'; export const FIELD_RECON_REPORT = 'designsafe.project.field_recon.report'; export const FIELD_RECON_SOCIAL_SCIENCE = @@ -72,6 +74,7 @@ export const PROJECT_COLORS: Record = [HYBRID_SIM_SIM_SUBSTRUCTURE]: { outline: '#BD5717', fill: '#EBCCB9' }, [HYBRID_SIM_SIM_OUTPUT]: { outline: '#B59300', fill: '#ECE4BF' }, + [FIELD_RECON_COLLECTION]: { outline: '#43A59D', fill: '#CAE9E6' }, [FIELD_RECON_REPORT]: { outline: '#cccccc', fill: '#f5f5f5' }, [FIELD_RECON_MISSION]: { outline: '#000000', fill: '#ffffff' }, [FIELD_RECON_GEOSCIENCE]: { outline: '#43A59D', fill: '#CAE9E6' }, @@ -191,6 +194,7 @@ export const DISPLAY_NAMES: Record = { [HYBRID_SIM_SIM_OUTPUT]: 'Simulation Output', // Field Recon [FIELD_RECON_MISSION]: 'Mission', + [FIELD_RECON_COLLECTION]: 'Collection', [FIELD_RECON_GEOSCIENCE]: 'Geoscience Collection', [FIELD_RECON_SOCIAL_SCIENCE]: 'Social Science Collection', [FIELD_RECON_REPORT]: 'Document Collection', diff --git a/client/modules/workspace/src/AppsWizard/AppsFormSchema.ts b/client/modules/workspace/src/AppsWizard/AppsFormSchema.ts index 4a04a58181..8a3fec01c3 100644 --- a/client/modules/workspace/src/AppsWizard/AppsFormSchema.ts +++ b/client/modules/workspace/src/AppsWizard/AppsFormSchema.ts @@ -56,8 +56,7 @@ export type TField = { parameterSet?: string; description?: string; options?: TFieldOptions[]; - tapisFile?: boolean; - tapisFileSelectionMode?: string; + fileSettings?: TAppFileSettings; placeholder?: string; readOnly?: boolean; }; @@ -96,7 +95,7 @@ export type TAppFormSchema = { }; }; -export const inputFileRegex = /^tapis:\/\/(?[^/]+)/; +export const tapisInputFileRegex = /^tapis:\/\/(?[^/]+)/; export const fieldDisplayOrder: Record = { configuration: [ @@ -109,6 +108,14 @@ export const fieldDisplayOrder: Record = { outputs: ['name', 'archiveSystemId', 'archiveSystemDir'], }; +export type TAppFilePathRepresentation = 'FullTapisPath' | 'NameOnly'; +export type TAppFileSelectionMode = 'both' | 'file' | 'directory'; + +export type TAppFileSettings = { + fileNameRepresentation: TAppFilePathRepresentation; + fileSelectionMode: TAppFileSelectionMode; +}; + // See https://github.com/colinhacks/zod/issues/310 for Zod issue const emptyStringToUndefined = z.literal('').transform(() => undefined); function asOptionalField(schema: T) { @@ -316,6 +323,12 @@ const FormSchema = ( name: `parameters.${parameterSet}.${label}`, key: paramId, type: 'text', + ...(param.notes?.inputType === 'fileInput' && { + fileSettings: { + fileNameRepresentation: 'NameOnly', + fileSelectionMode: 'file', + }, + }), }; if (param.notes?.enum_values) { @@ -399,11 +412,14 @@ const FormSchema = ( required: input.inputMode === 'REQUIRED', name: `inputs.${input.name}`, key: `inputs.${input.name}`, - tapisFile: true, type: 'text', placeholder: 'Browse Data Files', readOnly: input.inputMode === 'FIXED', - tapisFileSelectionMode: input.notes?.selectionMode ?? 'both', + fileSettings: { + fileNameRepresentation: 'FullTapisPath', + fileSelectionMode: + (input.notes?.selectionMode as TAppFileSelectionMode) ?? 'both', + }, }; appFields.fileInputs.schema[input.name] = z.string(); diff --git a/client/modules/workspace/src/AppsWizard/FormField.tsx b/client/modules/workspace/src/AppsWizard/FormField.tsx index 1ec10262f0..8979c0b017 100644 --- a/client/modules/workspace/src/AppsWizard/FormField.tsx +++ b/client/modules/workspace/src/AppsWizard/FormField.tsx @@ -2,30 +2,32 @@ import React, { useState, useEffect } from 'react'; import { Button, Form, Input, Select } from 'antd'; import { FormItem } from 'react-hook-form-antd'; import { useFormContext, useWatch } from 'react-hook-form'; -import { TFieldOptions, inputFileRegex } from '../AppsWizard/AppsFormSchema'; +import { + TFieldOptions, + tapisInputFileRegex, + TAppFileSettings, +} from '../AppsWizard/AppsFormSchema'; import { SecondaryButton } from '@client/common-components'; import { SelectModal } from '../SelectModal/SelectModal'; export const FormField: React.FC<{ name: string; - tapisFile?: boolean; parameterSet?: string; description?: string; label: string; required?: boolean; type: string; - tapisFileSelectionMode?: string; + fileSettings?: TAppFileSettings; placeholder?: string; options?: TFieldOptions[]; }> = ({ name, - tapisFile = false, parameterSet = null, description, label, required = false, type, - tapisFileSelectionMode = null, + fileSettings = null, ...props }) => { const { resetField, control, getValues, setValue, trigger } = @@ -39,16 +41,17 @@ export const FormField: React.FC<{ setIsModalOpen(true); }; useEffect(() => { - if (tapisFile) { + setStorageSystem(null); + + if (fileSettings?.fileNameRepresentation === 'FullTapisPath') { const inputFileValue = getValues(name); - const match = inputFileValue?.match(inputFileRegex); - if (match && match.groups) { + const match = inputFileValue?.match(tapisInputFileRegex); + + if (match?.groups) { setStorageSystem(match.groups.storageSystem); - } else { - setStorageSystem(null); } } - }, [tapisFile, name, fieldState]); + }, [fileSettings, name, fieldState]); if (parameterSet) { parameterSetLabel = ( @@ -89,7 +92,7 @@ export const FormField: React.FC<{ /> ) : (
- {tapisFile && ( + {fileSettings && ( Select @@ -133,11 +136,11 @@ export const FormField: React.FC<{ {/* Select Modal has Form and input which cause state sharing with above FormItem So, SelectModal is outside FormItem. */} - {tapisFile && ( + {fileSettings && ( setIsModalOpen(false)} onSelect={(value: string) => { diff --git a/client/modules/workspace/src/JobsListing/JobsListing.tsx b/client/modules/workspace/src/JobsListing/JobsListing.tsx index d2dcf3404d..6aaae7aa54 100644 --- a/client/modules/workspace/src/JobsListing/JobsListing.tsx +++ b/client/modules/workspace/src/JobsListing/JobsListing.tsx @@ -1,4 +1,5 @@ import React, { useMemo, useState, useEffect } from 'react'; +import useWebSocket from 'react-use-websocket'; import { TableProps, Row, Flex, Button as AntButton } from 'antd'; import type { ButtonSize } from 'antd/es/button'; import { useQueryClient } from '@tanstack/react-query'; @@ -96,12 +97,16 @@ export const JobsListing: React.FC> = ({ markRead: false, }); const { mutate: readNotifications } = useReadNotifications(); + const { sendMessage } = useWebSocket( + `wss://${window.location.host}/ws/websockets/` + ); // mark all as read on component mount useEffect(() => { readNotifications({ eventTypes: ['interactive_session_ready', 'job'], }); + sendMessage('markAllNotificationsAsRead'); // update unread count state queryClient.setQueryData( diff --git a/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx b/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx index d3d11deb8e..4cffd2b64f 100644 --- a/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx +++ b/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx @@ -91,9 +91,11 @@ export const JobsListingTable: React.FC< isLoading, ]); - const lastNotificationJobUUID = lastMessage - ? (JSON.parse(lastMessage.data) as TJobStatusNotification).extra.uuid - : ''; + const lastMessageJSON = lastMessage?.data + ? (JSON.parse(lastMessage.data) as TJobStatusNotification) + : null; + const lastNotificationJobUUID = + lastMessageJSON?.event_type === 'job' ? lastMessageJSON.extra.uuid : ''; const unreadJobUUIDs = unreadNotifs?.notifs.map((x) => x.extra.uuid) ?? []; /* RENDER THE TABLE */ diff --git a/client/modules/workspace/src/SelectModal/SelectModal.tsx b/client/modules/workspace/src/SelectModal/SelectModal.tsx index fbbb4906f2..01a6cdd719 100644 --- a/client/modules/workspace/src/SelectModal/SelectModal.tsx +++ b/client/modules/workspace/src/SelectModal/SelectModal.tsx @@ -23,6 +23,8 @@ import { TFileListing, } from '@client/hooks'; +import { TAppFileSettings } from '../AppsWizard/AppsFormSchema'; + import { BaseFileListingBreadcrumb, FileListingTable, @@ -152,7 +154,7 @@ const getParentFolder = ( function getFilesColumns( api: string, path: string, - selectionMode: string, + appFileSettings: TAppFileSettings, searchTerm: string | null, clearSearchTerm: () => void, selectionCallback: (path: string) => void, @@ -233,9 +235,11 @@ function getFilesColumns( title: '', render: (_, record, index) => { const selectionModeAllowed = - (record.type === 'dir' && selectionMode === 'directory') || - (record.type === 'file' && selectionMode === 'file') || - selectionMode === 'both'; + (record.type === 'dir' && + appFileSettings.fileSelectionMode === 'directory') || + (record.type === 'file' && + appFileSettings.fileSelectionMode === 'file') || + appFileSettings.fileSelectionMode === 'both'; const isNotRoot = index > 0 || record.system.startsWith(projectPrefix) || @@ -244,9 +248,15 @@ function getFilesColumns( return shouldRenderSelectButton ? ( - selectionCallback(`${api}://${record.system}${record.path}`) - } + onClick={() => { + const lastPartOfPath = record.path.split('/').pop() ?? ''; + const filePath = + appFileSettings.fileNameRepresentation === 'FullTapisPath' + ? `${api}://${record.system}${record.path}` + : lastPartOfPath; + + selectionCallback(filePath); + }} > Select @@ -259,11 +269,11 @@ function getFilesColumns( export const SelectModal: React.FC<{ inputLabel: string; system: string | null; - selectionMode: string; + appFileSettings: TAppFileSettings; isOpen: boolean; onClose: () => void; onSelect: (value: string) => void; -}> = ({ inputLabel, system, selectionMode, isOpen, onClose, onSelect }) => { +}> = ({ inputLabel, system, appFileSettings, isOpen, onClose, onSelect }) => { const [isModalOpen, setIsModalOpen] = useState(false); const [searchTerm, setSearchTerm] = useState(null); const [form] = Form.useForm(); @@ -420,7 +430,7 @@ export const SelectModal: React.FC<{ getFilesColumns( selectedApi, selectedPath, - selectionMode, + appFileSettings, searchTerm, clearSearchTerm, (selection: string) => selectCallback(selection), @@ -435,7 +445,7 @@ export const SelectModal: React.FC<{ selectedSystem, selectedPath, systemLabel, - selectionMode, + appFileSettings, selectCallback, ] ); diff --git a/client/modules/workspace/src/Toast/Notifications.module.css b/client/modules/workspace/src/Toast/Notifications.module.css index 44beb6563c..44ca35802c 100644 --- a/client/modules/workspace/src/Toast/Notifications.module.css +++ b/client/modules/workspace/src/Toast/Notifications.module.css @@ -1,3 +1,13 @@ +.root { + cursor: pointer; + background: #f4f4f4; + border: 1px solid #222222; + &:hover { + border-color: #5695c4; + background: #aac7ff; + } +} + .toast-is-error { color: #eb6e6e; } diff --git a/client/modules/workspace/src/Toast/Toast.tsx b/client/modules/workspace/src/Toast/Toast.tsx index 1956cd1102..576476be92 100644 --- a/client/modules/workspace/src/Toast/Toast.tsx +++ b/client/modules/workspace/src/Toast/Toast.tsx @@ -1,10 +1,14 @@ import React, { useEffect } from 'react'; import useWebSocket from 'react-use-websocket'; import { useQueryClient } from '@tanstack/react-query'; -import { notification } from 'antd'; +import { notification, Flex } from 'antd'; +import { RightOutlined } from '@ant-design/icons'; import { useNavigate } from 'react-router-dom'; import { Icon } from '@client/common-components'; -import { TJobStatusNotification } from '@client/hooks'; +import { + TJobStatusNotification, + TGetNotificationsResponse, +} from '@client/hooks'; import { getToastMessage } from '../utils'; import styles from './Notifications.module.css'; @@ -31,19 +35,43 @@ const Notifications = () => { queryKey: ['workspace', 'jobsListing'], }); api.open({ - message: getToastMessage(notification), + message: ( + + {getToastMessage(notification)} + + + ), placement: 'bottomLeft', icon: , className: `${ notification.extra.status === 'FAILED' && styles['toast-is-error'] - }`, + } ${styles.root}`, closeIcon: false, duration: 5, onClick: () => { navigate('/history'); }, - style: { cursor: 'pointer' }, }); + } else if (notification.event_type === 'markAllNotificationsAsRead') { + // update unread count state + queryClient.setQueryData( + [ + 'workspace', + 'notifications', + { + eventTypes: ['interactive_session_ready', 'job'], + read: false, + markRead: false, + }, + ], + (oldData: TGetNotificationsResponse) => { + return { + ...oldData, + notifs: [], + unread: 0, + }; + } + ); } }; diff --git a/client/modules/workspace/src/utils/jobs.ts b/client/modules/workspace/src/utils/jobs.ts index 89b58a00fd..3915fe9923 100644 --- a/client/modules/workspace/src/utils/jobs.ts +++ b/client/modules/workspace/src/utils/jobs.ts @@ -68,7 +68,7 @@ export function getOutputPath(job: TTapisJob) { } export function isInteractiveJob(job: TTapisJob) { - return job.tags.includes('isInteractive'); + return job.tags?.includes('isInteractive'); } export function getJobInteractiveSessionInfo( diff --git a/client/src/workspace/workspaceRouter.tsx b/client/src/workspace/workspaceRouter.tsx index d9b3bf7c95..ad21288298 100644 --- a/client/src/workspace/workspaceRouter.tsx +++ b/client/src/workspace/workspaceRouter.tsx @@ -7,6 +7,13 @@ import { JobsListingLayout } from './layouts/JobsListingLayout'; import { AppsViewLayout } from './layouts/AppsViewLayout'; import { AppsPlaceholderLayout } from './layouts/AppsPlaceholderLayout'; +const getBaseName = () => { + if (window.location.pathname.startsWith('/rw/workspace')) { + return '/rw/workspace'; + } + return '/workspace'; +}; + const workspaceRouter = createBrowserRouter( [ { @@ -44,7 +51,7 @@ const workspaceRouter = createBrowserRouter( ], }, ], - { basename: '/rw/workspace' } + { basename: getBaseName() } ); export default workspaceRouter; diff --git a/conf/docker/docker-compose-dev.all.debug.m1.yml b/conf/docker/docker-compose-dev.all.debug.m1.yml index edbe69b19c..8337c2219f 100644 --- a/conf/docker/docker-compose-dev.all.debug.m1.yml +++ b/conf/docker/docker-compose-dev.all.debug.m1.yml @@ -77,7 +77,10 @@ services: django: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -98,7 +101,10 @@ services: websockets: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false volumes: - ../../.:/srv/www/designsafe - ../../data/media:/srv/www/designsafe/media @@ -108,7 +114,10 @@ services: workers: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -125,6 +134,20 @@ services: container_name: des_workers hostname: des_workers + ngrok: + image: ngrok/ngrok:latest + platform: "linux/amd64" + environment: + NGROK_AUTHTOKEN: ${NGROK_AUTHTOKEN} + command: + - "http" + - --url=${NGROK_DOMAIN} + - "https://host.docker.internal:443" + ports: + - 4040:4040 + container_name: des_ngrok + hostname: des_ngrok + volumes: redis_data_v3: des_postgres_data_v3: diff --git a/conf/docker/docker-compose-dev.all.debug.yml b/conf/docker/docker-compose-dev.all.debug.yml index 22d5c62155..9fb100962a 100644 --- a/conf/docker/docker-compose-dev.all.debug.yml +++ b/conf/docker/docker-compose-dev.all.debug.yml @@ -73,7 +73,10 @@ services: django: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -94,7 +97,10 @@ services: websockets: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false volumes: - ../../.:/srv/www/designsafe - ../../data/media:/srv/www/designsafe/media @@ -104,7 +110,10 @@ services: workers: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -121,6 +130,19 @@ services: container_name: des_workers hostname: des_workers + ngrok: + image: ngrok/ngrok:latest + environment: + NGROK_AUTHTOKEN: ${NGROK_AUTHTOKEN} + command: + - "http" + - --url=${NGROK_DOMAIN} + - "https://host.docker.internal:443" + ports: + - 4040:4040 + container_name: des_ngrok + hostname: des_ngrok + volumes: redis_data_v3: des_postgres_data_v3: diff --git a/conf/env_files/designsafe.sample.env b/conf/env_files/designsafe.sample.env index 5c13942f68..c9abbb3530 100644 --- a/conf/env_files/designsafe.sample.env +++ b/conf/env_files/designsafe.sample.env @@ -80,7 +80,6 @@ AGAVE_CLIENT_SECRET= AGAVE_SUPER_TOKEN= AGAVE_STORAGE_SYSTEM= -AGAVE_WORKING_SYSTEM= AGAVE_JWT_PUBKEY= AGAVE_JWT_ISSUER= AGAVE_JWT_HEADER= diff --git a/conf/env_files/ngrok.sample.env b/conf/env_files/ngrok.sample.env new file mode 100644 index 0000000000..65ae75620a --- /dev/null +++ b/conf/env_files/ngrok.sample.env @@ -0,0 +1,5 @@ +# Get authtoken from https://dashboard.ngrok.com/get-started/your-authtoken +NGROK_AUTHTOKEN=your_ngrok_authtoken + +# Get static domain from https://dashboard.ngrok.com/domains +NGROK_DOMAIN=your-ngrok-subdomain.ngrok-free.app diff --git a/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html b/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html index e0c074ed59..b70f0557f4 100644 --- a/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html +++ b/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html @@ -9,7 +9,7 @@

{{title}}

    {% url 'designsafe_accounts:manage_profile' as item_url %} -
  • Account Profile
    • +
  • Manage Account
    • {% url 'designsafe_accounts:manage_authentication' as item_url %}
  • Authentication
    • diff --git a/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html b/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html index 85ba529baf..0f41e9f442 100644 --- a/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html +++ b/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html @@ -1,11 +1,11 @@ {% extends "designsafe/apps/accounts/base.html" %} -{% block title %}Account Profile{% endblock %} +{% block title %}Manage Account{% endblock %} {% block panel_content %}
    -

    Account Profile

    +

    Manage Account

    diff --git a/designsafe/apps/accounts/urls.py b/designsafe/apps/accounts/urls.py index 18e1585e55..7d54f11d58 100644 --- a/designsafe/apps/accounts/urls.py +++ b/designsafe/apps/accounts/urls.py @@ -30,7 +30,7 @@ def menu_items(**kwargs): if 'type' in kwargs and kwargs['type'] == 'account': return [ { - 'label': _('Account Profile'), + 'label': _('Manage Account'), 'url': reverse('designsafe_accounts:index'), 'children': [], }, diff --git a/designsafe/apps/accounts/views.py b/designsafe/apps/accounts/views.py index 346fb04764..69765b0335 100644 --- a/designsafe/apps/accounts/views.py +++ b/designsafe/apps/accounts/views.py @@ -49,7 +49,7 @@ def manage_profile(request): logger.info('exception e:{} {}'.format(type(e), e)) context = { - 'title': 'Account Profile', + 'title': 'Manage Account', 'profile': user_profile, 'ds_profile': ds_profile, 'demographics': demographics, @@ -353,7 +353,7 @@ def profile_edit(request): form = forms.UserProfileForm(initial=tas_user) context = { - 'title': 'Account Profile', + 'title': 'Manage Account', 'form': form, 'pro_form': pro_form } diff --git a/designsafe/apps/api/notifications/receivers.py b/designsafe/apps/api/notifications/receivers.py index 316840a699..eada12fd0a 100644 --- a/designsafe/apps/api/notifications/receivers.py +++ b/designsafe/apps/api/notifications/receivers.py @@ -1,52 +1,47 @@ +"""Signal receivers for notifications""" + +import logging +import json +from django.db.models.signals import post_save from django.dispatch import receiver from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from django.db.models.signals import post_save from designsafe.apps.api.notifications.models import Notification, Broadcast -import logging -import json logger = logging.getLogger(__name__) -WEBSOCKETS_FACILITY = 'websockets' +WEBSOCKETS_FACILITY = "websockets" + + +@receiver(post_save, sender=Notification, dispatch_uid="notification_msg") +def send_notification_ws(instance, created, **kwargs): + """Send a websocket message to the user when a new notification is created.""" -@receiver(post_save, sender=Notification, dispatch_uid='notification_msg') -def send_notification_ws(sender, instance, created, **kwargs): - #Only send WS message if it's a new notification not if we're updating. - logger.debug('receiver received something.') if not created: return - try: - channel_layer = get_channel_layer() - instance_dict = json.dumps(instance.to_dict()) - logger.debug(instance_dict) - async_to_sync(channel_layer.group_send)(f"ds_{instance.user}", {"type": "ds.notification", "message": instance_dict}) + channel_layer = get_channel_layer() + instance_dict = json.dumps(instance.to_dict()) + async_to_sync(channel_layer.group_send)( + f"ds_{instance.user}", {"type": "ds.notification", "message": instance_dict} + ) - # logger.debug('WS socket msg sent: {}'.format(instance_dict)) - except Exception as e: - # logger.debug('Exception sending websocket message', - # exc_info=True, - # extra = instance.to_dict()) - logger.debug('Exception sending websocket message', - exc_info=True) return -@receiver(post_save, sender=Broadcast, dispatch_uid='broadcast_msg') -def send_broadcast_ws(sender, instance, created, **kwargs): + +@receiver(post_save, sender=Broadcast, dispatch_uid="broadcast_msg") +def send_broadcast_ws(instance, created, **kwargs): + """Send a websocket message to all users when a new broadcast is created.""" + if not created: return - try: - event_type, user, body = decompose_message(instance) - #rp = RedisPublisher(facility = WEBSOCKETS_FACILITY,broadcast=True) - channel_layer = get_channel_layer() - instance_dict = json.dumps(instance.to_dict()) - - async_to_sync(channel_layer.group_send)("ds_broadcast", {"type": "ds.notification", "message": instance_dict}) - logger.debug('WS socket msg sent: {}'.format(instance_dict)) - except Exception as e: - logger.debug('Exception sending websocket message', - exc_info=True, - extra = instance.to_dict()) + + channel_layer = get_channel_layer() + instance_dict = json.dumps(instance.to_dict()) + + async_to_sync(channel_layer.group_send)( + "ds_broadcast", {"type": "ds.notification", "message": instance_dict} + ) + return diff --git a/designsafe/apps/auth/tasks.py b/designsafe/apps/auth/tasks.py index 26eb935be3..b97f22709b 100644 --- a/designsafe/apps/auth/tasks.py +++ b/designsafe/apps/auth/tasks.py @@ -1,68 +1,83 @@ +""" Celery tasks for user onboarding and other user-related tasks. """ + +import logging from datetime import datetime, timedelta -import requests from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from django.core.mail import send_mail +from celery import shared_task +from pytas.http import TASClient +from tapipy.errors import NotFoundError, BaseTapyException, ForbiddenError from designsafe.apps.api.agave import get_service_account_client, get_tg458981_client from designsafe.apps.api.tasks import agave_indexer from designsafe.apps.api.notifications.models import Notification -from celery import shared_task -from django.contrib.auth import get_user_model -from pytas.http import TASClient -from tapipy.errors import NotFoundError, BaseTapyException -from designsafe.utils.system_access import register_public_key, create_system_credentials +from designsafe.utils.system_access import ( + register_public_key, + create_system_credentials, +) from designsafe.utils.encryption import createKeyPair -from django.contrib.auth import get_user_model -import logging logger = logging.getLogger(__name__) def get_systems_to_configure(username): - """ Get systems to configure either during startup or for new user """ + """Get systems to configure either during startup or for new user""" systems = [] for system in settings.TAPIS_SYSTEMS_TO_CONFIGURE: system_copy = system.copy() - system_copy['path'] = system_copy['path'].format(username=username) + system_copy["path"] = system_copy["path"].format(username=username) systems.append(system_copy) return systems -@shared_task(default_retry_delay=30, max_retries=3, queue='onboarding', bind=True) -def check_or_configure_system_and_user_directory(self, username, system_id, path, create_path): +@shared_task(default_retry_delay=30, max_retries=3, queue="onboarding", bind=True) +def check_or_configure_system_and_user_directory( + self, username, system_id, path, create_path +): + """Check if user has access to system and path, if not, configure it.""" try: user_client = get_user_model().objects.get(username=username).tapis_oauth.client - user_client.files.listFiles( - systemId=system_id, path=path + user_client.files.listFiles(systemId=system_id, path=path) + logger.info( + f"System Works: " + f"Checked and there is no need to configure system:{system_id} path:{path} for {username}" ) - logger.info(f"System Works: " - f"Checked and there is no need to configure system:{system_id} path:{path} for {username}") return except ObjectDoesNotExist: # User is missing; handling email confirmation process where user has not logged in - logger.info(f"New User: " - f"Checked and there is a need to configure system:{system_id} path:{path} for {username} ") + logger.info( + f"New User: " + f"Checked and there is a need to configure system:{system_id} path:{path} for {username} " + ) except BaseTapyException as e: - logger.info(f"Unable to list system/files: " - f"Checked and there is a need to configure system:{system_id} path:{path} for {username}: {e}") + logger.info( + f"Unable to list system/files: " + f"Checked and there is a need to configure system:{system_id} path:{path} for {username}: {e}" + ) try: if create_path: tg458981_client = get_tg458981_client() try: - # User tg account to check if path exists - tg458981_client.files.listFiles(systemId=system_id, path=path) - logger.info(f"Directory for user={username} on system={system_id}/{path} exists and works. ") - except NotFoundError: - logger.info("Creating the directory for user=%s then going to run setfacl on system=%s path=%s", - username, - system_id, - path) - - tg458981_client.files.mkdir(systemId=system_id, path=path) + # Use user account to check if path exists and is accessible + user_client.files.listFiles(systemId=system_id, path=path) + logger.info( + f"Directory for user={username} on system={system_id}/{path} exists and works. " + ) + except (NotFoundError, ForbiddenError) as e: + logger.info( + "Ensuring directory exists for user=%s then going to run setfacl on system=%s path=%s", + username, + system_id, + path, + ) + + if isinstance(e, NotFoundError): + tg458981_client.files.mkdir(systemId=system_id, path=path) + tg458981_client.files.setFacl( systemId=system_id, path=path, @@ -76,31 +91,45 @@ def check_or_configure_system_and_user_directory(self, username, system_id, path ) # create keys, push to key service and use as credential for Tapis system - logger.info("Creating credentials for user=%s on system=%s", username, system_id) + logger.info( + "Creating credentials for user=%s on system=%s", username, system_id + ) (private_key, public_key) = createKeyPair() register_public_key(username, public_key, system_id) service_account = get_service_account_client() - create_system_credentials(service_account, - username, - public_key, - private_key, - system_id) + create_system_credentials( + service_account, username, public_key, private_key, system_id + ) except BaseTapyException as exc: - logger.exception('Failed to configure system (i.e. create directory, set acl, create credentials).', - extra={'user': username, - 'systemId': system_id, - 'path': path}) + logger.exception( + "Failed to configure system (i.e. create directory, set acl, create credentials).", + extra={"user": username, "systemId": system_id, "path": path}, + ) raise self.retry(exc=exc) @shared_task(default_retry_delay=30, max_retries=3) def new_user_alert(username): user = get_user_model().objects.get(username=username) - send_mail('New User in DesignSafe, need Slack', 'Username: ' + user.username + '\n' + - 'Email: ' + user.email + '\n' + - 'Name: ' + user.first_name + ' ' + user.last_name + '\n' + - 'Id: ' + str(user.id) + '\n', - settings.DEFAULT_FROM_EMAIL, settings.NEW_ACCOUNT_ALERT_EMAILS.split(','),) + send_mail( + "New User in DesignSafe, need Slack", + "Username: " + + user.username + + "\n" + + "Email: " + + user.email + + "\n" + + "Name: " + + user.first_name + + " " + + user.last_name + + "\n" + + "Id: " + + str(user.id) + + "\n", + settings.DEFAULT_FROM_EMAIL, + settings.NEW_ACCOUNT_ALERT_EMAILS.split(","), + ) # Auto-add user to TRAM allocation # tram_headers = {"tram-services-key": settings.TRAM_SERVICES_KEY} @@ -127,5 +156,5 @@ def update_institution_from_tas(self, username): tas_model = TASClient().get_user(username=username) except Exception as exc: raise self.retry(exc=exc) - user_model.profile.institution = tas_model.get('institution', None) + user_model.profile.institution = tas_model.get("institution", None) user_model.profile.save() diff --git a/designsafe/apps/auth/test_tasks.py b/designsafe/apps/auth/test_tasks.py new file mode 100644 index 0000000000..576ca0a87a --- /dev/null +++ b/designsafe/apps/auth/test_tasks.py @@ -0,0 +1,152 @@ +import pytest +from unittest import mock +from django.core.exceptions import ObjectDoesNotExist +from tapipy.errors import NotFoundError, BaseTapyException, ForbiddenError +from designsafe.apps.auth.tasks import check_or_configure_system_and_user_directory + + +@pytest.fixture +def mock_get_user_model(authenticated_user): + with mock.patch("designsafe.apps.auth.tasks.get_user_model") as mock_get_user_model: + mock_get_user_model().objects.get.return_value = authenticated_user + yield mock_get_user_model + + +@pytest.fixture +def mock_get_tg458981_client(): + with mock.patch( + "designsafe.apps.auth.tasks.get_tg458981_client" + ) as mock_get_tg458981_client: + yield mock_get_tg458981_client + + +@pytest.fixture +def mock_get_service_account_client(): + with mock.patch( + "designsafe.apps.auth.tasks.get_service_account_client" + ) as mock_get_service_account_client: + yield mock_get_service_account_client + + +@pytest.fixture +def mock_createKeyPair(): + with mock.patch("designsafe.apps.auth.tasks.createKeyPair") as mock_createKeyPair: + mock_createKeyPair.return_value = ("private_key", "public_key") + yield mock_createKeyPair + + +@pytest.fixture +def mock_register_public_key(): + with mock.patch( + "designsafe.apps.auth.tasks.register_public_key" + ) as mock_register_public_key: + yield mock_register_public_key + + +@pytest.fixture +def mock_create_system_credentials(): + with mock.patch( + "designsafe.apps.auth.tasks.create_system_credentials" + ) as mock_create_system_credentials: + yield mock_create_system_credentials + + +@pytest.fixture +def mock_agave_indexer(): + with mock.patch("designsafe.apps.auth.tasks.agave_indexer") as mock_agave_indexer: + yield mock_agave_indexer + + +def test_check_or_configure_system_and_user_directory_no_configuration_needed( + mock_get_user_model, authenticated_user +): + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", False + ) + authenticated_user.tapis_oauth.client.files.listFiles.assert_called_once_with( + systemId="testsystem", path="/testpath" + ) + + +def test_check_or_configure_system_and_user_directory_user_missing( + mock_get_user_model, + mock_register_public_key, + mock_create_system_credentials, + mock_createKeyPair, + mock_get_service_account_client, +): + mock_get_user_model().objects.get.side_effect = ObjectDoesNotExist + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", False + ) + mock_get_user_model().objects.get.assert_called_once_with(username="testuser") + + +def test_check_or_configure_system_and_user_directory_base_tapy_exception( + mock_get_user_model, + authenticated_user, + mock_register_public_key, + mock_create_system_credentials, + mock_createKeyPair, + mock_get_service_account_client, +): + authenticated_user.tapis_oauth.client.files.listFiles.side_effect = ( + BaseTapyException + ) + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", False + ) + authenticated_user.tapis_oauth.client.files.listFiles.assert_called_once_with( + systemId="testsystem", path="/testpath" + ) + + +def test_check_or_configure_system_and_user_directory_create_path( + mock_get_user_model, + authenticated_user, + mock_get_tg458981_client, + mock_createKeyPair, + mock_register_public_key, + mock_get_service_account_client, + mock_create_system_credentials, + mock_agave_indexer, +): + authenticated_user.tapis_oauth.client.files.listFiles.side_effect = NotFoundError + tg458981_client = mock_get_tg458981_client() + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", True + ) + tg458981_client.files.mkdir.assert_called_once_with( + systemId="testsystem", path="/testpath" + ) + tg458981_client.files.setFacl.assert_called_once() + mock_createKeyPair.assert_called_once() + mock_register_public_key.assert_called_once_with( + "testuser", "public_key", "testsystem" + ) + mock_create_system_credentials.assert_called_once() + mock_agave_indexer.apply_async.assert_called_once() + + +def test_check_or_configure_system_and_user_directory_forbidden_error( + mock_get_user_model, + authenticated_user, + mock_get_tg458981_client, + mock_createKeyPair, + mock_register_public_key, + mock_get_service_account_client, + mock_create_system_credentials, + mock_agave_indexer, +): + authenticated_user.tapis_oauth.client.files.listFiles.side_effect = ForbiddenError + tg458981_client = mock_get_tg458981_client() + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", True + ) + tg458981_client.files.setFacl.assert_called_once() + mock_createKeyPair.assert_called_once() + mock_register_public_key.assert_called_once_with( + "testuser", "public_key", "testsystem" + ) + mock_create_system_credentials.assert_called_once() + mock_agave_indexer.apply_async.assert_called_once() diff --git a/designsafe/apps/box_integration/tests/test_views.py b/designsafe/apps/box_integration/tests/test_views.py index eac35a7de6..eb9396659c 100644 --- a/designsafe/apps/box_integration/tests/test_views.py +++ b/designsafe/apps/box_integration/tests/test_views.py @@ -1,12 +1,13 @@ from django.test import TestCase -from django.contrib.auth import get_user_model, signals +from django.contrib.auth import get_user_model from django.urls import reverse +from unittest import skip from boxsdk.object.user import User from designsafe.apps.box_integration.models import BoxUserToken -from designsafe.apps.auth.signals import on_user_logged_in import mock +@skip('This test is not working') class BoxInitializationTestCase(TestCase): fixtures = ['user-data.json'] @@ -16,9 +17,6 @@ def setUp(self): user.set_password('password') user.save() - # disconnect user_logged_in signal - signals.user_logged_in.disconnect(on_user_logged_in) - def test_index_view_not_enabled(self): """ Should render as not enabled @@ -84,6 +82,7 @@ def test_oauth2_callback(self, m_box_oauth_authenticate, m_user_get): 'Box.com as DS User (ds_user@designsafe-ci.org)') +@skip('This test is not working') class BoxDisconnectTestCase(TestCase): fixtures = ['user-data.json', 'box-user-token.json'] @@ -93,9 +92,6 @@ def setUp(self): user.set_password('password') user.save() - # disconnect user_logged_in signal - signals.user_logged_in.disconnect(on_user_logged_in) - def test_disconnect(self): """ Test disconnecting Box.com. diff --git a/designsafe/apps/djangoRT/forms.py b/designsafe/apps/djangoRT/forms.py index a3f468a805..51b5e0aee3 100755 --- a/designsafe/apps/djangoRT/forms.py +++ b/designsafe/apps/djangoRT/forms.py @@ -34,7 +34,6 @@ class BaseTicketForm(forms.Form): """ first_name = forms.CharField(widget=forms.TextInput(), max_length=255, required=True) last_name = forms.CharField(widget=forms.TextInput(), max_length=255, required=True) - email = forms.EmailField(widget=forms.EmailInput(), required=True) subject = forms.CharField(widget=forms.TextInput(), max_length=255, required=True) category_choices = (('', 'Choose one'),) + settings.TICKET_CATEGORIES diff --git a/designsafe/apps/djangoRT/views.py b/designsafe/apps/djangoRT/views.py index 9ecbe785c7..a26d252f1b 100755 --- a/designsafe/apps/djangoRT/views.py +++ b/designsafe/apps/djangoRT/views.py @@ -5,6 +5,7 @@ from designsafe.apps.djangoRT import rtUtil, forms, rtModels from django.contrib.auth.decorators import login_required from django.contrib import messages +from django.core.files.base import ContentFile from django.conf import settings from designsafe.apps.api.exceptions import ApiException from designsafe.apps.api.views import BaseApiView @@ -73,7 +74,8 @@ def ticketcreate(request): requestor_meta = '%s %s <%s>' % ( form.cleaned_data['first_name'], form.cleaned_data['last_name'], - form.cleaned_data['email']) + request.user.email + ) meta = ( ('Opened by', request.user), @@ -95,7 +97,7 @@ def ticketcreate(request): ticket = rtModels.Ticket(subject=form.cleaned_data['subject'], problem_description="\n ".join(ticket_body.splitlines()), - requestor=form.cleaned_data['email'], + requestor=request.user.email, cc=form.cleaned_data.get('cc', '')) logger.debug('Creating ticket for user: %s' % form.cleaned_data) @@ -105,12 +107,12 @@ def ticketcreate(request): if ticket_id > -1: messages.success(request, 'Your ticket has been submitted.') if 'attachment' in request.FILES: + f = request.FILES['attachment'] rt.replyToTicket( ticket_id, - files=([request.FILES['attachment'].name, - request.FILES['attachment'], - mimetypes.guess_type(request.FILES['attachment'].name)], - )) + files=[(f.name, + ContentFile(f.read()), f.content_type) + ]) if request.user.is_authenticated: return HttpResponseRedirect(reverse('djangoRT:ticketdetail', @@ -163,8 +165,8 @@ def ticketreply(request, ticketId): ( request.FILES['attachment'].name, request.FILES['attachment'], - mimetypes.guess_type(request.FILES['attachment'].name), - )) + request.FILES['attachment'].content_type), + ) if rt.replyToTicket(ticketId, text=reply_text, files=attachments): return HttpResponseRedirect(reverse('djangoRT:ticketdetail', args=[ticketId])) diff --git a/designsafe/apps/signals/websocket_consumers.py b/designsafe/apps/signals/websocket_consumers.py index 2353bad28e..d762a13aff 100644 --- a/designsafe/apps/signals/websocket_consumers.py +++ b/designsafe/apps/signals/websocket_consumers.py @@ -1,28 +1,37 @@ -import json +"""Websocket consumers""" -from channels.generic.websocket import WebsocketConsumer, AsyncWebsocketConsumer +import json +from channels.generic.websocket import AsyncWebsocketConsumer +from channels.layers import get_channel_layer class DesignsafeWebsocketConsumer(AsyncWebsocketConsumer): + """Websocket consumer for DesignSafe notifications""" + async def connect(self): - self.user_channel = f"ds_{self.scope['user']}" - self.broadcast_channel = "ds_broadcast" await self.channel_layer.group_add( - self.user_channel, self.channel_name + f"ds_{self.scope['user']}", self.channel_name ) await self.accept() - async def disconnect(self, close_code): - await self.channel_layer.group_discard( - self.user_channel, self.channel_name - ) + async def disconnect(self, code): await self.channel_layer.group_discard( - self.broadcast_channel, self.channel_name + f"ds_{self.scope['user']}", self.channel_name ) + await self.channel_layer.group_discard("ds_broadcast", self.channel_name) - async def receive(self, text_data): - pass + async def receive(self, text_data=None, bytes_data=None): + if text_data == "markAllNotificationsAsRead": + channel_layer = get_channel_layer() + await channel_layer.group_send( + f"ds_{self.scope['user']}", + { + "type": "ds_notification", + "message": json.dumps({"event_type": "markAllNotificationsAsRead"}), + }, + ) async def ds_notification(self, event): + """Send notification to user""" message = event["message"] await self.send(text_data=message) diff --git a/designsafe/apps/workspace/api/views.py b/designsafe/apps/workspace/api/views.py index c6bdf76acd..6d8e746398 100644 --- a/designsafe/apps/workspace/api/views.py +++ b/designsafe/apps/workspace/api/views.py @@ -5,6 +5,7 @@ import logging import json +from urllib.parse import urlparse from django.http import JsonResponse from django.conf import settings from django.core.exceptions import ObjectDoesNotExist @@ -758,14 +759,18 @@ def _submit_job(self, request, body, tapis, username): return {"execSys": system_needs_keys} if settings.DEBUG: - wh_base_url = settings.WEBHOOK_POST_URL + reverse( + parsed_url = urlparse(settings.NGROK_DOMAIN) + if not parsed_url.scheme: + webhook_base_url = f"https://{settings.NGROK_DOMAIN}" + else: + webhook_base_url = settings.NGROK_DOMAIN + + interactive_wh_url = webhook_base_url + reverse( "webhooks:interactive_wh_handler" ) - jobs_wh_url = settings.WEBHOOK_POST_URL + reverse( - "webhooks:jobs_wh_handler" - ) + jobs_wh_url = webhook_base_url + reverse("webhooks:jobs_wh_handler") else: - wh_base_url = request.build_absolute_uri( + interactive_wh_url = request.build_absolute_uri( reverse("webhooks:interactive_wh_handler") ) jobs_wh_url = request.build_absolute_uri( @@ -785,7 +790,7 @@ def _submit_job(self, request, body, tapis, username): projects = request.user.projects.order_by("-last_updated") entry["value"] = " ".join( f"{project.uuid},{project.value['projectId'] if project.value['projectId'] != 'None' else project.uuid}" - for project in projects[:settings.USER_PROJECTS_LIMIT] + for project in projects[: settings.USER_PROJECTS_LIMIT] ) job_post["parameterSet"]["envVariables"] = env_variables break @@ -795,7 +800,7 @@ def _submit_job(self, request, body, tapis, username): # Add webhook URL environment variable for interactive apps job_post["parameterSet"]["envVariables"] = job_post["parameterSet"].get( "envVariables", [] - ) + [{"key": "_INTERACTIVE_WEBHOOK_URL", "value": wh_base_url}] + ) + [{"key": "_INTERACTIVE_WEBHOOK_URL", "value": interactive_wh_url}] job_post["tags"].append("isInteractive") # Make sure $HOME/.tap directory exists for user when running interactive apps on TACC HPC Systems diff --git a/designsafe/apps/workspace/models/app_entries.py b/designsafe/apps/workspace/models/app_entries.py index f8cf2938ae..1477572aff 100644 --- a/designsafe/apps/workspace/models/app_entries.py +++ b/designsafe/apps/workspace/models/app_entries.py @@ -233,8 +233,8 @@ def href(self): """Retrieve the app's URL in the Tools & Applications space""" if self.external_href: return self.external_href - - app_href = f"/rw/workspace/{self.app_id}" + + app_href = f"/workspace/{self.app_id}" if self.version: app_href += f"?appVersion={self.version}" return app_href diff --git a/designsafe/apps/workspace/urls.py b/designsafe/apps/workspace/urls.py index 362316ed6a..00cc09812a 100644 --- a/designsafe/apps/workspace/urls.py +++ b/designsafe/apps/workspace/urls.py @@ -4,5 +4,6 @@ from designsafe.apps.workspace import views urlpatterns = [ + re_path('history', views.WorkspaceView.as_view(), name="history"), re_path('^', views.WorkspaceView.as_view(), name="workspace"), ] diff --git a/designsafe/middleware.py b/designsafe/middleware.py index 0f2a5e7387..266103d53f 100644 --- a/designsafe/middleware.py +++ b/designsafe/middleware.py @@ -30,7 +30,7 @@ class DesignsafeProfileUpdateMiddleware(MiddlewareMixin): def process_request(self, request): blocked_path = request.path.startswith( - ("/data", "/applications", "/rw/workspace", "/recon-portal", "/dashboard") + ("/data", "/applications", "/rw/workspace", "/workspace", "/recon-portal", "/dashboard") ) if request.user.is_authenticated and request.user.profile.update_required and blocked_path: messages.warning( @@ -65,7 +65,7 @@ def process_request(self, request): 'Use is required for continued use of DesignSafe ' 'resources.' % accept_url) return None - + class SiteMessageMiddleware(MiddlewareMixin): def process_request(self, request): diff --git a/designsafe/settings/common_settings.py b/designsafe/settings/common_settings.py index ca1f6ada5d..98e05e81ee 100644 --- a/designsafe/settings/common_settings.py +++ b/designsafe/settings/common_settings.py @@ -533,7 +533,6 @@ AGAVE_TOKEN_SESSION_ID = os.environ.get('AGAVE_TOKEN_SESSION_ID', 'agave_token') AGAVE_STORAGE_SYSTEM = os.environ.get('AGAVE_STORAGE_SYSTEM') -AGAVE_WORKING_SYSTEM = os.environ.get('AGAVE_WORKING_SYSTEM', 'designsafe.storage.frontera.work') AGAVE_JWT_PUBKEY = os.environ.get('AGAVE_JWT_PUBKEY') AGAVE_JWT_ISSUER = os.environ.get('AGAVE_JWT_ISSUER') @@ -546,7 +545,6 @@ TAPIS_SYSTEMS_TO_CONFIGURE = [ {"system_id": AGAVE_STORAGE_SYSTEM, "path": "{username}", "create_path": True}, - {"system_id": AGAVE_WORKING_SYSTEM, "path": "{username}", "create_path": True}, {"system_id": "cloud.data", "path": "/ ", "create_path": False}, ] @@ -670,7 +668,7 @@ '.java', '.js', '.less', '.m', '.make', '.md', '.ml', '.mm', '.msg', '.php', '.pl', '.properties', '.py', '.rb', '.sass', '.scala', '.script', '.sh', '.sml', '.sql', '.txt', '.vi', '.vim', '.xml', '.xsd', '.xsl', '.yaml', '.yml', '.tcl', - '.json', '.out', '.err', '.geojson', '.do', '.sas', '.hazmapper' + '.json', '.out', '.err', '.geojson', '.do', '.sas', '.hazmapper', ".log" ] SUPPORTED_OBJECT_PREVIEW_EXTS = [ @@ -706,7 +704,7 @@ FEDORA_CONTAINER= os.environ.get('FEDORA_CONTAINER', 'designsafe-publications-dev') CSRF_TRUSTED_ORIGINS = [f"https://{os.environ.get('SESSION_COOKIE_DOMAIN')}"] -WEBHOOK_POST_URL = os.environ.get('WEBHOOK_POST_URL', '') +NGROK_DOMAIN = os.environ.get('NGROK_DOMAIN', os.environ.get('WEBHOOK_POST_URL', '')) STAFF_VPN_IP_PREFIX = os.environ.get("STAFF_VPN_IP_PREFIX", "129.114") USER_PROJECTS_LIMIT = os.environ.get("USER_PROJECTS_LIMIT", 500) diff --git a/designsafe/settings/external_resource_secrets.sample.py b/designsafe/settings/external_resource_secrets.sample.py deleted file mode 100644 index 09b002818a..0000000000 --- a/designsafe/settings/external_resource_secrets.sample.py +++ /dev/null @@ -1,8 +0,0 @@ -### -# Secrets for Google Auth flow -# - -GOOGLE_OAUTH2_CLIENT_SECRET = "CHANGE_ME" -GOOGLE_OAUTH2_CLIENT_ID = "CHANGE_ME" - -WEBHOOK_POST_URL = "CHANGE_ME" \ No newline at end of file diff --git a/designsafe/settings/test_settings.py b/designsafe/settings/test_settings.py index 10ab3fa3d9..3786a69572 100644 --- a/designsafe/settings/test_settings.py +++ b/designsafe/settings/test_settings.py @@ -451,7 +451,6 @@ AGAVE_TOKEN_SESSION_ID = os.environ.get('AGAVE_TOKEN_SESSION_ID', 'agave_token') AGAVE_SUPER_TOKEN = os.environ.get('AGAVE_SUPER_TOKEN') AGAVE_STORAGE_SYSTEM = os.environ.get('AGAVE_STORAGE_SYSTEM') -AGAVE_WORKING_SYSTEM = os.environ.get('AGAVE_WORKING_SYSTEM') AGAVE_JWT_PUBKEY = os.environ.get('AGAVE_JWT_PUBKEY') AGAVE_JWT_ISSUER = os.environ.get('AGAVE_JWT_ISSUER') @@ -511,7 +510,7 @@ GOOGLE_OAUTH2_CLIENT_SECRET = "CHANGE_ME" GOOGLE_OAUTH2_CLIENT_ID = "CHANGE_ME" -WEBHOOK_POST_URL = "http://8cb9afb3.ngrok.io" +NGROK_DOMAIN = "https://8cb9afb3.ngrok.io" # Box sync BOX_APP_CLIENT_ID = 'boxappclientid' @@ -548,11 +547,9 @@ AGAVE_CLIENT_SECRET = 'example_com_client_secret' AGAVE_SUPER_TOKEN = 'example_com_client_token' AGAVE_STORAGE_SYSTEM = 'storage.example.com' -AGAVE_WORKING_SYSTEM = 'storage.example.work' TAPIS_SYSTEMS_TO_CONFIGURE = [ {"system_id": AGAVE_STORAGE_SYSTEM, "path": "{username}", "create_path": True}, - {"system_id": AGAVE_WORKING_SYSTEM, "path": "{username}", "create_path": True}, {"system_id": "cloud.data", "path": "/ ", "create_path": False}, ] diff --git a/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html b/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html index 45340a215d..64adf97513 100644 --- a/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html +++ b/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html @@ -37,7 +37,7 @@
    Quick Links
    - +
    diff --git a/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html b/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html index bd706a3cba..93ec82e040 100644 --- a/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html +++ b/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html @@ -1,17 +1,18 @@ -