From 4a5e0f8c53712f9e9773ad76c50817d71bbff0e2 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 24 Sep 2024 13:36:58 -0500 Subject: [PATCH 1/9] Fix display bug for generic FR collections from 2019 (#1448) --- .../src/projects/ProjectCollapser/ProjectCollapser.tsx | 7 ++++--- .../datafiles/src/projects/ProjectTree/ProjectTree.tsx | 4 ++-- client/modules/datafiles/src/projects/constants.ts | 4 ++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx b/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx index aacdf9250..aac0067b4 100644 --- a/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx +++ b/client/modules/datafiles/src/projects/ProjectCollapser/ProjectCollapser.tsx @@ -17,8 +17,8 @@ export const ProjectCollapse: React.FC< if (header.length > 0) { const headerElement = header[0] as HTMLElement; headerElement.style.backgroundColor = - PROJECT_COLORS[entityName]['fill']; - headerElement.style.border = `1px solid ${PROJECT_COLORS[entityName]['outline']}`; + PROJECT_COLORS[entityName]?.['fill']; + headerElement.style.border = `1px solid ${PROJECT_COLORS[entityName]?.['outline']}`; } } }, @@ -36,7 +36,8 @@ export const ProjectCollapse: React.FC< forceRender: true, label: ( - {DISPLAY_NAMES[entityName]} | {title} + {DISPLAY_NAMES[entityName] ?? 'Collection'} |{' '} + {title} ), children, diff --git a/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx b/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx index 76208dfee..b05ab1915 100644 --- a/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx +++ b/client/modules/datafiles/src/projects/ProjectTree/ProjectTree.tsx @@ -137,8 +137,8 @@ const RecursiveTree: React.FC<{
{DISPLAY_NAMES[treeData.name]} diff --git a/client/modules/datafiles/src/projects/constants.ts b/client/modules/datafiles/src/projects/constants.ts index bda51640b..51eb5d2cd 100644 --- a/client/modules/datafiles/src/projects/constants.ts +++ b/client/modules/datafiles/src/projects/constants.ts @@ -17,6 +17,8 @@ export const SIMULATION_MODEL = 'designsafe.project.simulation.model'; export const SIMULATION_INPUT = 'designsafe.project.simulation.input'; export const SIMULATION_OUTPUT = 'designsafe.project.simulation.output'; // Field Research +export const FIELD_RECON_COLLECTION = + 'designsafe.project.field_recon.collection'; export const FIELD_RECON_MISSION = 'designsafe.project.field_recon.mission'; export const FIELD_RECON_REPORT = 'designsafe.project.field_recon.report'; export const FIELD_RECON_SOCIAL_SCIENCE = @@ -72,6 +74,7 @@ export const PROJECT_COLORS: Record = [HYBRID_SIM_SIM_SUBSTRUCTURE]: { outline: '#BD5717', fill: '#EBCCB9' }, [HYBRID_SIM_SIM_OUTPUT]: { outline: '#B59300', fill: '#ECE4BF' }, + [FIELD_RECON_COLLECTION]: { outline: '#43A59D', fill: '#CAE9E6' }, [FIELD_RECON_REPORT]: { outline: '#cccccc', fill: '#f5f5f5' }, [FIELD_RECON_MISSION]: { outline: '#000000', fill: '#ffffff' }, [FIELD_RECON_GEOSCIENCE]: { outline: '#43A59D', fill: '#CAE9E6' }, @@ -191,6 +194,7 @@ export const DISPLAY_NAMES: Record = { [HYBRID_SIM_SIM_OUTPUT]: 'Simulation Output', // Field Recon [FIELD_RECON_MISSION]: 'Mission', + [FIELD_RECON_COLLECTION]: 'Collection', [FIELD_RECON_GEOSCIENCE]: 'Geoscience Collection', [FIELD_RECON_SOCIAL_SCIENCE]: 'Social Science Collection', [FIELD_RECON_REPORT]: 'Document Collection', From cdde74b72af40407eb702b78500d7474212ee652 Mon Sep 17 00:00:00 2001 From: Sal Tijerina Date: Tue, 1 Oct 2024 13:36:38 -0500 Subject: [PATCH 2/9] bug/WP-683: Fix setfacl race condition (#1444) * handle race condition where dir exists but user doesn't exist on system to apply acls * remove AGAVE_WORKING_SYSTEM from TAPIS_SYSTEMS_TO_CONFIGURE * formatting * add unit tests * cleanup * remove AGAVE_WORKING_SYSTEM everywhere else * support test_*.py files for pytest * skip bad old tests --- .pylintrc | 2 +- conf/env_files/designsafe.sample.env | 1 - designsafe/apps/auth/tasks.py | 121 ++++++++------ designsafe/apps/auth/test_tasks.py | 152 ++++++++++++++++++ .../apps/box_integration/tests/test_views.py | 12 +- designsafe/settings/common_settings.py | 2 - designsafe/settings/test_settings.py | 3 - pytest.ini | 2 +- 8 files changed, 233 insertions(+), 62 deletions(-) create mode 100644 designsafe/apps/auth/test_tasks.py diff --git a/.pylintrc b/.pylintrc index d8f0f8022..3458e3124 100644 --- a/.pylintrc +++ b/.pylintrc @@ -52,7 +52,7 @@ ignore=CVS,tests.py # ignore-list. The regex matches against paths and can be in Posix or Windows # format. Because '\\' represents the directory delimiter on Windows systems, # it can't be used as an escape character. -ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$ +ignore-paths=^.*migrations/.*$,^.*_tests/.*$,^.*unit_test.*$,^.*test_.*$ # Files or directories matching the regular expression patterns are skipped. # The regex matches against base names, not paths. The default value ignores diff --git a/conf/env_files/designsafe.sample.env b/conf/env_files/designsafe.sample.env index 5c13942f6..c9abbb353 100644 --- a/conf/env_files/designsafe.sample.env +++ b/conf/env_files/designsafe.sample.env @@ -80,7 +80,6 @@ AGAVE_CLIENT_SECRET= AGAVE_SUPER_TOKEN= AGAVE_STORAGE_SYSTEM= -AGAVE_WORKING_SYSTEM= AGAVE_JWT_PUBKEY= AGAVE_JWT_ISSUER= AGAVE_JWT_HEADER= diff --git a/designsafe/apps/auth/tasks.py b/designsafe/apps/auth/tasks.py index 26eb935be..b97f22709 100644 --- a/designsafe/apps/auth/tasks.py +++ b/designsafe/apps/auth/tasks.py @@ -1,68 +1,83 @@ +""" Celery tasks for user onboarding and other user-related tasks. """ + +import logging from datetime import datetime, timedelta -import requests from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ObjectDoesNotExist from django.core.mail import send_mail +from celery import shared_task +from pytas.http import TASClient +from tapipy.errors import NotFoundError, BaseTapyException, ForbiddenError from designsafe.apps.api.agave import get_service_account_client, get_tg458981_client from designsafe.apps.api.tasks import agave_indexer from designsafe.apps.api.notifications.models import Notification -from celery import shared_task -from django.contrib.auth import get_user_model -from pytas.http import TASClient -from tapipy.errors import NotFoundError, BaseTapyException -from designsafe.utils.system_access import register_public_key, create_system_credentials +from designsafe.utils.system_access import ( + register_public_key, + create_system_credentials, +) from designsafe.utils.encryption import createKeyPair -from django.contrib.auth import get_user_model -import logging logger = logging.getLogger(__name__) def get_systems_to_configure(username): - """ Get systems to configure either during startup or for new user """ + """Get systems to configure either during startup or for new user""" systems = [] for system in settings.TAPIS_SYSTEMS_TO_CONFIGURE: system_copy = system.copy() - system_copy['path'] = system_copy['path'].format(username=username) + system_copy["path"] = system_copy["path"].format(username=username) systems.append(system_copy) return systems -@shared_task(default_retry_delay=30, max_retries=3, queue='onboarding', bind=True) -def check_or_configure_system_and_user_directory(self, username, system_id, path, create_path): +@shared_task(default_retry_delay=30, max_retries=3, queue="onboarding", bind=True) +def check_or_configure_system_and_user_directory( + self, username, system_id, path, create_path +): + """Check if user has access to system and path, if not, configure it.""" try: user_client = get_user_model().objects.get(username=username).tapis_oauth.client - user_client.files.listFiles( - systemId=system_id, path=path + user_client.files.listFiles(systemId=system_id, path=path) + logger.info( + f"System Works: " + f"Checked and there is no need to configure system:{system_id} path:{path} for {username}" ) - logger.info(f"System Works: " - f"Checked and there is no need to configure system:{system_id} path:{path} for {username}") return except ObjectDoesNotExist: # User is missing; handling email confirmation process where user has not logged in - logger.info(f"New User: " - f"Checked and there is a need to configure system:{system_id} path:{path} for {username} ") + logger.info( + f"New User: " + f"Checked and there is a need to configure system:{system_id} path:{path} for {username} " + ) except BaseTapyException as e: - logger.info(f"Unable to list system/files: " - f"Checked and there is a need to configure system:{system_id} path:{path} for {username}: {e}") + logger.info( + f"Unable to list system/files: " + f"Checked and there is a need to configure system:{system_id} path:{path} for {username}: {e}" + ) try: if create_path: tg458981_client = get_tg458981_client() try: - # User tg account to check if path exists - tg458981_client.files.listFiles(systemId=system_id, path=path) - logger.info(f"Directory for user={username} on system={system_id}/{path} exists and works. ") - except NotFoundError: - logger.info("Creating the directory for user=%s then going to run setfacl on system=%s path=%s", - username, - system_id, - path) - - tg458981_client.files.mkdir(systemId=system_id, path=path) + # Use user account to check if path exists and is accessible + user_client.files.listFiles(systemId=system_id, path=path) + logger.info( + f"Directory for user={username} on system={system_id}/{path} exists and works. " + ) + except (NotFoundError, ForbiddenError) as e: + logger.info( + "Ensuring directory exists for user=%s then going to run setfacl on system=%s path=%s", + username, + system_id, + path, + ) + + if isinstance(e, NotFoundError): + tg458981_client.files.mkdir(systemId=system_id, path=path) + tg458981_client.files.setFacl( systemId=system_id, path=path, @@ -76,31 +91,45 @@ def check_or_configure_system_and_user_directory(self, username, system_id, path ) # create keys, push to key service and use as credential for Tapis system - logger.info("Creating credentials for user=%s on system=%s", username, system_id) + logger.info( + "Creating credentials for user=%s on system=%s", username, system_id + ) (private_key, public_key) = createKeyPair() register_public_key(username, public_key, system_id) service_account = get_service_account_client() - create_system_credentials(service_account, - username, - public_key, - private_key, - system_id) + create_system_credentials( + service_account, username, public_key, private_key, system_id + ) except BaseTapyException as exc: - logger.exception('Failed to configure system (i.e. create directory, set acl, create credentials).', - extra={'user': username, - 'systemId': system_id, - 'path': path}) + logger.exception( + "Failed to configure system (i.e. create directory, set acl, create credentials).", + extra={"user": username, "systemId": system_id, "path": path}, + ) raise self.retry(exc=exc) @shared_task(default_retry_delay=30, max_retries=3) def new_user_alert(username): user = get_user_model().objects.get(username=username) - send_mail('New User in DesignSafe, need Slack', 'Username: ' + user.username + '\n' + - 'Email: ' + user.email + '\n' + - 'Name: ' + user.first_name + ' ' + user.last_name + '\n' + - 'Id: ' + str(user.id) + '\n', - settings.DEFAULT_FROM_EMAIL, settings.NEW_ACCOUNT_ALERT_EMAILS.split(','),) + send_mail( + "New User in DesignSafe, need Slack", + "Username: " + + user.username + + "\n" + + "Email: " + + user.email + + "\n" + + "Name: " + + user.first_name + + " " + + user.last_name + + "\n" + + "Id: " + + str(user.id) + + "\n", + settings.DEFAULT_FROM_EMAIL, + settings.NEW_ACCOUNT_ALERT_EMAILS.split(","), + ) # Auto-add user to TRAM allocation # tram_headers = {"tram-services-key": settings.TRAM_SERVICES_KEY} @@ -127,5 +156,5 @@ def update_institution_from_tas(self, username): tas_model = TASClient().get_user(username=username) except Exception as exc: raise self.retry(exc=exc) - user_model.profile.institution = tas_model.get('institution', None) + user_model.profile.institution = tas_model.get("institution", None) user_model.profile.save() diff --git a/designsafe/apps/auth/test_tasks.py b/designsafe/apps/auth/test_tasks.py new file mode 100644 index 000000000..576ca0a87 --- /dev/null +++ b/designsafe/apps/auth/test_tasks.py @@ -0,0 +1,152 @@ +import pytest +from unittest import mock +from django.core.exceptions import ObjectDoesNotExist +from tapipy.errors import NotFoundError, BaseTapyException, ForbiddenError +from designsafe.apps.auth.tasks import check_or_configure_system_and_user_directory + + +@pytest.fixture +def mock_get_user_model(authenticated_user): + with mock.patch("designsafe.apps.auth.tasks.get_user_model") as mock_get_user_model: + mock_get_user_model().objects.get.return_value = authenticated_user + yield mock_get_user_model + + +@pytest.fixture +def mock_get_tg458981_client(): + with mock.patch( + "designsafe.apps.auth.tasks.get_tg458981_client" + ) as mock_get_tg458981_client: + yield mock_get_tg458981_client + + +@pytest.fixture +def mock_get_service_account_client(): + with mock.patch( + "designsafe.apps.auth.tasks.get_service_account_client" + ) as mock_get_service_account_client: + yield mock_get_service_account_client + + +@pytest.fixture +def mock_createKeyPair(): + with mock.patch("designsafe.apps.auth.tasks.createKeyPair") as mock_createKeyPair: + mock_createKeyPair.return_value = ("private_key", "public_key") + yield mock_createKeyPair + + +@pytest.fixture +def mock_register_public_key(): + with mock.patch( + "designsafe.apps.auth.tasks.register_public_key" + ) as mock_register_public_key: + yield mock_register_public_key + + +@pytest.fixture +def mock_create_system_credentials(): + with mock.patch( + "designsafe.apps.auth.tasks.create_system_credentials" + ) as mock_create_system_credentials: + yield mock_create_system_credentials + + +@pytest.fixture +def mock_agave_indexer(): + with mock.patch("designsafe.apps.auth.tasks.agave_indexer") as mock_agave_indexer: + yield mock_agave_indexer + + +def test_check_or_configure_system_and_user_directory_no_configuration_needed( + mock_get_user_model, authenticated_user +): + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", False + ) + authenticated_user.tapis_oauth.client.files.listFiles.assert_called_once_with( + systemId="testsystem", path="/testpath" + ) + + +def test_check_or_configure_system_and_user_directory_user_missing( + mock_get_user_model, + mock_register_public_key, + mock_create_system_credentials, + mock_createKeyPair, + mock_get_service_account_client, +): + mock_get_user_model().objects.get.side_effect = ObjectDoesNotExist + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", False + ) + mock_get_user_model().objects.get.assert_called_once_with(username="testuser") + + +def test_check_or_configure_system_and_user_directory_base_tapy_exception( + mock_get_user_model, + authenticated_user, + mock_register_public_key, + mock_create_system_credentials, + mock_createKeyPair, + mock_get_service_account_client, +): + authenticated_user.tapis_oauth.client.files.listFiles.side_effect = ( + BaseTapyException + ) + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", False + ) + authenticated_user.tapis_oauth.client.files.listFiles.assert_called_once_with( + systemId="testsystem", path="/testpath" + ) + + +def test_check_or_configure_system_and_user_directory_create_path( + mock_get_user_model, + authenticated_user, + mock_get_tg458981_client, + mock_createKeyPair, + mock_register_public_key, + mock_get_service_account_client, + mock_create_system_credentials, + mock_agave_indexer, +): + authenticated_user.tapis_oauth.client.files.listFiles.side_effect = NotFoundError + tg458981_client = mock_get_tg458981_client() + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", True + ) + tg458981_client.files.mkdir.assert_called_once_with( + systemId="testsystem", path="/testpath" + ) + tg458981_client.files.setFacl.assert_called_once() + mock_createKeyPair.assert_called_once() + mock_register_public_key.assert_called_once_with( + "testuser", "public_key", "testsystem" + ) + mock_create_system_credentials.assert_called_once() + mock_agave_indexer.apply_async.assert_called_once() + + +def test_check_or_configure_system_and_user_directory_forbidden_error( + mock_get_user_model, + authenticated_user, + mock_get_tg458981_client, + mock_createKeyPair, + mock_register_public_key, + mock_get_service_account_client, + mock_create_system_credentials, + mock_agave_indexer, +): + authenticated_user.tapis_oauth.client.files.listFiles.side_effect = ForbiddenError + tg458981_client = mock_get_tg458981_client() + check_or_configure_system_and_user_directory( + "testuser", "testsystem", "/testpath", True + ) + tg458981_client.files.setFacl.assert_called_once() + mock_createKeyPair.assert_called_once() + mock_register_public_key.assert_called_once_with( + "testuser", "public_key", "testsystem" + ) + mock_create_system_credentials.assert_called_once() + mock_agave_indexer.apply_async.assert_called_once() diff --git a/designsafe/apps/box_integration/tests/test_views.py b/designsafe/apps/box_integration/tests/test_views.py index eac35a7de..eb9396659 100644 --- a/designsafe/apps/box_integration/tests/test_views.py +++ b/designsafe/apps/box_integration/tests/test_views.py @@ -1,12 +1,13 @@ from django.test import TestCase -from django.contrib.auth import get_user_model, signals +from django.contrib.auth import get_user_model from django.urls import reverse +from unittest import skip from boxsdk.object.user import User from designsafe.apps.box_integration.models import BoxUserToken -from designsafe.apps.auth.signals import on_user_logged_in import mock +@skip('This test is not working') class BoxInitializationTestCase(TestCase): fixtures = ['user-data.json'] @@ -16,9 +17,6 @@ def setUp(self): user.set_password('password') user.save() - # disconnect user_logged_in signal - signals.user_logged_in.disconnect(on_user_logged_in) - def test_index_view_not_enabled(self): """ Should render as not enabled @@ -84,6 +82,7 @@ def test_oauth2_callback(self, m_box_oauth_authenticate, m_user_get): 'Box.com as DS User (ds_user@designsafe-ci.org)') +@skip('This test is not working') class BoxDisconnectTestCase(TestCase): fixtures = ['user-data.json', 'box-user-token.json'] @@ -93,9 +92,6 @@ def setUp(self): user.set_password('password') user.save() - # disconnect user_logged_in signal - signals.user_logged_in.disconnect(on_user_logged_in) - def test_disconnect(self): """ Test disconnecting Box.com. diff --git a/designsafe/settings/common_settings.py b/designsafe/settings/common_settings.py index ca1f6ada5..9672aa616 100644 --- a/designsafe/settings/common_settings.py +++ b/designsafe/settings/common_settings.py @@ -533,7 +533,6 @@ AGAVE_TOKEN_SESSION_ID = os.environ.get('AGAVE_TOKEN_SESSION_ID', 'agave_token') AGAVE_STORAGE_SYSTEM = os.environ.get('AGAVE_STORAGE_SYSTEM') -AGAVE_WORKING_SYSTEM = os.environ.get('AGAVE_WORKING_SYSTEM', 'designsafe.storage.frontera.work') AGAVE_JWT_PUBKEY = os.environ.get('AGAVE_JWT_PUBKEY') AGAVE_JWT_ISSUER = os.environ.get('AGAVE_JWT_ISSUER') @@ -546,7 +545,6 @@ TAPIS_SYSTEMS_TO_CONFIGURE = [ {"system_id": AGAVE_STORAGE_SYSTEM, "path": "{username}", "create_path": True}, - {"system_id": AGAVE_WORKING_SYSTEM, "path": "{username}", "create_path": True}, {"system_id": "cloud.data", "path": "/ ", "create_path": False}, ] diff --git a/designsafe/settings/test_settings.py b/designsafe/settings/test_settings.py index 10ab3fa3d..845cf61b0 100644 --- a/designsafe/settings/test_settings.py +++ b/designsafe/settings/test_settings.py @@ -451,7 +451,6 @@ AGAVE_TOKEN_SESSION_ID = os.environ.get('AGAVE_TOKEN_SESSION_ID', 'agave_token') AGAVE_SUPER_TOKEN = os.environ.get('AGAVE_SUPER_TOKEN') AGAVE_STORAGE_SYSTEM = os.environ.get('AGAVE_STORAGE_SYSTEM') -AGAVE_WORKING_SYSTEM = os.environ.get('AGAVE_WORKING_SYSTEM') AGAVE_JWT_PUBKEY = os.environ.get('AGAVE_JWT_PUBKEY') AGAVE_JWT_ISSUER = os.environ.get('AGAVE_JWT_ISSUER') @@ -548,11 +547,9 @@ AGAVE_CLIENT_SECRET = 'example_com_client_secret' AGAVE_SUPER_TOKEN = 'example_com_client_token' AGAVE_STORAGE_SYSTEM = 'storage.example.com' -AGAVE_WORKING_SYSTEM = 'storage.example.work' TAPIS_SYSTEMS_TO_CONFIGURE = [ {"system_id": AGAVE_STORAGE_SYSTEM, "path": "{username}", "create_path": True}, - {"system_id": AGAVE_WORKING_SYSTEM, "path": "{username}", "create_path": True}, {"system_id": "cloud.data", "path": "/ ", "create_path": False}, ] diff --git a/pytest.ini b/pytest.ini index 285191f76..2f22a5121 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ [pytest] DJANGO_SETTINGS_MODULE = designsafe.settings.test_settings -python_files = tests.py *unit_test.py +python_files = tests.py *unit_test.py test_*.py addopts = -p no:warnings From 56b76080bd2e7d1f9259b8c5cec8b1c4ae11d2b2 Mon Sep 17 00:00:00 2001 From: Sal Tijerina Date: Tue, 1 Oct 2024 15:51:39 -0500 Subject: [PATCH 3/9] Integrate `ngrok` service in docker compose stack (#1443) * enable docker ngrok * unify ngrok webhook settings, and move to conf/env_files/ngrok.env * backwards compatibility support for WEBHOOK_POST_URL * handle missing ngrok.env; handle missing http scheme --- .gitignore | 1 + Makefile | 14 +++- README.md | 77 ++++++++----------- .../docker-compose-dev.all.debug.m1.yml | 29 ++++++- conf/docker/docker-compose-dev.all.debug.yml | 28 ++++++- conf/env_files/ngrok.sample.env | 5 ++ designsafe/apps/workspace/api/views.py | 19 +++-- designsafe/settings/common_settings.py | 2 +- .../external_resource_secrets.sample.py | 8 -- designsafe/settings/test_settings.py | 2 +- 10 files changed, 112 insertions(+), 73 deletions(-) create mode 100644 conf/env_files/ngrok.sample.env delete mode 100644 designsafe/settings/external_resource_secrets.sample.py diff --git a/.gitignore b/.gitignore index dbcc6364e..bcd4a23c8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ designsafe/templates/react-assets.html designsafe.env mysql.env rabbitmq.env +ngrok.env rabbitmq.conf mysql.cnf diff --git a/Makefile b/Makefile index 0104e5c8b..dc85626ec 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,9 @@ +NGROK_ENV_FILE = ./conf/env_files/ngrok.env +ifeq ("$(wildcard $(NGROK_ENV_FILE))","") + NGROK_ENV_FILE = ./conf/env_files/ngrok.sample.env +endif + + .PHONY: build build: docker compose -f ./conf/docker/docker-compose.yml build @@ -8,16 +14,16 @@ build-dev: .PHONY: start start: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml up + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml up .PHONY: stop stop: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.yml down + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.yml down .PHONY: start-m1 start-m1: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml up .PHONY: stop-m1 stop-m1: - docker compose -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down + docker compose --env-file $(NGROK_ENV_FILE) -f ./conf/docker/docker-compose-dev.all.debug.m1.yml down diff --git a/README.md b/README.md index b2e91f8b7..b5de1b005 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ -[![Build Status](https://travis-ci.org/DesignSafe-CI/portal.svg?branch=master)](https://travis-ci.org/DesignSafe-CI/portal) -[![codecov](https://codecov.io/gh/DesignSafe-CI/portal/branch/master/graph/badge.svg)](https://codecov.io/gh/DesignSafe-CI/portal) - # DesignSafe-CI Portal ## Prequisites for running the portal application @@ -11,8 +8,8 @@ on. - [Install Docker][3] - [Install Docker Compose][4] -- [Install Make][12] -- [Node.js][13] 16.x +- [Install Make][11] +- [Node.js][12] 20.x If you are on a Mac or a Windows machine, the recommended method is to install [Docker Desktop][5], which will install both Docker and Docker Compose, which is required to run Docker on Mac/Windows hosts. @@ -37,7 +34,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install - `DJANGO_DEBUG`: should be set to `True` for development - `DJANGO_SECRET`: should be changed for production - `TAS_*`: should be set to enable direct access to `django.contrib.admin` - - `AGAVE_*`: should be set to enable Agave API integration (authentication, etc.) + - `TAPIS_*`: should be set to enable Tapis API integration (authentication, etc.) - `RT_*`: should be set to enable ticketing Make a copy of [rabbitmq.sample.env](conf/env_files/rabbitmq.sample.env) @@ -46,7 +43,15 @@ If you are on a Mac or a Windows machine, the recommended method is to install Make a copy of [external_resource_secrets.sample.py](designsafe/settings/external_resource_secrets.sample.py) and rename it to `external_resource_secrets.py`. -3. Build the containers and frontend packages +3. Configure ngrok + + a. Install [ngrok](https://ngrok.com/docs/getting-started/), and create an ngrok account. + + b. Copy [conf/env_files/ngrok.sample.env](conf/env_files/ngrok.sample.env) to `conf/env_files/ngrok.env`. + + c. In `conf/env_files/ngrok.env`, set the `NGROK_AUTHTOKEN` and `NGROK_DOMAIN` variables using your authtoken and static ngrok domain found in your [ngrok dashboard](https://dashboard.ngrok.com/). + +4. Build the containers and frontend packages 1. Containers: ```sh @@ -72,7 +77,7 @@ If you are on a Mac or a Windows machine, the recommended method is to install npm run start ``` -4. Start local containers +5. Start local containers ``` $ make start @@ -89,11 +94,11 @@ If you are on a Mac or a Windows machine, the recommended method is to install $ ./manage.py createsuperuser ``` -5. Setup local access to the portal: +6. Setup local access to the portal: Add a record to your local hosts file for `127.0.0.1 designsafe.dev` ``` - sudo vim /etc/hosts + $ sudo vim /etc/hosts ``` Now you can navigate to [designsafe.dev](designsafe.dev) in your browser. @@ -178,11 +183,10 @@ See the [DesignSafe Styles Reference][7] for style reference and custom CSS docu ### Updating Python dependencies -For simplicity the Dockerfile uses a `requirements.txt` exported from Poetry. To add a new dependency: +This project uses [Python Poetry](https://python-poetry.org/docs/) to manage dependencies. To add a new dependency: 1. Run `poetry add $NEW_DEPENDENCY`. -2. Run `poetry export > requirements.txt --dev --without-hashes` in the repository root. -3. Rebuild the dev image with `docker-compose -f conf/docker/docker-compose.yml build` +2. Rebuild the dev image with `make build-dev` ## Testing @@ -200,16 +204,14 @@ Django tests should be written according to standard [Django testing procedures] You can run Django tests with the following command: ```shell -$ docker exec -it des_django pytest designsafe +$ docker exec -it des_django pytest -ra designsafe ``` ### Frontend tests -Frontend tests are [Jasmine][9] tests executed using the [Karma engine][10]. Testing -guidelines can be found in the [AngularJS Developer Guide on Unit Testing][11]. +Frontend tests are [Vitest][9] tests executed using [Nx][10]. -To run frontend tests, ensure that all scripts and test scripts are configured in -[`karma-conf.js`](karma-conf.js) and then run the command: +To run frontend tests, run the command: ```shell $ npm run test @@ -217,47 +219,31 @@ $ npm run test ## Development setup -Use `docker-compose` to run the portal in development. The default compose file, -[`docker-compose.yml`](docker-compose.yml) runs the main django server in development +Use `docker compose` to run the portal in development. The default compose file, +[`docker-compose.yml`](conf/docker/docker-compose.yml) runs the main django server in development mode with a redis service for websockets support. You can optionally enable the EF sites for testing. ```shell -$ docker-compose -f conf/docker/docker-compose.yml build -$ docker-compose -f conf/docker/docker-compose-dev.all.debug.yml up -$ npm run dev +$ make build-dev +$ make start +$ npm run start +$ docker run -v `pwd`:`pwd` -w `pwd` -it node:16 /bin/bash -c "npm run dev" ``` When using this compose file, your Tapis Client should be configured with a `callback_url` of `http://$DOCKER_HOST_IP:8000/auth/tapis/callback/`. -For developing some services, e.g. Box.com integration, https support is required. To -enable an Nginx http proxy run using the [`docker-compose-http.yml`](docker-compose-http.yml) -file. This file configures the same services as the default compose file, but it also sets -up an Nginx proxy secured by a self-signed certificate. ```shell $ docker-compose -f docker-compose-http.yml build $ docker-compose -f docker-compose-http.yml up ``` -### Agave filesystem setup -1. Delete all of the old metadata objects using this command: - - `metadata-list Q '{"name": "designsafe metadata"}' | while read x; do metadata-delete $x; done;` -2. Run `dsapi/agave/tools/bin/walker.py` to create the metadata objects for the existing files in your FS. - - `python portal/dsapi/agave/tools/bin/walker.py ` - - `base_folder` is your username, if you want to fix everything under your home dir. - - `command`: - - `files`: Walk through the files and print their path. - - `meta`: Walk through the metadata objs in a filesystem-like manner and print their path. - - `files-fix`: Check if there's a meta obj for every file, if not create the meta obj. - - `meta-fix`: Check if there's a file for every meta obj, if not delete the meta obj. ## Production setup -Production deployment is managed by ansible. See https://github.com/designsafe-ci/ansible. +Production deployment is managed by Camino. See https://github.com/TACC/Camino. [1]: https://docs.docker.com/ @@ -267,8 +253,7 @@ Production deployment is managed by ansible. See https://github.com/designsafe-c [5]: https://docs.docker.com/desktop/ [7]: https://github.com/DesignSafe-CI/portal/wiki/CSS-Styles-Reference [8]: https://docs.djangoproject.com/en/dev/topics/testing/ -[9]: http://jasmine.github.io/1.3/introduction.html -[10]: http://karma-runner.github.io/0.12/intro/installation.html -[11]: https://docs.angularjs.org/guide/unit-testing -[12]: https://www.gnu.org/software/make/ -[13]: https://nodejs.org/ +[9]: https://vitest.dev/ +[10]: https://nx.dev/getting-started/intro +[11]: https://www.gnu.org/software/make/ +[12]: https://nodejs.org/ diff --git a/conf/docker/docker-compose-dev.all.debug.m1.yml b/conf/docker/docker-compose-dev.all.debug.m1.yml index edbe69b19..8337c2219 100644 --- a/conf/docker/docker-compose-dev.all.debug.m1.yml +++ b/conf/docker/docker-compose-dev.all.debug.m1.yml @@ -77,7 +77,10 @@ services: django: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -98,7 +101,10 @@ services: websockets: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false volumes: - ../../.:/srv/www/designsafe - ../../data/media:/srv/www/designsafe/media @@ -108,7 +114,10 @@ services: workers: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -125,6 +134,20 @@ services: container_name: des_workers hostname: des_workers + ngrok: + image: ngrok/ngrok:latest + platform: "linux/amd64" + environment: + NGROK_AUTHTOKEN: ${NGROK_AUTHTOKEN} + command: + - "http" + - --url=${NGROK_DOMAIN} + - "https://host.docker.internal:443" + ports: + - 4040:4040 + container_name: des_ngrok + hostname: des_ngrok + volumes: redis_data_v3: des_postgres_data_v3: diff --git a/conf/docker/docker-compose-dev.all.debug.yml b/conf/docker/docker-compose-dev.all.debug.yml index 22d5c6215..9fb100962 100644 --- a/conf/docker/docker-compose-dev.all.debug.yml +++ b/conf/docker/docker-compose-dev.all.debug.yml @@ -73,7 +73,10 @@ services: django: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -94,7 +97,10 @@ services: websockets: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false volumes: - ../../.:/srv/www/designsafe - ../../data/media:/srv/www/designsafe/media @@ -104,7 +110,10 @@ services: workers: image: designsafeci/portal:tapis-v3 - env_file: ../env_files/designsafe.env + env_file: + - path: ../env_files/designsafe.env + - path: ../env_files/ngrok.env + required: false links: - memcached:memcached - rabbitmq:rabbitmq @@ -121,6 +130,19 @@ services: container_name: des_workers hostname: des_workers + ngrok: + image: ngrok/ngrok:latest + environment: + NGROK_AUTHTOKEN: ${NGROK_AUTHTOKEN} + command: + - "http" + - --url=${NGROK_DOMAIN} + - "https://host.docker.internal:443" + ports: + - 4040:4040 + container_name: des_ngrok + hostname: des_ngrok + volumes: redis_data_v3: des_postgres_data_v3: diff --git a/conf/env_files/ngrok.sample.env b/conf/env_files/ngrok.sample.env new file mode 100644 index 000000000..65ae75620 --- /dev/null +++ b/conf/env_files/ngrok.sample.env @@ -0,0 +1,5 @@ +# Get authtoken from https://dashboard.ngrok.com/get-started/your-authtoken +NGROK_AUTHTOKEN=your_ngrok_authtoken + +# Get static domain from https://dashboard.ngrok.com/domains +NGROK_DOMAIN=your-ngrok-subdomain.ngrok-free.app diff --git a/designsafe/apps/workspace/api/views.py b/designsafe/apps/workspace/api/views.py index c6bdf76ac..6d8e74639 100644 --- a/designsafe/apps/workspace/api/views.py +++ b/designsafe/apps/workspace/api/views.py @@ -5,6 +5,7 @@ import logging import json +from urllib.parse import urlparse from django.http import JsonResponse from django.conf import settings from django.core.exceptions import ObjectDoesNotExist @@ -758,14 +759,18 @@ def _submit_job(self, request, body, tapis, username): return {"execSys": system_needs_keys} if settings.DEBUG: - wh_base_url = settings.WEBHOOK_POST_URL + reverse( + parsed_url = urlparse(settings.NGROK_DOMAIN) + if not parsed_url.scheme: + webhook_base_url = f"https://{settings.NGROK_DOMAIN}" + else: + webhook_base_url = settings.NGROK_DOMAIN + + interactive_wh_url = webhook_base_url + reverse( "webhooks:interactive_wh_handler" ) - jobs_wh_url = settings.WEBHOOK_POST_URL + reverse( - "webhooks:jobs_wh_handler" - ) + jobs_wh_url = webhook_base_url + reverse("webhooks:jobs_wh_handler") else: - wh_base_url = request.build_absolute_uri( + interactive_wh_url = request.build_absolute_uri( reverse("webhooks:interactive_wh_handler") ) jobs_wh_url = request.build_absolute_uri( @@ -785,7 +790,7 @@ def _submit_job(self, request, body, tapis, username): projects = request.user.projects.order_by("-last_updated") entry["value"] = " ".join( f"{project.uuid},{project.value['projectId'] if project.value['projectId'] != 'None' else project.uuid}" - for project in projects[:settings.USER_PROJECTS_LIMIT] + for project in projects[: settings.USER_PROJECTS_LIMIT] ) job_post["parameterSet"]["envVariables"] = env_variables break @@ -795,7 +800,7 @@ def _submit_job(self, request, body, tapis, username): # Add webhook URL environment variable for interactive apps job_post["parameterSet"]["envVariables"] = job_post["parameterSet"].get( "envVariables", [] - ) + [{"key": "_INTERACTIVE_WEBHOOK_URL", "value": wh_base_url}] + ) + [{"key": "_INTERACTIVE_WEBHOOK_URL", "value": interactive_wh_url}] job_post["tags"].append("isInteractive") # Make sure $HOME/.tap directory exists for user when running interactive apps on TACC HPC Systems diff --git a/designsafe/settings/common_settings.py b/designsafe/settings/common_settings.py index 9672aa616..648a99e0f 100644 --- a/designsafe/settings/common_settings.py +++ b/designsafe/settings/common_settings.py @@ -704,7 +704,7 @@ FEDORA_CONTAINER= os.environ.get('FEDORA_CONTAINER', 'designsafe-publications-dev') CSRF_TRUSTED_ORIGINS = [f"https://{os.environ.get('SESSION_COOKIE_DOMAIN')}"] -WEBHOOK_POST_URL = os.environ.get('WEBHOOK_POST_URL', '') +NGROK_DOMAIN = os.environ.get('NGROK_DOMAIN', os.environ.get('WEBHOOK_POST_URL', '')) STAFF_VPN_IP_PREFIX = os.environ.get("STAFF_VPN_IP_PREFIX", "129.114") USER_PROJECTS_LIMIT = os.environ.get("USER_PROJECTS_LIMIT", 500) diff --git a/designsafe/settings/external_resource_secrets.sample.py b/designsafe/settings/external_resource_secrets.sample.py deleted file mode 100644 index 09b002818..000000000 --- a/designsafe/settings/external_resource_secrets.sample.py +++ /dev/null @@ -1,8 +0,0 @@ -### -# Secrets for Google Auth flow -# - -GOOGLE_OAUTH2_CLIENT_SECRET = "CHANGE_ME" -GOOGLE_OAUTH2_CLIENT_ID = "CHANGE_ME" - -WEBHOOK_POST_URL = "CHANGE_ME" \ No newline at end of file diff --git a/designsafe/settings/test_settings.py b/designsafe/settings/test_settings.py index 845cf61b0..3786a6957 100644 --- a/designsafe/settings/test_settings.py +++ b/designsafe/settings/test_settings.py @@ -510,7 +510,7 @@ GOOGLE_OAUTH2_CLIENT_SECRET = "CHANGE_ME" GOOGLE_OAUTH2_CLIENT_ID = "CHANGE_ME" -WEBHOOK_POST_URL = "http://8cb9afb3.ngrok.io" +NGROK_DOMAIN = "https://8cb9afb3.ngrok.io" # Box sync BOX_APP_CLIENT_ID = 'boxappclientid' From 06a068a2fb57c8151170670968d7517e3ed2b366 Mon Sep 17 00:00:00 2001 From: Jake Rosenberg Date: Tue, 1 Oct 2024 16:32:09 -0500 Subject: [PATCH 4/9] filter citation authors to remove those with authorship=false (#1453) --- .../src/projects/ProjectCitation/ProjectCitation.tsx | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx b/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx index 28c9fe72e..6fd4f8748 100644 --- a/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx +++ b/client/modules/datafiles/src/projects/ProjectCitation/ProjectCitation.tsx @@ -14,7 +14,9 @@ export const ProjectCitation: React.FC<{ const { data } = useProjectDetail(projectId); const entityDetails = data?.entities.find((e) => e.uuid === entityUuid); const authors = - entityDetails?.value.authors?.filter((a) => a.fname && a.lname) ?? []; + entityDetails?.value.authors?.filter( + (a) => a.fname && a.lname && a.authorship !== false + ) ?? []; if (!data || !entityDetails) return null; return (
@@ -44,7 +46,8 @@ export const PublishedCitation: React.FC<{ (child) => child.uuid === entityUuid && child.version === version ); - const authors = entityDetails?.value.authors ?? []; + const authors = + entityDetails?.value.authors?.filter((a) => a.authorship !== false) ?? []; if (!data || !entityDetails) return null; const doi = From 1295c87af90d328be66ce3d34d076c6934af1f13 Mon Sep 17 00:00:00 2001 From: Sal Tijerina Date: Wed, 2 Oct 2024 12:14:54 -0500 Subject: [PATCH 5/9] task/DSAPP-46, task/WP-611: Tools & Apps Workspace Changes; Notifications Enhancements (#1442) * support .log preview * change "Account Profile" to "Manage Account * add workspace links to account dropdown * change /rw/workspace to /workspace; backwards compatible * add "Job Status" label to notification bell * enable docker ngrok * remove notifications link; adjust dropdown header * formatting * sync job notification states * update m1 docker-compose; linting * adjust toast UI to look like secondary button * unify ngrok webhook settings, and move to conf/env_files/ngrok.env * Revert "unify ngrok webhook settings, and move to conf/env_files/ngrok.env" This reverts commit 940be993cbe6137433caf9c1b123bb4efb5f1ca3. * Revert "enable docker ngrok" This reverts commit 793b6d0eb0e2f0cb812de784f8899db717d60114. --- .../src/notifications/useNotifications.ts | 6 +- .../workspace/src/JobsListing/JobsListing.tsx | 5 ++ .../JobsListingTable/JobsListingTable.tsx | 8 ++- .../src/Toast/Notifications.module.css | 10 +++ client/modules/workspace/src/Toast/Toast.tsx | 38 +++++++++-- client/src/workspace/workspaceRouter.tsx | 9 ++- .../designsafe/apps/accounts/base.html | 2 +- .../designsafe/apps/accounts/profile.html | 4 +- designsafe/apps/accounts/urls.py | 2 +- designsafe/apps/accounts/views.py | 4 +- .../apps/api/notifications/receivers.py | 63 +++++++++---------- .../apps/signals/websocket_consumers.py | 33 ++++++---- .../apps/workspace/models/app_entries.py | 4 +- designsafe/apps/workspace/urls.py | 1 + designsafe/middleware.py | 4 +- designsafe/settings/common_settings.py | 2 +- .../dashboard/dashboard.component.html | 2 +- .../notification-badge.component.html | 19 +++--- .../controllers/notifications.js | 9 ++- .../providers/notifications-provider.js | 2 +- .../ng-designsafe/providers/ws-provider.js | 26 +++++--- .../static/scripts/notifications/app.js | 2 +- designsafe/static/styles/main.css | 4 ++ designsafe/templates/includes/header.html | 3 +- designsafe/urls.py | 5 +- 25 files changed, 176 insertions(+), 91 deletions(-) diff --git a/client/modules/_hooks/src/notifications/useNotifications.ts b/client/modules/_hooks/src/notifications/useNotifications.ts index 566a0633e..cfbcebe90 100644 --- a/client/modules/_hooks/src/notifications/useNotifications.ts +++ b/client/modules/_hooks/src/notifications/useNotifications.ts @@ -6,7 +6,11 @@ import { } from '@tanstack/react-query'; import apiClient from '../apiClient'; -type TPortalEventType = 'data_depot' | 'job' | 'interactive_session_ready'; +type TPortalEventType = + | 'data_depot' + | 'job' + | 'interactive_session_ready' + | 'markAllNotificationsAsRead'; export type TJobStatusNotification = { action_link: string; diff --git a/client/modules/workspace/src/JobsListing/JobsListing.tsx b/client/modules/workspace/src/JobsListing/JobsListing.tsx index d2dcf3404..6aaae7aa5 100644 --- a/client/modules/workspace/src/JobsListing/JobsListing.tsx +++ b/client/modules/workspace/src/JobsListing/JobsListing.tsx @@ -1,4 +1,5 @@ import React, { useMemo, useState, useEffect } from 'react'; +import useWebSocket from 'react-use-websocket'; import { TableProps, Row, Flex, Button as AntButton } from 'antd'; import type { ButtonSize } from 'antd/es/button'; import { useQueryClient } from '@tanstack/react-query'; @@ -96,12 +97,16 @@ export const JobsListing: React.FC> = ({ markRead: false, }); const { mutate: readNotifications } = useReadNotifications(); + const { sendMessage } = useWebSocket( + `wss://${window.location.host}/ws/websockets/` + ); // mark all as read on component mount useEffect(() => { readNotifications({ eventTypes: ['interactive_session_ready', 'job'], }); + sendMessage('markAllNotificationsAsRead'); // update unread count state queryClient.setQueryData( diff --git a/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx b/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx index d3d11deb8..4cffd2b64 100644 --- a/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx +++ b/client/modules/workspace/src/JobsListing/JobsListingTable/JobsListingTable.tsx @@ -91,9 +91,11 @@ export const JobsListingTable: React.FC< isLoading, ]); - const lastNotificationJobUUID = lastMessage - ? (JSON.parse(lastMessage.data) as TJobStatusNotification).extra.uuid - : ''; + const lastMessageJSON = lastMessage?.data + ? (JSON.parse(lastMessage.data) as TJobStatusNotification) + : null; + const lastNotificationJobUUID = + lastMessageJSON?.event_type === 'job' ? lastMessageJSON.extra.uuid : ''; const unreadJobUUIDs = unreadNotifs?.notifs.map((x) => x.extra.uuid) ?? []; /* RENDER THE TABLE */ diff --git a/client/modules/workspace/src/Toast/Notifications.module.css b/client/modules/workspace/src/Toast/Notifications.module.css index 44beb6563..44ca35802 100644 --- a/client/modules/workspace/src/Toast/Notifications.module.css +++ b/client/modules/workspace/src/Toast/Notifications.module.css @@ -1,3 +1,13 @@ +.root { + cursor: pointer; + background: #f4f4f4; + border: 1px solid #222222; + &:hover { + border-color: #5695c4; + background: #aac7ff; + } +} + .toast-is-error { color: #eb6e6e; } diff --git a/client/modules/workspace/src/Toast/Toast.tsx b/client/modules/workspace/src/Toast/Toast.tsx index 1956cd110..576476be9 100644 --- a/client/modules/workspace/src/Toast/Toast.tsx +++ b/client/modules/workspace/src/Toast/Toast.tsx @@ -1,10 +1,14 @@ import React, { useEffect } from 'react'; import useWebSocket from 'react-use-websocket'; import { useQueryClient } from '@tanstack/react-query'; -import { notification } from 'antd'; +import { notification, Flex } from 'antd'; +import { RightOutlined } from '@ant-design/icons'; import { useNavigate } from 'react-router-dom'; import { Icon } from '@client/common-components'; -import { TJobStatusNotification } from '@client/hooks'; +import { + TJobStatusNotification, + TGetNotificationsResponse, +} from '@client/hooks'; import { getToastMessage } from '../utils'; import styles from './Notifications.module.css'; @@ -31,19 +35,43 @@ const Notifications = () => { queryKey: ['workspace', 'jobsListing'], }); api.open({ - message: getToastMessage(notification), + message: ( + + {getToastMessage(notification)} + + + ), placement: 'bottomLeft', icon: , className: `${ notification.extra.status === 'FAILED' && styles['toast-is-error'] - }`, + } ${styles.root}`, closeIcon: false, duration: 5, onClick: () => { navigate('/history'); }, - style: { cursor: 'pointer' }, }); + } else if (notification.event_type === 'markAllNotificationsAsRead') { + // update unread count state + queryClient.setQueryData( + [ + 'workspace', + 'notifications', + { + eventTypes: ['interactive_session_ready', 'job'], + read: false, + markRead: false, + }, + ], + (oldData: TGetNotificationsResponse) => { + return { + ...oldData, + notifs: [], + unread: 0, + }; + } + ); } }; diff --git a/client/src/workspace/workspaceRouter.tsx b/client/src/workspace/workspaceRouter.tsx index d9b3bf7c9..ad2128829 100644 --- a/client/src/workspace/workspaceRouter.tsx +++ b/client/src/workspace/workspaceRouter.tsx @@ -7,6 +7,13 @@ import { JobsListingLayout } from './layouts/JobsListingLayout'; import { AppsViewLayout } from './layouts/AppsViewLayout'; import { AppsPlaceholderLayout } from './layouts/AppsPlaceholderLayout'; +const getBaseName = () => { + if (window.location.pathname.startsWith('/rw/workspace')) { + return '/rw/workspace'; + } + return '/workspace'; +}; + const workspaceRouter = createBrowserRouter( [ { @@ -44,7 +51,7 @@ const workspaceRouter = createBrowserRouter( ], }, ], - { basename: '/rw/workspace' } + { basename: getBaseName() } ); export default workspaceRouter; diff --git a/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html b/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html index e0c074ed5..b70f0557f 100644 --- a/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html +++ b/designsafe/apps/accounts/templates/designsafe/apps/accounts/base.html @@ -9,7 +9,7 @@

{{title}}

    {% url 'designsafe_accounts:manage_profile' as item_url %} -
  • Account Profile
    • +
  • Manage Account
    • {% url 'designsafe_accounts:manage_authentication' as item_url %}
  • Authentication
    • diff --git a/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html b/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html index 85ba529ba..0f41e9f44 100644 --- a/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html +++ b/designsafe/apps/accounts/templates/designsafe/apps/accounts/profile.html @@ -1,11 +1,11 @@ {% extends "designsafe/apps/accounts/base.html" %} -{% block title %}Account Profile{% endblock %} +{% block title %}Manage Account{% endblock %} {% block panel_content %}
    -

    Account Profile

    +

    Manage Account

    diff --git a/designsafe/apps/accounts/urls.py b/designsafe/apps/accounts/urls.py index 18e1585e5..7d54f11d5 100644 --- a/designsafe/apps/accounts/urls.py +++ b/designsafe/apps/accounts/urls.py @@ -30,7 +30,7 @@ def menu_items(**kwargs): if 'type' in kwargs and kwargs['type'] == 'account': return [ { - 'label': _('Account Profile'), + 'label': _('Manage Account'), 'url': reverse('designsafe_accounts:index'), 'children': [], }, diff --git a/designsafe/apps/accounts/views.py b/designsafe/apps/accounts/views.py index 346fb0476..69765b033 100644 --- a/designsafe/apps/accounts/views.py +++ b/designsafe/apps/accounts/views.py @@ -49,7 +49,7 @@ def manage_profile(request): logger.info('exception e:{} {}'.format(type(e), e)) context = { - 'title': 'Account Profile', + 'title': 'Manage Account', 'profile': user_profile, 'ds_profile': ds_profile, 'demographics': demographics, @@ -353,7 +353,7 @@ def profile_edit(request): form = forms.UserProfileForm(initial=tas_user) context = { - 'title': 'Account Profile', + 'title': 'Manage Account', 'form': form, 'pro_form': pro_form } diff --git a/designsafe/apps/api/notifications/receivers.py b/designsafe/apps/api/notifications/receivers.py index 316840a69..eada12fd0 100644 --- a/designsafe/apps/api/notifications/receivers.py +++ b/designsafe/apps/api/notifications/receivers.py @@ -1,52 +1,47 @@ +"""Signal receivers for notifications""" + +import logging +import json +from django.db.models.signals import post_save from django.dispatch import receiver from asgiref.sync import async_to_sync from channels.layers import get_channel_layer -from django.db.models.signals import post_save from designsafe.apps.api.notifications.models import Notification, Broadcast -import logging -import json logger = logging.getLogger(__name__) -WEBSOCKETS_FACILITY = 'websockets' +WEBSOCKETS_FACILITY = "websockets" + + +@receiver(post_save, sender=Notification, dispatch_uid="notification_msg") +def send_notification_ws(instance, created, **kwargs): + """Send a websocket message to the user when a new notification is created.""" -@receiver(post_save, sender=Notification, dispatch_uid='notification_msg') -def send_notification_ws(sender, instance, created, **kwargs): - #Only send WS message if it's a new notification not if we're updating. - logger.debug('receiver received something.') if not created: return - try: - channel_layer = get_channel_layer() - instance_dict = json.dumps(instance.to_dict()) - logger.debug(instance_dict) - async_to_sync(channel_layer.group_send)(f"ds_{instance.user}", {"type": "ds.notification", "message": instance_dict}) + channel_layer = get_channel_layer() + instance_dict = json.dumps(instance.to_dict()) + async_to_sync(channel_layer.group_send)( + f"ds_{instance.user}", {"type": "ds.notification", "message": instance_dict} + ) - # logger.debug('WS socket msg sent: {}'.format(instance_dict)) - except Exception as e: - # logger.debug('Exception sending websocket message', - # exc_info=True, - # extra = instance.to_dict()) - logger.debug('Exception sending websocket message', - exc_info=True) return -@receiver(post_save, sender=Broadcast, dispatch_uid='broadcast_msg') -def send_broadcast_ws(sender, instance, created, **kwargs): + +@receiver(post_save, sender=Broadcast, dispatch_uid="broadcast_msg") +def send_broadcast_ws(instance, created, **kwargs): + """Send a websocket message to all users when a new broadcast is created.""" + if not created: return - try: - event_type, user, body = decompose_message(instance) - #rp = RedisPublisher(facility = WEBSOCKETS_FACILITY,broadcast=True) - channel_layer = get_channel_layer() - instance_dict = json.dumps(instance.to_dict()) - - async_to_sync(channel_layer.group_send)("ds_broadcast", {"type": "ds.notification", "message": instance_dict}) - logger.debug('WS socket msg sent: {}'.format(instance_dict)) - except Exception as e: - logger.debug('Exception sending websocket message', - exc_info=True, - extra = instance.to_dict()) + + channel_layer = get_channel_layer() + instance_dict = json.dumps(instance.to_dict()) + + async_to_sync(channel_layer.group_send)( + "ds_broadcast", {"type": "ds.notification", "message": instance_dict} + ) + return diff --git a/designsafe/apps/signals/websocket_consumers.py b/designsafe/apps/signals/websocket_consumers.py index 2353bad28..d762a13af 100644 --- a/designsafe/apps/signals/websocket_consumers.py +++ b/designsafe/apps/signals/websocket_consumers.py @@ -1,28 +1,37 @@ -import json +"""Websocket consumers""" -from channels.generic.websocket import WebsocketConsumer, AsyncWebsocketConsumer +import json +from channels.generic.websocket import AsyncWebsocketConsumer +from channels.layers import get_channel_layer class DesignsafeWebsocketConsumer(AsyncWebsocketConsumer): + """Websocket consumer for DesignSafe notifications""" + async def connect(self): - self.user_channel = f"ds_{self.scope['user']}" - self.broadcast_channel = "ds_broadcast" await self.channel_layer.group_add( - self.user_channel, self.channel_name + f"ds_{self.scope['user']}", self.channel_name ) await self.accept() - async def disconnect(self, close_code): - await self.channel_layer.group_discard( - self.user_channel, self.channel_name - ) + async def disconnect(self, code): await self.channel_layer.group_discard( - self.broadcast_channel, self.channel_name + f"ds_{self.scope['user']}", self.channel_name ) + await self.channel_layer.group_discard("ds_broadcast", self.channel_name) - async def receive(self, text_data): - pass + async def receive(self, text_data=None, bytes_data=None): + if text_data == "markAllNotificationsAsRead": + channel_layer = get_channel_layer() + await channel_layer.group_send( + f"ds_{self.scope['user']}", + { + "type": "ds_notification", + "message": json.dumps({"event_type": "markAllNotificationsAsRead"}), + }, + ) async def ds_notification(self, event): + """Send notification to user""" message = event["message"] await self.send(text_data=message) diff --git a/designsafe/apps/workspace/models/app_entries.py b/designsafe/apps/workspace/models/app_entries.py index f8cf2938a..1477572af 100644 --- a/designsafe/apps/workspace/models/app_entries.py +++ b/designsafe/apps/workspace/models/app_entries.py @@ -233,8 +233,8 @@ def href(self): """Retrieve the app's URL in the Tools & Applications space""" if self.external_href: return self.external_href - - app_href = f"/rw/workspace/{self.app_id}" + + app_href = f"/workspace/{self.app_id}" if self.version: app_href += f"?appVersion={self.version}" return app_href diff --git a/designsafe/apps/workspace/urls.py b/designsafe/apps/workspace/urls.py index 362316ed6..00cc09812 100644 --- a/designsafe/apps/workspace/urls.py +++ b/designsafe/apps/workspace/urls.py @@ -4,5 +4,6 @@ from designsafe.apps.workspace import views urlpatterns = [ + re_path('history', views.WorkspaceView.as_view(), name="history"), re_path('^', views.WorkspaceView.as_view(), name="workspace"), ] diff --git a/designsafe/middleware.py b/designsafe/middleware.py index 0f2a5e738..266103d53 100644 --- a/designsafe/middleware.py +++ b/designsafe/middleware.py @@ -30,7 +30,7 @@ class DesignsafeProfileUpdateMiddleware(MiddlewareMixin): def process_request(self, request): blocked_path = request.path.startswith( - ("/data", "/applications", "/rw/workspace", "/recon-portal", "/dashboard") + ("/data", "/applications", "/rw/workspace", "/workspace", "/recon-portal", "/dashboard") ) if request.user.is_authenticated and request.user.profile.update_required and blocked_path: messages.warning( @@ -65,7 +65,7 @@ def process_request(self, request): 'Use is required for continued use of DesignSafe ' 'resources.' % accept_url) return None - + class SiteMessageMiddleware(MiddlewareMixin): def process_request(self, request): diff --git a/designsafe/settings/common_settings.py b/designsafe/settings/common_settings.py index 648a99e0f..98e05e81e 100644 --- a/designsafe/settings/common_settings.py +++ b/designsafe/settings/common_settings.py @@ -668,7 +668,7 @@ '.java', '.js', '.less', '.m', '.make', '.md', '.ml', '.mm', '.msg', '.php', '.pl', '.properties', '.py', '.rb', '.sass', '.scala', '.script', '.sh', '.sml', '.sql', '.txt', '.vi', '.vim', '.xml', '.xsd', '.xsl', '.yaml', '.yml', '.tcl', - '.json', '.out', '.err', '.geojson', '.do', '.sas', '.hazmapper' + '.json', '.out', '.err', '.geojson', '.do', '.sas', '.hazmapper', ".log" ] SUPPORTED_OBJECT_PREVIEW_EXTS = [ diff --git a/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html b/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html index 45340a215..64adf9751 100644 --- a/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html +++ b/designsafe/static/scripts/dashboard/components/dashboard/dashboard.component.html @@ -37,7 +37,7 @@
    Quick Links
    - +
    diff --git a/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html b/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html index bd706a3cb..93ec82e04 100644 --- a/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html +++ b/designsafe/static/scripts/ng-designsafe/components/notification-badge/notification-badge.component.html @@ -1,17 +1,18 @@ -