diff --git a/server/portal/apps/webhooks/views.py b/server/portal/apps/webhooks/views.py index 702045b27..54544b23d 100644 --- a/server/portal/apps/webhooks/views.py +++ b/server/portal/apps/webhooks/views.py @@ -20,6 +20,7 @@ validate_webhook, execute_callback ) +from portal.apps.workspace.api.utils import check_job_for_timeout from django.conf import settings @@ -54,6 +55,8 @@ def validate_tapis_job(job_uuid, job_owner, disallowed_states=[]): if job_data.status in disallowed_states: return None + job_data = check_job_for_timeout(job_data) + return job_data @@ -122,6 +125,7 @@ def post(self, request, *args, **kwargs): job_details = validate_tapis_job(job_uuid, username, disallowed_states=non_terminal_states) if job_details: event_data[Notification.EXTRA]['remoteOutcome'] = job_details.remoteOutcome + event_data[Notification.EXTRA]['status'] = job_details.status try: logger.info('Indexing job output for job={}'.format(job_uuid)) diff --git a/server/portal/apps/workspace/api/utils.py b/server/portal/apps/workspace/api/utils.py new file mode 100644 index 000000000..9d9923dfa --- /dev/null +++ b/server/portal/apps/workspace/api/utils.py @@ -0,0 +1,28 @@ +import json + + +def get_tapis_timeout_error_messages(job_id): + return [ + 'JOBS_EARLY_TERMINATION Job terminated by Tapis because: TIME_EXPIRED', + f'JOBS_USER_APP_FAILURE The user application ({job_id}) ended with remote status "TIMEOUT" and returned exit code: 0:0.' + ] + + +def check_job_for_timeout(job): + """ + Check an interactive job for timeout status and mark it as finished + since Tapis does not have native support for interactive jobs yet + """ + + if (hasattr(job, 'notes')): + notes = json.loads(job.notes) + + is_failed = job.status == 'FAILED' + is_interactive = notes.get('isInteractive', False) + has_timeout_message = job.lastMessage in get_tapis_timeout_error_messages(job.remoteJobId) + + if is_failed and is_interactive and has_timeout_message: + job.status = 'FINISHED' + job.remoteOutcome = 'FINISHED' + + return job diff --git a/server/portal/apps/workspace/api/views.py b/server/portal/apps/workspace/api/views.py index b79716428..9a94db829 100644 --- a/server/portal/apps/workspace/api/views.py +++ b/server/portal/apps/workspace/api/views.py @@ -23,6 +23,7 @@ from portal.apps.onboarding.steps.system_access_v3 import create_system_credentials from portal.apps.users.utils import get_user_data from .handlers.tapis_handlers import tapis_get_handler +from portal.apps.workspace.api.utils import check_job_for_timeout logger = logging.getLogger(__name__) METRICS = logging.getLogger('metrics.{}'.format(__name__)) @@ -138,6 +139,7 @@ def get(self, request, *args, **kwargs): @method_decorator(login_required, name='dispatch') class JobsView(BaseApiView): + def get(self, request, operation=None): allowed_actions = ['listing', 'search', 'select'] @@ -150,6 +152,12 @@ def get(self, request, operation=None): op = getattr(self, operation) data = op(tapis, request) + if (isinstance(data, list)): + for index, job in enumerate(data): + data[index] = check_job_for_timeout(job) + else: + data = check_job_for_timeout(data) + return JsonResponse( { 'status': 200,