From 73a44a6bc1598b039dd6b2e265af69e918f853f5 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 19 Dec 2022 22:36:56 +0000 Subject: [PATCH 001/158] Enables resubmissions for the k8s runner --- lib/galaxy/jobs/runners/kubernetes.py | 31 +++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 1f9e984605c3..87a28d815b8c 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -95,7 +95,8 @@ def __init__(self, app, nworkers, **kwargs): map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None ), k8s_cleanup_job=dict(map=str, valid=lambda s: s in {"onsuccess", "always", "never"}, default="always"), - k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=3), + k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=1), # note that if the backOffLimit is lower, this paramer will have not effect. + k8s_job_spec_back_off_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=0), # this means that it will stop retrying after 1 failure. k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800), k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: not x or int(x) >= 0, default=None), k8s_interactivetools_use_ssl=dict(map=bool, default=False), @@ -323,6 +324,7 @@ def __get_k8s_job_spec(self, ajs): job_ttl = self.runner_params["k8s_job_ttl_secs_after_finished"] if self.runner_params["k8s_cleanup_job"] != "never" and job_ttl is not None: k8s_job_spec["ttlSecondsAfterFinished"] = job_ttl + k8s_job_spec["backoffLimit"] = self.runner_params["k8s_job_spec_back_off_limit"] return k8s_job_spec def __force_label_conformity(self, value): @@ -526,7 +528,7 @@ def __get_k8s_containers(self, ajs): # command line execution, separated by ;, which is what Galaxy does # to assemble the command. "command": [ajs.job_wrapper.shell], - "args": ["-c", ajs.job_file], + "args": ["-c", f"{ajs.job_file}; exit $(cat {ajs.exit_code_file})"], "workingDir": ajs.job_wrapper.working_directory, "volumeMounts": deduplicate_entries(mounts), } @@ -715,6 +717,10 @@ def check_watched_item(self, job_state): else: max_pod_retries = 1 + # make sure that we don't have any conditions by which the runner + # would wait forever for a pod that never gets sent. + max_pod_retries = min(max_pod_retries, self.runner_params["k8s_job_spec_back_off_limit"]) + # Check if job.obj['status'] is empty, # return job_state unchanged if this is the case # as probably this means that the k8s API server hasn't @@ -736,7 +742,7 @@ def check_watched_item(self, job_state): job_state.running = False self.mark_as_finished(job_state) return None - elif active > 0 and failed <= max_pod_retries: + elif active > 0 and failed < max_pod_retries + 1: if not job_state.running: if self.__job_pending_due_to_unschedulable_pod(job_state): if self.runner_params.get("k8s_unschedulable_walltime_limit"): @@ -760,7 +766,10 @@ def check_watched_item(self, job_state): job_state.job_wrapper.cleanup() return None else: - return self._handle_job_failure(job, job_state) + self._handle_job_failure(job, job_state) + # changes for resubmission (removed self.mark_as_failed from handle_job_failure) + self.work_queue.put((self.mark_as_failed, job_state)) + return None elif len(jobs.response["items"]) == 0: if job_state.job_wrapper.get_job().state == model.Job.states.DELETED: @@ -798,6 +807,8 @@ def _handle_unschedulable_job(self, job, job_state): def _handle_job_failure(self, job, job_state): # Figure out why job has failed with open(job_state.error_file, "a") as error_file: + # TODO we need to remove probably these error_file.writes, as they remove the stderr / stdout capture + # from failed Galaxy k8s jobs. if self.__job_failed_due_to_low_memory(job_state): error_file.write("Job killed after running out of memory. Try with more memory.\n") job_state.fail_message = "Tool failed due to insufficient memory. Try with more memory." @@ -809,8 +820,9 @@ def _handle_job_failure(self, job, job_state): else: error_file.write("Exceeded max number of Kubernetes pod retries allowed for job\n") job_state.fail_message = "More pods failed than allowed. See stdout for pods details." - job_state.running = False - self.mark_as_failed(job_state) + # changes for resubmission, to mimick what happens in the LSF-cli runner + # job_state.running = False + # self.mark_as_failed(job_state) try: if self.__has_guest_ports(job_state.job_wrapper): self.__cleanup_k8s_guest_ports(job_state.job_wrapper, job) @@ -855,11 +867,12 @@ def __job_failed_due_to_low_memory(self, job_state): if not pods.response["items"]: return False - pod = self._get_pod_for_job(job_state) + # pod = self._get_pod_for_job(job_state) # this was always None + pod = pods.response["items"][0] if ( pod - and pod.obj["status"]["phase"] == "Failed" - and pod.obj["status"]["containerStatuses"][0]["state"]["terminated"]["reason"] == "OOMKilled" + and "terminated" in pod["status"]["containerStatuses"][0]["state"] + and pod["status"]["containerStatuses"][0]["state"]["terminated"]["reason"] == "OOMKilled" ): return True From 4ed002540cafc6ef7411cf4ca9838f2a4680d24f Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Mon, 27 Feb 2023 11:38:05 +0000 Subject: [PATCH 002/158] Fix detection of stderr / stdout and placement on UI --- lib/galaxy/jobs/runners/kubernetes.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 87a28d815b8c..4cb85284bfc7 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -741,10 +741,12 @@ def check_watched_item(self, job_state): if succeeded > 0 or job_state == model.Job.states.STOPPED: job_state.running = False self.mark_as_finished(job_state) + log.debug("k8s job succeeded") return None elif active > 0 and failed < max_pod_retries + 1: if not job_state.running: if self.__job_pending_due_to_unschedulable_pod(job_state): + log.debug("k8s job pending..") if self.runner_params.get("k8s_unschedulable_walltime_limit"): creation_time_str = job.obj["metadata"].get("creationTimestamp") creation_time = datetime.strptime(creation_time_str, "%Y-%m-%dT%H:%M:%SZ") @@ -756,19 +758,28 @@ def check_watched_item(self, job_state): else: pass else: + log.debug("k8s job is running..") job_state.running = True job_state.job_wrapper.change_state(model.Job.states.RUNNING) return job_state elif job_persisted_state == model.Job.states.DELETED: # Job has been deleted via stop_job and job has not been deleted, # remove from watched_jobs by returning `None` + log.debug("PP Job is DELETED..") if job_state.job_wrapper.cleanup_job in ("always", "onsuccess"): job_state.job_wrapper.cleanup() return None else: + log.debug("k8s job is failed and not deleted, looking at failure") self._handle_job_failure(job, job_state) # changes for resubmission (removed self.mark_as_failed from handle_job_failure) self.work_queue.put((self.mark_as_failed, job_state)) + # If the job was not resubmitted after being put in the failed queue, + # we mark it as finished as well for stderr / stdout detection. + # Otherwise, the user doesn't see any stdout/stderr in the UI. + if job_state.job_wrapper.get_state() != model.Job.states.RESUBMITTED: + self.mark_as_finished(job_state) + return None elif len(jobs.response["items"]) == 0: @@ -807,22 +818,21 @@ def _handle_unschedulable_job(self, job, job_state): def _handle_job_failure(self, job, job_state): # Figure out why job has failed with open(job_state.error_file, "a") as error_file: - # TODO we need to remove probably these error_file.writes, as they remove the stderr / stdout capture - # from failed Galaxy k8s jobs. + log.debug("Trying with error file in _handle_job_failure") if self.__job_failed_due_to_low_memory(job_state): + log.debug("OOM condition reached") error_file.write("Job killed after running out of memory. Try with more memory.\n") job_state.fail_message = "Tool failed due to insufficient memory. Try with more memory." job_state.runner_state = JobState.runner_states.MEMORY_LIMIT_REACHED elif self.__job_failed_due_to_walltime_limit(job): + log.debug("Walltime condition reached") error_file.write("DeadlineExceeded") job_state.fail_message = "Job was active longer than specified deadline" job_state.runner_state = JobState.runner_states.WALLTIME_REACHED else: - error_file.write("Exceeded max number of Kubernetes pod retries allowed for job\n") - job_state.fail_message = "More pods failed than allowed. See stdout for pods details." - # changes for resubmission, to mimick what happens in the LSF-cli runner - # job_state.running = False - # self.mark_as_failed(job_state) + log.debug("Runner cannot detect a specific reason for failure, must be a tool failure.") + error_file.write("Exceeded max number of job retries allowed for job\n") + job_state.fail_message = "More job retries failed than allowed. See standard output and standard error within the info section for details." try: if self.__has_guest_ports(job_state.job_wrapper): self.__cleanup_k8s_guest_ports(job_state.job_wrapper, job) @@ -962,6 +972,7 @@ def recover(self, job, job_wrapper): ajs.old_state = model.Job.states.QUEUED ajs.running = False self.monitor_queue.put(ajs) + def finish_job(self, job_state): self._handle_metadata_externally(job_state.job_wrapper, resolve_requirements=True) From e4aeae0892ad88858ed228861ad99f295e74557e Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 8 Mar 2023 18:06:44 +0000 Subject: [PATCH 003/158] Everything working --- lib/galaxy/jobs/runners/kubernetes.py | 196 ++++++++++++++++++++------ 1 file changed, 152 insertions(+), 44 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 4cb85284bfc7..da212d2a68e6 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -3,14 +3,19 @@ """ import logging +import json # for debugging of API objects import math import os import re +import time from datetime import datetime import yaml from galaxy import model +from galaxy.util import ( + unicodify, +) from galaxy.jobs.runners import ( AsynchronousJobRunner, AsynchronousJobState, @@ -214,7 +219,11 @@ def queue_job(self, job_wrapper): self.monitor_queue.put(ajs) def __has_guest_ports(self, job_wrapper): - return bool(job_wrapper.guest_ports) + # Check if job has guest ports or interactive tool entry points that would signal that + # this is an interactive tool. + log.debug(f"Checking if job {job_wrapper.get_id_tag()} has guest ports: {job_wrapper.guest_ports}") + log.debug(f"Checking if job {job_wrapper.get_id_tag()} has interactive entry points: {job_wrapper.guest_ports}") + return bool(job_wrapper.guest_ports) or bool(job_wrapper.get_job().interactivetool_entry_points) def __configure_port_routing(self, ajs): # Configure interactive tool entry points first @@ -231,9 +240,19 @@ def __configure_port_routing(self, ajs): k8s_service_obj = service_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_service_spec(ajs)) k8s_ingress_obj = ingress_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_ingress_spec(ajs)) + # pretty print the objects for debugging + log.debug(f"Kubernetes service object: {json.dumps(k8s_service_obj, indent=4)}") + log.debug(f"Kubernetes ingress object: {json.dumps(k8s_ingress_obj, indent=4)}") + # We avoid creating service and ingress if they already exist (e.g. when Galaxy is restarted or resubmitting a job) service = Service(self._pykube_api, k8s_service_obj) + # if service.exists(): + # log.debug(f"Service {k8s_job_name} already exists, skipping creation") + # else: service.create() ingress = Ingress(self._pykube_api, k8s_ingress_obj) + # if ingress.exists(): + # log.debug(f"Ingress {k8s_job_name} already exists, skipping creation") + # else: ingress.create() def __get_overridable_params(self, job_wrapper, param_key): @@ -456,26 +475,27 @@ def __get_k8s_ingress_spec(self, ajs): "annotations": {"app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id}, }, "spec": { - "rules": [ - { - "host": ep["domain"], - "http": { - "paths": [ - { - "backend": { - "serviceName": self.__get_k8s_job_name( - self.__produce_k8s_job_prefix(), ajs.job_wrapper - ), - "servicePort": int(ep["tool_port"]), - }, - "path": ep.get("entry_path", "/"), - "pathType": "Prefix", - } - ] - }, - } - for ep in entry_points - ] + "ingressClassName": "nginx", + "rules":[ { + "host": ep["domain"], + "http": { + "paths": [ { + "backend": { + "service": { + "name": self.__get_k8s_job_name( + self.__produce_k8s_job_prefix(), ajs.job_wrapper + ), + "port": { "number": int(ep["tool_port"])} + } + }, + "path": ep.get("entry_path", "/"), + "pathType": "Prefix" + } + ] + }, + } + for ep in entry_points + ] }, } if self.runner_params.get("k8s_interactivetools_use_ssl"): @@ -741,12 +761,12 @@ def check_watched_item(self, job_state): if succeeded > 0 or job_state == model.Job.states.STOPPED: job_state.running = False self.mark_as_finished(job_state) - log.debug("k8s job succeeded") + log.debug("Job succeeded") return None elif active > 0 and failed < max_pod_retries + 1: if not job_state.running: if self.__job_pending_due_to_unschedulable_pod(job_state): - log.debug("k8s job pending..") + log.debug("PP Job pending..") if self.runner_params.get("k8s_unschedulable_walltime_limit"): creation_time_str = job.obj["metadata"].get("creationTimestamp") creation_time = datetime.strptime(creation_time_str, "%Y-%m-%dT%H:%M:%SZ") @@ -758,39 +778,39 @@ def check_watched_item(self, job_state): else: pass else: - log.debug("k8s job is running..") + log.debug("Job set to running..") job_state.running = True job_state.job_wrapper.change_state(model.Job.states.RUNNING) return job_state elif job_persisted_state == model.Job.states.DELETED: # Job has been deleted via stop_job and job has not been deleted, # remove from watched_jobs by returning `None` - log.debug("PP Job is DELETED..") + log.debug("Job is DELETED..") if job_state.job_wrapper.cleanup_job in ("always", "onsuccess"): job_state.job_wrapper.cleanup() return None else: - log.debug("k8s job is failed and not deleted, looking at failure") + log.debug(f"Job is failed and not deleted, looking at failure") + log.debug(f"Job state before handle job failure: {job_state.job_wrapper.get_state()}") self._handle_job_failure(job, job_state) # changes for resubmission (removed self.mark_as_failed from handle_job_failure) self.work_queue.put((self.mark_as_failed, job_state)) - # If the job was not resubmitted after being put in the failed queue, - # we mark it as finished as well for stderr / stdout detection. - # Otherwise, the user doesn't see any stdout/stderr in the UI. - if job_state.job_wrapper.get_state() != model.Job.states.RESUBMITTED: - self.mark_as_finished(job_state) return None elif len(jobs.response["items"]) == 0: if job_state.job_wrapper.get_job().state == model.Job.states.DELETED: - # Job has been deleted via stop_job and job has been deleted, - # cleanup and remove from watched_jobs by returning `None` if job_state.job_wrapper.cleanup_job in ("always", "onsuccess"): job_state.job_wrapper.cleanup() return None + if job_state.job_wrapper.get_job().state == model.Job.states.STOPPED and self.__has_guest_ports(job_state.job_wrapper): + # Interactive job has been stopped via stop_job (most likely by the user), + # cleanup and remove from watched_jobs by returning `None`. STOPPED jobs are cleaned up elsewhere. + # Marking as finished makes sure that the interactive job output is available in the UI. + self.mark_as_finished(job_state) + return None # there is no job responding to this job_id, it is either lost or something happened. - log.error("No Jobs are available under expected selector app=%s", job_state.job_id) + log.error(f"No Jobs are available under expected selector app={job_state.job_id} and they are not deleted or stopped either.") self.mark_as_failed(job_state) # job is no longer viable - remove from watched jobs return None @@ -818,21 +838,24 @@ def _handle_unschedulable_job(self, job, job_state): def _handle_job_failure(self, job, job_state): # Figure out why job has failed with open(job_state.error_file, "a") as error_file: - log.debug("Trying with error file in _handle_job_failure") + log.debug("PP Trying with error file in _handle_job_failure") if self.__job_failed_due_to_low_memory(job_state): - log.debug("OOM condition reached") + log.debug("PP OOM reached!") error_file.write("Job killed after running out of memory. Try with more memory.\n") job_state.fail_message = "Tool failed due to insufficient memory. Try with more memory." job_state.runner_state = JobState.runner_states.MEMORY_LIMIT_REACHED elif self.__job_failed_due_to_walltime_limit(job): - log.debug("Walltime condition reached") + log.debug("PP checking for walltime") error_file.write("DeadlineExceeded") job_state.fail_message = "Job was active longer than specified deadline" job_state.runner_state = JobState.runner_states.WALLTIME_REACHED else: - log.debug("Runner cannot detect a specific reason for failure, must be a tool failure.") + log.debug("PP no idea!") error_file.write("Exceeded max number of job retries allowed for job\n") - job_state.fail_message = "More job retries failed than allowed. See standard output and standard error within the info section for details." + job_state.fail_message = "More job retries failed than allowed. See standard output within the info section for details." + # changes for resubmission + # job_state.running = False + # self.mark_as_failed(job_state) try: if self.__has_guest_ports(job_state.job_wrapper): self.__cleanup_k8s_guest_ports(job_state.job_wrapper, job) @@ -845,11 +868,11 @@ def __cleanup_k8s_job(self, job): k8s_cleanup_job = self.runner_params["k8s_cleanup_job"] delete_job(job, k8s_cleanup_job) - def __cleanup_k8s_ingress(self, ingress, job_failed): + def __cleanup_k8s_ingress(self, ingress, job_failed=False): k8s_cleanup_job = self.runner_params["k8s_cleanup_job"] delete_ingress(ingress, k8s_cleanup_job, job_failed) - def __cleanup_k8s_service(self, service, job_failed): + def __cleanup_k8s_service(self, service, job_failed=False): k8s_cleanup_job = self.runner_params["k8s_cleanup_job"] delete_service(service, k8s_cleanup_job, job_failed) @@ -903,32 +926,48 @@ def __cleanup_k8s_guest_ports(self, job_wrapper, k8s_job): k8s_job_prefix = self.__produce_k8s_job_prefix() k8s_job_name = f"{k8s_job_prefix}-{self.__force_label_conformity(job_wrapper.get_id_tag())}" log.debug(f"Deleting service/ingress for job with ID {job_wrapper.get_id_tag()}") - job_failed = k8s_job.obj["status"]["failed"] > 0 if "failed" in k8s_job.obj["status"] else False ingress_to_delete = find_ingress_object_by_name( self._pykube_api, k8s_job_name, self.runner_params["k8s_namespace"] ) if ingress_to_delete and len(ingress_to_delete.response["items"]) > 0: k8s_ingress = Ingress(self._pykube_api, ingress_to_delete.response["items"][0]) - self.__cleanup_k8s_ingress(k8s_ingress, job_failed) + self.__cleanup_k8s_ingress(k8s_ingress) + else: + log.debug(f"No ingress found for job with k8s_job_name {k8s_job_name}") service_to_delete = find_service_object_by_name( self._pykube_api, k8s_job_name, self.runner_params["k8s_namespace"] ) if service_to_delete and len(service_to_delete.response["items"]) > 0: k8s_service = Service(self._pykube_api, service_to_delete.response["items"][0]) - self.__cleanup_k8s_service(k8s_service, job_failed) + self.__cleanup_k8s_service(k8s_service) + else: + log.debug(f"No service found for job with k8s_job_name {k8s_job_name}") + # remove the interactive environment entrypoints + eps = job_wrapper.get_job().interactivetool_entry_points + if eps: + log.debug(f"Removing entry points for job with ID {job_wrapper.get_id_tag()}") + self.app.interactivetool_manager.remove_entry_points(eps) def stop_job(self, job_wrapper): """Attempts to delete a dispatched job to the k8s cluster""" job = job_wrapper.get_job() try: + log.debug(f"Attempting to stop job {job.id} ({job.job_runner_external_id})") job_to_delete = find_job_object_by_name( self._pykube_api, job.get_job_runner_external_id(), self.runner_params["k8s_namespace"] ) if job_to_delete and len(job_to_delete.response["items"]) > 0: k8s_job = Job(self._pykube_api, job_to_delete.response["items"][0]) + log.debug(f"Found job with id {job.get_job_runner_external_id()} to delete") + # For interactive jobs, at this point because the job stopping has been partly handled by the + # interactive tool manager, the job wrapper no longer shows any guest ports. We need another way + # to check if the job is an interactive job. if self.__has_guest_ports(job_wrapper): + log.debug(f"Job {job.id} ({job.job_runner_external_id}) has guest ports, cleaning them up") self.__cleanup_k8s_guest_ports(job_wrapper, k8s_job) self.__cleanup_k8s_job(k8s_job) + else: + log.debug(f"Could not find job with id {job.get_job_runner_external_id()} to delete") # TODO assert whether job parallelism == 0 # assert not job_to_delete.exists(), "Could not delete job,"+job.job_runner_external_id+" it still exists" log.debug(f"({job.id}/{job.job_runner_external_id}) Terminated at user's request") @@ -972,6 +1011,75 @@ def recover(self, job, job_wrapper): ajs.old_state = model.Job.states.QUEUED ajs.running = False self.monitor_queue.put(ajs) + + # added to make sure that stdout and stderr is captured for Kubernetes + def fail_job(self, job_state: "JobState", exception=False, message="Job failed", full_status=None): + log.debug("PP Getting into fail_job in k8s runner") + job = job_state.job_wrapper.get_job() + + #### Get STDOUT and STDERR from the job and tool to be stored in the database #### + #### This is needed because when calling finish_job on a failed job, the check_output method + #### overrides the job error state and tries to figure it out from the job output files + #### breaking OOM resubmissions. + # To ensure that files below are readable, ownership must be reclaimed first + job_state.job_wrapper.reclaim_ownership() + + # wait for the files to appear + which_try = 0 + while which_try < self.app.config.retry_job_output_collection + 1: + try: + with open(job_state.output_file, "rb") as stdout_file, open(job_state.error_file, "rb") as stderr_file: + job_stdout = self._job_io_for_db(stdout_file) + job_stderr = self._job_io_for_db(stderr_file) + break + except Exception as e: + if which_try == self.app.config.retry_job_output_collection: + job_stdout = "" + job_stderr = job_state.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER + log.error(f"{job.id}/{job.job_runner_external_id} {job_stderr}: {unicodify(e)}") + else: + time.sleep(1) + which_try += 1 + + # get stderr and stdout to database + outputs_directory = os.path.join(job_state.job_wrapper.working_directory, "outputs") + if not os.path.exists(outputs_directory): + outputs_directory = job_state.job_wrapper.working_directory + + tool_stdout_path = os.path.join(outputs_directory, "tool_stdout") + tool_stderr_path = os.path.join(outputs_directory, "tool_stderr") + + # TODO: These might not exist for running jobs at the upgrade to 19.XX, remove that + # assumption in 20.XX. + tool_stderr = None + if os.path.exists(tool_stdout_path): + with open(tool_stdout_path, "rb") as stdout_file: + tool_stdout = self._job_io_for_db(stdout_file) + else: + # Legacy job, were getting a merged output - assume it is mostly tool output. + tool_stdout = job_stdout + job_stdout = None + + tool_stdout = None + if os.path.exists(tool_stderr_path): + with open(tool_stderr_path, "rb") as stdout_file: + tool_stderr = self._job_io_for_db(stdout_file) + else: + # Legacy job, were getting a merged output - assume it is mostly tool output. + tool_stderr = job_stderr + job_stderr = None + + #### END Get STDOUT and STDERR from the job and tool to be stored in the database #### + + # full status empty leaves the UI without stderr/stdout + full_status = { "stderr" : tool_stderr, "stdout" : tool_stdout} + log.debug(f"({job.id}/{job.job_runner_external_id}) tool_stdout: {tool_stdout}") + log.debug(f"({job.id}/{job.job_runner_external_id}) tool_stderr: {tool_stderr}") + log.debug(f"({job.id}/{job.job_runner_external_id}) job_stdout: {job_stdout}") + log.debug(f"({job.id}/{job.job_runner_external_id}) job_stderr: {job_stderr}") + + # run super method + super().fail_job(job_state, exception, message, full_status) def finish_job(self, job_state): From 8100e6966f3709c886c2d51f72af758d83f71027 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Tue, 3 Jan 2023 12:56:44 +0000 Subject: [PATCH 004/158] Move to pykube-ng (cherry picked from commit 652c04f315e45626570da6420606f74541f0117b) (cherry picked from commit 7d4b0b4b4e15654323aa30835ff307b12d4b40d1) --- lib/galaxy/dependencies/conditional-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 17b16118f5a8..392cbb23f7c2 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -35,7 +35,8 @@ custos-sdk chronos-python==1.2.1 # Kubernetes job runner -pykube==0.15.0 +# pykube==0.15.0 +pykube-ng==22.9.0 # Synnefo / Pithos+ object store client kamaki From 8ba802083dd12647bbbd92ca8b2cc392672fccaf Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Tue, 3 Jan 2023 22:41:12 +0000 Subject: [PATCH 005/158] Change ingress API version to current one (cherry picked from commit f1b92d911827c897cb8e65a060e99b44f9d4ebf5) --- lib/galaxy/jobs/runners/util/pykube_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/util/pykube_util.py b/lib/galaxy/jobs/runners/util/pykube_util.py index 7c3f32d87b09..28020a8c8dde 100644 --- a/lib/galaxy/jobs/runners/util/pykube_util.py +++ b/lib/galaxy/jobs/runners/util/pykube_util.py @@ -31,7 +31,7 @@ DEFAULT_JOB_API_VERSION = "batch/v1" DEFAULT_SERVICE_API_VERSION = "v1" -DEFAULT_INGRESS_API_VERSION = "extensions/v1beta1" +DEFAULT_INGRESS_API_VERSION = "networking.k8s.io/v1" DEFAULT_NAMESPACE = "default" INSTANCE_ID_INVALID_MESSAGE = ( "Galaxy instance [%s] is either too long " From c75b8123f7223c3ed9c6bf4ec521f14980ae9743 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Thu, 9 Mar 2023 08:58:03 +0000 Subject: [PATCH 006/158] Missing stdout (cherry picked from commit a201abbb08bd855ecf85fe8250384e972077cb9b) --- lib/galaxy/jobs/runners/kubernetes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index da212d2a68e6..7c05d094635b 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -1051,7 +1051,8 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", # TODO: These might not exist for running jobs at the upgrade to 19.XX, remove that # assumption in 20.XX. - tool_stderr = None + tool_stderr = "Galaxy issue: Stderr failed to be retrieved from the job working directory." + tool_stdout = "Galaxy issue: Stdout failed to be retrieved from the job working directory." if os.path.exists(tool_stdout_path): with open(tool_stdout_path, "rb") as stdout_file: tool_stdout = self._job_io_for_db(stdout_file) @@ -1060,7 +1061,6 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", tool_stdout = job_stdout job_stdout = None - tool_stdout = None if os.path.exists(tool_stderr_path): with open(tool_stderr_path, "rb") as stdout_file: tool_stderr = self._job_io_for_db(stdout_file) From 63360bd5690bbf28d371e96be533da2a22a63e3a Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 15 Mar 2023 09:12:10 +0000 Subject: [PATCH 007/158] Apply suggestions from code review Mostly cleanups from Nuwan and Pablo. Co-authored-by: Nuwan Goonasekera <2070605+nuwang@users.noreply.github.com> --- .../dependencies/conditional-requirements.txt | 1 - lib/galaxy/jobs/runners/kubernetes.py | 32 +++++++------------ 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 392cbb23f7c2..40861584fc09 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -35,7 +35,6 @@ custos-sdk chronos-python==1.2.1 # Kubernetes job runner -# pykube==0.15.0 pykube-ng==22.9.0 # Synnefo / Pithos+ object store client diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 7c05d094635b..6d209a1b5b4d 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -100,7 +100,7 @@ def __init__(self, app, nworkers, **kwargs): map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None ), k8s_cleanup_job=dict(map=str, valid=lambda s: s in {"onsuccess", "always", "never"}, default="always"), - k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=1), # note that if the backOffLimit is lower, this paramer will have not effect. + k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=1), # note that if the backOffLimit is lower, this paramer will have no effect. k8s_job_spec_back_off_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=0), # this means that it will stop retrying after 1 failure. k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800), k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: not x or int(x) >= 0, default=None), @@ -220,9 +220,7 @@ def queue_job(self, job_wrapper): def __has_guest_ports(self, job_wrapper): # Check if job has guest ports or interactive tool entry points that would signal that - # this is an interactive tool. - log.debug(f"Checking if job {job_wrapper.get_id_tag()} has guest ports: {job_wrapper.guest_ports}") - log.debug(f"Checking if job {job_wrapper.get_id_tag()} has interactive entry points: {job_wrapper.guest_ports}") + log.debug(f"Checking if job {job_wrapper.get_id_tag()} is an interactive tool. guest ports: {job_wrapper.guest_ports}. interactive entry points: {job_wrapper.interactivetool_entry_points}") return bool(job_wrapper.guest_ports) or bool(job_wrapper.get_job().interactivetool_entry_points) def __configure_port_routing(self, ajs): @@ -245,14 +243,8 @@ def __configure_port_routing(self, ajs): log.debug(f"Kubernetes ingress object: {json.dumps(k8s_ingress_obj, indent=4)}") # We avoid creating service and ingress if they already exist (e.g. when Galaxy is restarted or resubmitting a job) service = Service(self._pykube_api, k8s_service_obj) - # if service.exists(): - # log.debug(f"Service {k8s_job_name} already exists, skipping creation") - # else: service.create() ingress = Ingress(self._pykube_api, k8s_ingress_obj) - # if ingress.exists(): - # log.debug(f"Ingress {k8s_job_name} already exists, skipping creation") - # else: ingress.create() def __get_overridable_params(self, job_wrapper, param_key): @@ -766,7 +758,7 @@ def check_watched_item(self, job_state): elif active > 0 and failed < max_pod_retries + 1: if not job_state.running: if self.__job_pending_due_to_unschedulable_pod(job_state): - log.debug("PP Job pending..") + log.debug(f"Job id: {job_state.job_id} with k8s id: {job.name} pending...") if self.runner_params.get("k8s_unschedulable_walltime_limit"): creation_time_str = job.obj["metadata"].get("creationTimestamp") creation_time = datetime.strptime(creation_time_str, "%Y-%m-%dT%H:%M:%SZ") @@ -778,20 +770,20 @@ def check_watched_item(self, job_state): else: pass else: - log.debug("Job set to running..") + log.debug("Job set to running...") job_state.running = True job_state.job_wrapper.change_state(model.Job.states.RUNNING) return job_state elif job_persisted_state == model.Job.states.DELETED: # Job has been deleted via stop_job and job has not been deleted, # remove from watched_jobs by returning `None` - log.debug("Job is DELETED..") + log.debug(f"Job id: {job_state.job_id} has been already deleted...") if job_state.job_wrapper.cleanup_job in ("always", "onsuccess"): job_state.job_wrapper.cleanup() return None else: log.debug(f"Job is failed and not deleted, looking at failure") - log.debug(f"Job state before handle job failure: {job_state.job_wrapper.get_state()}") + log.debug(f"Job id: {job_state.job_id} failed but has not been deleted yet. Current state: {job_state.job_wrapper.get_state()}") self._handle_job_failure(job, job_state) # changes for resubmission (removed self.mark_as_failed from handle_job_failure) self.work_queue.put((self.mark_as_failed, job_state)) @@ -838,19 +830,19 @@ def _handle_unschedulable_job(self, job, job_state): def _handle_job_failure(self, job, job_state): # Figure out why job has failed with open(job_state.error_file, "a") as error_file: - log.debug("PP Trying with error file in _handle_job_failure") + log.debug("Trying with error file in _handle_job_failure") if self.__job_failed_due_to_low_memory(job_state): - log.debug("PP OOM reached!") + log.debug(f"OOM detected for job: {job_state.job_id}") error_file.write("Job killed after running out of memory. Try with more memory.\n") job_state.fail_message = "Tool failed due to insufficient memory. Try with more memory." job_state.runner_state = JobState.runner_states.MEMORY_LIMIT_REACHED elif self.__job_failed_due_to_walltime_limit(job): - log.debug("PP checking for walltime") + log.debug(f"Walltime limit reached for job: {job_state.job_id}") error_file.write("DeadlineExceeded") job_state.fail_message = "Job was active longer than specified deadline" job_state.runner_state = JobState.runner_states.WALLTIME_REACHED else: - log.debug("PP no idea!") + log.debug(f"Unknown error detected in job: {job_state.job_id}") error_file.write("Exceeded max number of job retries allowed for job\n") job_state.fail_message = "More job retries failed than allowed. See standard output within the info section for details." # changes for resubmission @@ -1051,8 +1043,8 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", # TODO: These might not exist for running jobs at the upgrade to 19.XX, remove that # assumption in 20.XX. - tool_stderr = "Galaxy issue: Stderr failed to be retrieved from the job working directory." - tool_stdout = "Galaxy issue: Stdout failed to be retrieved from the job working directory." + tool_stderr = "Galaxy issue: stderr could not be retrieved from the job working directory." + tool_stdout = "Galaxy issue: stdout could not be retrieved from the job working directory." if os.path.exists(tool_stdout_path): with open(tool_stdout_path, "rb") as stdout_file: tool_stdout = self._job_io_for_db(stdout_file) From 5c3ccb01191e6dd6378593ec36c79c07ff3decaa Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 15 Mar 2023 09:18:07 +0000 Subject: [PATCH 008/158] Please linter --- lib/galaxy/jobs/runners/kubernetes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 6d209a1b5b4d..af9d5551ec66 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -782,8 +782,7 @@ def check_watched_item(self, job_state): job_state.job_wrapper.cleanup() return None else: - log.debug(f"Job is failed and not deleted, looking at failure") - log.debug(f"Job id: {job_state.job_id} failed but has not been deleted yet. Current state: {job_state.job_wrapper.get_state()}") + log.debug(f"Job id: {job_state.job_id} failed and it is not a deletion case. Current state: {job_state.job_wrapper.get_state()}") self._handle_job_failure(job, job_state) # changes for resubmission (removed self.mark_as_failed from handle_job_failure) self.work_queue.put((self.mark_as_failed, job_state)) From bc2ac6b15004f8baa1d680027040f2415160d97d Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 15 Mar 2023 10:03:33 +0000 Subject: [PATCH 009/158] More linter pleasing --- lib/galaxy/jobs/runners/kubernetes.py | 59 +++++++++++++-------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index af9d5551ec66..2f30496432b5 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -3,7 +3,7 @@ """ import logging -import json # for debugging of API objects +import json # for debugging of API objects import math import os import re @@ -100,8 +100,8 @@ def __init__(self, app, nworkers, **kwargs): map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None ), k8s_cleanup_job=dict(map=str, valid=lambda s: s in {"onsuccess", "always", "never"}, default="always"), - k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=1), # note that if the backOffLimit is lower, this paramer will have no effect. - k8s_job_spec_back_off_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=0), # this means that it will stop retrying after 1 failure. + k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=1), # note that if the backOffLimit is lower, this paramer will have no effect. + k8s_job_spec_back_off_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=0), # this means that it will stop retrying after 1 failure. k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800), k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: not x or int(x) >= 0, default=None), k8s_interactivetools_use_ssl=dict(map=bool, default=False), @@ -468,26 +468,26 @@ def __get_k8s_ingress_spec(self, ajs): }, "spec": { "ingressClassName": "nginx", - "rules":[ { - "host": ep["domain"], - "http": { - "paths": [ { - "backend": { - "service": { - "name": self.__get_k8s_job_name( - self.__produce_k8s_job_prefix(), ajs.job_wrapper - ), - "port": { "number": int(ep["tool_port"])} - } - }, - "path": ep.get("entry_path", "/"), - "pathType": "Prefix" - } - ] - }, + "rules": [ { + "host": ep["domain"], + "http": { + "paths": [ { + "backend": { + "service": { + "name": self.__get_k8s_job_name( + self.__produce_k8s_job_prefix(), ajs.job_wrapper + ), + "port": { "number": int(ep["tool_port"])} + } + }, + "path": ep.get("entry_path", "/"), + "pathType": "Prefix" + } + ] + }, } for ep in entry_points - ] + ] }, } if self.runner_params.get("k8s_interactivetools_use_ssl"): @@ -1007,11 +1007,11 @@ def recover(self, job, job_wrapper): def fail_job(self, job_state: "JobState", exception=False, message="Job failed", full_status=None): log.debug("PP Getting into fail_job in k8s runner") job = job_state.job_wrapper.get_job() - - #### Get STDOUT and STDERR from the job and tool to be stored in the database #### - #### This is needed because when calling finish_job on a failed job, the check_output method - #### overrides the job error state and tries to figure it out from the job output files - #### breaking OOM resubmissions. + + # Get STDOUT and STDERR from the job and tool to be stored in the database # + # This is needed because when calling finish_job on a failed job, the check_output method + # overrides the job error state and tries to figure it out from the job output files + # breaking OOM resubmissions. # To ensure that files below are readable, ownership must be reclaimed first job_state.job_wrapper.reclaim_ownership() @@ -1039,7 +1039,7 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", tool_stdout_path = os.path.join(outputs_directory, "tool_stdout") tool_stderr_path = os.path.join(outputs_directory, "tool_stderr") - + # TODO: These might not exist for running jobs at the upgrade to 19.XX, remove that # assumption in 20.XX. tool_stderr = "Galaxy issue: stderr could not be retrieved from the job working directory." @@ -1060,10 +1060,10 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", tool_stderr = job_stderr job_stderr = None - #### END Get STDOUT and STDERR from the job and tool to be stored in the database #### + # END Get STDOUT and STDERR from the job and tool to be stored in the database # # full status empty leaves the UI without stderr/stdout - full_status = { "stderr" : tool_stderr, "stdout" : tool_stdout} + full_status = {"stderr" : tool_stderr, "stdout" : tool_stdout} log.debug(f"({job.id}/{job.job_runner_external_id}) tool_stdout: {tool_stdout}") log.debug(f"({job.id}/{job.job_runner_external_id}) tool_stderr: {tool_stderr}") log.debug(f"({job.id}/{job.job_runner_external_id}) job_stdout: {job_stdout}") @@ -1071,7 +1071,6 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", # run super method super().fail_job(job_state, exception, message, full_status) - def finish_job(self, job_state): self._handle_metadata_externally(job_state.job_wrapper, resolve_requirements=True) From 80e254a4f2b7df22855a764a349804f55cd89ba7 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 15 Mar 2023 10:25:18 +0000 Subject: [PATCH 010/158] Black + isort --- lib/galaxy/jobs/runners/kubernetes.py | 76 ++++++++++++++++----------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 2f30496432b5..a1281e7cea54 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -2,8 +2,8 @@ Offload jobs to a Kubernetes cluster. """ -import logging import json # for debugging of API objects +import logging import math import os import re @@ -13,9 +13,6 @@ import yaml from galaxy import model -from galaxy.util import ( - unicodify, -) from galaxy.jobs.runners import ( AsynchronousJobRunner, AsynchronousJobState, @@ -48,6 +45,7 @@ Service, service_object_dict, ) +from galaxy.util import unicodify from galaxy.util.bytesize import ByteSize log = logging.getLogger(__name__) @@ -100,8 +98,12 @@ def __init__(self, app, nworkers, **kwargs): map=str, valid=lambda s: s == "$gid" or isinstance(s, int) or not s or s.isdigit(), default=None ), k8s_cleanup_job=dict(map=str, valid=lambda s: s in {"onsuccess", "always", "never"}, default="always"), - k8s_pod_retries=dict(map=int, valid=lambda x: int(x) >= 0, default=1), # note that if the backOffLimit is lower, this paramer will have no effect. - k8s_job_spec_back_off_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=0), # this means that it will stop retrying after 1 failure. + k8s_pod_retries=dict( + map=int, valid=lambda x: int(x) >= 0, default=1 + ), # note that if the backOffLimit is lower, this paramer will have no effect. + k8s_job_spec_back_off_limit=dict( + map=int, valid=lambda x: int(x) >= 0, default=0 + ), # this means that it will stop retrying after 1 failure. k8s_walltime_limit=dict(map=int, valid=lambda x: int(x) >= 0, default=172800), k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: not x or int(x) >= 0, default=None), k8s_interactivetools_use_ssl=dict(map=bool, default=False), @@ -220,7 +222,9 @@ def queue_job(self, job_wrapper): def __has_guest_ports(self, job_wrapper): # Check if job has guest ports or interactive tool entry points that would signal that - log.debug(f"Checking if job {job_wrapper.get_id_tag()} is an interactive tool. guest ports: {job_wrapper.guest_ports}. interactive entry points: {job_wrapper.interactivetool_entry_points}") + log.debug( + f"Checking if job {job_wrapper.get_id_tag()} is an interactive tool. guest ports: {job_wrapper.guest_ports}. interactive entry points: {job_wrapper.interactivetool_entry_points}" + ) return bool(job_wrapper.guest_ports) or bool(job_wrapper.get_job().interactivetool_entry_points) def __configure_port_routing(self, ajs): @@ -468,26 +472,28 @@ def __get_k8s_ingress_spec(self, ajs): }, "spec": { "ingressClassName": "nginx", - "rules": [ { - "host": ep["domain"], - "http": { - "paths": [ { - "backend": { - "service": { - "name": self.__get_k8s_job_name( - self.__produce_k8s_job_prefix(), ajs.job_wrapper - ), - "port": { "number": int(ep["tool_port"])} - } - }, - "path": ep.get("entry_path", "/"), - "pathType": "Prefix" - } - ] + "rules": [ + { + "host": ep["domain"], + "http": { + "paths": [ + { + "backend": { + "service": { + "name": self.__get_k8s_job_name( + self.__produce_k8s_job_prefix(), ajs.job_wrapper + ), + "port": {"number": int(ep["tool_port"])}, + } }, - } - for ep in entry_points - ] + "path": ep.get("entry_path", "/"), + "pathType": "Prefix", + } + ] + }, + } + for ep in entry_points + ], }, } if self.runner_params.get("k8s_interactivetools_use_ssl"): @@ -782,7 +788,9 @@ def check_watched_item(self, job_state): job_state.job_wrapper.cleanup() return None else: - log.debug(f"Job id: {job_state.job_id} failed and it is not a deletion case. Current state: {job_state.job_wrapper.get_state()}") + log.debug( + f"Job id: {job_state.job_id} failed and it is not a deletion case. Current state: {job_state.job_wrapper.get_state()}" + ) self._handle_job_failure(job, job_state) # changes for resubmission (removed self.mark_as_failed from handle_job_failure) self.work_queue.put((self.mark_as_failed, job_state)) @@ -794,14 +802,18 @@ def check_watched_item(self, job_state): if job_state.job_wrapper.cleanup_job in ("always", "onsuccess"): job_state.job_wrapper.cleanup() return None - if job_state.job_wrapper.get_job().state == model.Job.states.STOPPED and self.__has_guest_ports(job_state.job_wrapper): + if job_state.job_wrapper.get_job().state == model.Job.states.STOPPED and self.__has_guest_ports( + job_state.job_wrapper + ): # Interactive job has been stopped via stop_job (most likely by the user), # cleanup and remove from watched_jobs by returning `None`. STOPPED jobs are cleaned up elsewhere. # Marking as finished makes sure that the interactive job output is available in the UI. self.mark_as_finished(job_state) return None # there is no job responding to this job_id, it is either lost or something happened. - log.error(f"No Jobs are available under expected selector app={job_state.job_id} and they are not deleted or stopped either.") + log.error( + f"No Jobs are available under expected selector app={job_state.job_id} and they are not deleted or stopped either." + ) self.mark_as_failed(job_state) # job is no longer viable - remove from watched jobs return None @@ -843,7 +855,9 @@ def _handle_job_failure(self, job, job_state): else: log.debug(f"Unknown error detected in job: {job_state.job_id}") error_file.write("Exceeded max number of job retries allowed for job\n") - job_state.fail_message = "More job retries failed than allowed. See standard output within the info section for details." + job_state.fail_message = ( + "More job retries failed than allowed. See standard output within the info section for details." + ) # changes for resubmission # job_state.running = False # self.mark_as_failed(job_state) @@ -1063,7 +1077,7 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", # END Get STDOUT and STDERR from the job and tool to be stored in the database # # full status empty leaves the UI without stderr/stdout - full_status = {"stderr" : tool_stderr, "stdout" : tool_stdout} + full_status = {"stderr": tool_stderr, "stdout": tool_stdout} log.debug(f"({job.id}/{job.job_runner_external_id}) tool_stdout: {tool_stdout}") log.debug(f"({job.id}/{job.job_runner_external_id}) tool_stderr: {tool_stderr}") log.debug(f"({job.id}/{job.job_runner_external_id}) job_stdout: {job_stdout}") From 5dd8751c0579199caa6ce45176123b439edd01f5 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Tue, 28 Mar 2023 14:15:25 +0100 Subject: [PATCH 011/158] Fix its issue on logging (cherry picked from commit d5e73b8130ea29d3da9c073e1d74d295e2c4c03a) --- lib/galaxy/jobs/runners/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index a1281e7cea54..dab97fabacf3 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -223,7 +223,7 @@ def queue_job(self, job_wrapper): def __has_guest_ports(self, job_wrapper): # Check if job has guest ports or interactive tool entry points that would signal that log.debug( - f"Checking if job {job_wrapper.get_id_tag()} is an interactive tool. guest ports: {job_wrapper.guest_ports}. interactive entry points: {job_wrapper.interactivetool_entry_points}" + f"Checking if job {job_wrapper.get_id_tag()} is an interactive tool. guest ports: {job_wrapper.guest_ports}. interactive entry points: {job_wrapper.get_job().interactivetool_entry_points}" ) return bool(job_wrapper.guest_ports) or bool(job_wrapper.get_job().interactivetool_entry_points) From 5ca868703b394f7efbc2e9968cd6a29545d72e2f Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Tue, 28 Mar 2023 14:35:01 +0100 Subject: [PATCH 012/158] trace for larger json dump logs --- lib/galaxy/jobs/runners/kubernetes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index dab97fabacf3..7c4a04972142 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -243,8 +243,8 @@ def __configure_port_routing(self, ajs): k8s_ingress_obj = ingress_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_ingress_spec(ajs)) # pretty print the objects for debugging - log.debug(f"Kubernetes service object: {json.dumps(k8s_service_obj, indent=4)}") - log.debug(f"Kubernetes ingress object: {json.dumps(k8s_ingress_obj, indent=4)}") + log.trace(f"Kubernetes service object: {json.dumps(k8s_service_obj, indent=4)}") + log.trace(f"Kubernetes ingress object: {json.dumps(k8s_ingress_obj, indent=4)}") # We avoid creating service and ingress if they already exist (e.g. when Galaxy is restarted or resubmitting a job) service = Service(self._pykube_api, k8s_service_obj) service.create() From a9cc2ff450887dee825fddaec10fda3ce4119f00 Mon Sep 17 00:00:00 2001 From: Pablo Moreno Date: Wed, 29 Mar 2023 21:37:13 +0100 Subject: [PATCH 013/158] Make ingress class configurable for ITs --- lib/galaxy/jobs/runners/kubernetes.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 7c4a04972142..3ee12585d071 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -108,6 +108,7 @@ def __init__(self, app, nworkers, **kwargs): k8s_unschedulable_walltime_limit=dict(map=int, valid=lambda x: not x or int(x) >= 0, default=None), k8s_interactivetools_use_ssl=dict(map=bool, default=False), k8s_interactivetools_ingress_annotations=dict(map=str), + k8s_interactivetools_ingress_class=dict(map=str, default=None), ) if "runner_param_specs" not in kwargs: @@ -471,7 +472,6 @@ def __get_k8s_ingress_spec(self, ajs): "annotations": {"app.galaxyproject.org/tool_id": ajs.job_wrapper.tool.id}, }, "spec": { - "ingressClassName": "nginx", "rules": [ { "host": ep["domain"], @@ -496,6 +496,9 @@ def __get_k8s_ingress_spec(self, ajs): ], }, } + default_ingress_class = self.runner_params.get("k8s_interactivetools_ingress_class") + if default_ingress_class: + k8s_spec_template["spec"]["ingressClassName"] = default_ingress_class if self.runner_params.get("k8s_interactivetools_use_ssl"): domains = list({e["domain"] for e in entry_points}) k8s_spec_template["spec"]["tls"] = [ From 6ca1a021f45ee9be07dc63c18124f3ae245a132c Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Tue, 6 Jun 2023 00:51:06 +0530 Subject: [PATCH 014/158] Remove extra comments and minor tweaks to debug logs --- lib/galaxy/jobs/runners/kubernetes.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 3ee12585d071..f02608811cae 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -243,7 +243,6 @@ def __configure_port_routing(self, ajs): k8s_service_obj = service_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_service_spec(ajs)) k8s_ingress_obj = ingress_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_ingress_spec(ajs)) - # pretty print the objects for debugging log.trace(f"Kubernetes service object: {json.dumps(k8s_service_obj, indent=4)}") log.trace(f"Kubernetes ingress object: {json.dumps(k8s_ingress_obj, indent=4)}") # We avoid creating service and ingress if they already exist (e.g. when Galaxy is restarted or resubmitting a job) @@ -549,6 +548,7 @@ def __get_k8s_containers(self, ajs): # command line execution, separated by ;, which is what Galaxy does # to assemble the command. "command": [ajs.job_wrapper.shell], + # Make sure that the exit code is propagated to k8s, so k8s knows why the tool failed (e.g. OOM) "args": ["-c", f"{ajs.job_file}; exit $(cat {ajs.exit_code_file})"], "workingDir": ajs.job_wrapper.working_directory, "volumeMounts": deduplicate_entries(mounts), @@ -762,7 +762,7 @@ def check_watched_item(self, job_state): if succeeded > 0 or job_state == model.Job.states.STOPPED: job_state.running = False self.mark_as_finished(job_state) - log.debug("Job succeeded") + log.debug(f"Job id: {job_state.job_id} with k8s id: {job.name} succeeded") return None elif active > 0 and failed < max_pod_retries + 1: if not job_state.running: @@ -1077,8 +1077,6 @@ def fail_job(self, job_state: "JobState", exception=False, message="Job failed", tool_stderr = job_stderr job_stderr = None - # END Get STDOUT and STDERR from the job and tool to be stored in the database # - # full status empty leaves the UI without stderr/stdout full_status = {"stderr": tool_stderr, "stdout": tool_stdout} log.debug(f"({job.id}/{job.job_runner_external_id}) tool_stdout: {tool_stdout}") From 8276216fa8752718ab26517d8ae0ed02016e8cd3 Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Tue, 6 Jun 2023 12:18:55 +0530 Subject: [PATCH 015/158] Change trace to debug --- lib/galaxy/jobs/runners/kubernetes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index f02608811cae..6ef015f0d812 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -243,8 +243,8 @@ def __configure_port_routing(self, ajs): k8s_service_obj = service_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_service_spec(ajs)) k8s_ingress_obj = ingress_object_dict(self.runner_params, k8s_job_name, self.__get_k8s_ingress_spec(ajs)) - log.trace(f"Kubernetes service object: {json.dumps(k8s_service_obj, indent=4)}") - log.trace(f"Kubernetes ingress object: {json.dumps(k8s_ingress_obj, indent=4)}") + log.debug(f"Kubernetes service object: {json.dumps(k8s_service_obj, indent=4)}") + log.debug(f"Kubernetes ingress object: {json.dumps(k8s_ingress_obj, indent=4)}") # We avoid creating service and ingress if they already exist (e.g. when Galaxy is restarted or resubmitting a job) service = Service(self._pykube_api, k8s_service_obj) service.create() From eeb4d25987a002fef154b66c2663751bfffefd0d Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Wed, 21 Jun 2023 22:17:53 +0530 Subject: [PATCH 016/158] Don't mark job as failed if unknown exit code --- lib/galaxy/authnz/custos_authnz.py | 2 +- lib/galaxy/jobs/runners/kubernetes.py | 46 +++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/authnz/custos_authnz.py b/lib/galaxy/authnz/custos_authnz.py index 4b4d40ec6860..75d3187cc508 100644 --- a/lib/galaxy/authnz/custos_authnz.py +++ b/lib/galaxy/authnz/custos_authnz.py @@ -204,7 +204,7 @@ def callback(self, state_token, authz_code, trans, login_redirect_url): ): user = existing_user else: - message = f"There already exists a user with email {email}. To associate this external login, you must first be logged in as that existing account." + message = f"There already exists a user with email {email}. To associate this external login, you must first be logged in as that existing account." log.info(message) login_redirect_url = ( f"{login_redirect_url}login/start" diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 6ef015f0d812..6f033a1cd57f 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -794,9 +794,14 @@ def check_watched_item(self, job_state): log.debug( f"Job id: {job_state.job_id} failed and it is not a deletion case. Current state: {job_state.job_wrapper.get_state()}" ) - self._handle_job_failure(job, job_state) - # changes for resubmission (removed self.mark_as_failed from handle_job_failure) - self.work_queue.put((self.mark_as_failed, job_state)) + if self._handle_job_failure(job, job_state): + # changes for resubmission (removed self.mark_as_failed from handle_job_failure) + self.work_queue.put((self.mark_as_failed, job_state)) + else: + # Job failure was not due to a k8s issue or something that k8s can handle, so it's a tool error. + job_state.running = False + self.mark_as_finished(job_state) + return None return None @@ -843,6 +848,7 @@ def _handle_unschedulable_job(self, job, job_state): def _handle_job_failure(self, job, job_state): # Figure out why job has failed + mark_failed = True with open(job_state.error_file, "a") as error_file: log.debug("Trying with error file in _handle_job_failure") if self.__job_failed_due_to_low_memory(job_state): @@ -855,11 +861,18 @@ def _handle_job_failure(self, job, job_state): error_file.write("DeadlineExceeded") job_state.fail_message = "Job was active longer than specified deadline" job_state.runner_state = JobState.runner_states.WALLTIME_REACHED + elif self.__job_failed_due_to_unknown_exit_code(job_state): + msg = f"Job: {job_state.job_id} failed due to an unknown exit code from the tool." + log.debug(msg) + job_state.fail_message = msg + job_state.runner_state = JobState.runner_states.TOOL_DETECT_ERROR + mark_failed = False else: - log.debug(f"Unknown error detected in job: {job_state.job_id}") - error_file.write("Exceeded max number of job retries allowed for job\n") + msg = f"An unknown error occurred in this job and the maximum number of retries have been exceeded for job: {job_state.job_id}." + log.debug(msg) + error_file.write(msg) job_state.fail_message = ( - "More job retries failed than allowed. See standard output within the info section for details." + "An unknown error occurered with this job. See standard output within the info section for details." ) # changes for resubmission # job_state.running = False @@ -870,7 +883,7 @@ def _handle_job_failure(self, job, job_state): self.__cleanup_k8s_job(job) except Exception: log.exception("Could not clean up k8s batch job. Ignoring...") - return None + return mark_failed def __cleanup_k8s_job(self, job): k8s_cleanup_job = self.runner_params["k8s_cleanup_job"] @@ -930,6 +943,25 @@ def __job_pending_due_to_unschedulable_pod(self, job_state): pod = Pod(self._pykube_api, pods.response["items"][0]) return is_pod_unschedulable(self._pykube_api, pod, self.runner_params["k8s_namespace"]) + def __job_failed_due_to_unknown_exit_code(self, job_state): + """ + checks whether the pod exited prematurely due to an unknown exit code (i.e. not an exit code like OOM that + we can handle). This would mean that the tool failed, but the job should be considered to have succeeded. + """ + pods = find_pod_object_by_name(self._pykube_api, job_state.job_id, self.runner_params["k8s_namespace"]) + if not pods.response["items"]: + return False + + pod = pods.response["items"][0] + if ( + pod + and "terminated" in pod["status"]["containerStatuses"][0]["state"] + and pod["status"]["containerStatuses"][0]["state"].get("exitCode") + ): + return True + + return False + def __cleanup_k8s_guest_ports(self, job_wrapper, k8s_job): k8s_job_prefix = self.__produce_k8s_job_prefix() k8s_job_name = f"{k8s_job_prefix}-{self.__force_label_conformity(job_wrapper.get_id_tag())}" From 7724435df87af319351de967074e707290e1c1a0 Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Wed, 21 Jun 2023 23:58:29 +0530 Subject: [PATCH 017/158] Get exitCode from correct dict entry --- lib/galaxy/jobs/runners/kubernetes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 6f033a1cd57f..7c6c6f153f84 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -956,7 +956,7 @@ def __job_failed_due_to_unknown_exit_code(self, job_state): if ( pod and "terminated" in pod["status"]["containerStatuses"][0]["state"] - and pod["status"]["containerStatuses"][0]["state"].get("exitCode") + and pod["status"]["containerStatuses"][0]["state"]["terminated"].get("exitCode") ): return True From c1b3b275344feaef63be73f0696b9d5dc4455c15 Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Thu, 22 Jun 2023 02:49:37 +0530 Subject: [PATCH 018/158] Bump pykube version --- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 40861584fc09..1bea9abfe5a2 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -35,7 +35,7 @@ custos-sdk chronos-python==1.2.1 # Kubernetes job runner -pykube-ng==22.9.0 +pykube-ng==23.6.0 # Synnefo / Pithos+ object store client kamaki From 452a042d672d11a76ab666eca8b52bb064b3f8a7 Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Thu, 22 Jun 2023 17:41:31 +0530 Subject: [PATCH 019/158] Fix conditional requirement for pykube-ng --- lib/galaxy/dependencies/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index e46c6fee60c9..091fb4da57dc 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -203,7 +203,7 @@ def check_total_perspective_vortex(self): def check_pbs_python(self): return "galaxy.jobs.runners.pbs:PBSJobRunner" in self.job_runners - def check_pykube(self): + def check_pykube_ng(self): return "galaxy.jobs.runners.kubernetes:KubernetesJobRunner" in self.job_runners or which("kubectl") def check_chronos_python(self): From 250be5db661ee7f8a6f43ca8086dacbbf6f7e7e1 Mon Sep 17 00:00:00 2001 From: nuwang <2070605+nuwang@users.noreply.github.com> Date: Thu, 22 Jun 2023 17:53:36 +0530 Subject: [PATCH 020/158] Set a pykube version that's available --- lib/galaxy/dependencies/conditional-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 1bea9abfe5a2..4d6eccc54007 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -35,7 +35,7 @@ custos-sdk chronos-python==1.2.1 # Kubernetes job runner -pykube-ng==23.6.0 +pykube-ng==21.3.0 # Synnefo / Pithos+ object store client kamaki From 85f20770c1205e728740d14a3b1539a2f4d3a29d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 19 Oct 2023 10:06:14 +0200 Subject: [PATCH 021/158] Add back 1.1.0 version of Filtering1 tool Which is the same as 1.1.1 but hard to explain to users why the run form shows a different version than what is shown in the user interface. --- lib/galaxy/config/sample/tool_conf.xml.sample | 1 + tools/stats/filtering_1_1_0.xml | 103 ++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 tools/stats/filtering_1_1_0.xml diff --git a/lib/galaxy/config/sample/tool_conf.xml.sample b/lib/galaxy/config/sample/tool_conf.xml.sample index f6f060739291..f3818f88f46c 100644 --- a/lib/galaxy/config/sample/tool_conf.xml.sample +++ b/lib/galaxy/config/sample/tool_conf.xml.sample @@ -79,6 +79,7 @@
+ diff --git a/tools/stats/filtering_1_1_0.xml b/tools/stats/filtering_1_1_0.xml new file mode 100644 index 000000000000..1a20cf40c947 --- /dev/null +++ b/tools/stats/filtering_1_1_0.xml @@ -0,0 +1,103 @@ + + data on any column using simple expressions + + operation_0335 + + + python '$__tool_directory__/filtering.py' '$input' '$out_file1' '$inputs' ${input.metadata.columns} "${input.metadata.column_types}" $header_lines + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +.. class:: warningmark + +Double equal signs, ==, must be used as *"equal to"* (e.g., **c1 == 'chr22'**) + +.. class:: infomark + +**TIP:** Attempting to apply a filtering condition may throw exceptions if the data type (e.g., string, integer) in every line of the columns being filtered is not appropriate for the condition (e.g., attempting certain numerical calculations on strings). If an exception is thrown when applying the condition to a line, that line is skipped as invalid for the filter condition. The number of invalid skipped lines is documented in the resulting history item as a "Condition/data issue". + +.. class:: infomark + +**TIP:** If your data is not TAB delimited, use *Text Manipulation->Convert* + +----- + +**Syntax** + +The filter tool allows you to restrict the dataset using simple conditional statements. + +- Columns are referenced with **c** and a **number**. For example, **c1** refers to the first column of a tab-delimited file +- Make sure that multi-character operators contain no white space ( e.g., **<=** is valid while **< =** is not valid ) +- When using 'equal-to' operator **double equal sign '==' must be used** ( e.g., **c1=='chr1'** ) +- Non-numerical values must be included in single or double quotes ( e.g., **c6=='+'** ) +- Filtering condition can include logical operators, but **make sure operators are all lower case** ( e.g., **(c1!='chrX' and c1!='chrY') or not c6=='+'** ) + +----- + +**Example** + +- **c1=='chr1'** selects lines in which the first column is chr1 +- **c3-c2<100*c4** selects lines where subtracting column 3 from column 2 is less than the value of column 4 times 100 +- **len(c2.split(',')) < 4** will select lines where the second column has less than four comma separated elements +- **c2>=1** selects lines in which the value of column 2 is greater than or equal to 1 +- Numbers should not contain commas - **c2<=44,554,350** will not work, but **c2<=44554350** will +- Some words in the data can be used, but must be single or double quoted ( e.g., **c3=='exon'** ) + + + + From efa2f58d3feb0ac272493df85d876750e7fd87d9 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 1 Dec 2023 11:53:08 +0100 Subject: [PATCH 022/158] also remove duplicates in sections --- lib/galaxy/tool_util/toolbox/views/static.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/toolbox/views/static.py b/lib/galaxy/tool_util/toolbox/views/static.py index 2eba8eef3f4b..7972cc84e835 100644 --- a/lib/galaxy/tool_util/toolbox/views/static.py +++ b/lib/galaxy/tool_util/toolbox/views/static.py @@ -105,7 +105,7 @@ def definition_with_items_to_panel(definition, allow_sections: bool = True, item f"Failed to find matching section for (id, name) = ({section_def.id}, {section_def.name})" ) continue - section = closest_section.copy() + section = closest_section.copy(merge_tools=True) if section_def.id is not None: section.id = section_def.id if section_def.name is not None: From 2ecf789058510ef88a2e44fea30d05bd2c4e08a0 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Dec 2023 13:16:22 -0500 Subject: [PATCH 023/158] Convert sample object store conf to YAML --- config/object_store_conf.sample.yml | 1 + config/object_store_conf.xml.sample | 1 - .../sample/object_store_conf.sample.yml | 523 ++++++++++++++++++ 3 files changed, 524 insertions(+), 1 deletion(-) create mode 120000 config/object_store_conf.sample.yml delete mode 120000 config/object_store_conf.xml.sample create mode 100644 lib/galaxy/config/sample/object_store_conf.sample.yml diff --git a/config/object_store_conf.sample.yml b/config/object_store_conf.sample.yml new file mode 120000 index 000000000000..62e2d9d9dfce --- /dev/null +++ b/config/object_store_conf.sample.yml @@ -0,0 +1 @@ +../lib/galaxy/config/sample/object_store_conf.sample.yml \ No newline at end of file diff --git a/config/object_store_conf.xml.sample b/config/object_store_conf.xml.sample deleted file mode 120000 index 71edcdea144a..000000000000 --- a/config/object_store_conf.xml.sample +++ /dev/null @@ -1 +0,0 @@ -../lib/galaxy/config/sample/object_store_conf.xml.sample \ No newline at end of file diff --git a/lib/galaxy/config/sample/object_store_conf.sample.yml b/lib/galaxy/config/sample/object_store_conf.sample.yml new file mode 100644 index 000000000000..5db8d6ffa08a --- /dev/null +++ b/lib/galaxy/config/sample/object_store_conf.sample.yml @@ -0,0 +1,523 @@ +--- +### +### Sample Object Store configuration file +### + +# There should only be one root object store, multiple are shown here to show different configuration options. Any +# object store can be used as backends to the distributed and hierarchical object stores (including distributed and +# hierarchical themselves). + +# Most non-disk object store types have a `cache` option like: +# +# cache: +# # local path to store cached data, +# path: "database/object_store_cache" +# # size of the cache in gigabytes +# size: 1000 +# # optional parameter that allows to control data is being sent directly to an object store without storing it in the +# # cache. By default (true) data is also copied to the cache. +# cache_updated_data: true +# +# Most object store types have a `store_by` option which can be set to either `uuid` or `id`. Older Galaxy servers +# stored datasets by their numeric id (000/dataset_1.dat, 00/dataset_2.dat, ...), whereas newer Galaxy servers store +# them by UUID (b/5/e/dataset_b5e0301c-4c2e-41ac-b2c1-3c243f91b6ec.dat, ...). Storing by UUID is preferred, storing by +# ID should only be used for preexisting store-by-id backends. + +# +# Sample Disk Object Store configuration +# + +# This mirrors the default configuration if there is no object store configuration file. The default uses the values of +# file_path, new_file_path, and job_working_directory in galaxy.yml. + +type: disk +store_by: uuid +files_dir: database/objects +extra_dirs: + - type: job_work + path: database/jobs_directory + + +# +# Sample Hierarchical Object Store with disk backends configuration +# + +# In the hierarchical object store, existing datasets will be searched for in backends in the order of the list of +# specified backends, until the dataset is found. New datasets are always created in the first backend in the list. + +type: hierarchical +backends: + - type: disk + store_by: uuid + files_dir: /new-fs/galaxy/files + extra_dirs: + - type: job_work + path: /new-fs/galaxy/jobs + - type: disk + store_by: id + files_dir: /old-fs/galaxy/files + +# +# Sample Distributed Object Store with disk backends configuration +# + +# In the distributed object store, existing dataests will be located by the `object_store_id` column in the `dataset` +# table of the Galaxy database, which corresponds to the `id` option on the backend. New datasets are created based on +# the `weight` option: a backend with weight "2" has twice the chance of being (randomly) selected for new datasets as a +# backend with weight "1". A weight of "0" will still allow datasets in that backend to be read, but no new datasets +# will be written to that backend. +# +# In distributed and hierarchical configurations, you can choose that some backends are automatically unused whenever +# they become too full. Setting the `global_max_percent_full` option on the top level backends config enables this, or +# it can be applied to individual backends with `max_percent_full` to override the global setting. This only applies to +# disk based backends and not remote object stores. +# +# By default, if a dataset should exist but its object_store_id is null, all backends will be searched until it is +# found. This is to aid in Galaxy servers moving from non-distributed to distributed object stores, but this behavior +# can be disabled by setting `search_for_missing` to "false" on the top level backends config. + +type: distributed +global_max_percent_full: 90 +search_for_missing: true +backends: + - id: new-big + type: disk + store_by: uuid + weight: 3 + files_dir: /new-big-fs/galaxy/files + extra_dirs: + - type: job_work + path: /new-big-fs/galaxy/jobs + - id: new-small + type: disk + store_by: uuid + weight: 1 + max_percent_full: 80 + files_dir: /new-small-fs/galaxy/files + extra_dirs: + - type: job_work + path: /new-small-fs/galaxy/jobs + - id: old + type: disk + store_by: uuid + weight: 0 + files_dir: /old-fs/galaxy/files + +# +# Sample Nested (Distributed in Hierarchical) Object Store configuration +# + +# These object stores support nesting object stores inside object stores. In this example, new data are created in the +# distributed object store, but old data will be searched for in a disk object store. This is useful if moving from +# non-distributed to distributed since you don't have to set `object_store_id` for old data in the database. +# +# In this example, new dataset creation is distributed evenly between two backends. + +type: hierarchical +backends: + - type: distributed + backends: + - id: new1 + store_by: uuid + weight: 1 + files_dir: /new-fs/galaxy/files1 + extra_dirs: + - type: job_work + path: /new-fs/galaxy/jobs1 + - id: new2 + store_by: uuid + weight: 1 + files_dir: /new-fs/galaxy/files2 + extra_dirs: + - type: job_work + path: /new-fs/galaxy/jobs2 + - type: disk + store_by: uuid + files_dir: /old-fs/galaxy/files + +# +# Sample AWS S3 Object Store configuration +# + +type: aws_s3 +auth: + access_key: ... + secret_key: ... +bucket: + name: unique_bucket_name_all_lowercase + use_reduced_redundancy: false + max_chunk_size: 250 +cache: + path: database/object_store_cache_s3 + size: 1000 + cache_updated_data: true +extra_dirs: + - type: job_work + path: database/job_working_directory_s3 + +# +# Sample iRODS Object Store configuration +# + +type: irods +auth: + username: rods + password: rods +resource: + name: demoResc +zone: + name: tempZone +connection: + host: localhost + port: 1247 + timeout: 30 + refresh_time: 300 + connection_pool_monitor_interval: 3600 +cache: + path: database/object_store_cache_irods + size: 1000 + cache_updated_data: true +extra_dirs: + - type: job_work + path: database/job_working_directory_irods + +# +# Sample non-AWS S3 Object Store (e.g. swift) configuration +# + +type: generic_s3 +auth: + access_key: ... + secret_key: ... +bucket: + name: unique_bucket_name_all_lowercase + use_reduced_redundancy: false + max_chunk_size: 250 +connection: + host: swift.example.org + port: 6000 + conn_path: / + multipart: true +cache: + path: database/object_store_cache_swift + size: 1000 + cache_updated_data: true +extra_dirs: + - type: job_work + path: database/job_working_directory_swift + +# +# Sample Azure Object Store configuration +# + +type: azure_blob +auth: + account_name: ... + account_key: ... +container: + name: unique_container_name + max_chunk_size: 250 +cache: + path: database/object_store_cache_azure + size: 1000 + cache_updated_data: true +extra_dirs: + - type: job_work + path: database/job_working_directory_azure + +# +# Cloud Object Store +# + +# The Cloud Object Store uses the cloud-provider-agnostic CloudBridge (http://cloudbridge.cloudve.org/) library to +# interface with cloud storage rather than the cloud-native libraries directly. + +# Sample Cloud Object Store with Amazon Simple Storage Service (S3) configuration + +type: cloud +provider: aws +auth: + access_key: ... + secret_key: ... +bucket: + name: unique_bucket_name + use_reduced_redundancy: false +cache: + path: database/object_store_cache_cloud + size: 1000 + cache_updated_data: true +extra_dirs: + - type: job_work + path: database/job_working_directory_cloud + +# Sample Cloud Object Store with Microsoft Azure Blob Storage configuration + +type: cloud +provider: azure +auth: + subscription_id: ... + client_id: ... + secret: ... + tenant: ... +bucket: + name: unique_bucket_name + use_reduced_redundancy: false +cache: + path: database/object_store_cache_cloud + size: 1000 + cache_updated_data: true +extra_dirs: + - type: job_work + path: database/job_working_directory_cloud + +# Sample Cloud Object Store with Google Compute Platform (GCP) configuration + +type: cloud +provider: google +auth: + credentials_file: ... +bucket: + name: unique_bucket_name + use_reduced_redundancy: false +cache: + path: database/object_store_cache_cloud + size: 1000 + cache_updated_data: true +extra_dirs: + - type: job_work + path: database/job_working_directory_cloud + +# +# Sample User-Selectable Scratch Storage configuration +# + +# This distributed object store will default to a normal path on disk using the default quota but sets up a second path +# with user-private storage a larger quota and warns the user the disk is routinely cleaned. Relative speed and +# stability differences are communicated to the user using object store badges - as well as how data is backed up (in +# the default case) and not backed up for scratch storage. +# +# The admin is responsible for routinely cleaning that storage using Galaxy's admin scripts - this object store +# configuration just allows the user selection and communicates expectations to the user. Training related to Galaxy +# cleanup scripts can be found in the Galaxy Training Network: +# +# Slides: https://gxy.io/GTN:S00103 +# Tutorial: https://gxy.io/GTN:T00324 +# +# In this example, the scratch storage is marked as user-private by setting the `private` option to "true" on the +# backend definition. This means it cannot be used in public datasets, shared between users, etc.. This is more example +# purposes - you may very well not want scratch storage to be defined as private as it prevents a lot of regular +# functionality and Galaxy handles regularly cleaned datasets fairly gracefully when the appropriate admin scripts are +# used. + +type: distributed +backends: + - id: default + type: disk + weight: 1 + allow_selection: true + name: Default Galaxy Storage + description: > + This is Galaxy's default object store - this disk space is regularly backed up and all users have a default quota + of 200 GB. + files_dir: database/objects/default + badges: + - type: slower + - type: more_stable + - type: backed_up + message: > + Backed up to Galaxy's institutional long term tape drive nightly. More information about our tape drive can be + found on our [Archive Tier Storage](https://www.msi.umn.edu/content/archive-tier-storage) page. + - id: scratch + type: disk + weight: 0 + allow_selection: true + private: true + name: Scratch Storage + description: > + This object store is connected to institutional scratch storage. This disk space is not backed up and private to + your user, and datasets belonging to this storage will be automatically deleted after one month. + quota: + source: second_tier + files_dir: database/objects/temp + badges: + - type: faster + - type: less_stable + - type: not_backed_up + - type: short_term + message: The data stored here is purged after a month. + +# +# Sample User-Selectable Experimental Storage configuration +# + +# This distributed object store will default to a normal path on disk using the default quota but sets up a second path +# with more experimental storage (here iRODS) and a higher quota. The different backup strategies for normal disk and +# iRODS as well as their respective stability are communicated to the user using object store badges. + +type: distributed +backends: + - id: default + type: disk + weight: 1 + allow_selection: true + name: Default Galaxy Storage + description: > + This is Galaxy's default object store - this disk space is regularly backed up and all users have a default quota + of 200 GB. + files_dir: database/objects/default + badges: + - type: more_stable + - type: backed_up + message: > + Backed up to Galaxy's institutional long term tape drive nightly. More information about our tape drive can be + found on our [Archive Tier Storage](https://www.msi.umn.edu/content/archive-tier-storage) page. + - id: experimental + type: irods + weight: 0 + allow_selection: true + name: Experimental iRODS Storage + description: > + This object store uses our experimental instituional iRODS service. This disk has larger quotas but is more + experimental and expected job failure rates are higher. + quota: + source: irods_quota + auth: + username: rods + password: rods + resource: + name: demoResc + zone: + name: tempZone + connection: + host: localhost + port: 1247 + timeout: 30 + refresh_time: 300 + connection_pool_monitor_interval: 3600 + cache: + path: database/object_store_cache_irods + size: 1000 + cache_updated_data: true + badges: + - type: less_stable + - type: backed_up + message: > + This data is backed up using iRODS native hierarchal storage management mechanisms. The rules describing how + data is stored and backed up in iRODS can be found in our institutional + [iRODS documentation](https://irods.org/uploads/2018/Saum-SURFsara-Data_Archiving_in_iRODS-slides.pdf) + +# +# User-Selectable Storage - A Complex Institutional Example +# + +# Huge chunks of text were stolen wholesale from MSI's data storage website +# (https://www.msi.umn.edu/content/data-storage). Large changes were made and adapted this for demonstration purposes - +# none of the text or policies or guarantees reflect actual current MSI or UMN policies. + +type: distributed +backends: + - id: high_performance + type: disk + weight: 1 + allow_selection: true + name: High Performance Storage + description: > + All MSI researchers have access to a high-performance, high capacity primary storage platform. This system + currently provides 3.5 PB (petabytes) of storage. The integrity of the data is protected by daily snapshots and + tape backups. It has sustained read and write speeds of up to 25 GB/sec. + + There is default access to this storage by any MSI group with an active account. Very large needs can be also met, + but need to be approved by the MSI HPC Allocation Committee. More details are available on the + [Storage Allocations](https://www.msi.umn.edu/content/storage-allocations) page. + files_dir: /hps/galaxy/objects + badges: + - type: faster + - type: more_stable + - type: backed_up + message: > + Backed up to MSI's long term tape drive nightly. More information about our tape drive can be found on our + [Archive Tier Storage](https://www.msi.umn.edu/content/archive-tier-storage) page. + - id: second_tier + type: disk + weight: 0 + allow_selection: true + name: Second Tier Storage + description: > + MSI first added a Ceph object storage system in November 2014 as a second tier storage option. The system + currently has around 10 PB of usable storage installed. + + MSI's second tier storage is designed to address the growing need for resources that support data-intensive + research. It is tightly integrated with other MSI storage and computing resources in order to support a wide + variety of research data life cycles and data analysis workflows. In addition, this object storage platform offers + new access modes, such as Amazon’s S3 (Simple Storage Service) interface, so that researchers can better manage + their data and more seamlessly share data with other researchers whether or not the other researcher has an MSI + account or is at the University of Minnesota. + + More information about MSI Storage can be found [here](https://www.msi.umn.edu/content/data-storage). + quota: + source: second_tier_quota + files_dir: /ceph/galaxy/objects + badges: + - type: faster + - type: less_stable + - type: not_backed_up + - type: less_secure + message: > + MSI's enterprise level data security policies and montioring have not yet been integrated with Ceph storage. + - type: short_term + message: > + The data stored here is purged after a month. + - id: experimental + type: disk + weight: 0 + allow_selection: true + private: true + name: Experimental Scratch + description: > + MSI Ceph storage that is purged more aggressively (weekly instead of monthly) and so it only appropriate for short + term methods development and such. The rapid deletion of stored data enables us to provide this storage without a + quota. + + More information about MSI Storage can be found [here](https://www.msi.umn.edu/content/data-storage). + quota: + enabled: false + files_dir: /ceph/galaxy/scratch + badges: + - type: faster + - type: less_stable + - type: not_backed_up + - type: less_secure + message: > + MSI's enterprise level data security policies and montioring have not yet been integrated with Ceph storage. + - type: short_term + message: > + The data stored here is purged after a week. + - id: surfs + type: disk + weight: 0 + allow_selection: true + name: SURFS + description: > + Much of the data analysis conducted on MSI’s high-performance computing resources uses data gathered from UMN + shared research facilities (SRFs). In recognition of the need for short to medium term storage for this data, MSI + provides a service, Shared User Research Facilities Storage (SURFS), enabling SRFs to deliver data directly to MSI + users. By providing a designated location for this data, MSI can focus data backup and other processes to these + key datasets. As part of this service, MSI will provide the storage of the data for one year from its delivery + date. + + It's expected that the consumers of these data sets will be responsible for discerning which data they may wish to + keep past the 1-year term, and finding an appropriate place to keep it. There are several possible storage options + both at MSI and the wider university. You can explore your options using OIT’s digital [storage options chooser + tool](https://it.umn.edu/services-technologies/comparisons/select-digital-storage-options). + + More information about MSI Storage can be found [here](https://www.msi.umn.edu/content/data-storage). + quota: + source: umn_surfs + files_dir: /surfs/galaxy/objecs + badges: + - type: slower + - type: more_stable + - type: backed_up + - type: more_secure + message: > + University of Minnesota data security analysist's have authorized this storage for the storage of human data. From a65dd8ae49ae2c778f17b024ba20ab51fdf098ad Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 20 Dec 2023 13:47:45 -0500 Subject: [PATCH 024/158] Support configuring object store inline --- doc/source/admin/galaxy_options.rst | 18 ++++++++++++++++++ lib/galaxy/config/sample/galaxy.yml.sample | 10 ++++++++++ lib/galaxy/config/schemas/config_schema.yml | 14 ++++++++++++++ lib/galaxy/model/unittest_utils/data_app.py | 1 + lib/galaxy/objectstore/__init__.py | 6 ++++++ 5 files changed, 49 insertions(+) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 7ee4d6243cd8..342933fe168b 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1442,6 +1442,24 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~ +``object_store_config`` +~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Rather than specifying an object_store_config_file, the object + store configuration can be embedded into Galaxy's config with this + option. + This option has no effect if the file specified by + object_store_config_file exists. Otherwise, if this option is set, + it overrides any other objectstore settings. + The syntax, available instrumenters, and documentation of their + options is explained in detail in the object store sample + configuration file, `object_store_conf.sample.yml` +:Default: ``None`` +:Type: seq + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``object_store_cache_monitor_driver`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 8d1149c1e445..b5efed9397a9 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1025,6 +1025,16 @@ galaxy: # . #object_store_config_file: object_store_conf.xml + # Rather than specifying an object_store_config_file, the object store + # configuration can be embedded into Galaxy's config with this option. + # This option has no effect if the file specified by + # object_store_config_file exists. Otherwise, if this option is set, + # it overrides any other objectstore settings. + # The syntax, available instrumenters, and documentation of their + # options is explained in detail in the object store sample + # configuration file, `object_store_conf.sample.yml` + #object_store_config: null + # Specify where cache monitoring is driven for caching object stores # such as S3, Azure, and iRODS. This option has no affect on disk # object stores. For production instances, the cache should be diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 52456679891c..cb05212c149d 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -1034,6 +1034,20 @@ mapping: Configuration file for the object store If this is set and exists, it overrides any other objectstore settings. + object_store_config: + type: seq + sequence: + - type: any + desc: | + Rather than specifying an object_store_config_file, the object store configuration can be embedded into + Galaxy's config with this option. + + This option has no effect if the file specified by object_store_config_file exists. Otherwise, if this option + is set, it overrides any other objectstore settings. + + The syntax, available instrumenters, and documentation of their options is explained in detail in the + object store sample configuration file, `object_store_conf.sample.yml` + object_store_cache_monitor_driver: type: str default: 'auto' diff --git a/lib/galaxy/model/unittest_utils/data_app.py b/lib/galaxy/model/unittest_utils/data_app.py index ac981782e153..288fa56c693a 100644 --- a/lib/galaxy/model/unittest_utils/data_app.py +++ b/lib/galaxy/model/unittest_utils/data_app.py @@ -56,6 +56,7 @@ def __init__(self, root=None, **kwd): # objectstore config values... self.object_store_config_file = "" + self.object_store_config = None self.object_store = "disk" self.object_store_check_old_style = False self.object_store_cache_path = "/tmp/cache" diff --git a/lib/galaxy/objectstore/__init__.py b/lib/galaxy/objectstore/__init__.py index 31bc280244b1..6a5447eb3364 100644 --- a/lib/galaxy/objectstore/__init__.py +++ b/lib/galaxy/objectstore/__init__.py @@ -1437,6 +1437,7 @@ def build_object_store_from_config( if config_xml is None and config_dict is None: config_file = config.object_store_config_file if os.path.exists(config_file): + log.debug("Reading object store config from file: %s", config_file) if config_file.endswith(".xml") or config_file.endswith(".xml.sample"): # This is a top level invocation of build_object_store_from_config, and # we have an object_store_conf.xml -- read the .xml and build @@ -1448,6 +1449,11 @@ def build_object_store_from_config( config_dict = yaml.safe_load(f) from_object = "dict" store = config_dict.get("type") + elif config.object_store_config: + log.debug("Reading object store config from object_store_config option") + from_object = "dict" + config_dict = config.object_store_config + store = config_dict.get("type") else: store = config.object_store elif config_xml is not None: From 7cd4fc6dc4e3a792abedbd6d021c85669d7334f8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 22 Dec 2023 10:23:23 -0500 Subject: [PATCH 025/158] Update lib/galaxy/config/schemas/config_schema.yml --- lib/galaxy/config/schemas/config_schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index cb05212c149d..35c4a6df6a19 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -1045,7 +1045,7 @@ mapping: This option has no effect if the file specified by object_store_config_file exists. Otherwise, if this option is set, it overrides any other objectstore settings. - The syntax, available instrumenters, and documentation of their options is explained in detail in the + The syntax, available storage plugins, and documentation of their options is explained in detail in the object store sample configuration file, `object_store_conf.sample.yml` object_store_cache_monitor_driver: From 4e761100aab3f2bece1d57fc0d622a9c669ebc75 Mon Sep 17 00:00:00 2001 From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com> Date: Fri, 5 Jan 2024 12:00:07 +0100 Subject: [PATCH 026/158] increase default extra offset --- client/src/components/Workflow/Editor/_nodeTerminalStyle.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/components/Workflow/Editor/_nodeTerminalStyle.scss b/client/src/components/Workflow/Editor/_nodeTerminalStyle.scss index fc7cf9f03cb5..e4b2fdb2e8b7 100644 --- a/client/src/components/Workflow/Editor/_nodeTerminalStyle.scss +++ b/client/src/components/Workflow/Editor/_nodeTerminalStyle.scss @@ -1,6 +1,6 @@ @mixin node-terminal-style($side) { --size: 11px; - --offset-extra: 0px; + --offset-extra: 4px; position: absolute; #{$side}: calc(var(--offset-extra) * -1 - 0.65rem); From 43cd19f520d43b83d64cf1ebb7c83821440c64db Mon Sep 17 00:00:00 2001 From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:07:33 +0100 Subject: [PATCH 027/158] convert userTags composable to store --- .../TagsMultiselect/StatelessTags.test.js | 5 ++-- .../TagsMultiselect/StatelessTags.vue | 4 +-- .../Workflow/Editor/Attributes.test.js | 5 ++-- .../components/Workflow/WorkflowList.test.js | 5 ++-- client/src/composables/user.ts | 27 +------------------ client/src/stores/userTagsStore.ts | 27 +++++++++++++++++++ 6 files changed, 39 insertions(+), 34 deletions(-) create mode 100644 client/src/stores/userTagsStore.ts diff --git a/client/src/components/TagsMultiselect/StatelessTags.test.js b/client/src/components/TagsMultiselect/StatelessTags.test.js index 5468a153a8e9..cb2c2db73b17 100644 --- a/client/src/components/TagsMultiselect/StatelessTags.test.js +++ b/client/src/components/TagsMultiselect/StatelessTags.test.js @@ -1,9 +1,10 @@ import { mount } from "@vue/test-utils"; import { useToast } from "composables/toast"; -import { useUserTags } from "composables/user"; import { getLocalVue } from "tests/jest/helpers"; import { computed } from "vue"; +import { useUserTagsStore } from "@/stores/userTagsStore"; + import StatelessTags from "./StatelessTags"; const autocompleteTags = ["#named_user_tag", "abc", "my_tag"]; @@ -19,7 +20,7 @@ const mountWithProps = (props) => { jest.mock("composables/user"); const addLocalTagMock = jest.fn((tag) => tag); -useUserTags.mockReturnValue({ +useUserTagsStore.mockReturnValue({ userTags: computed(() => autocompleteTags), addLocalTag: addLocalTagMock, }); diff --git a/client/src/components/TagsMultiselect/StatelessTags.vue b/client/src/components/TagsMultiselect/StatelessTags.vue index a00f45bafcda..71a944340753 100644 --- a/client/src/components/TagsMultiselect/StatelessTags.vue +++ b/client/src/components/TagsMultiselect/StatelessTags.vue @@ -8,8 +8,8 @@ import Multiselect from "vue-multiselect"; import { useToast } from "@/composables/toast"; import { useMultiselect } from "@/composables/useMultiselect"; -import { useUserTags } from "@/composables/user"; import { useUid } from "@/composables/utils/uid"; +import { useUserTagsStore } from "@/stores/userTagsStore"; import Tag from "./Tag.vue"; @@ -39,7 +39,7 @@ const emit = defineEmits<{ library.add(faTags, faCheck, faTimes, faPlus); -const { userTags, addLocalTag } = useUserTags(); +const { userTags, addLocalTag } = useUserTagsStore(); const { warning } = useToast(); function onAddTag(tag: string) { diff --git a/client/src/components/Workflow/Editor/Attributes.test.js b/client/src/components/Workflow/Editor/Attributes.test.js index 544cbe7281d3..af85f253e775 100644 --- a/client/src/components/Workflow/Editor/Attributes.test.js +++ b/client/src/components/Workflow/Editor/Attributes.test.js @@ -1,8 +1,9 @@ import { createLocalVue, mount } from "@vue/test-utils"; -import { useUserTags } from "composables/user"; import { isDate } from "date-fns"; import { computed } from "vue"; +import { useUserTagsStore } from "@/stores/userTagsStore"; + import Attributes from "./Attributes"; import { UntypedParameters } from "./modules/parameters"; @@ -17,7 +18,7 @@ const TEST_VERSIONS = [ const autocompleteTags = ["#named_uer_tag", "abc", "my_tag"]; jest.mock("composables/user"); -useUserTags.mockReturnValue({ +useUserTagsStore.mockReturnValue({ userTags: computed(() => autocompleteTags), addLocalTag: jest.fn(), }); diff --git a/client/src/components/Workflow/WorkflowList.test.js b/client/src/components/Workflow/WorkflowList.test.js index 6bb58612251a..7bdd4537e4ba 100644 --- a/client/src/components/Workflow/WorkflowList.test.js +++ b/client/src/components/Workflow/WorkflowList.test.js @@ -2,13 +2,14 @@ import { createTestingPinia } from "@pinia/testing"; import { mount } from "@vue/test-utils"; import axios from "axios"; import MockAdapter from "axios-mock-adapter"; -import { useUserTags } from "composables/user"; import { formatDistanceToNow, parseISO } from "date-fns"; import flushPromises from "flush-promises"; import { PiniaVuePlugin } from "pinia"; import { getLocalVue, wait } from "tests/jest/helpers"; import { computed } from "vue"; +import { useUserTagsStore } from "@/stores/userTagsStore"; + import Tag from "../TagsMultiselect/Tag"; import Workflows from "../Workflow/WorkflowList"; @@ -17,7 +18,7 @@ localVue.use(PiniaVuePlugin); const autocompleteTags = ["#named_user_tags", "abc", "my_tag"]; jest.mock("composables/user"); -useUserTags.mockReturnValue({ +useUserTagsStore.mockReturnValue({ userTags: computed(() => autocompleteTags), addLocalTag: jest.fn(), }); diff --git a/client/src/composables/user.ts b/client/src/composables/user.ts index ecc6bddadcfa..9182934a2a7a 100644 --- a/client/src/composables/user.ts +++ b/client/src/composables/user.ts @@ -1,5 +1,4 @@ -import { storeToRefs } from "pinia"; -import { computed, ref } from "vue"; +import { computed } from "vue"; import { useUserStore } from "@/stores/userStore"; @@ -14,27 +13,3 @@ export function useCurrentTheme() { setCurrentTheme, }; } - -// temporarily stores tags which have not yet been fetched from the backend -const localTags = ref([]); - -/** - * Keeps tracks of the tags the current user has used. - */ -export function useUserTags() { - const { currentUser } = storeToRefs(useUserStore()); - const userTags = computed(() => { - let tags: string[]; - if (currentUser.value && !currentUser.value.isAnonymous) { - tags = [...currentUser.value.tags_used, ...localTags.value]; - } else { - tags = localTags.value; - } - const tagSet = new Set(tags); - return Array.from(tagSet).map((tag) => tag.replace(/^name:/, "#")); - }); - const addLocalTag = (tag: string) => { - localTags.value.push(tag); - }; - return { userTags, addLocalTag }; -} diff --git a/client/src/stores/userTagsStore.ts b/client/src/stores/userTagsStore.ts new file mode 100644 index 000000000000..3c009c8f46d9 --- /dev/null +++ b/client/src/stores/userTagsStore.ts @@ -0,0 +1,27 @@ +import { defineStore, storeToRefs } from "pinia"; +import { computed, ref } from "vue"; + +import { useUserStore } from "./userStore"; + +export const useUserTagsStore = defineStore("userTagsStore", () => { + const localTags = ref([]); + + const { currentUser } = storeToRefs(useUserStore()); + + const userTags = computed(() => { + let tags: string[]; + if (currentUser.value && !currentUser.value.isAnonymous) { + tags = [...currentUser.value.tags_used, ...localTags.value]; + } else { + tags = localTags.value; + } + const tagSet = new Set(tags); + return Array.from(tagSet).map((tag) => tag.replace(/^name:/, "#")); + }); + + const addLocalTag = (tag: string) => { + localTags.value.push(tag); + }; + + return { userTags, addLocalTag }; +}); From e6c1c8fd402c761485dd01bbe079c350702107e8 Mon Sep 17 00:00:00 2001 From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com> Date: Tue, 9 Jan 2024 16:07:51 +0100 Subject: [PATCH 028/158] fix jest mocks --- client/src/components/TagsMultiselect/StatelessTags.test.js | 2 +- client/src/components/Workflow/Editor/Attributes.test.js | 2 +- client/src/components/Workflow/WorkflowList.test.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/src/components/TagsMultiselect/StatelessTags.test.js b/client/src/components/TagsMultiselect/StatelessTags.test.js index cb2c2db73b17..09b0226b61b2 100644 --- a/client/src/components/TagsMultiselect/StatelessTags.test.js +++ b/client/src/components/TagsMultiselect/StatelessTags.test.js @@ -18,7 +18,7 @@ const mountWithProps = (props) => { }); }; -jest.mock("composables/user"); +jest.mock("@/stores/userTagsStore"); const addLocalTagMock = jest.fn((tag) => tag); useUserTagsStore.mockReturnValue({ userTags: computed(() => autocompleteTags), diff --git a/client/src/components/Workflow/Editor/Attributes.test.js b/client/src/components/Workflow/Editor/Attributes.test.js index af85f253e775..2f477bca6b9f 100644 --- a/client/src/components/Workflow/Editor/Attributes.test.js +++ b/client/src/components/Workflow/Editor/Attributes.test.js @@ -17,7 +17,7 @@ const TEST_VERSIONS = [ ]; const autocompleteTags = ["#named_uer_tag", "abc", "my_tag"]; -jest.mock("composables/user"); +jest.mock("@/stores/userTagsStore"); useUserTagsStore.mockReturnValue({ userTags: computed(() => autocompleteTags), addLocalTag: jest.fn(), diff --git a/client/src/components/Workflow/WorkflowList.test.js b/client/src/components/Workflow/WorkflowList.test.js index 7bdd4537e4ba..4b07236eb091 100644 --- a/client/src/components/Workflow/WorkflowList.test.js +++ b/client/src/components/Workflow/WorkflowList.test.js @@ -17,7 +17,7 @@ const localVue = getLocalVue(); localVue.use(PiniaVuePlugin); const autocompleteTags = ["#named_user_tags", "abc", "my_tag"]; -jest.mock("composables/user"); +jest.mock("@/stores/userTagsStore"); useUserTagsStore.mockReturnValue({ userTags: computed(() => autocompleteTags), addLocalTag: jest.fn(), From e81619e321457328a7078ff57e27e170ee99f98c Mon Sep 17 00:00:00 2001 From: Laila Los <44241786+ElectronicBlueberry@users.noreply.github.com> Date: Wed, 10 Jan 2024 11:04:17 +0100 Subject: [PATCH 029/158] fix tags_used potentially undefined in store --- client/src/stores/userTagsStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/stores/userTagsStore.ts b/client/src/stores/userTagsStore.ts index 3c009c8f46d9..47f717ddea0c 100644 --- a/client/src/stores/userTagsStore.ts +++ b/client/src/stores/userTagsStore.ts @@ -11,7 +11,7 @@ export const useUserTagsStore = defineStore("userTagsStore", () => { const userTags = computed(() => { let tags: string[]; if (currentUser.value && !currentUser.value.isAnonymous) { - tags = [...currentUser.value.tags_used, ...localTags.value]; + tags = [...(currentUser.value.tags_used ?? []), ...localTags.value]; } else { tags = localTags.value; } From db9cd1cd49effb4b31d36afe15fb2c6148dafc78 Mon Sep 17 00:00:00 2001 From: Alexander OSTROVSKY Date: Wed, 10 Jan 2024 15:56:31 -0800 Subject: [PATCH 030/158] add binary datatypes for intermediate output of fastk tools --- lib/galaxy/config/sample/datatypes_conf.xml.sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index f97bf1f54c73..e2d80eae608d 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -210,6 +210,9 @@ + + + From 103f6bdba550d0778d6e40fedc0a942218a7aafc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Jan 2024 11:00:42 +0100 Subject: [PATCH 031/158] Always copy datasets in collection builder modals Fixes https://github.com/galaxyproject/galaxy/issues/17249 --- .../src/components/Collections/ListCollectionCreatorModal.js | 4 +--- .../src/components/Collections/PairCollectionCreatorModal.js | 4 +--- .../Collections/PairedListCollectionCreatorModal.js | 4 +--- .../components/Collections/RuleBasedCollectionCreatorModal.js | 3 +-- .../src/components/History/adapters/buildCollectionModal.js | 3 +-- 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/client/src/components/Collections/ListCollectionCreatorModal.js b/client/src/components/Collections/ListCollectionCreatorModal.js index 7f0ee56abc21..41b84c896345 100644 --- a/client/src/components/Collections/ListCollectionCreatorModal.js +++ b/client/src/components/Collections/ListCollectionCreatorModal.js @@ -28,7 +28,6 @@ function listCollectionCreatorModal(elements, options) { */ function createListCollection(contents, defaultHideSourceItems = true) { const elements = contents.toJSON(); - let copyElements; const promise = listCollectionCreatorModal(elements, { defaultHideSourceItems: defaultHideSourceItems, creationFn: function (elements, name, hideSourceItems) { @@ -38,8 +37,7 @@ function createListCollection(contents, defaultHideSourceItems = true) { //TODO: this allows for list:list even if the filter above does not - reconcile src: element.src || (element.history_content_type == "dataset" ? "hda" : "hdca"), })); - copyElements = !hideSourceItems; - return contents.createHDCA(elements, "list", name, hideSourceItems, copyElements); + return contents.createHDCA(elements, "list", name, hideSourceItems); }, }); return promise; diff --git a/client/src/components/Collections/PairCollectionCreatorModal.js b/client/src/components/Collections/PairCollectionCreatorModal.js index 7d9fe752d0f9..4eb362b2ad44 100644 --- a/client/src/components/Collections/PairCollectionCreatorModal.js +++ b/client/src/components/Collections/PairCollectionCreatorModal.js @@ -24,7 +24,6 @@ function pairCollectionCreatorModal(elements, options) { } function createPairCollection(contents, defaultHideSourceItems = true) { var elements = contents.toJSON(); - var copyElements; var promise = pairCollectionCreatorModal(elements, { defaultHideSourceItems: defaultHideSourceItems, creationFn: function (elements, name, hideSourceItems) { @@ -32,8 +31,7 @@ function createPairCollection(contents, defaultHideSourceItems = true) { { name: "forward", src: elements[0].src || "hda", id: elements[0].id }, { name: "reverse", src: elements[1].src || "hda", id: elements[1].id }, ]; - copyElements = !hideSourceItems; - return contents.createHDCA(elements, "paired", name, hideSourceItems, copyElements); + return contents.createHDCA(elements, "paired", name, hideSourceItems); }, }); return promise; diff --git a/client/src/components/Collections/PairedListCollectionCreatorModal.js b/client/src/components/Collections/PairedListCollectionCreatorModal.js index d455128f46a8..12f0a59944be 100644 --- a/client/src/components/Collections/PairedListCollectionCreatorModal.js +++ b/client/src/components/Collections/PairedListCollectionCreatorModal.js @@ -27,7 +27,6 @@ function pairedListCollectionCreatorModal(elements, options) { */ function createPairedListCollection(contents, defaultHideSourceItems) { const elements = contents.toJSON(); - var copyElements; const promise = pairedListCollectionCreatorModal(elements, { defaultHideSourceItems: defaultHideSourceItems, creationFn: function (elements, name, hideSourceItems) { @@ -48,8 +47,7 @@ function createPairedListCollection(contents, defaultHideSourceItems) { }, ], })); - copyElements = !hideSourceItems; - return contents.createHDCA(elements, "list:paired", name, hideSourceItems, copyElements); + return contents.createHDCA(elements, "list:paired", name, hideSourceItems); }, }); return promise; diff --git a/client/src/components/Collections/RuleBasedCollectionCreatorModal.js b/client/src/components/Collections/RuleBasedCollectionCreatorModal.js index 46fb48e879b2..b29778db23cd 100644 --- a/client/src/components/Collections/RuleBasedCollectionCreatorModal.js +++ b/client/src/components/Collections/RuleBasedCollectionCreatorModal.js @@ -47,7 +47,6 @@ function createCollectionViaRules(selection, defaultHideSourceItems = true) { let elementsType; let importType; const selectionType = selection.selectionType; - const copyElements = !defaultHideSourceItems; if (!selectionType) { // Have HDAs from the history panel. elements = selection.toJSON(); @@ -81,7 +80,7 @@ function createCollectionViaRules(selection, defaultHideSourceItems = true) { ftpUploadSite: selection.ftpUploadSite, defaultHideSourceItems: defaultHideSourceItems, creationFn: function (elements, collectionType, name, hideSourceItems) { - return selection.createHDCA(elements, collectionType, name, hideSourceItems, copyElements); + return selection.createHDCA(elements, collectionType, name, hideSourceItems); }, }); return promise; diff --git a/client/src/components/History/adapters/buildCollectionModal.js b/client/src/components/History/adapters/buildCollectionModal.js index 355204c9675c..1be6cc19f605 100644 --- a/client/src/components/History/adapters/buildCollectionModal.js +++ b/client/src/components/History/adapters/buildCollectionModal.js @@ -45,12 +45,11 @@ const createBackboneContent = (historyId, selection) => { toJSON: () => selectionJson, // result must be a $.Deferred object instead of a promise because // that's the kind of deprecated data format that backbone likes to use. - createHDCA(element_identifiers, collection_type, name, hide_source_items, copy_elements, options = {}) { + createHDCA(element_identifiers, collection_type, name, hide_source_items, options = {}) { const def = jQuery.Deferred(); return def.resolve(null, { collection_type, name, - copy_elements, hide_source_items, element_identifiers, options, From e4f8315c2bacce38f3e37e7cde559a727cc548f5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 11 Jan 2024 12:33:12 +0100 Subject: [PATCH 032/158] Adjust selenium tests for new numbering --- .../selenium/test_collection_builders.py | 12 +++++++----- lib/galaxy_test/selenium/test_collection_edit.py | 6 +++--- lib/galaxy_test/selenium/test_uploads.py | 16 ++++++++++------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/galaxy_test/selenium/test_collection_builders.py b/lib/galaxy_test/selenium/test_collection_builders.py index 40de27bab006..ec03563d0343 100644 --- a/lib/galaxy_test/selenium/test_collection_builders.py +++ b/lib/galaxy_test/selenium/test_collection_builders.py @@ -17,7 +17,7 @@ def test_build_list_simple_hidden(self): self.collection_builder_set_name("my cool list") self.screenshot("collection_builder_list") self.collection_builder_create() - self._wait_for_hid_visible(2) + self._wait_for_hid_visible(3) @selenium_test @managed_history @@ -30,7 +30,6 @@ def test_build_list_and_show_items(self): self.collection_builder_hide_originals() self.collection_builder_set_name("my cool list") self.collection_builder_create() - self.home() # this shouldn't be necessary, and it isn't with a real browser. self._wait_for_hid_visible(3) @selenium_test @@ -43,7 +42,7 @@ def test_build_pair_simple_hidden(self): self.collection_builder_set_name("my awesome pair") self.screenshot("collection_builder_pair") self.collection_builder_create() - self._wait_for_hid_visible(3) + self._wait_for_hid_visible(5) @selenium_test @managed_history @@ -58,11 +57,13 @@ def test_build_paired_list_simple(self): self.collection_builder_set_name("my awesome paired list") self.screenshot("collection_builder_paired_list") self.collection_builder_create() - self._wait_for_hid_visible(3) + self._wait_for_hid_visible(5) # switch to hidden filters to see the hidden datasets appear self._show_hidden_content() self._wait_for_hid_visible(1) self._wait_for_hid_visible(2) + self._wait_for_hid_visible(3) + self._wait_for_hid_visible(4) @selenium_test @managed_history @@ -98,9 +99,10 @@ def test_build_simple_list_via_rules_hidden(self): self.collection_builder_set_name("my cool list") self.screenshot("collection_builder_rules_list") self.collection_builder_create() - self._wait_for_hid_visible(2) + self._wait_for_hid_visible(3) self._show_hidden_content() self._wait_for_hid_visible(1) + self._wait_for_hid_visible(2) def _wait_for_hid_visible(self, hid, state="ok"): # takes a little while for these things to upload and end up in the history diff --git a/lib/galaxy_test/selenium/test_collection_edit.py b/lib/galaxy_test/selenium/test_collection_edit.py index 037f7d84d644..09ff53d71d55 100644 --- a/lib/galaxy_test/selenium/test_collection_edit.py +++ b/lib/galaxy_test/selenium/test_collection_edit.py @@ -20,7 +20,7 @@ def test_change_dbkey_simple_list(self): self.check_current_data_value(dataValue) dataNew = "hg17" self.change_dbkey_value_and_click_submit(dataValue, dataNew) - self.history_panel_wait_for_hid_ok(4) + self.history_panel_wait_for_hid_ok(5) self.open_collection_edit_view() self.navigate_to_database_tab() self.check_current_data_value(dataNew) @@ -42,7 +42,7 @@ def test_change_datatype_simple_list(self): self.change_datatype_value_and_click_submit(dataValue, dataNew) self.check_current_data_value(dataNew) self.wait_for_history() - self.history_panel_expand_collection(2) + self.history_panel_expand_collection(3) self.history_panel_ensure_showing_item_details(1) item = self.history_panel_item_component(hid=1) item.datatype.wait_for_visible() @@ -56,7 +56,7 @@ def _create_simple_list_collection(self, filename, ext): self.collection_builder_set_name("my cool list") self.collection_builder_create() - self._wait_for_hid_visible(2) + self._wait_for_hid_visible(3) def open_collection_edit_view(self): self.components.history_panel.collection_menu_edit_attributes.wait_for_and_click() diff --git a/lib/galaxy_test/selenium/test_uploads.py b/lib/galaxy_test/selenium/test_uploads.py index c2c45a564c92..ddaddff0873b 100644 --- a/lib/galaxy_test/selenium/test_uploads.py +++ b/lib/galaxy_test/selenium/test_uploads.py @@ -118,11 +118,11 @@ def test_upload_deferred(self): @selenium_test def test_upload_list(self): self.upload_list([self.get_filename("1.tabular")], name="Test List") - self.history_panel_wait_for_hid_ok(2) + self.history_panel_wait_for_hid_ok(3) # Make sure modals disappeared - both List creator (TODO: upload). self.wait_for_selector_absent_or_hidden(".collection-creator") - self.assert_item_name(2, "Test List") + self.assert_item_name(3, "Test List") # Make sure source item is hidden when the collection is created. self.history_panel_wait_for_hid_hidden(1) @@ -130,15 +130,17 @@ def test_upload_list(self): @selenium_test def test_upload_pair(self): self.upload_list([self.get_filename("1.tabular"), self.get_filename("2.tabular")], name="Test Pair") - self.history_panel_wait_for_hid_ok(3) + self.history_panel_wait_for_hid_ok(5) # Make sure modals disappeared - both collection creator (TODO: upload). self.wait_for_selector_absent_or_hidden(".collection-creator") - self.assert_item_name(3, "Test Pair") + self.assert_item_name(5, "Test Pair") # Make sure source items are hidden when the collection is created. self.history_panel_wait_for_hid_hidden(1) self.history_panel_wait_for_hid_hidden(2) + self.history_panel_wait_for_hid_hidden(3) + self.history_panel_wait_for_hid_hidden(4) @selenium_test def test_upload_pair_specify_extension(self): @@ -161,14 +163,16 @@ def test_upload_paired_list(self): self.upload_paired_list( [self.get_filename("1.tabular"), self.get_filename("2.tabular")], name="Test Paired List" ) - self.history_panel_wait_for_hid_ok(3) + self.history_panel_wait_for_hid_ok(5) # Make sure modals disappeared - both collection creator (TODO: upload). self.wait_for_selector_absent_or_hidden(".collection-creator") - self.assert_item_name(3, "Test Paired List") + self.assert_item_name(5, "Test Paired List") # Make sure source items are hidden when the collection is created. self.history_panel_wait_for_hid_hidden(1) self.history_panel_wait_for_hid_hidden(2) + self.history_panel_wait_for_hid_hidden(3) + self.history_panel_wait_for_hid_hidden(4) @selenium_test @pytest.mark.gtn_screenshot From cf79d14db18a4d4505cbb2205cbdba4bde8caf0e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 30 Nov 2023 13:31:54 -0500 Subject: [PATCH 033/158] Render useful Markdown components for mapped over steps. API functionality to fetch and query ImplicitCollectionJobs' jobs. --- client/src/api/schema/schema.ts | 7 +++ .../src/components/JobMetrics/JobMetrics.vue | 11 +++- .../JobParameters/JobParameters.vue | 24 ++++++--- .../Markdown/Elements/JobMetrics.vue | 51 +++++++++++++++---- .../Markdown/Elements/JobParameters.vue | 43 +++++++++++++--- .../Markdown/Elements/JobSelection.vue | 44 ++++++++++++++++ .../Markdown/Elements/handlesMappingJobs.ts | 51 +++++++++++++++++++ .../components/Markdown/MarkdownContainer.vue | 2 + lib/galaxy/managers/collections_util.py | 5 ++ lib/galaxy/managers/jobs.py | 21 +++++++- lib/galaxy/managers/markdown_parse.py | 8 +-- lib/galaxy/managers/markdown_util.py | 12 +++-- lib/galaxy/schema/schema.py | 5 ++ lib/galaxy/webapps/galaxy/api/jobs.py | 8 +++ lib/galaxy/webapps/galaxy/services/jobs.py | 10 +++- lib/galaxy_test/api/test_jobs.py | 16 ++++++ 16 files changed, 279 insertions(+), 39 deletions(-) create mode 100644 client/src/components/Markdown/Elements/JobSelection.vue create mode 100644 client/src/components/Markdown/Elements/handlesMappingJobs.ts diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 6175fdc94d4b..f42c45fedc52 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -5495,6 +5495,11 @@ export interface components { * @example 0123456789ABCDEF */ id: string; + /** + * Implicit Collection Jobs Id + * @description Encoded ID for the ICJ object describing the collection of jobs corresponding to this collection + */ + implicit_collection_jobs_id?: string | null; /** * Job Source ID * @description The encoded ID of the Job that produced this dataset collection. Used to track the state of the job. @@ -16846,6 +16851,7 @@ export interface operations { /** @description Limit listing of jobs to those that match the history_id. If none, jobs from any history may be returned. */ /** @description Limit listing of jobs to those that match the specified workflow ID. If none, jobs from any workflow (or from no workflows) may be returned. */ /** @description Limit listing of jobs to those that match the specified workflow invocation ID. If none, jobs from any workflow invocation (or from no workflows) may be returned. */ + /** @description Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned. */ /** @description Sort results by specified field. */ /** * @description A mix of free text and GitHub-style tags used to filter the index operation. @@ -16897,6 +16903,7 @@ export interface operations { history_id?: string | null; workflow_id?: string | null; invocation_id?: string | null; + implicit_collection_jobs_id?: string | null; order_by?: components["schemas"]["JobIndexSortByEnum"]; search?: string | null; limit?: number; diff --git a/client/src/components/JobMetrics/JobMetrics.vue b/client/src/components/JobMetrics/JobMetrics.vue index 508de897f543..de1cbe9a1d6e 100644 --- a/client/src/components/JobMetrics/JobMetrics.vue +++ b/client/src/components/JobMetrics/JobMetrics.vue @@ -1,5 +1,5 @@ + + diff --git a/client/src/components/Markdown/Elements/handlesMappingJobs.ts b/client/src/components/Markdown/Elements/handlesMappingJobs.ts new file mode 100644 index 000000000000..fa257872dd69 --- /dev/null +++ b/client/src/components/Markdown/Elements/handlesMappingJobs.ts @@ -0,0 +1,51 @@ +import { format, parseISO } from "date-fns"; +import { computed, Ref, ref, watch } from "vue"; + +import { fetcher } from "@/api/schema"; + +const jobsFetcher = fetcher.path("/api/jobs").method("get").create(); + +export interface SelectOption { + value: string; + text: string; +} + +interface Job { + id: string; + create_time: string; +} + +export function useMappingJobs( + singleJobId: Ref, + implicitCollectionJobsId: Ref +) { + const selectJobOptions = ref([]); + const selectedJob = ref(undefined); + const targetJobId = computed(() => { + if (singleJobId.value) { + return singleJobId.value; + } else { + return selectedJob.value; + } + }); + watch( + implicitCollectionJobsId, + async () => { + if (implicitCollectionJobsId.value) { + const response = await jobsFetcher({ implicit_collection_jobs_id: implicitCollectionJobsId.value }); + const jobs: Job[] = response.data as unknown as Job[]; + selectJobOptions.value = jobs.map((value, index) => { + const isoCreateTime = parseISO(`${value.create_time}Z`); + const prettyTime = format(isoCreateTime, "eeee MMM do H:mm:ss yyyy zz"); + return { value: value.id, text: `${index + 1}: ${prettyTime}` }; + }); + if (jobs[0]) { + const job: Job = jobs[0]; + selectedJob.value = job.id; + } + } + }, + { immediate: true } + ); + return { selectJobOptions, selectedJob, targetJobId }; +} diff --git a/client/src/components/Markdown/MarkdownContainer.vue b/client/src/components/Markdown/MarkdownContainer.vue index a76c465c10d0..9eb19973390c 100644 --- a/client/src/components/Markdown/MarkdownContainer.vue +++ b/client/src/components/Markdown/MarkdownContainer.vue @@ -151,11 +151,13 @@ function argToBoolean(args, name, booleanDefault) { diff --git a/lib/galaxy/managers/collections_util.py b/lib/galaxy/managers/collections_util.py index b10a282c44d7..f5af1c1eeb63 100644 --- a/lib/galaxy/managers/collections_util.py +++ b/lib/galaxy/managers/collections_util.py @@ -144,6 +144,11 @@ def dictify_dataset_collection_instance( else: element_func = dictify_element_reference dict_value["elements"] = [element_func(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements] + icj = dataset_collection_instance.implicit_collection_jobs + if icj: + dict_value["implicit_collection_jobs_id"] = icj.id + else: + dict_value["implicit_collection_jobs_id"] = None return dict_value diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index c8e6df486340..601979de8a8f 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -41,6 +41,7 @@ from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager from galaxy.model import ( + ImplicitCollectionJobs, ImplicitCollectionJobsJobAssociation, Job, JobParameter, @@ -105,12 +106,18 @@ def __init__(self, app: StructuredApp): self.dataset_manager = DatasetManager(app) def index_query(self, trans, payload: JobIndexQueryPayload) -> sqlalchemy.engine.Result: + """The caller is responsible for security checks on the resulting job if + history_id, invocation_id, or implicit_collection_jobs_id is set. + Otherwise this will only return the user's jobs or all jobs if the requesting + user is acting as an admin. + """ is_admin = trans.user_is_admin user_details = payload.user_details decoded_user_id = payload.user_id history_id = payload.history_id workflow_id = payload.workflow_id invocation_id = payload.invocation_id + implicit_collection_jobs_id = payload.implicit_collection_jobs_id search = payload.search order_by = payload.order_by @@ -200,7 +207,9 @@ def add_search_criteria(stmt): if user_details: stmt = stmt.outerjoin(Job.user) else: - stmt = stmt.where(Job.user_id == trans.user.id) + if history_id is None and invocation_id is None and implicit_collection_jobs_id is None: + stmt = stmt.where(Job.user_id == trans.user.id) + # caller better check security stmt = build_and_apply_filters(stmt, payload.states, lambda s: model.Job.state == s) stmt = build_and_apply_filters(stmt, payload.tool_ids, lambda t: model.Job.tool_id == t) @@ -214,7 +223,15 @@ def add_search_criteria(stmt): order_by_columns = Job if workflow_id or invocation_id: stmt, order_by_columns = add_workflow_jobs() - + elif implicit_collection_jobs_id: + stmt = ( + stmt.join(ImplicitCollectionJobsJobAssociation, ImplicitCollectionJobsJobAssociation.job_id == Job.id) + .join( + ImplicitCollectionJobs, + ImplicitCollectionJobs.id == ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id, + ) + .where(ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id == implicit_collection_jobs_id) + ) if search: stmt = add_search_criteria(stmt) diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index 4d0f20b5d2ca..a0931a73485a 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -50,10 +50,10 @@ class DynamicArguments: "workflow_display": ["workflow_id", "workflow_checkpoint"], "workflow_license": ["workflow_id"], "workflow_image": ["workflow_id", "size", "workflow_checkpoint"], - "job_metrics": ["step", "job_id"], - "job_parameters": ["step", "job_id"], - "tool_stderr": ["step", "job_id"], - "tool_stdout": ["step", "job_id"], + "job_metrics": ["step", "job_id", "implicit_collection_jobs_id"], + "job_parameters": ["step", "job_id", "implicit_collection_jobs_id"], + "tool_stderr": ["step", "job_id", "implicit_collection_jobs_id"], + "tool_stdout": ["step", "job_id", "implicit_collection_jobs_id"], "generate_galaxy_version": [], "generate_time": [], "instance_access_link": [], diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index c573b630287f..8b5aac2fdf70 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -73,10 +73,10 @@ SIZE_PATTERN = re.compile(r"size=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX) # STEP_OUTPUT_LABEL_PATTERN = re.compile(r'step_output=([\w_\-]+)/([\w_\-]+)') UNENCODED_ID_PATTERN = re.compile( - r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([\d]+)" + r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|implicit_collection_jobs_id|invocation_id)=([\d]+)" ) ENCODED_ID_PATTERN = re.compile( - r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([a-z0-9]+)" + r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|implicit_collection_jobs_id|invocation_id)=([a-z0-9]+)" ) INVOCATION_SECTION_MARKDOWN_CONTAINER_LINE_PATTERN = re.compile(r"```\s*galaxy\s*") GALAXY_FENCED_BLOCK = re.compile(r"^```\s*galaxy\s*(.*?)^```", re.MULTILINE ^ re.DOTALL) @@ -930,9 +930,13 @@ def find_non_empty_group(match): elif step_match: target_match = step_match name = find_non_empty_group(target_match) - ref_object_type = "job" invocation_step = invocation.step_invocation_for_label(name) - ref_object = invocation_step and invocation_step.job + if invocation_step and invocation_step.job: + ref_object_type = "job" + ref_object = invocation_step.job + elif invocation_step and invocation_step.implicit_collection_jobs: + ref_object_type = "implicit_collection_jobs" + ref_object = invocation_step.implicit_collection_jobs else: target_match = None ref_object = None diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index b68f8cb9e216..7e59383d133a 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -1057,6 +1057,10 @@ class HDCADetailed(HDCASummary): elements_datatypes: Set[str] = Field( ..., description="A set containing all the different element datatypes in the collection." ) + implicit_collection_jobs_id: Optional[EncodedDatabaseIdField] = Field( + None, + description="Encoded ID for the ICJ object describing the collection of jobs corresponding to this collection", + ) class HistoryBase(Model): @@ -1411,6 +1415,7 @@ class JobIndexQueryPayload(Model): history_id: Optional[DecodedDatabaseIdField] = None workflow_id: Optional[DecodedDatabaseIdField] = None invocation_id: Optional[DecodedDatabaseIdField] = None + implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = None order_by: JobIndexSortByEnum = JobIndexSortByEnum.update_time search: Optional[str] = None limit: int = 500 diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 1550ceb5de57..7d0d2757b333 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -149,6 +149,12 @@ description="Limit listing of jobs to those that match the specified workflow invocation ID. If none, jobs from any workflow invocation (or from no workflows) may be returned.", ) +ImplicitCollectionJobsIdQueryParam: Optional[DecodedDatabaseIdField] = Query( + default=None, + title="Implicit Collection Jobs ID", + description="Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned.", +) + SortByQueryParam: JobIndexSortByEnum = Query( default=JobIndexSortByEnum.update_time, title="Sort By", @@ -215,6 +221,7 @@ def index( history_id: Optional[DecodedDatabaseIdField] = HistoryIdQueryParam, workflow_id: Optional[DecodedDatabaseIdField] = WorkflowIdQueryParam, invocation_id: Optional[DecodedDatabaseIdField] = InvocationIdQueryParam, + implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = ImplicitCollectionJobsIdQueryParam, order_by: JobIndexSortByEnum = SortByQueryParam, search: Optional[str] = SearchQueryParam, limit: int = LimitQueryParam, @@ -232,6 +239,7 @@ def index( history_id=history_id, workflow_id=workflow_id, invocation_id=invocation_id, + implicit_collection_jobs_id=implicit_collection_jobs_id, order_by=order_by, search=search, limit=limit, diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index 6bc6146a66a9..240d841a67ea 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -11,6 +11,7 @@ model, ) from galaxy.managers import hdas +from galaxy.managers.base import security_check from galaxy.managers.context import ProvidesUserContext from galaxy.managers.jobs import ( JobManager, @@ -71,13 +72,20 @@ def index( payload.user_details = True user_details = payload.user_details decoded_user_id = payload.user_id - if not is_admin: self._check_nonadmin_access(view, user_details, decoded_user_id, trans.user and trans.user.id) + check_security_of_jobs = ( + payload.invocation_id is not None + or payload.implicit_collection_jobs_id is not None + or payload.history_id is not None + ) jobs = self.job_manager.index_query(trans, payload) out = [] for job in jobs.yield_per(model.YIELD_PER_ROWS): + # TODO: optimize if this crucial + if check_security_of_jobs and not security_check(trans, job.history, check_accessible=True): + raise exceptions.ItemAccessibilityException("Cannot access the request job objects.") job_dict = job.to_dict(view, system_details=is_admin) if view == JobIndexViewEnum.admin_job_list: job_dict["decoded_job_id"] = job.id diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 5b295742a116..69a5bb520ccd 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -741,6 +741,22 @@ def test_search_delete_outputs(self, history_id): search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) self._search(search_payload, expected_search_count=0) + def test_implicit_collection_jobs(self, history_id): + run_response = self._run_map_over_error(history_id) + implicit_collection_id = run_response["implicit_collections"][0]["id"] + failed_hdca = self.dataset_populator.get_history_collection_details( + history_id=history_id, + content_id=implicit_collection_id, + assert_ok=False, + ) + job_id = run_response["jobs"][0]["id"] + icj_id = failed_hdca["implicit_collection_jobs_id"] + assert icj_id + index = self.__jobs_index(data=dict(implicit_collection_jobs_id=icj_id)) + assert len(index) == 1 + assert index[0]["id"] == job_id + assert index[0]["state"] == "error", index + @pytest.mark.require_new_history def test_search_with_hdca_list_input(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type="list", history_id=history_id) From 1ccc7a196327e5a294213c6c30b574ae7141d61f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Dec 2023 11:21:04 -0500 Subject: [PATCH 034/158] ToolStd -> typescript --- .../components/Markdown/Elements/ToolStd.vue | 52 ++++++++++--------- .../components/Markdown/MarkdownContainer.vue | 6 ++- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/client/src/components/Markdown/Elements/ToolStd.vue b/client/src/components/Markdown/Elements/ToolStd.vue index cebf0036e5d9..522d9709ba1e 100644 --- a/client/src/components/Markdown/Elements/ToolStd.vue +++ b/client/src/components/Markdown/Elements/ToolStd.vue @@ -1,35 +1,37 @@ + + - \ No newline at end of file diff --git a/client/src/components/Grid/GridList.vue b/client/src/components/Grid/GridList.vue index c5de3f921608..ad088d5adf19 100644 --- a/client/src/components/Grid/GridList.vue +++ b/client/src/components/Grid/GridList.vue @@ -10,6 +10,7 @@ import { useRouter } from "vue-router/composables"; import { FieldHandler, GridConfig, Operation, RowData } from "./configs/types"; import GridBoolean from "./GridElements/GridBoolean.vue"; +import GridDatasets from "./GridElements/GridDatasets.vue"; import GridLink from "./GridElements/GridLink.vue"; import GridOperations from "./GridElements/GridOperations.vue"; import GridText from "./GridElements/GridText.vue"; @@ -273,6 +274,7 @@ watch(operationMessage, () => { :title="rowData[fieldEntry.key]" @execute="onOperation($event, rowData)" /> + | void; export type RowData = Record; -type validTypes = "boolean" | "date" | "link" | "operations" | "sharing" | "tags" | "text"; +type validTypes = "boolean" | "date" | "datasets" | "link" | "operations" | "sharing" | "tags" | "text"; diff --git a/client/src/style/scss/unsorted.scss b/client/src/style/scss/unsorted.scss index 4c3799859c8f..f330afc362d3 100644 --- a/client/src/style/scss/unsorted.scss +++ b/client/src/style/scss/unsorted.scss @@ -384,16 +384,6 @@ div.debug { .grid .current { background-color: lighten($brand-success, 20%); } - -// Pulled out of grid base -.count-box { - min-width: 1.1em; - padding: 5px; - border-width: 1px; - border-style: solid; - text-align: center; - display: inline-block; -} .text-filter-val { border: solid 1px #aaaaaa; padding: 1px 2px 1px 3px; From 441c0a0f15971ec6f1e1d981faf2d07b6ba51e85 Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 13 Dec 2023 07:47:31 +0300 Subject: [PATCH 047/158] Add tags handler, adjust dataset state box --- .../Grid/GridElements/GridDatasets.vue | 60 +++++++++++-------- .../src/components/Grid/configs/histories.ts | 22 ++++--- .../components/Grid/configs/visualizations.ts | 8 +-- 3 files changed, 52 insertions(+), 38 deletions(-) diff --git a/client/src/components/Grid/GridElements/GridDatasets.vue b/client/src/components/Grid/GridElements/GridDatasets.vue index 967d4b27565e..21b7d764c6f5 100644 --- a/client/src/components/Grid/GridElements/GridDatasets.vue +++ b/client/src/components/Grid/GridElements/GridDatasets.vue @@ -1,8 +1,9 @@ - \ No newline at end of file + diff --git a/client/src/components/Grid/configs/histories.ts b/client/src/components/Grid/configs/histories.ts index eb11f291883a..e4d0a469f0b3 100644 --- a/client/src/components/Grid/configs/histories.ts +++ b/client/src/components/Grid/configs/histories.ts @@ -11,10 +11,11 @@ import { import { useEventBus } from "@vueuse/core"; import { deleteHistory, historiesQuery, purgeHistory, undeleteHistory } from "@/api/histories"; +import { updateTags } from "@/api/tags"; import { useHistoryStore } from "@/stores/historyStore"; import Filtering, { contains, equals, expandNameTag, toBool, type ValidFilter } from "@/utils/filtering"; import _l from "@/utils/localization"; -import { errorMessageAsString } from "@/utils/simple-error"; +import { errorMessageAsString, rethrowSimple } from "@/utils/simple-error"; import type { ActionArray, FieldArray, GridConfig } from "./types"; @@ -129,7 +130,7 @@ const fields: FieldArray = [ { title: "Delete Permanently", icon: faTrash, - condition: (data: HistoryEntry) => !!data.deleted, + condition: (data: HistoryEntry) => !data.deleted, handler: async (data: HistoryEntry) => { if (confirm(_l("Are you sure that you want to delete this history?"))) { try { @@ -173,15 +174,22 @@ const fields: FieldArray = [ title: "Items", type: "text", }, + { + key: "id", + title: "Size", + type: "datasets", + }, { key: "tags", title: "Tags", type: "tags", - }, - { - key: "id", - title: "Datasets", - type: "datasets", + handler: async (data: HistoryEntry) => { + try { + await updateTags(data.id as string, "History", data.tags as Array); + } catch (e) { + rethrowSimple(e); + } + }, }, { key: "create_time", diff --git a/client/src/components/Grid/configs/visualizations.ts b/client/src/components/Grid/configs/visualizations.ts index 61a1f947a4cf..e93eb60268d7 100644 --- a/client/src/components/Grid/configs/visualizations.ts +++ b/client/src/components/Grid/configs/visualizations.ts @@ -3,6 +3,7 @@ import { useEventBus } from "@vueuse/core"; import axios from "axios"; import { fetcher } from "@/api/schema"; +import { updateTags } from "@/api/tags"; import { getGalaxyInstance } from "@/app"; import Filtering, { contains, equals, expandNameTag, toBool, type ValidFilter } from "@/utils/filtering"; import { withPrefix } from "@/utils/redirect"; @@ -16,7 +17,6 @@ const { emit } = useEventBus("grid-router-push"); * Api endpoint handlers */ const getVisualizations = fetcher.path("/api/visualizations").method("get").create(); -const updateTags = fetcher.path("/api/tags").method("put").create(); /** * Local types @@ -172,11 +172,7 @@ const fields: FieldArray = [ type: "tags", handler: async (data: VisualizationEntry) => { try { - await updateTags({ - item_id: data.id as string, - item_class: "Visualization", - item_tags: data.tags as Array, - }); + await updateTags(data.id as string, "Visualization", data.tags as Array); } catch (e) { rethrowSimple(e); } From ffcce81bd65c4f6b90a137885f6c3bbb6b0b62d1 Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 13 Dec 2023 08:16:48 +0300 Subject: [PATCH 048/158] Fix naming, prep username filter Search by username? --- client/src/entry/analysis/router.js | 11 +++++++---- lib/galaxy/managers/histories.py | 16 +++++++++++++--- lib/galaxy/managers/visualizations.py | 14 +++++++------- lib/galaxy/model/__init__.py | 8 +++++++- lib/galaxy/webapps/galaxy/api/histories.py | 12 ++++++++++-- 5 files changed, 44 insertions(+), 17 deletions(-) diff --git a/client/src/entry/analysis/router.js b/client/src/entry/analysis/router.js index e9b5bd6bdf11..94f7564f5d5c 100644 --- a/client/src/entry/analysis/router.js +++ b/client/src/entry/analysis/router.js @@ -8,9 +8,9 @@ import DatasetDetails from "components/DatasetInformation/DatasetDetails"; import DatasetError from "components/DatasetInformation/DatasetError"; import FormGeneric from "components/Form/FormGeneric"; import historiesGridConfig from "components/Grid/configs/histories"; +import historiesSharedGridConfig from "components/Grid/configs/historiesShared"; import visualizationsGridConfig from "components/Grid/configs/visualizations"; import visualizationsPublishedGridConfig from "components/Grid/configs/visualizationsPublished"; -import GridHistory from "components/Grid/GridHistory"; import GridList from "components/Grid/GridList"; import HistoryExportTasks from "components/History/Export/HistoryExport"; import HistoryPublished from "components/History/HistoryPublished"; @@ -298,11 +298,14 @@ export function getRouter(Galaxy) { props: { gridConfig: historiesGridConfig, }, + redirect: redirectAnon(), }, { - path: "histories/:actionId", - component: GridHistory, - props: true, + path: "histories/list_shared", + component: GridList, + props: { + gridConfig: historiesSharedGridConfig, + }, redirect: redirectAnon(), }, { diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 8d7d83c9ede8..31a3e628bbd4 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -124,18 +124,25 @@ def index_query( self, trans: ProvidesUserContext, payload: HistoryIndexQueryPayload, include_total_count: bool = False ) -> Tuple[List[model.History], int]: show_deleted = False + show_own = payload.show_own show_published = payload.show_published + show_shared = payload.show_shared is_admin = trans.user_is_admin user = trans.user + if not user: + message = "Requires user to log in." + raise exceptions.RequestParameterInvalidException(message) + query = trans.sa_session.query(self.model_class) + query = query.outerjoin(self.model_class.user) filters = [] - if not show_published: + if show_own or (not show_published and not is_admin): filters = [self.model_class.user == user] - else: + if show_published: filters.append(self.model_class.published == true()) - if user and show_published: + if show_shared: filters.append(self.user_share_model.user == user) query = query.outerjoin(self.model_class.users_shared_with) query = query.filter(or_(*filters)) @@ -188,6 +195,9 @@ def p_tag_filter(term_text: str, quoted: bool): ) ) + if show_published and not is_admin: + deleted = False + query = query.filter(self.model_class.deleted == (true() if show_deleted else false())) if include_total_count: diff --git a/lib/galaxy/managers/visualizations.py b/lib/galaxy/managers/visualizations.py index 6a1db77d3570..52a104932858 100644 --- a/lib/galaxy/managers/visualizations.py +++ b/lib/galaxy/managers/visualizations.py @@ -75,17 +75,14 @@ def index_query( self, trans: ProvidesUserContext, payload: VisualizationIndexQueryPayload, include_total_count: bool = False ) -> Tuple[List[model.Visualization], int]: show_deleted = payload.deleted - show_shared = payload.show_shared - show_published = payload.show_published show_own = payload.show_own + show_published = payload.show_published + show_shared = payload.show_shared is_admin = trans.user_is_admin user = trans.user - if show_shared is None: - show_shared = not show_deleted - - if show_shared and show_deleted: - message = "show_shared and show_deleted cannot both be specified as true" + if not user: + message = "Requires user to log in." raise exceptions.RequestParameterInvalidException(message) query = trans.sa_session.query(self.model_class) @@ -154,6 +151,9 @@ def p_tag_filter(term_text: str, quoted: bool): ) ) + if show_published and not is_admin: + deleted = False + query = query.filter(self.model_class.deleted == (true() if show_deleted else false())) if include_total_count: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 377870ef9c78..6df2a015d00d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3033,7 +3033,7 @@ class History(Base, HasTags, Dictifiable, UsesAnnotations, HasName, Serializable "slug", "tags", "update_time", - "user", + "username", ] default_name = "Unnamed history" @@ -3067,6 +3067,12 @@ def stage_addition(self, items): def empty(self): return self.hid_counter is None or self.hid_counter == 1 + @property + def username(self): + if self.user: + return self.user.username + return None + @property def count(self): return self.hid_counter - 1 diff --git a/lib/galaxy/webapps/galaxy/api/histories.py b/lib/galaxy/webapps/galaxy/api/histories.py index 3c288a89ab2a..760bb4e9796e 100644 --- a/lib/galaxy/webapps/galaxy/api/histories.py +++ b/lib/galaxy/webapps/galaxy/api/histories.py @@ -116,8 +116,12 @@ free_text_fields=["title", "description", "slug", "tag"], ) -ShowPublishedQueryParam: bool = Query( - default=False, title="Restrict to published histories and those shared with authenticated user.", description="" +ShowOwnQueryParam: bool = Query(default=True, title="Show histories owned by user.", description="") + +ShowPublishedQueryParam: bool = Query(default=True, title="Include published histories.", description="") + +ShowSharedQueryParam: bool = Query( + default=False, title="Include histories shared with authenticated user.", description="" ) SortByQueryParam: HistorySortByEnum = Query( @@ -179,13 +183,17 @@ async def index_query( trans: ProvidesUserContext = DependsOnTrans, limit: Optional[int] = LimitQueryParam, offset: Optional[int] = OffsetQueryParam, + show_own: bool = ShowOwnQueryParam, show_published: bool = ShowPublishedQueryParam, + show_shared: bool = ShowSharedQueryParam, sort_by: HistorySortByEnum = SortByQueryParam, sort_desc: bool = SortDescQueryParam, search: Optional[str] = SearchQueryParam, ) -> HistoryQueryResultList: payload = HistoryIndexQueryPayload.construct( + show_own=show_own, show_published=show_published, + show_shared=show_shared, sort_by=sort_by, sort_desc=sort_desc, limit=limit, From beccfd3182fa07e903b4a196a3045f48d4d4c78b Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 14 Dec 2023 22:07:56 +0300 Subject: [PATCH 049/158] Add initial draft of history shared grid config --- .../Grid/configs/historiesShared.ts | 127 ++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 client/src/components/Grid/configs/historiesShared.ts diff --git a/client/src/components/Grid/configs/historiesShared.ts b/client/src/components/Grid/configs/historiesShared.ts new file mode 100644 index 000000000000..1fdd13d94538 --- /dev/null +++ b/client/src/components/Grid/configs/historiesShared.ts @@ -0,0 +1,127 @@ +import { + faExchangeAlt, + faEye, + faPlus, + faShareAlt, + faSignature, + faTrash, + faTrashRestore, + faUsers, +} from "@fortawesome/free-solid-svg-icons"; +import { useEventBus } from "@vueuse/core"; + +import { deleteHistory, historiesQuery, purgeHistory, undeleteHistory } from "@/api/histories"; +import { updateTags } from "@/api/tags"; +import { useHistoryStore } from "@/stores/historyStore"; +import Filtering, { contains, equals, expandNameTag, toBool, type ValidFilter } from "@/utils/filtering"; +import _l from "@/utils/localization"; +import { errorMessageAsString, rethrowSimple } from "@/utils/simple-error"; + +import type { ActionArray, FieldArray, GridConfig } from "./types"; + +const { emit } = useEventBus("grid-router-push"); + +/** + * Local types + */ +type HistoryEntry = Record; +type SortKeyLiteral = "create_time" | "name" | "update_time" | undefined; + +/** + * Request and return data from server + */ +async function getData(offset: number, limit: number, search: string, sort_by: string, sort_desc: boolean) { + const { data, headers } = await historiesQuery({ + limit, + offset, + search, + sort_by: sort_by as SortKeyLiteral, + sort_desc, + show_published: false, + }); + const totalMatches = parseInt(headers.get("total_matches") ?? "0"); + return [data, totalMatches]; +} + +/** + * Declare columns to be displayed + */ +const fields: FieldArray = [ + { + key: "name", + title: "Name", + type: "operations", + operations: [ + { + title: "View", + icon: faEye, + condition: (data: HistoryEntry) => !data.deleted, + handler: (data: HistoryEntry) => { + emit(`/histories/view?id=${data.id}`); + }, + }, + ], + }, + { + key: "id", + title: "Size", + type: "datasets", + }, + { + key: "tags", + title: "Tags", + type: "tags", + handler: async (data: HistoryEntry) => { + try { + await updateTags(data.id as string, "History", data.tags as Array); + } catch (e) { + rethrowSimple(e); + } + }, + }, + { + key: "create_time", + title: "Created", + type: "date", + }, + { + key: "update_time", + title: "Updated", + type: "date", + }, + { + key: "username", + title: "Username", + type: "text", + }, +]; + +/** + * Declare filter options + */ +const validFilters: Record> = { + title: { placeholder: "name", type: String, handler: contains("name"), menuItem: true }, + tag: { + placeholder: "tag(s)", + type: "MultiTags", + handler: contains("tag", "tag", expandNameTag), + menuItem: true, + }, +}; + +/** + * Grid configuration + */ +const gridConfig: GridConfig = { + id: "histories-shared-grid", + fields: fields, + filtering: new Filtering(validFilters, undefined, false, false), + getData: getData, + plural: "Histories", + sortBy: "name", + sortDesc: true, + sortKeys: ["create_time", "name", "update_time", "username"], + title: "Shared Histories", +}; + +export default gridConfig; From ad06e7077877e0228c65e670cdea1ee4228e3276 Mon Sep 17 00:00:00 2001 From: guerler Date: Tue, 19 Dec 2023 20:00:29 +0300 Subject: [PATCH 050/158] Remove false condition from visualizations grid --- client/src/components/Grid/configs/visualizations.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/client/src/components/Grid/configs/visualizations.ts b/client/src/components/Grid/configs/visualizations.ts index e93eb60268d7..940d065ae2bc 100644 --- a/client/src/components/Grid/configs/visualizations.ts +++ b/client/src/components/Grid/configs/visualizations.ts @@ -69,7 +69,6 @@ const fields: FieldArray = [ key: "title", type: "operations", width: 40, - condition: (data: VisualizationEntry) => !data.deleted, operations: [ { title: "Open", From 3ba70f1af4d89deb1bd2a8229b98a24762adcb03 Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 20 Dec 2023 06:53:10 +0300 Subject: [PATCH 051/158] Lint and remove unused imports --- .../src/components/Grid/configs/histories.ts | 2 +- .../Grid/configs/historiesShared.ts | 22 +++++------------- lib/galaxy/managers/histories.py | 23 ++++++++++--------- lib/galaxy/managers/visualizations.py | 2 +- 4 files changed, 20 insertions(+), 29 deletions(-) diff --git a/client/src/components/Grid/configs/histories.ts b/client/src/components/Grid/configs/histories.ts index e4d0a469f0b3..952b0de81be2 100644 --- a/client/src/components/Grid/configs/histories.ts +++ b/client/src/components/Grid/configs/histories.ts @@ -212,7 +212,7 @@ const fields: FieldArray = [ * Declare filter options */ const validFilters: Record> = { - title: { placeholder: "name", type: String, handler: contains("name"), menuItem: true }, + name: { placeholder: "name", type: String, handler: contains("name"), menuItem: true }, tag: { placeholder: "tag(s)", type: "MultiTags", diff --git a/client/src/components/Grid/configs/historiesShared.ts b/client/src/components/Grid/configs/historiesShared.ts index 1fdd13d94538..7be0f77e0dbe 100644 --- a/client/src/components/Grid/configs/historiesShared.ts +++ b/client/src/components/Grid/configs/historiesShared.ts @@ -1,23 +1,13 @@ -import { - faExchangeAlt, - faEye, - faPlus, - faShareAlt, - faSignature, - faTrash, - faTrashRestore, - faUsers, -} from "@fortawesome/free-solid-svg-icons"; +import { faEye } from "@fortawesome/free-solid-svg-icons"; import { useEventBus } from "@vueuse/core"; -import { deleteHistory, historiesQuery, purgeHistory, undeleteHistory } from "@/api/histories"; +import { historiesQuery } from "@/api/histories"; import { updateTags } from "@/api/tags"; -import { useHistoryStore } from "@/stores/historyStore"; -import Filtering, { contains, equals, expandNameTag, toBool, type ValidFilter } from "@/utils/filtering"; +import Filtering, { contains, expandNameTag, type ValidFilter } from "@/utils/filtering"; import _l from "@/utils/localization"; -import { errorMessageAsString, rethrowSimple } from "@/utils/simple-error"; +import { rethrowSimple } from "@/utils/simple-error"; -import type { ActionArray, FieldArray, GridConfig } from "./types"; +import type { FieldArray, GridConfig } from "./types"; const { emit } = useEventBus("grid-router-push"); @@ -100,7 +90,7 @@ const fields: FieldArray = [ * Declare filter options */ const validFilters: Record> = { - title: { placeholder: "name", type: String, handler: contains("name"), menuItem: true }, + name: { placeholder: "name", type: String, handler: contains("name"), menuItem: true }, tag: { placeholder: "tag(s)", type: "MultiTags", diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 31a3e628bbd4..239722d1f47f 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -29,9 +29,11 @@ from sqlalchemy.orm import aliased from typing_extensions import Literal -from galaxy import ( - exceptions as glx_exceptions, - model, +from galaxy import model +from galaxy.exceptions import ( + RequestParameterInvalidException, + MessageException, + ObjectNotFound, ) from galaxy.managers import ( deletable, @@ -91,7 +93,6 @@ INDEX_SEARCH_FILTERS = { "name": "name", - "annotation": "annotation", "tag": "tag", "is": "is", } @@ -132,7 +133,7 @@ def index_query( if not user: message = "Requires user to log in." - raise exceptions.RequestParameterInvalidException(message) + raise RequestParameterInvalidException(message) query = trans.sa_session.query(self.model_class) query = query.outerjoin(self.model_class.user) @@ -178,7 +179,7 @@ def p_tag_filter(term_text: str, quoted: bool): elif q == "shared_with_me": if not show_published: message = "Can only use tag is:shared_with_me if show_published parameter also true." - raise exceptions.RequestParameterInvalidException(message) + raise RequestParameterInvalidException(message) query = query.filter(self.user_share_model.user == user) elif isinstance(term, RawTextTerm): tf = p_tag_filter(term.text, False) @@ -196,7 +197,7 @@ def p_tag_filter(term_text: str, quoted: bool): ) if show_published and not is_admin: - deleted = False + show_deleted = False query = query.filter(self.model_class.deleted == (true() if show_deleted else false())) @@ -331,7 +332,7 @@ def parse_order_by(self, order_by_string, default=None): # TODO: add functional/non-orm orders (such as rating) if default: return self.parse_order_by(default) - raise glx_exceptions.RequestParameterInvalidException( + raise RequestParameterInvalidException( "Unknown order_by", order_by=order_by_string, available=["create_time", "update_time", "name", "size"] ) @@ -507,7 +508,7 @@ def restore_archived_history(self, history: model.History, force: bool = False): record to restore the history and its datasets as a new copy. """ if history.archive_export_id is not None and history.purged and not force: - raise glx_exceptions.RequestParameterInvalidException( + raise RequestParameterInvalidException( "Cannot restore an archived (and purged) history that is associated with an archive export record. " "Please try importing it back as a new copy from the associated archive export record instead. " "You can still force the un-archiving of the purged history by setting the 'force' parameter." @@ -717,11 +718,11 @@ def get_ready_jeha(self, trans, history_id: int, jeha_id: Union[int, Literal["la if jeha_id != "latest": matching_exports = [e for e in matching_exports if e.id == jeha_id] if len(matching_exports) == 0: - raise glx_exceptions.ObjectNotFound("Failed to find target history export") + raise ObjectNotFound("Failed to find target history export") jeha = matching_exports[0] if not jeha.ready: - raise glx_exceptions.MessageException("Export not available or not yet ready.") + raise MessageException("Export not available or not yet ready.") return jeha diff --git a/lib/galaxy/managers/visualizations.py b/lib/galaxy/managers/visualizations.py index 52a104932858..ef48eb383d7b 100644 --- a/lib/galaxy/managers/visualizations.py +++ b/lib/galaxy/managers/visualizations.py @@ -152,7 +152,7 @@ def p_tag_filter(term_text: str, quoted: bool): ) if show_published and not is_admin: - deleted = False + show_deleted = False query = query.filter(self.model_class.deleted == (true() if show_deleted else false())) From 7acb3a7ef8e9cbefe18c1ce6ccd4c8b00827ac93 Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 20 Dec 2023 07:12:37 +0300 Subject: [PATCH 052/158] Add history schema --- client/src/api/schema/schema.ts | 50 +++++++++++++++++++ lib/galaxy/managers/histories.py | 2 +- lib/galaxy/schema/history.py | 83 ++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/schema/history.py diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 726e4d0c9fc0..13cfaae2efb3 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -14932,6 +14932,56 @@ export interface operations { extra_files_history_api_histories__history_id__contents__history_content_id__extra_files_get: { /** Get the list of extra files/directories associated with a dataset. */ parameters: { + index_query_api_histories_query_get: { + /** Returns histories available to the current user. */ + parameters?: { + /** @description The maximum number of items to return. */ + /** @description Starts at the beginning skip the first ( offset - 1 ) items and begin returning at the Nth item */ + /** @description Sort index by this specified attribute */ + /** @description Sort in descending order? */ + /** + * @description A mix of free text and GitHub-style tags used to filter the index operation. + * + * ## Query Structure + * + * GitHub-style filter tags (not be confused with Galaxy tags) are tags of the form + * `:` or `:''`. The tag name + * *generally* (but not exclusively) corresponds to the name of an attribute on the model + * being indexed (i.e. a column in the database). + * + * If the tag is quoted, the attribute will be filtered exactly. If the tag is unquoted, + * generally a partial match will be used to filter the query (i.e. in terms of the implementation + * this means the database operation `ILIKE` will typically be used). + * + * Once the tagged filters are extracted from the search query, the remaining text is just + * used to search various documented attributes of the object. + * + * ## GitHub-style Tags Available + * + * `name` + * : The history's name. + * + * `annotation` + * : The history's annotation. (The tag `a` can be used a short hand alias for this tag to filter on this attribute.) + * + * `tag` + * : The history's tags. (The tag `t` can be used a short hand alias for this tag to filter on this attribute.) + * + * ## Free Text + * + * Free text search terms will be searched against the following attributes of the + * Historys: `title`, `description`, `slug`, `tag`. + */ + query?: { + limit?: number; + offset?: number; + show_own?: boolean; + show_published?: boolean; + show_shared?: boolean; + sort_by?: "create_time" | "name" | "update_time" | "username"; + sort_desc?: boolean; + search?: string; + }; /** @description The user ID that will be used to effectively make this API call. Only admins and designated users can make API calls on behalf of other users. */ header?: { "run-as"?: string | null; diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 239722d1f47f..f032cc1688d7 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -31,9 +31,9 @@ from galaxy import model from galaxy.exceptions import ( - RequestParameterInvalidException, MessageException, ObjectNotFound, + RequestParameterInvalidException, ) from galaxy.managers import ( deletable, diff --git a/lib/galaxy/schema/history.py b/lib/galaxy/schema/history.py new file mode 100644 index 000000000000..fa926dd98571 --- /dev/null +++ b/lib/galaxy/schema/history.py @@ -0,0 +1,83 @@ +from datetime import datetime +from typing import ( + List, + Optional, +) + +from pydantic import ( + Extra, + Field, +) +from typing_extensions import Literal + +from galaxy.schema.fields import EncodedDatabaseIdField +from galaxy.schema.schema import ( + CreateTimeField, + Model, + TagCollection, + UpdateTimeField, +) + +HistorySortByEnum = Literal["create_time", "name", "update_time", "username"] + + +class HistoryIndexQueryPayload(Model): + show_own: Optional[bool] = None + show_published: Optional[bool] = None + show_shared: Optional[bool] = None + sort_by: HistorySortByEnum = Field("update_time", title="Sort By", description="Sort by this attribute.") + sort_desc: Optional[bool] = Field(default=True, title="Sort descending", description="Sort in descending order.") + search: Optional[str] = Field(default=None, title="Filter text", description="Freetext to search.") + limit: Optional[int] = Field( + default=100, lt=1000, title="Limit", description="Maximum number of entries to return." + ) + offset: Optional[int] = Field(default=0, title="Offset", description="Number of entries to skip.") + + +class HistoryQueryResult(Model): + id: EncodedDatabaseIdField = Field( + ..., + title="ID", + description="Encoded ID of the History.", + ) + annotation: Optional[str] = Field( + default=None, + title="Annotation", + description="The annotation of this History.", + ) + deleted: bool = Field( + ..., # Required + title="Deleted", + description="Whether this History has been deleted.", + ) + importable: bool = Field( + ..., # Required + title="Importable", + description="Whether this History can be imported.", + ) + published: bool = Field( + ..., # Required + title="Published", + description="Whether this History has been published.", + ) + tags: Optional[TagCollection] = Field( + ..., + title="Tags", + description="A list of tags to add to this item.", + ) + name: str = Field( + title="Name", + description="The name of the History.", + ) + create_time: Optional[datetime] = CreateTimeField + update_time: Optional[datetime] = UpdateTimeField + + class Config: + extra = Extra.allow # Allow any other extra fields + + +class HistoryQueryResultList(Model): + __root__: List[HistoryQueryResult] = Field( + default=[], + title="List with detailed information of Histories.", + ) From d645a7b7d2e334829cbf8d19d4518a13041e3afe Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 23 Dec 2023 12:25:20 +0300 Subject: [PATCH 053/158] Switch user manager from session query to stmt --- .../Grid/configs/historiesShared.ts | 3 ++ lib/galaxy/managers/histories.py | 45 ++++++++++--------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/client/src/components/Grid/configs/historiesShared.ts b/client/src/components/Grid/configs/historiesShared.ts index 7be0f77e0dbe..6e7be23e0ab6 100644 --- a/client/src/components/Grid/configs/historiesShared.ts +++ b/client/src/components/Grid/configs/historiesShared.ts @@ -28,6 +28,8 @@ async function getData(offset: number, limit: number, search: string, sort_by: s sort_by: sort_by as SortKeyLiteral, sort_desc, show_published: false, + show_own: false, + show_shared: true, }); const totalMatches = parseInt(headers.get("total_matches") ?? "0"); return [data, totalMatches]; @@ -91,6 +93,7 @@ const fields: FieldArray = [ */ const validFilters: Record> = { name: { placeholder: "name", type: String, handler: contains("name"), menuItem: true }, + user: { placeholder: "user", type: String, handler: contains("username"), menuItem: true }, tag: { placeholder: "tag(s)", type: "MultiTags", diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index f032cc1688d7..41c12212d475 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -93,6 +93,7 @@ INDEX_SEARCH_FILTERS = { "name": "name", + "user": "user", "tag": "tag", "is": "is", } @@ -135,8 +136,7 @@ def index_query( message = "Requires user to log in." raise RequestParameterInvalidException(message) - query = trans.sa_session.query(self.model_class) - query = query.outerjoin(self.model_class.user) + stmt = select(self.model_class).outerjoin(self.model_class.user) filters = [] if show_own or (not show_published and not is_admin): @@ -145,17 +145,17 @@ def index_query( filters.append(self.model_class.published == true()) if show_shared: filters.append(self.user_share_model.user == user) - query = query.outerjoin(self.model_class.users_shared_with) - query = query.filter(or_(*filters)) + stmt = stmt.outerjoin(self.model_class.users_shared_with) + stmt = stmt.where(or_(*filters)) if payload.search: search_query = payload.search parsed_search = parse_filters_structured(search_query, INDEX_SEARCH_FILTERS) def p_tag_filter(term_text: str, quoted: bool): - nonlocal query + nonlocal stmt alias = aliased(model.HistoryTagAssociation) - query = query.outerjoin(self.model_class.tags.of_type(alias)) + stmt = stmt.outerjoin(self.model_class.tags.of_type(alias)) return tag_filter(alias, term_text, quoted) for term in parsed_search.terms: @@ -164,28 +164,28 @@ def p_tag_filter(term_text: str, quoted: bool): q = term.text if key == "tag": pg = p_tag_filter(term.text, term.quoted) - query = query.filter(pg) + stmt = stmt.where(pg) elif key == "name": - query = query.filter(text_column_filter(self.model_class.name, term)) + stmt = stmt.where(text_column_filter(self.model_class.name, term)) elif key == "user": - query = append_user_filter(query, self.model_class, term) + stmt = append_user_filter(stmt, self.model_class, term) elif key == "is": if q == "deleted": show_deleted = True if q == "published": - query = query.filter(self.model_class.published == true()) + stmt = stmt.where(self.model_class.published == true()) if q == "importable": - query = query.filter(self.model_class.importable == true()) + stmt = stmt.where(self.model_class.importable == true()) elif q == "shared_with_me": if not show_published: message = "Can only use tag is:shared_with_me if show_published parameter also true." raise RequestParameterInvalidException(message) - query = query.filter(self.user_share_model.user == user) + stmt = stmt.where(self.user_share_model.user == user) elif isinstance(term, RawTextTerm): tf = p_tag_filter(term.text, False) alias = aliased(model.User) - query = query.outerjoin(self.model_class.user.of_type(alias)) - query = query.filter( + stmt = stmt.outerjoin(self.model_class.user.of_type(alias)) + stmt = stmt.where( raw_text_column_filter( [ self.model_class.name, @@ -199,21 +199,21 @@ def p_tag_filter(term_text: str, quoted: bool): if show_published and not is_admin: show_deleted = False - query = query.filter(self.model_class.deleted == (true() if show_deleted else false())) + stmt = stmt.where(self.model_class.deleted == (true() if show_deleted else false())) if include_total_count: - total_matches = query.count() + total_matches = get_count(trans.sa_session, stmt) else: total_matches = None sort_column = getattr(model.History, payload.sort_by) if payload.sort_desc: sort_column = sort_column.desc() - query = query.order_by(sort_column) + stmt = stmt.order_by(sort_column) if payload.limit is not None: - query = query.limit(payload.limit) + stmt = stmt.limit(payload.limit) if payload.offset is not None: - query = query.offset(payload.offset) - return query, total_matches + stmt = stmt.offset(payload.offset) + return trans.sa_session.scalars(stmt), total_matches def copy(self, history, user, **kwargs): """ @@ -1019,3 +1019,8 @@ def username_eq(self, item, val: str) -> bool: def username_contains(self, item, val: str) -> bool: return val.lower() in str(item.user.username).lower() + + +def get_count(session, statement): + stmt = select(func.count()).select_from(statement) + return session.scalar(stmt) From b33f4d98c8a71e7c3270736511d7db854057334a Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 23 Dec 2023 12:47:45 +0300 Subject: [PATCH 054/158] Adjust query allow for sorting by username --- client/src/components/Grid/configs/histories.ts | 1 + .../src/components/Grid/configs/historiesShared.ts | 2 +- lib/galaxy/managers/histories.py | 13 ++++++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/client/src/components/Grid/configs/histories.ts b/client/src/components/Grid/configs/histories.ts index 952b0de81be2..916da8fdd89e 100644 --- a/client/src/components/Grid/configs/histories.ts +++ b/client/src/components/Grid/configs/histories.ts @@ -37,6 +37,7 @@ async function getData(offset: number, limit: number, search: string, sort_by: s search, sort_by: sort_by as SortKeyLiteral, sort_desc, + show_own: true, show_published: false, }); const totalMatches = parseInt(headers.get("total_matches") ?? "0"); diff --git a/client/src/components/Grid/configs/historiesShared.ts b/client/src/components/Grid/configs/historiesShared.ts index 6e7be23e0ab6..2d9616110e1e 100644 --- a/client/src/components/Grid/configs/historiesShared.ts +++ b/client/src/components/Grid/configs/historiesShared.ts @@ -27,8 +27,8 @@ async function getData(offset: number, limit: number, search: string, sort_by: s search, sort_by: sort_by as SortKeyLiteral, sort_desc, - show_published: false, show_own: false, + show_published: false, show_shared: true, }); const totalMatches = parseInt(headers.get("total_matches") ?? "0"); diff --git a/lib/galaxy/managers/histories.py b/lib/galaxy/managers/histories.py index 41c12212d475..280a4e64393f 100644 --- a/lib/galaxy/managers/histories.py +++ b/lib/galaxy/managers/histories.py @@ -136,10 +136,10 @@ def index_query( message = "Requires user to log in." raise RequestParameterInvalidException(message) - stmt = select(self.model_class).outerjoin(self.model_class.user) + stmt = select(self.model_class).outerjoin(model.User) filters = [] - if show_own or (not show_published and not is_admin): + if show_own or (not show_published and not show_shared and not is_admin): filters = [self.model_class.user == user] if show_published: filters.append(self.model_class.published == true()) @@ -177,8 +177,8 @@ def p_tag_filter(term_text: str, quoted: bool): if q == "importable": stmt = stmt.where(self.model_class.importable == true()) elif q == "shared_with_me": - if not show_published: - message = "Can only use tag is:shared_with_me if show_published parameter also true." + if not show_shared: + message = "Can only use tag is:shared_with_me if show_shared parameter also true." raise RequestParameterInvalidException(message) stmt = stmt.where(self.user_share_model.user == user) elif isinstance(term, RawTextTerm): @@ -205,7 +205,10 @@ def p_tag_filter(term_text: str, quoted: bool): total_matches = get_count(trans.sa_session, stmt) else: total_matches = None - sort_column = getattr(model.History, payload.sort_by) + if payload.sort_by == "username": + sort_column = model.User.username + else: + sort_column = getattr(model.History, payload.sort_by) if payload.sort_desc: sort_column = sort_column.desc() stmt = stmt.order_by(sort_column) From 591d8e5bf867d866eebe931906ca0e268d7654d3 Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 23 Dec 2023 16:40:40 +0300 Subject: [PATCH 055/158] Fix alignment of history states --- .../Grid/GridElements/GridDatasets.vue | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/client/src/components/Grid/GridElements/GridDatasets.vue b/client/src/components/Grid/GridElements/GridDatasets.vue index 21b7d764c6f5..6890a5e6b39c 100644 --- a/client/src/components/Grid/GridElements/GridDatasets.vue +++ b/client/src/components/Grid/GridElements/GridDatasets.vue @@ -18,10 +18,11 @@ interface HistoryStats { active?: number; }; contents_states: { + error?: number; ok?: number; + new?: number; running?: number; queued?: number; - new?: number; }; } const historyStats: Ref = ref(null); @@ -49,22 +50,18 @@ onMounted(() => { {{ historyStats.nice_size }} - - - {{ stateCount }} - + + {{ stateCount }} + + + {{ historyStats.contents_active.deleted }} - - {{ historyStats.contents_active.deleted }} - - - {{ historyStats.contents_active.hidden }} + + {{ historyStats.contents_active.hidden }} @@ -72,14 +69,14 @@ onMounted(() => { From c4873ea8823c07f5c45b3244be27cdccbaf184ca Mon Sep 17 00:00:00 2001 From: guerler Date: Sat, 23 Dec 2023 17:18:11 +0300 Subject: [PATCH 056/158] Adjust selenium tests --- client/src/components/Grid/GridList.vue | 8 +- client/src/utils/navigation/navigation.yml | 13 ++- lib/galaxy/selenium/navigates_galaxy.py | 5 +- .../selenium/test_histories_list.py | 82 +++++-------------- .../selenium/test_history_sharing.py | 3 +- 5 files changed, 36 insertions(+), 75 deletions(-) diff --git a/client/src/components/Grid/GridList.vue b/client/src/components/Grid/GridList.vue index ad088d5adf19..2117bd4244d7 100644 --- a/client/src/components/Grid/GridList.vue +++ b/client/src/components/Grid/GridList.vue @@ -300,13 +300,7 @@ watch(operationMessage, () => {
- +
diff --git a/client/src/utils/navigation/navigation.yml b/client/src/utils/navigation/navigation.yml index e1e473a59c75..9cf01615cc78 100644 --- a/client/src/utils/navigation/navigation.yml +++ b/client/src/utils/navigation/navigation.yml @@ -408,8 +408,8 @@ published_histories: shared_histories: selectors: - _: ".histories-shared-with-you-by-others" - histories: "#grid-table-body tr" + _: '#histories-shared-grid' + histories: '.grid-table tr' history_copy_elements: selectors: @@ -430,8 +430,13 @@ collection_builders: name: "input.collection-name" histories: - labels: - import_button: 'Import history' + selectors: + advanced_search_toggle: '#histories-grid [data-description="toggle advanced search"]' + advanced_search_name_input: '#histories-advanced-filter-name' + advanced_search_tag_input: '#histories-advanced-filter-tag .stateless-tags' + advanced_search_submit: '#histories-advanced-filter-submit' + import_button: '[data-description="grid action import new history"]' + histories: '.grid-table tr' sharing: selectors: unshare_user_button: '.share_with_view .multiselect__tag-icon' diff --git a/lib/galaxy/selenium/navigates_galaxy.py b/lib/galaxy/selenium/navigates_galaxy.py index 246955f8fc50..148524e2b6c7 100644 --- a/lib/galaxy/selenium/navigates_galaxy.py +++ b/lib/galaxy/selenium/navigates_galaxy.py @@ -1706,12 +1706,11 @@ def histories_click_advanced_search(self): def histories_get_history_names(self): self.sleep_for(self.wait_types.UX_RENDER) names = [] - grid = self.wait_for_selector("#grid-table-body") + grid = self.wait_for_selector("#histories-grid") for row in grid.find_elements(By.TAG_NAME, "tr"): td = row.find_elements(By.TAG_NAME, "td") name = td[1].text if td[0].text == "" else td[0].text - if name != "No items" and not name.startswith("No matching entries found"): - names.append(name) + names.append(name) return names @edit_details diff --git a/lib/galaxy_test/selenium/test_histories_list.py b/lib/galaxy_test/selenium/test_histories_list.py index fe52b8ec1208..e7142bbd94b2 100644 --- a/lib/galaxy_test/selenium/test_histories_list.py +++ b/lib/galaxy_test/selenium/test_histories_list.py @@ -1,3 +1,5 @@ +from selenium.webdriver.common.by import By + from .framework import ( retry_assertion_during_transitions, selenium_test, @@ -110,35 +112,6 @@ def test_permanently_delete_history(self): self.assert_histories_in_grid([self.history4_name]) - @selenium_test - def test_delete_and_undelete_multiple_histories(self): - self._login() - self.navigate_to_histories_page() - - delete_button_selector = 'input[type="button"][value="Delete"]' - undelete_button_selector = 'input[type="button"][value="Undelete"]' - - # Delete multiple histories - self.check_histories([self.history2_name, self.history3_name]) - self.wait_for_and_click_selector(delete_button_selector) - - self.assert_histories_in_grid([self.history2_name, self.history3_name], False) - - self.histories_click_advanced_search() - self.select_filter("status", "deleted") - self.sleep_for(self.wait_types.UX_RENDER) - - # Restore multiple histories - self.check_histories([self.history2_name, self.history3_name]) - self.wait_for_and_click_selector(undelete_button_selector) - - self.assert_grid_histories_are([]) - # Following msg popups but goes away and so can cause transient errors. - # self.wait_for_selector_visible('.donemessage') - self.select_filter("status", "active") - - self.assert_histories_in_grid([self.history2_name, self.history3_name]) - @selenium_test def test_sort_by_name(self): self._login() @@ -183,29 +156,27 @@ def test_standard_search(self): def test_advanced_search(self): self._login() self.navigate_to_histories_page() + self.sleep_for(self.wait_types.UX_RENDER) + self.components.histories.advanced_search_toggle.wait_for_and_click() + # search by tag and name + self.components.histories.advanced_search_name_input.wait_for_and_send_keys(self.history3_name) + self.components.histories.advanced_search_submit.wait_for_and_click() + self.assert_histories_present([self.history3_name]) - self.histories_click_advanced_search() - - name_filter_selector = "#input-name-filter" - tags_filter_selector = "#input-tags-filter" - - # Search by name - self.set_filter(name_filter_selector, self.history2_name) - self.assert_grid_histories_are([self.history2_name]) - self.unset_filter("name", self.history2_name) - - self.set_filter(name_filter_selector, self.history4_name) - self.assert_grid_histories_are([]) - self.unset_filter("name", self.history4_name) - - # Search by tags - self.set_filter(tags_filter_selector, self.history3_tags[0]) - self.assert_grid_histories_are([self.history3_name]) - self.unset_filter("tags", self.history3_tags[0]) - - self.set_filter(tags_filter_selector, self.history4_tags[0]) - self.assert_grid_histories_are([]) - self.unset_filter("tags", self.history4_tags[0]) + @retry_assertion_during_transitions + def assert_histories_present(self, expected_histories, sort_by_matters=False): + present_histories = self.get_present_histories() + assert len(present_histories) == len(expected_histories) + for index, row in enumerate(present_histories): + cell = row.find_elements(By.TAG_NAME, "td")[0] + if not sort_by_matters: + assert cell.text in expected_histories + else: + assert cell.text == expected_histories[index] + + def get_present_histories(self): + self.sleep_for(self.wait_types.UX_RENDER) + return self.components.histories.histories.all() @selenium_test def test_tags(self): @@ -300,12 +271,3 @@ def get_history_tags_cell(self, history_name): raise AssertionError(f"Failed to find history with name [{history_name}]") return tags_cell - - def check_histories(self, histories): - grid = self.wait_for_selector("#grid-table-body") - for row in grid.find_elements(self.by.CSS_SELECTOR, "tr"): - td = row.find_elements(self.by.CSS_SELECTOR, "td") - history_name = td[1].text - if history_name in histories: - checkbox = td[0].find_element(self.by.CSS_SELECTOR, "input") - checkbox.click() diff --git a/lib/galaxy_test/selenium/test_history_sharing.py b/lib/galaxy_test/selenium/test_history_sharing.py index eb53a0297649..0b39a8d0cbab 100644 --- a/lib/galaxy_test/selenium/test_history_sharing.py +++ b/lib/galaxy_test/selenium/test_history_sharing.py @@ -79,9 +79,10 @@ def test_shared_with_me(self): self.submit_login(user2_email, retries=VALID_LOGIN_RETRIES) self.navigate_to_histories_shared_with_me_page() self.components.shared_histories.selector.wait_for_present() + self.components.shared_histories.histories.wait_for_present() rows = self.components.shared_histories.histories.all() assert len(rows) > 0 - assert any(user1_email in row.text for row in rows) + assert any(user1_email.split("@")[0] in row.text for row in rows) def setup_two_users_with_one_shared_history(self, share_by_id=False): user1_email = self._get_random_email() From 104f223663c8d86b87e8addec2dae75d88d82ef2 Mon Sep 17 00:00:00 2001 From: guerler Date: Wed, 27 Dec 2023 08:31:49 +0300 Subject: [PATCH 057/158] Adjust histories advanced filter selenium tests --- .../Grid/GridElements/GridOperations.vue | 1 + client/src/utils/navigation/navigation.yml | 11 +++--- lib/galaxy/selenium/navigates_galaxy.py | 16 +++++---- .../selenium/test_histories_list.py | 36 ++++--------------- 4 files changed, 24 insertions(+), 40 deletions(-) diff --git a/client/src/components/Grid/GridElements/GridOperations.vue b/client/src/components/Grid/GridElements/GridOperations.vue index 8f1fae493d6b..62b5ddcadfbb 100644 --- a/client/src/components/Grid/GridElements/GridOperations.vue +++ b/client/src/components/Grid/GridElements/GridOperations.vue @@ -47,6 +47,7 @@ function hasCondition(conditionHandler: (rowData: RowData, config: GalaxyConfigu