From cf79d14db18a4d4505cbb2205cbdba4bde8caf0e Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 30 Nov 2023 13:31:54 -0500 Subject: [PATCH 1/4] Render useful Markdown components for mapped over steps. API functionality to fetch and query ImplicitCollectionJobs' jobs. --- client/src/api/schema/schema.ts | 7 +++ .../src/components/JobMetrics/JobMetrics.vue | 11 +++- .../JobParameters/JobParameters.vue | 24 ++++++--- .../Markdown/Elements/JobMetrics.vue | 51 +++++++++++++++---- .../Markdown/Elements/JobParameters.vue | 43 +++++++++++++--- .../Markdown/Elements/JobSelection.vue | 44 ++++++++++++++++ .../Markdown/Elements/handlesMappingJobs.ts | 51 +++++++++++++++++++ .../components/Markdown/MarkdownContainer.vue | 2 + lib/galaxy/managers/collections_util.py | 5 ++ lib/galaxy/managers/jobs.py | 21 +++++++- lib/galaxy/managers/markdown_parse.py | 8 +-- lib/galaxy/managers/markdown_util.py | 12 +++-- lib/galaxy/schema/schema.py | 5 ++ lib/galaxy/webapps/galaxy/api/jobs.py | 8 +++ lib/galaxy/webapps/galaxy/services/jobs.py | 10 +++- lib/galaxy_test/api/test_jobs.py | 16 ++++++ 16 files changed, 279 insertions(+), 39 deletions(-) create mode 100644 client/src/components/Markdown/Elements/JobSelection.vue create mode 100644 client/src/components/Markdown/Elements/handlesMappingJobs.ts diff --git a/client/src/api/schema/schema.ts b/client/src/api/schema/schema.ts index 6175fdc94d4b..f42c45fedc52 100644 --- a/client/src/api/schema/schema.ts +++ b/client/src/api/schema/schema.ts @@ -5495,6 +5495,11 @@ export interface components { * @example 0123456789ABCDEF */ id: string; + /** + * Implicit Collection Jobs Id + * @description Encoded ID for the ICJ object describing the collection of jobs corresponding to this collection + */ + implicit_collection_jobs_id?: string | null; /** * Job Source ID * @description The encoded ID of the Job that produced this dataset collection. Used to track the state of the job. @@ -16846,6 +16851,7 @@ export interface operations { /** @description Limit listing of jobs to those that match the history_id. If none, jobs from any history may be returned. */ /** @description Limit listing of jobs to those that match the specified workflow ID. If none, jobs from any workflow (or from no workflows) may be returned. */ /** @description Limit listing of jobs to those that match the specified workflow invocation ID. If none, jobs from any workflow invocation (or from no workflows) may be returned. */ + /** @description Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned. */ /** @description Sort results by specified field. */ /** * @description A mix of free text and GitHub-style tags used to filter the index operation. @@ -16897,6 +16903,7 @@ export interface operations { history_id?: string | null; workflow_id?: string | null; invocation_id?: string | null; + implicit_collection_jobs_id?: string | null; order_by?: components["schemas"]["JobIndexSortByEnum"]; search?: string | null; limit?: number; diff --git a/client/src/components/JobMetrics/JobMetrics.vue b/client/src/components/JobMetrics/JobMetrics.vue index 508de897f543..de1cbe9a1d6e 100644 --- a/client/src/components/JobMetrics/JobMetrics.vue +++ b/client/src/components/JobMetrics/JobMetrics.vue @@ -1,5 +1,5 @@ + + diff --git a/client/src/components/Markdown/Elements/handlesMappingJobs.ts b/client/src/components/Markdown/Elements/handlesMappingJobs.ts new file mode 100644 index 000000000000..fa257872dd69 --- /dev/null +++ b/client/src/components/Markdown/Elements/handlesMappingJobs.ts @@ -0,0 +1,51 @@ +import { format, parseISO } from "date-fns"; +import { computed, Ref, ref, watch } from "vue"; + +import { fetcher } from "@/api/schema"; + +const jobsFetcher = fetcher.path("/api/jobs").method("get").create(); + +export interface SelectOption { + value: string; + text: string; +} + +interface Job { + id: string; + create_time: string; +} + +export function useMappingJobs( + singleJobId: Ref, + implicitCollectionJobsId: Ref +) { + const selectJobOptions = ref([]); + const selectedJob = ref(undefined); + const targetJobId = computed(() => { + if (singleJobId.value) { + return singleJobId.value; + } else { + return selectedJob.value; + } + }); + watch( + implicitCollectionJobsId, + async () => { + if (implicitCollectionJobsId.value) { + const response = await jobsFetcher({ implicit_collection_jobs_id: implicitCollectionJobsId.value }); + const jobs: Job[] = response.data as unknown as Job[]; + selectJobOptions.value = jobs.map((value, index) => { + const isoCreateTime = parseISO(`${value.create_time}Z`); + const prettyTime = format(isoCreateTime, "eeee MMM do H:mm:ss yyyy zz"); + return { value: value.id, text: `${index + 1}: ${prettyTime}` }; + }); + if (jobs[0]) { + const job: Job = jobs[0]; + selectedJob.value = job.id; + } + } + }, + { immediate: true } + ); + return { selectJobOptions, selectedJob, targetJobId }; +} diff --git a/client/src/components/Markdown/MarkdownContainer.vue b/client/src/components/Markdown/MarkdownContainer.vue index a76c465c10d0..9eb19973390c 100644 --- a/client/src/components/Markdown/MarkdownContainer.vue +++ b/client/src/components/Markdown/MarkdownContainer.vue @@ -151,11 +151,13 @@ function argToBoolean(args, name, booleanDefault) { diff --git a/lib/galaxy/managers/collections_util.py b/lib/galaxy/managers/collections_util.py index b10a282c44d7..f5af1c1eeb63 100644 --- a/lib/galaxy/managers/collections_util.py +++ b/lib/galaxy/managers/collections_util.py @@ -144,6 +144,11 @@ def dictify_dataset_collection_instance( else: element_func = dictify_element_reference dict_value["elements"] = [element_func(_, rank_fuzzy_counts=rest_fuzzy_counts) for _ in elements] + icj = dataset_collection_instance.implicit_collection_jobs + if icj: + dict_value["implicit_collection_jobs_id"] = icj.id + else: + dict_value["implicit_collection_jobs_id"] = None return dict_value diff --git a/lib/galaxy/managers/jobs.py b/lib/galaxy/managers/jobs.py index c8e6df486340..601979de8a8f 100644 --- a/lib/galaxy/managers/jobs.py +++ b/lib/galaxy/managers/jobs.py @@ -41,6 +41,7 @@ from galaxy.managers.hdas import HDAManager from galaxy.managers.lddas import LDDAManager from galaxy.model import ( + ImplicitCollectionJobs, ImplicitCollectionJobsJobAssociation, Job, JobParameter, @@ -105,12 +106,18 @@ def __init__(self, app: StructuredApp): self.dataset_manager = DatasetManager(app) def index_query(self, trans, payload: JobIndexQueryPayload) -> sqlalchemy.engine.Result: + """The caller is responsible for security checks on the resulting job if + history_id, invocation_id, or implicit_collection_jobs_id is set. + Otherwise this will only return the user's jobs or all jobs if the requesting + user is acting as an admin. + """ is_admin = trans.user_is_admin user_details = payload.user_details decoded_user_id = payload.user_id history_id = payload.history_id workflow_id = payload.workflow_id invocation_id = payload.invocation_id + implicit_collection_jobs_id = payload.implicit_collection_jobs_id search = payload.search order_by = payload.order_by @@ -200,7 +207,9 @@ def add_search_criteria(stmt): if user_details: stmt = stmt.outerjoin(Job.user) else: - stmt = stmt.where(Job.user_id == trans.user.id) + if history_id is None and invocation_id is None and implicit_collection_jobs_id is None: + stmt = stmt.where(Job.user_id == trans.user.id) + # caller better check security stmt = build_and_apply_filters(stmt, payload.states, lambda s: model.Job.state == s) stmt = build_and_apply_filters(stmt, payload.tool_ids, lambda t: model.Job.tool_id == t) @@ -214,7 +223,15 @@ def add_search_criteria(stmt): order_by_columns = Job if workflow_id or invocation_id: stmt, order_by_columns = add_workflow_jobs() - + elif implicit_collection_jobs_id: + stmt = ( + stmt.join(ImplicitCollectionJobsJobAssociation, ImplicitCollectionJobsJobAssociation.job_id == Job.id) + .join( + ImplicitCollectionJobs, + ImplicitCollectionJobs.id == ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id, + ) + .where(ImplicitCollectionJobsJobAssociation.implicit_collection_jobs_id == implicit_collection_jobs_id) + ) if search: stmt = add_search_criteria(stmt) diff --git a/lib/galaxy/managers/markdown_parse.py b/lib/galaxy/managers/markdown_parse.py index 4d0f20b5d2ca..a0931a73485a 100644 --- a/lib/galaxy/managers/markdown_parse.py +++ b/lib/galaxy/managers/markdown_parse.py @@ -50,10 +50,10 @@ class DynamicArguments: "workflow_display": ["workflow_id", "workflow_checkpoint"], "workflow_license": ["workflow_id"], "workflow_image": ["workflow_id", "size", "workflow_checkpoint"], - "job_metrics": ["step", "job_id"], - "job_parameters": ["step", "job_id"], - "tool_stderr": ["step", "job_id"], - "tool_stdout": ["step", "job_id"], + "job_metrics": ["step", "job_id", "implicit_collection_jobs_id"], + "job_parameters": ["step", "job_id", "implicit_collection_jobs_id"], + "tool_stderr": ["step", "job_id", "implicit_collection_jobs_id"], + "tool_stdout": ["step", "job_id", "implicit_collection_jobs_id"], "generate_galaxy_version": [], "generate_time": [], "instance_access_link": [], diff --git a/lib/galaxy/managers/markdown_util.py b/lib/galaxy/managers/markdown_util.py index c573b630287f..8b5aac2fdf70 100644 --- a/lib/galaxy/managers/markdown_util.py +++ b/lib/galaxy/managers/markdown_util.py @@ -73,10 +73,10 @@ SIZE_PATTERN = re.compile(r"size=\s*%s\s*" % ARG_VAL_CAPTURED_REGEX) # STEP_OUTPUT_LABEL_PATTERN = re.compile(r'step_output=([\w_\-]+)/([\w_\-]+)') UNENCODED_ID_PATTERN = re.compile( - r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([\d]+)" + r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|implicit_collection_jobs_id|invocation_id)=([\d]+)" ) ENCODED_ID_PATTERN = re.compile( - r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|invocation_id)=([a-z0-9]+)" + r"(history_id|workflow_id|history_dataset_id|history_dataset_collection_id|job_id|implicit_collection_jobs_id|invocation_id)=([a-z0-9]+)" ) INVOCATION_SECTION_MARKDOWN_CONTAINER_LINE_PATTERN = re.compile(r"```\s*galaxy\s*") GALAXY_FENCED_BLOCK = re.compile(r"^```\s*galaxy\s*(.*?)^```", re.MULTILINE ^ re.DOTALL) @@ -930,9 +930,13 @@ def find_non_empty_group(match): elif step_match: target_match = step_match name = find_non_empty_group(target_match) - ref_object_type = "job" invocation_step = invocation.step_invocation_for_label(name) - ref_object = invocation_step and invocation_step.job + if invocation_step and invocation_step.job: + ref_object_type = "job" + ref_object = invocation_step.job + elif invocation_step and invocation_step.implicit_collection_jobs: + ref_object_type = "implicit_collection_jobs" + ref_object = invocation_step.implicit_collection_jobs else: target_match = None ref_object = None diff --git a/lib/galaxy/schema/schema.py b/lib/galaxy/schema/schema.py index b68f8cb9e216..7e59383d133a 100644 --- a/lib/galaxy/schema/schema.py +++ b/lib/galaxy/schema/schema.py @@ -1057,6 +1057,10 @@ class HDCADetailed(HDCASummary): elements_datatypes: Set[str] = Field( ..., description="A set containing all the different element datatypes in the collection." ) + implicit_collection_jobs_id: Optional[EncodedDatabaseIdField] = Field( + None, + description="Encoded ID for the ICJ object describing the collection of jobs corresponding to this collection", + ) class HistoryBase(Model): @@ -1411,6 +1415,7 @@ class JobIndexQueryPayload(Model): history_id: Optional[DecodedDatabaseIdField] = None workflow_id: Optional[DecodedDatabaseIdField] = None invocation_id: Optional[DecodedDatabaseIdField] = None + implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = None order_by: JobIndexSortByEnum = JobIndexSortByEnum.update_time search: Optional[str] = None limit: int = 500 diff --git a/lib/galaxy/webapps/galaxy/api/jobs.py b/lib/galaxy/webapps/galaxy/api/jobs.py index 1550ceb5de57..7d0d2757b333 100644 --- a/lib/galaxy/webapps/galaxy/api/jobs.py +++ b/lib/galaxy/webapps/galaxy/api/jobs.py @@ -149,6 +149,12 @@ description="Limit listing of jobs to those that match the specified workflow invocation ID. If none, jobs from any workflow invocation (or from no workflows) may be returned.", ) +ImplicitCollectionJobsIdQueryParam: Optional[DecodedDatabaseIdField] = Query( + default=None, + title="Implicit Collection Jobs ID", + description="Limit listing of jobs to those that match the specified implicit collection job ID. If none, jobs from any implicit collection execution (or from no implicit collection execution) may be returned.", +) + SortByQueryParam: JobIndexSortByEnum = Query( default=JobIndexSortByEnum.update_time, title="Sort By", @@ -215,6 +221,7 @@ def index( history_id: Optional[DecodedDatabaseIdField] = HistoryIdQueryParam, workflow_id: Optional[DecodedDatabaseIdField] = WorkflowIdQueryParam, invocation_id: Optional[DecodedDatabaseIdField] = InvocationIdQueryParam, + implicit_collection_jobs_id: Optional[DecodedDatabaseIdField] = ImplicitCollectionJobsIdQueryParam, order_by: JobIndexSortByEnum = SortByQueryParam, search: Optional[str] = SearchQueryParam, limit: int = LimitQueryParam, @@ -232,6 +239,7 @@ def index( history_id=history_id, workflow_id=workflow_id, invocation_id=invocation_id, + implicit_collection_jobs_id=implicit_collection_jobs_id, order_by=order_by, search=search, limit=limit, diff --git a/lib/galaxy/webapps/galaxy/services/jobs.py b/lib/galaxy/webapps/galaxy/services/jobs.py index 6bc6146a66a9..240d841a67ea 100644 --- a/lib/galaxy/webapps/galaxy/services/jobs.py +++ b/lib/galaxy/webapps/galaxy/services/jobs.py @@ -11,6 +11,7 @@ model, ) from galaxy.managers import hdas +from galaxy.managers.base import security_check from galaxy.managers.context import ProvidesUserContext from galaxy.managers.jobs import ( JobManager, @@ -71,13 +72,20 @@ def index( payload.user_details = True user_details = payload.user_details decoded_user_id = payload.user_id - if not is_admin: self._check_nonadmin_access(view, user_details, decoded_user_id, trans.user and trans.user.id) + check_security_of_jobs = ( + payload.invocation_id is not None + or payload.implicit_collection_jobs_id is not None + or payload.history_id is not None + ) jobs = self.job_manager.index_query(trans, payload) out = [] for job in jobs.yield_per(model.YIELD_PER_ROWS): + # TODO: optimize if this crucial + if check_security_of_jobs and not security_check(trans, job.history, check_accessible=True): + raise exceptions.ItemAccessibilityException("Cannot access the request job objects.") job_dict = job.to_dict(view, system_details=is_admin) if view == JobIndexViewEnum.admin_job_list: job_dict["decoded_job_id"] = job.id diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index 5b295742a116..69a5bb520ccd 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -741,6 +741,22 @@ def test_search_delete_outputs(self, history_id): search_payload = self._search_payload(history_id=history_id, tool_id="cat1", inputs=inputs) self._search(search_payload, expected_search_count=0) + def test_implicit_collection_jobs(self, history_id): + run_response = self._run_map_over_error(history_id) + implicit_collection_id = run_response["implicit_collections"][0]["id"] + failed_hdca = self.dataset_populator.get_history_collection_details( + history_id=history_id, + content_id=implicit_collection_id, + assert_ok=False, + ) + job_id = run_response["jobs"][0]["id"] + icj_id = failed_hdca["implicit_collection_jobs_id"] + assert icj_id + index = self.__jobs_index(data=dict(implicit_collection_jobs_id=icj_id)) + assert len(index) == 1 + assert index[0]["id"] == job_id + assert index[0]["state"] == "error", index + @pytest.mark.require_new_history def test_search_with_hdca_list_input(self, history_id): list_id_a = self.__history_with_ok_collection(collection_type="list", history_id=history_id) From 1ccc7a196327e5a294213c6c30b574ae7141d61f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 19 Dec 2023 11:21:04 -0500 Subject: [PATCH 2/4] ToolStd -> typescript --- .../components/Markdown/Elements/ToolStd.vue | 52 ++++++++++--------- .../components/Markdown/MarkdownContainer.vue | 6 ++- 2 files changed, 32 insertions(+), 26 deletions(-) diff --git a/client/src/components/Markdown/Elements/ToolStd.vue b/client/src/components/Markdown/Elements/ToolStd.vue index cebf0036e5d9..522d9709ba1e 100644 --- a/client/src/components/Markdown/Elements/ToolStd.vue +++ b/client/src/components/Markdown/Elements/ToolStd.vue @@ -1,35 +1,37 @@ + + -