From 6c9a77c32ad4df0edd2e5ccbd2b3ee628256912e Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 17 Aug 2023 17:08:55 -0400 Subject: [PATCH 01/26] feat: add dataset workflow input capability --- src/components/manager/RunSetupInputForm.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/components/manager/RunSetupInputForm.js b/src/components/manager/RunSetupInputForm.js index f04238771..5a92a4144 100644 --- a/src/components/manager/RunSetupInputForm.js +++ b/src/components/manager/RunSetupInputForm.js @@ -13,6 +13,7 @@ import {BENTO_DROP_BOX_FS_BASE_PATH} from "../../config"; import {workflowPropTypesShape} from "../../propTypes"; import {nop} from "../../utils/misc"; +import DatasetTreeSelect from "./DatasetTreeSelect"; import DropBoxTreeSelect from "./DropBoxTreeSelect"; @@ -33,6 +34,9 @@ const getInputComponent = ({type, extensions, values}) => { // TODO: directory + case "dataset": + return ; + case "enum": // TODO: enum[] - need to be able to reselect return ; From ccb61e742c6aacba5c594621a9d010519b45d312 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 17 Aug 2023 17:17:49 -0400 Subject: [PATCH 02/26] chore: allow dataset tree select to not include project id --- src/components/manager/DatasetTreeSelect.js | 25 +++++++++++++++------ src/components/manager/RunSetupInputForm.js | 4 ++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/components/manager/DatasetTreeSelect.js b/src/components/manager/DatasetTreeSelect.js index 0ade47abc..39357c5cf 100644 --- a/src/components/manager/DatasetTreeSelect.js +++ b/src/components/manager/DatasetTreeSelect.js @@ -4,7 +4,10 @@ import PropTypes from "prop-types"; import {Spin, TreeSelect} from "antd"; -const DatasetTreeSelect = ({value, onChange, style}) => { +export const ID_FORMAT_PROJECT_DATASET = "project:dataset"; +export const ID_FORMAT_DATASET = "dataset"; + +const DatasetTreeSelect = ({value, onChange, style, idFormat}) => { const {items: projectItems, isFetching: projectsFetching} = useSelector((state) => state.projects); const servicesFetching = useSelector((state) => state.services.isFetchingAll); @@ -27,12 +30,15 @@ const DatasetTreeSelect = ({value, onChange, style}) => { selectable: false, key: p.identifier, value: p.identifier, - children: p.datasets.map(d => ({ - title: d.title, - key: `${p.identifier}:${d.identifier}`, - value: `${p.identifier}:${d.identifier}`, - })), - })), [projectItems]); + children: p.datasets.map(d => { + const key = idFormat === ID_FORMAT_PROJECT_DATASET ? `${p.identifier}:${d.identifier}` : d.identifier; + return { + title: d.title, + key, + value: key, + }; + }), + })), [idFormat, projectItems]); return { ; }; +DatasetTreeSelect.defaultProps = { + idFormat: ID_FORMAT_PROJECT_DATASET, +} + DatasetTreeSelect.propTypes = { style: PropTypes.object, value: PropTypes.string, onChange: PropTypes.func, + valueFormat: PropTypes.oneOf([ID_FORMAT_PROJECT_DATASET, ID_FORMAT_DATASET]), }; export default DatasetTreeSelect; diff --git a/src/components/manager/RunSetupInputForm.js b/src/components/manager/RunSetupInputForm.js index 5a92a4144..9e445321e 100644 --- a/src/components/manager/RunSetupInputForm.js +++ b/src/components/manager/RunSetupInputForm.js @@ -13,7 +13,7 @@ import {BENTO_DROP_BOX_FS_BASE_PATH} from "../../config"; import {workflowPropTypesShape} from "../../propTypes"; import {nop} from "../../utils/misc"; -import DatasetTreeSelect from "./DatasetTreeSelect"; +import DatasetTreeSelect, {ID_FORMAT_DATASET} from "./DatasetTreeSelect"; import DropBoxTreeSelect from "./DropBoxTreeSelect"; @@ -35,7 +35,7 @@ const getInputComponent = ({type, extensions, values}) => { // TODO: directory case "dataset": - return ; + return ; case "enum": // TODO: enum[] - need to be able to reselect From e91415d5040b2a0bb989d58af76fea133873f4f6 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 17 Aug 2023 17:21:22 -0400 Subject: [PATCH 03/26] feat: add project-dataset input type for workflows --- src/components/manager/RunSetupInputForm.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/components/manager/RunSetupInputForm.js b/src/components/manager/RunSetupInputForm.js index 9e445321e..0f6967528 100644 --- a/src/components/manager/RunSetupInputForm.js +++ b/src/components/manager/RunSetupInputForm.js @@ -13,7 +13,7 @@ import {BENTO_DROP_BOX_FS_BASE_PATH} from "../../config"; import {workflowPropTypesShape} from "../../propTypes"; import {nop} from "../../utils/misc"; -import DatasetTreeSelect, {ID_FORMAT_DATASET} from "./DatasetTreeSelect"; +import DatasetTreeSelect, {ID_FORMAT_DATASET, ID_FORMAT_PROJECT_DATASET} from "./DatasetTreeSelect"; import DropBoxTreeSelect from "./DropBoxTreeSelect"; @@ -34,6 +34,9 @@ const getInputComponent = ({type, extensions, values}) => { // TODO: directory + case "project:dataset": + return ; + case "dataset": return ; From 41d2b6849b35dff5af9bfc6ae1343e6ae5e7163c Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 23 Aug 2023 10:17:07 -0400 Subject: [PATCH 04/26] lint --- src/components/manager/DatasetTreeSelect.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/manager/DatasetTreeSelect.js b/src/components/manager/DatasetTreeSelect.js index 39357c5cf..f28f46c87 100644 --- a/src/components/manager/DatasetTreeSelect.js +++ b/src/components/manager/DatasetTreeSelect.js @@ -54,13 +54,13 @@ const DatasetTreeSelect = ({value, onChange, style, idFormat}) => { DatasetTreeSelect.defaultProps = { idFormat: ID_FORMAT_PROJECT_DATASET, -} +}; DatasetTreeSelect.propTypes = { style: PropTypes.object, value: PropTypes.string, onChange: PropTypes.func, - valueFormat: PropTypes.oneOf([ID_FORMAT_PROJECT_DATASET, ID_FORMAT_DATASET]), + idFormat: PropTypes.oneOf([ID_FORMAT_PROJECT_DATASET, ID_FORMAT_DATASET]), }; export default DatasetTreeSelect; From 3715fffdeabf767f42de8946ca470bce66d13d3e Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 13 Oct 2023 10:59:11 -0400 Subject: [PATCH 05/26] feat(manager): support new workflow input types --- src/components/manager/RunSetupInputForm.js | 104 +++++++++++++------- src/propTypes.js | 13 ++- 2 files changed, 74 insertions(+), 43 deletions(-) diff --git a/src/components/manager/RunSetupInputForm.js b/src/components/manager/RunSetupInputForm.js index 0f6967528..3dc2da212 100644 --- a/src/components/manager/RunSetupInputForm.js +++ b/src/components/manager/RunSetupInputForm.js @@ -1,7 +1,7 @@ import React, {useCallback} from "react"; import PropTypes from "prop-types"; -import {Button, Form, Icon, Input, Select} from "antd"; +import { Button, Checkbox, Form, Icon, Input, Select } from "antd"; import { FORM_LABEL_COL, @@ -13,45 +13,72 @@ import {BENTO_DROP_BOX_FS_BASE_PATH} from "../../config"; import {workflowPropTypesShape} from "../../propTypes"; import {nop} from "../../utils/misc"; -import DatasetTreeSelect, {ID_FORMAT_DATASET, ID_FORMAT_PROJECT_DATASET} from "./DatasetTreeSelect"; +import DatasetTreeSelect, { ID_FORMAT_PROJECT_DATASET } from "./DatasetTreeSelect"; import DropBoxTreeSelect from "./DropBoxTreeSelect"; -const getInputComponent = ({type, extensions, values}) => { - const dropBoxTreeNodeEnabled = ({name, contents}) => contents !== undefined || - extensions.find(e => name.endsWith(e)) !== undefined; +const getInputComponentAndOptions = ({ type, pattern, values, required, repeatable }) => { + const dropBoxTreeNodeEnabled = ({ name, contents }) => + contents !== undefined || !pattern || (new RegExp(pattern)).test(name); - switch (type) { - case "file": - case "file[]": - // TODO: What about non-unique files? - // TODO: Don't hard-code configured filesystem path for input files - return ; + const options = { + // Default to requiring the field unless the "required" property is set on the input + rules: [{ required: required ?? true }], + }; - // TODO: directory + const isArray = type.endsWith("[]"); - case "project:dataset": - return ; + switch (type) { + case "string": + return [, options]; + case "string[]": { + // TODO: string[] - need to be able to reselect if repeatable + return [, options]; + // case "number[]": - case "dataset": - return ; + case "boolean": + return [, { ...options, valuePropName: "checked" }]; case "enum": - // TODO: enum[] - need to be able to reselect - return ; + case "enum[]": { + const mode = (isArray && !repeatable) ? "multiple" : "default"; - case "number": - return ; + // TODO: enum[] - need to be able to reselect if repeatable + return [ + , + options, + ]; + } + + case "file": + case "file[]": + // TODO: What about non-unique files? + // TODO: Don't hard-code configured filesystem path for input files + return [ + , + options, + ]; + + case "directory": + case "directory[]": + return [ + , + options, + ]; - // TODO: string[], enum[], number[] - // TODO: drsObject, drsObject[] + case "project:dataset": + return [, options]; default: - return ; + return [, options]; } }; @@ -68,15 +95,20 @@ const RunSetupInputForm = ({initialValues, form, onSubmit, workflow, onBack}) => return
{[ - ...workflow.inputs.filter(i => !i.hidden).map(i => ( - - {form.getFieldDecorator(i.id, { - initialValue: initialValues[i.id], // undefined if not set - // Default to requiring the field unless the "required" property is set on the input - rules: [{required: i.required === undefined ? true : i.required}], - })(getInputComponent(i))} - - )), + ...workflow.inputs.filter(i => !i.hidden && !i.injected).map(i => { + const [component, options] = getInputComponentAndOptions(i); + + return ( + + {form.getFieldDecorator(i.id, { + initialValue: initialValues[i.id], // undefined if not set + // Default to requiring the field unless the "required" property is set on the input + rules: [{ required: i.required === undefined ? true : i.required }], + ...options, + })(component)} + + ); + }), {onBack ? : null} diff --git a/src/propTypes.js b/src/propTypes.js index b55eebd5f..801bbb0ff 100644 --- a/src/propTypes.js +++ b/src/propTypes.js @@ -189,13 +189,12 @@ export const workflowPropTypesShape = PropTypes.shape({ description: PropTypes.string, data_type: PropTypes.string, inputs: PropTypes.arrayOf(PropTypes.shape({ - type: PropTypes.string, - id: PropTypes.string, - extensions: PropTypes.arrayOf(PropTypes.string), // File type only - })), - outputs: PropTypes.arrayOf(PropTypes.shape({ - type: PropTypes.string, - value: PropTypes.string, + type: PropTypes.string.isRequired, + id: PropTypes.string.isRequired, + pattern: PropTypes.string, // File type only + required: PropTypes.bool, + injected: PropTypes.bool, + repeatable: PropTypes.bool, })), }); From 208a9f1601a0f5b6e402077ffde682b65dee33ab Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 13 Oct 2023 10:59:40 -0400 Subject: [PATCH 06/26] chore(manager): remove workflow outputs from list item --- src/components/manager/WorkflowListItem.js | 33 ++++------------------ 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/src/components/manager/WorkflowListItem.js b/src/components/manager/WorkflowListItem.js index c0a1ecef7..d8ed4ebec 100644 --- a/src/components/manager/WorkflowListItem.js +++ b/src/components/manager/WorkflowListItem.js @@ -33,7 +33,7 @@ const ioTagWithType = (id, ioType, typeContent = "") => ( ); const WorkflowListItem = ({onClick, workflow}) => { - const {inputs, outputs, name, description, data_type: dt} = workflow; + const {inputs, name, description, data_type: dt} = workflow; const typeTag = dt ? {dt} : null; @@ -41,28 +41,6 @@ const WorkflowListItem = ({onClick, workflow}) => { .filter(i => !i.hidden) // Filter out hidden (often injected/FROM_CONFIG) inputs .map(i => ioTagWithType(i.id, i.type, i.type.startsWith("file") ? i.extensions.join(" / ") : "")); - const inputExtensions = Object.fromEntries(inputs - .filter(i => i.type.startsWith("file")) - .map(i => [i.id, i.extensions[0]])); // TODO: What to do with more than one? - - const outputTags = outputs.map(o => { - if (!o.value) console.error("Missing or invalid value prop for workflow output: ", o); - - if (!o.type.startsWith("file")) return ioTagWithType(o.id, o.type); - - const outputValue = o.value || ""; - let formattedOutput = outputValue; - - [...outputValue.matchAll(/{(.*)}/g)].forEach(([_, id]) => { - formattedOutput = formattedOutput.replace(`{${id}}`, { - ...inputExtensions, - "": o.hasOwnProperty("map_from_input") ? inputExtensions[o.map_from_input] : undefined, - }[id]); - }); - - return ioTagWithType(o.id, o.type, formattedOutput); - }); - const selectable = !!onClick; // Can be selected if a click handler exists return @@ -80,10 +58,11 @@ const WorkflowListItem = ({onClick, workflow}) => { {inputTags} -
- Outputs: - {outputTags} -
+ {/* TODO: parse outputs from WDL. For now, we cannot list them, so we just don't show anything */} + {/*
*/} + {/* Outputs:*/} + {/* {outputTags}*/} + {/*
*/} ; }; From ff135aa51389957dc33121fcb9344dd6da944e80 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 13 Oct 2023 11:06:26 -0400 Subject: [PATCH 07/26] lint --- src/components/manager/RunSetupInputForm.js | 25 +++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/components/manager/RunSetupInputForm.js b/src/components/manager/RunSetupInputForm.js index 3dc2da212..271d6fe4c 100644 --- a/src/components/manager/RunSetupInputForm.js +++ b/src/components/manager/RunSetupInputForm.js @@ -17,7 +17,7 @@ import DatasetTreeSelect, { ID_FORMAT_PROJECT_DATASET } from "./DatasetTreeSelec import DropBoxTreeSelect from "./DropBoxTreeSelect"; -const getInputComponentAndOptions = ({ type, pattern, values, required, repeatable }) => { +const getInputComponentAndOptions = ({ id, type, pattern, values, required, repeatable }) => { const dropBoxTreeNodeEnabled = ({ name, contents }) => contents !== undefined || !pattern || (new RegExp(pattern)).test(name); @@ -26,22 +26,23 @@ const getInputComponentAndOptions = ({ type, pattern, values, required, repeatab rules: [{ required: required ?? true }], }; + const key = `input-${id}`; const isArray = type.endsWith("[]"); switch (type) { case "string": - return [, options]; + return [, options]; case "string[]": { // TODO: string[] - need to be able to reselect if repeatable - return [, options]; } case "number": - return [, options]; + return [, options]; // case "number[]": case "boolean": - return [, { ...options, valuePropName: "checked" }]; + return [, { ...options, valuePropName: "checked" }]; case "enum": case "enum[]": { @@ -49,7 +50,7 @@ const getInputComponentAndOptions = ({ type, pattern, values, required, repeatab // TODO: enum[] - need to be able to reselect if repeatable return [ - , + , options, ]; } @@ -60,6 +61,7 @@ const getInputComponentAndOptions = ({ type, pattern, values, required, repeatab // TODO: Don't hard-code configured filesystem path for input files return [ , + , options, ]; case "project:dataset": - return [, options]; + return [, options]; default: - return [, options]; + return [, options]; } }; From 3284f50b2e275de1b2bb93728a0a6e0fbabee9a8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 13 Oct 2023 15:43:07 -0400 Subject: [PATCH 08/26] feat(manager): WIP new workflow selection / input fields interface --- .../manager/ManagerAnalysisContent.js | 36 +----- .../manager/ManagerIngestionContent.js | 112 ++--------------- src/components/manager/RunSetupInputForm.js | 48 ++++++- src/components/manager/RunSetupInputsTable.js | 64 ++++++---- src/components/manager/RunSetupWizard.js | 31 +++-- src/components/manager/WorkflowListItem.js | 54 +++++--- src/components/manager/WorkflowSelection.js | 117 ++++++++++++++++++ src/components/manager/runs/RunRequest.js | 25 +--- src/components/manager/workflowCommon.js | 5 +- src/propTypes.js | 2 + 10 files changed, 274 insertions(+), 220 deletions(-) create mode 100644 src/components/manager/WorkflowSelection.js diff --git a/src/components/manager/ManagerAnalysisContent.js b/src/components/manager/ManagerAnalysisContent.js index d54ab76f4..6d29cd49d 100644 --- a/src/components/manager/ManagerAnalysisContent.js +++ b/src/components/manager/ManagerAnalysisContent.js @@ -2,9 +2,8 @@ import React from "react"; import {useDispatch, useSelector} from "react-redux"; import PropTypes from "prop-types"; -import {Button, Form, List, Skeleton, Spin} from "antd"; +import { Button, Form, List } from "antd"; -import {workflowsStateToPropsMixin} from "../../propTypes"; import WorkflowListItem from "./WorkflowListItem"; import {FORM_BUTTON_COL, FORM_LABEL_COL, FORM_WRAPPER_COL} from "./workflowCommon"; import {useHistory} from "react-router-dom"; @@ -12,32 +11,6 @@ import RunSetupWizard from "./RunSetupWizard"; import RunSetupInputsTable from "./RunSetupInputsTable"; import {submitAnalysisWorkflowRun} from "../../modules/wes/actions"; -const AnalysisWorkflowSelection = ({handleWorkflowClick}) => { - const {workflows, workflowsLoading} = useSelector(workflowsStateToPropsMixin); - - const workflowItems = workflows.analysis.map(w => - handleWorkflowClick(w)} - />, - ); - - return - - - {workflowsLoading - ? - : {workflowItems}} - - - ; -}; -AnalysisWorkflowSelection.propTypes = { - handleWorkflowClick: PropTypes.func, -}; - const AnalysisConfirmDisplay = ({selectedWorkflow, inputs, handleRunWorkflow}) => { const isSubmittingAnalysisRun = useSelector(state => state.runs.isSubmittingAnalysisRun); @@ -74,15 +47,14 @@ const ManagerAnalysisContent = () => { const history = useHistory(); return ( - - )} + workflowType="analysis" confirmDisplay={props => } onSubmit={({selectedWorkflow, inputs}) => { if (!selectedWorkflow) { - // TODO: GUI error message + message.error(`Missing workflow selection; cannot submit run!`); return; } + dispatch(submitAnalysisWorkflowRun( selectedWorkflow.service_base_url, selectedWorkflow, diff --git a/src/components/manager/ManagerIngestionContent.js b/src/components/manager/ManagerIngestionContent.js index 7122d1cd4..b1b2914ce 100644 --- a/src/components/manager/ManagerIngestionContent.js +++ b/src/components/manager/ManagerIngestionContent.js @@ -1,9 +1,9 @@ -import React, {useCallback} from "react"; +import React from "react"; import {useDispatch, useSelector} from "react-redux"; import {useHistory} from "react-router-dom"; import PropTypes from "prop-types"; -import {Button, Empty, Form, List, Skeleton, Spin, message} from "antd"; +import {Button, Form, List, message} from "antd"; import WorkflowListItem from "./WorkflowListItem"; @@ -15,72 +15,9 @@ import { FORM_BUTTON_COL, } from "./workflowCommon"; -import DatasetTreeSelect from "./DatasetTreeSelect"; - -import {workflowTarget, workflowsStateToPropsMixin} from "../../propTypes"; +import { workflowTarget } from "../../propTypes"; import RunSetupWizard from "./RunSetupWizard"; import RunSetupInputsTable from "./RunSetupInputsTable"; -import DataTypeSelect from "./DataTypeSelect"; - - -const IngestWorkflowSelection = ({values, setValues, handleWorkflowClick}) => { - const {workflows, workflowsLoading} = useSelector(workflowsStateToPropsMixin); - const {selectedProject, selectedDataset, selectedDataType} = values; - - const workflowItems = workflows.ingestion - .filter(w => selectedDataset && selectedDataType && w.data_type === selectedDataType) - .map(w => - handleWorkflowClick(w)} - />, - ); - - const onChange = useCallback(({ - project = selectedProject, - dataset = selectedDataset, - dataType = selectedDataType, - }) => { - setValues({ - selectedProject: project, - selectedDataset: dataset, - selectedDataType: dataType, - }); - }, [selectedDataset, selectedDataType, setValues]); - - return
- - onChange({project: p, dataset: d})} - value={selectedDataset} - /> - - - onChange({dataType: dt})} - value={selectedDataType} - /> - - - {selectedDataset && selectedDataType - ? - {workflowsLoading - ? - : {workflowItems}} - - : - } - -
; -}; -IngestWorkflowSelection.propTypes = { - values: workflowTarget, - setValues: PropTypes.func, - handleWorkflowClick: PropTypes.func, -}; const TitleAndID = React.memo(({title, id}) => title ? {title} ({id}) : {id}); TitleAndID.propTypes = { @@ -90,24 +27,11 @@ TitleAndID.propTypes = { const STYLE_RUN_INGESTION = {marginTop: "16px", float: "right"}; -const IngestConfirmDisplay = ({target, selectedWorkflow, inputs, handleRunWorkflow}) => { - const projectsByID = useSelector(state => state.projects.itemsByID); +const IngestConfirmDisplay = ({selectedWorkflow, inputs, handleRunWorkflow}) => { const isSubmittingIngestionRun = useSelector(state => state.runs.isSubmittingIngestionRun); - const datasetsByID = useSelector((state) => state.projects.datasetsByID); - - const {selectedProject, selectedDataset} = target; - - const projectTitle = projectsByID[selectedProject]?.title || null; - const datasetTitle = datasetsByID[selectedDataset]?.title || null; return (
- - - - - - @@ -141,38 +65,24 @@ const ManagerIngestionContent = () => { const history = useHistory(); return ( - - )} - workflowSelectionTitle="Dataset & Workflow" - workflowSelectionDescription={ - Choose a dataset and ingestion workflow. - } - confirmDisplay={({selectedWorkflow, workflowSelectionValues, inputs, handleRunWorkflow}) => ( + workflowType="ingestion" + workflowSelectionTitle="Workflow" + workflowSelectionDescription="Choose an ingestion workflow." + confirmDisplay={({selectedWorkflow, inputs, handleRunWorkflow}) => ( )} - onSubmit={({workflowSelectionValues, selectedWorkflow, inputs}) => { - const {selectedProject, selectedDataset, selectedDataType} = workflowSelectionValues; - - if (!selectedDataset || !selectedWorkflow) { - message.error(`Missing ${selectedDataset ? "workflow" : "dataset"} selection; cannot submit run!`); + onSubmit={({selectedWorkflow, inputs}) => { + if (!selectedWorkflow) { + message.error(`Missing workflow selection; cannot submit run!`); return; } dispatch(submitIngestionWorkflowRun( selectedWorkflow.service_base_url, - selectedProject, - selectedDataset, - selectedDataType, selectedWorkflow, inputs, "/admin/data/manager/runs", diff --git a/src/components/manager/RunSetupInputForm.js b/src/components/manager/RunSetupInputForm.js index 271d6fe4c..1be9b07ad 100644 --- a/src/components/manager/RunSetupInputForm.js +++ b/src/components/manager/RunSetupInputForm.js @@ -1,7 +1,7 @@ -import React, {useCallback} from "react"; +import React, { forwardRef, useCallback, useEffect, useState } from "react"; import PropTypes from "prop-types"; -import { Button, Checkbox, Form, Icon, Input, Select } from "antd"; +import { Button, Checkbox, Form, Icon, Input, Select, Spin } from "antd"; import { FORM_LABEL_COL, @@ -17,6 +17,45 @@ import DatasetTreeSelect, { ID_FORMAT_PROJECT_DATASET } from "./DatasetTreeSelec import DropBoxTreeSelect from "./DropBoxTreeSelect"; +const EnumSelect = forwardRef(({ mode, onChange, values: valuesConfig, value }, ref) => { + const isUrl = typeof valuesConfig === "string"; + + const [values, setValues] = useState(isUrl ? [] : valuesConfig); + const [fetching, setFetching] = useState(false); + + useEffect(() => { + if (isUrl) { + setFetching(true); + fetch(valuesConfig) + .then(r => r.json()) + .then(data => { + if (Array.isArray(data)) { + setValues(data); + } + setFetching(false); + }) + .catch(err => { + console.error(err); + setValues([]); + setFetching(false); + }); + } + }, [isUrl]); + + return ( + + ); +}); + + const getInputComponentAndOptions = ({ id, type, pattern, values, required, repeatable }) => { const dropBoxTreeNodeEnabled = ({ name, contents }) => contents !== undefined || !pattern || (new RegExp(pattern)).test(name); @@ -49,10 +88,7 @@ const getInputComponentAndOptions = ({ id, type, pattern, values, required, repe const mode = (isArray && !repeatable) ? "multiple" : "default"; // TODO: enum[] - need to be able to reselect if repeatable - return [ - , - options, - ]; + return [, options]; } case "file": diff --git a/src/components/manager/RunSetupInputsTable.js b/src/components/manager/RunSetupInputsTable.js index 4cb506ae6..976f8210c 100644 --- a/src/components/manager/RunSetupInputsTable.js +++ b/src/components/manager/RunSetupInputsTable.js @@ -1,41 +1,59 @@ import React, {useMemo} from "react"; +import { useSelector } from "react-redux"; import PropTypes from "prop-types"; import { Table } from "antd"; import { EM_DASH } from "../../constants"; -const RUN_SETUP_INPUTS_COLUMNS = [ - { - title: "ID", - dataIndex: "id", - render: iID => {iID}, - }, - { - title: "Value", - dataIndex: "value", - render: value => - value === undefined - ? EM_DASH - : ( - value instanceof Array - ?
    {value.map(v =>
  • {v.toString()}
    • )}
- : value.toString() - ), - }, -]; - const RunSetupInputsTable = ({ selectedWorkflow, inputs }) => { + const projectsByID = useSelector(state => state.projects.itemsByID); + const datasetsByID = useSelector((state) => state.projects.datasetsByID); + + const columns = useMemo(() => [ + { + title: "ID", + dataIndex: "id", + render: iID => {iID}, + }, + { + title: "Value", + dataIndex: "value", + render: (value, input) => { + if (value === undefined) { + return EM_DASH; + } + + if (input.inputConfig.type === "project:dataset") { + const [projectID, datasetID] = value.split(":"); + return
+ Project: {projectsByID[projectID]?.title}
+ Dataset: {datasetsByID[datasetID]?.title} +
; + } + + if (Array.isArray(value)) { + return
    + {value.map(v =>
  • {v.toString()}
    • )} +
; + } + + return value.toString(); + }, + }, + ], []); + const dataSource = useMemo( () => selectedWorkflow.inputs - .filter(i => !(i.hidden ?? false)) - .map(i => ({ id: i.id, value: inputs[i.id] })), + .filter(i => !(i.hidden ?? false) && !i.injected) + .map(i => ({ id: i.id, value: inputs[i.id], inputConfig: i })), [inputs]); + return ( diff --git a/src/components/manager/RunSetupWizard.js b/src/components/manager/RunSetupWizard.js index 24cfebe35..2c6509c93 100644 --- a/src/components/manager/RunSetupWizard.js +++ b/src/components/manager/RunSetupWizard.js @@ -12,9 +12,11 @@ import { STEP_INPUT, STEP_CONFIRM, } from "./workflowCommon"; +import WorkflowSelection from "./WorkflowSelection"; +import { workflowTypePropType } from "../../propTypes"; const RunSetupWizard = ({ - workflowSelection, + workflowType, workflowSelectionTitle, workflowSelectionDescription, confirmDisplay, @@ -25,17 +27,15 @@ const RunSetupWizard = ({ const [step, setStep] = useState(STEP_WORKFLOW_SELECTION); const [selectedWorkflow, setSelectedWorkflow] = useState(null); - // Extra values (tables etc. for ingestion) - const [workflowSelectionValues, setWorkflowSelectionValues] = useState({}); - const [inputs, setInputs] = useState({}); + const [initialWorkflowFilterValues, setInitialWorkflowFilterValues] = useState(undefined); const [initialInputValues, setInitialInputValues] = useState({}); const [inputFormFields, setInputFormFields] = useState({}); useEffect(() => { const { step: newStep, - workflowSelectionValues: newWorkflowSelectionValues, + initialWorkflowFilterValues: newInitialWorkflowFilterValues, selectedWorkflow: newSelectedWorkflow, initialInputValues: newInitialInputValues, } = location?.state ?? {}; @@ -43,8 +43,8 @@ const RunSetupWizard = ({ if (newStep !== undefined) { setStep(newStep); } - if (newWorkflowSelectionValues !== undefined) { - setWorkflowSelectionValues(newWorkflowSelectionValues); + if (newInitialWorkflowFilterValues !== undefined) { + setInitialWorkflowFilterValues(newInitialWorkflowFilterValues); } if (newSelectedWorkflow !== undefined) { setSelectedWorkflow(newSelectedWorkflow); @@ -65,17 +65,17 @@ const RunSetupWizard = ({ }, []); const handleRunWorkflow = useCallback(() => { - onSubmit({ - workflowSelectionValues, - selectedWorkflow, - inputs, - }); + onSubmit({ selectedWorkflow, inputs }); }, [selectedWorkflow, inputs]); const getStepContents = useCallback(() => { switch (step) { case STEP_WORKFLOW_SELECTION: - return workflowSelection({workflowSelectionValues, setWorkflowSelectionValues, handleWorkflowClick}); + return ; case STEP_INPUT: return setStep(STEP_WORKFLOW_SELECTION)} />; case STEP_CONFIRM: - return confirmDisplay({selectedWorkflow, workflowSelectionValues, inputs, handleRunWorkflow}); + return confirmDisplay({ selectedWorkflow, inputs, handleRunWorkflow }); default: return
; } }, [ step, - workflowSelectionValues, inputs, selectedWorkflow, initialInputValues, @@ -125,7 +124,7 @@ const RunSetupWizard = ({ ; }; RunSetupWizard.propTypes = { - workflowSelection: PropTypes.func, + workflowType: workflowTypePropType, workflowSelectionTitle: PropTypes.oneOfType([PropTypes.string, PropTypes.node]), workflowSelectionDescription: PropTypes.oneOfType([PropTypes.string, PropTypes.node]), confirmDisplay: PropTypes.func, diff --git a/src/components/manager/WorkflowListItem.js b/src/components/manager/WorkflowListItem.js index d8ed4ebec..4efe0f717 100644 --- a/src/components/manager/WorkflowListItem.js +++ b/src/components/manager/WorkflowListItem.js @@ -7,15 +7,7 @@ import {nop} from "../../utils/misc"; import {workflowPropTypesShape} from "../../propTypes"; const TYPE_TAG_DISPLAY = { - file: { - color: "volcano", - icon: "file", - }, - enum: { - color: "blue", - icon: "menu", - }, - number: { // TODO: Break into int and float? + number: { color: "green", icon: "number", }, @@ -23,6 +15,26 @@ const TYPE_TAG_DISPLAY = { color: "purple", icon: "font-size", }, + boolean: { + color: "cyan", + icon: "check-square", + }, + enum: { + color: "blue", + icon: "menu", + }, + "project:dataset": { + color: "magenta", + icon: "database", + }, + file: { + color: "volcano", + icon: "file", + }, + directory: { + color: "orange", + icon: "folder", + }, }; const ioTagWithType = (id, ioType, typeContent = "") => ( @@ -38,18 +50,30 @@ const WorkflowListItem = ({onClick, workflow}) => { const typeTag = dt ? {dt} : null; const inputTags = inputs - .filter(i => !i.hidden) // Filter out hidden (often injected/FROM_CONFIG) inputs - .map(i => ioTagWithType(i.id, i.type, i.type.startsWith("file") ? i.extensions.join(" / ") : "")); + .filter(i => !i.hidden && !i.injected) // Filter out hidden (often injected/FROM_CONFIG) inputs + .map(i => + ioTagWithType( + i.id, + i.type, + i.type.startsWith("file") + ? i.extensions ? (i.extensions.join(" / ")) : (i.pattern ?? "") + : "", + )); const selectable = !!onClick; // Can be selected if a click handler exists return (onClick || nop)()}> - {typeTag} {name} - - : {typeTag} {name}} + ? (onClick || nop)()} style={{ display: "flex" }}> + + {name} + + {typeTag} + : + {name} + {typeTag} + } description={description || ""} /> diff --git a/src/components/manager/WorkflowSelection.js b/src/components/manager/WorkflowSelection.js new file mode 100644 index 000000000..446b80de1 --- /dev/null +++ b/src/components/manager/WorkflowSelection.js @@ -0,0 +1,117 @@ +import React, { useCallback, useEffect, useMemo, useState } from "react"; +import { useSelector } from "react-redux"; +import PropTypes from "prop-types"; + +import { Col, Form, Input, List, Row, Select, Skeleton, Spin } from "antd"; +import WorkflowListItem from "./WorkflowListItem"; + +import { workflowsStateToPropsMixin, workflowTypePropType } from "../../propTypes"; +import { FORM_LABEL_COL, FORM_WRAPPER_COL } from "./workflowCommon"; + +const filterValuesPropType = PropTypes.shape({ + text: PropTypes.string, + tags: PropTypes.arrayOf(PropTypes.string), +}); + +const WorkflowFilter = ({ loading, tags, value, onChange }) => { + const onChangeText = useCallback(e => onChange({ ...value, text: e.target.value }), [onChange]); + const onChangeTags = useCallback(tags => onChange({ ...value, tags }), [onChange]); + + return +
+ + + + + + ; +}; +WorkflowFilter.propTypes = { + loading: PropTypes.bool, + tags: PropTypes.arrayOf(PropTypes.string), + value: filterValuesPropType, +}; + +const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowClick }) => { + const { workflows, workflowsLoading } = useSelector(workflowsStateToPropsMixin); + + const workflowsOfType = workflows[workflowType] ?? []; + const tags = useMemo( + () => Array.from(new Set(workflowsOfType.map(w => w.data_type))), + [workflowsOfType], + ); + + const [filterValues, setFilterValues] = useState({ + text: "", + tags: [], + }); + + useEffect(() => { + if (filterValues.text === "" && !filterValues.tags.length) { + setFilterValues(initialFilterValues); + } + }, [initialFilterValues]); + + /** @type {React.ReactNode[]} */ + const workflowItems = useMemo( + () => { + const ftLower = filterValues.text.toLowerCase().trim(); + const ftTags = filterValues.tags; + + return workflowsOfType + .filter(w => + ( + !ftLower || + w.name.toLowerCase().includes(ftLower) || + w.description.toLowerCase().includes(ftLower) || + w.data_type.includes(ftLower) + ) && (ftTags.length === 0 || ftTags.includes(w.data_type)) + ) // TODO: tags too, properly + .map(w => + handleWorkflowClick(w)} + />, + ) + }, + [workflowsOfType, filterValues], + ); + + return + + + + + + {workflowsLoading + ? + : {workflowItems}} + + + ; +}; +WorkflowSelection.propTypes = { + workflowType: workflowTypePropType.isRequired, + initialFilterValues: filterValuesPropType, + handleWorkflowClick: PropTypes.func, +}; + +export default WorkflowSelection; diff --git a/src/components/manager/runs/RunRequest.js b/src/components/manager/runs/RunRequest.js index 3dc56e985..846fabc7b 100644 --- a/src/components/manager/runs/RunRequest.js +++ b/src/components/manager/runs/RunRequest.js @@ -9,37 +9,14 @@ import ReactJson from "react-json-view"; import WorkflowListItem from "../WorkflowListItem"; const RunRequest = ({run}) => { - const projectsById = useSelector((state) => state.projects.itemsByID); - const details = run?.details; if (!details) return
; const runDataType = details.request.tags.workflow_metadata.data_type; - const {project_id: projectId, dataset_id: datasetId} = details.request.tags; - - const project = projectsById[projectId] ?? null; - const dataset = project ? (project.datasets.find(d => d.identifier === datasetId) ?? null) : null; - - const projectTitle = project?.title ?? null; - const datasetTitle = dataset?.title ?? null; - - const projectIdFragment = {projectId}; - const datasetIdFragment = {datasetId}; - return - {project !== null && ( - - {projectTitle ? <>{projectTitle} ({projectIdFragment}) : projectIdFragment} - - )} - {dataset !== null && ( - - {datasetTitle ? <>{datasetTitle} ({datasetIdFragment}) : datasetIdFragment} - - )} - {dataset !== null && ( + {runDataType && ( {runDataType} diff --git a/src/components/manager/workflowCommon.js b/src/components/manager/workflowCommon.js index 1e98f8946..6f43409c2 100644 --- a/src/components/manager/workflowCommon.js +++ b/src/components/manager/workflowCommon.js @@ -1,10 +1,9 @@ -export const FORM_LABEL_COL = {md: {span: 24}, lg: {span: 4}, xl: {span: 6}, xxl: {span: 7}}; -export const FORM_WRAPPER_COL = {md: {span: 24}, lg: {span: 16}, xl: {span: 12}, xxl: {span: 10}}; +export const FORM_LABEL_COL = {md: {span: 24}, lg: {span: 4}, xl: {span: 6}}; +export const FORM_WRAPPER_COL = {md: {span: 24}, lg: {span: 16}, xl: {span: 12}}; export const FORM_BUTTON_COL = { md: {span: 24}, lg: {offset: 4, span: 16}, xl: {offset: 6, span: 12}, - xxl: {offset: 7, span: 10}, }; export const STEP_WORKFLOW_SELECTION = 0; diff --git a/src/propTypes.js b/src/propTypes.js index 801bbb0ff..9f4738682 100644 --- a/src/propTypes.js +++ b/src/propTypes.js @@ -198,6 +198,8 @@ export const workflowPropTypesShape = PropTypes.shape({ })), }); +export const workflowTypePropType = PropTypes.oneOf(["ingestion", "analysis", "export"]); + // Any components which include workflowStateToPropsMixin should include this as well in their prop types. export const workflowsStateToPropsMixinPropTypes = { workflows: PropTypes.shape({ From 492e7453e40c6da81cc7f4dedee496a7b8abd7ad Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 25 Oct 2023 10:11:58 -0400 Subject: [PATCH 09/26] lint --- src/components/manager/ManagerAnalysisContent.js | 14 +++++++------- src/components/manager/ManagerIngestionContent.js | 2 +- src/components/manager/RunSetupInputForm.js | 6 ++++++ src/components/manager/WorkflowSelection.js | 5 +++-- src/components/manager/runs/RunRequest.js | 1 - 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/components/manager/ManagerAnalysisContent.js b/src/components/manager/ManagerAnalysisContent.js index 6d29cd49d..8923ae63c 100644 --- a/src/components/manager/ManagerAnalysisContent.js +++ b/src/components/manager/ManagerAnalysisContent.js @@ -1,15 +1,15 @@ import React from "react"; -import {useDispatch, useSelector} from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; +import { useHistory } from "react-router-dom"; import PropTypes from "prop-types"; -import { Button, Form, List } from "antd"; +import { Button, Form, List, message } from "antd"; +import { submitAnalysisWorkflowRun } from "../../modules/wes/actions"; import WorkflowListItem from "./WorkflowListItem"; -import {FORM_BUTTON_COL, FORM_LABEL_COL, FORM_WRAPPER_COL} from "./workflowCommon"; -import {useHistory} from "react-router-dom"; import RunSetupWizard from "./RunSetupWizard"; import RunSetupInputsTable from "./RunSetupInputsTable"; -import {submitAnalysisWorkflowRun} from "../../modules/wes/actions"; +import { FORM_BUTTON_COL, FORM_LABEL_COL, FORM_WRAPPER_COL } from "./workflowCommon"; const AnalysisConfirmDisplay = ({selectedWorkflow, inputs, handleRunWorkflow}) => { const isSubmittingAnalysisRun = useSelector(state => state.runs.isSubmittingAnalysisRun); @@ -27,7 +27,7 @@ const AnalysisConfirmDisplay = ({selectedWorkflow, inputs, handleRunWorkflow}) = {/* TODO: Back button like the last one */} : null} - + <> {/* Funny hack to make the type warning for multipe children in a Form.Item go away */} + {onBack ? : null} + + , ]} ; From a83f9a72536dccdb641a44a5e930d74992c23e46 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Mon, 13 Nov 2023 22:15:58 -0500 Subject: [PATCH 11/26] fix: issues with ingestion --- src/components/manager/DatasetTreeSelect.js | 10 +++--- src/components/manager/WorkflowSelection.js | 2 +- src/modules/wes/actions.js | 34 +++++++++------------ 3 files changed, 20 insertions(+), 26 deletions(-) diff --git a/src/components/manager/DatasetTreeSelect.js b/src/components/manager/DatasetTreeSelect.js index f28f46c87..2fb206201 100644 --- a/src/components/manager/DatasetTreeSelect.js +++ b/src/components/manager/DatasetTreeSelect.js @@ -1,4 +1,4 @@ -import React, {useCallback, useEffect, useMemo, useState} from "react"; +import React, {forwardRef, useCallback, useEffect, useMemo, useState} from "react"; import {useSelector} from "react-redux"; import PropTypes from "prop-types"; @@ -7,7 +7,7 @@ import {Spin, TreeSelect} from "antd"; export const ID_FORMAT_PROJECT_DATASET = "project:dataset"; export const ID_FORMAT_DATASET = "dataset"; -const DatasetTreeSelect = ({value, onChange, style, idFormat}) => { +const DatasetTreeSelect = forwardRef(({value, onChange, style, idFormat}, ref) => { const {items: projectItems, isFetching: projectsFetching} = useSelector((state) => state.projects); const servicesFetching = useSelector((state) => state.services.isFetchingAll); @@ -20,8 +20,7 @@ const DatasetTreeSelect = ({value, onChange, style, idFormat}) => { const onChangeInner = useCallback((newSelected) => { if (!value) setSelected(newSelected); if (onChange) { - const [project, dataset] = newSelected.split(":"); - onChange(project, dataset); + onChange(newSelected); } }, [value, onChange, selected]); @@ -42,6 +41,7 @@ const DatasetTreeSelect = ({value, onChange, style, idFormat}) => { return { treeDefaultExpandAll={true} /> ; -}; +}); DatasetTreeSelect.defaultProps = { idFormat: ID_FORMAT_PROJECT_DATASET, diff --git a/src/components/manager/WorkflowSelection.js b/src/components/manager/WorkflowSelection.js index bddb707f1..fe2952058 100644 --- a/src/components/manager/WorkflowSelection.js +++ b/src/components/manager/WorkflowSelection.js @@ -64,7 +64,7 @@ const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowCl }); useEffect(() => { - if (filterValues.text === "" && !filterValues.tags.length) { + if (filterValues.text === "" && !filterValues.tags.length && initialFilterValues) { setFilterValues(initialFilterValues); } }, [initialFilterValues]); diff --git a/src/modules/wes/actions.js b/src/modules/wes/actions.js index f4ee8bf98..2be14fb8c 100644 --- a/src/modules/wes/actions.js +++ b/src/modules/wes/actions.js @@ -130,26 +130,20 @@ export const submitWorkflowRun = networkAction( }); -export const submitIngestionWorkflowRun = - (serviceBaseUrl, projectID, datasetID, dataType, workflow, inputs, redirect, hist) => - (dispatch) => - dispatch(submitWorkflowRun( - SUBMIT_INGESTION_RUN, - serviceBaseUrl, - workflow, - {projectID, datasetID, dataType}, // params - inputs, - { // tags - project_id: projectID, - dataset_id: datasetID, - data_type: dataType, - }, - run => { // onSuccess - message.success(`Ingestion with run ID "${run.run_id}" submitted!`); - if (redirect) hist.push(redirect); - }, - "Error submitting ingestion workflow", // errorMessage - )); +export const submitIngestionWorkflowRun = (serviceBaseUrl, workflow, inputs, redirect, hist) => (dispatch) => + dispatch(submitWorkflowRun( + SUBMIT_INGESTION_RUN, + serviceBaseUrl, + workflow, + {}, // params + inputs, + {}, // tags + run => { // onSuccess + message.success(`Ingestion with run ID "${run.run_id}" submitted!`); + if (redirect) hist.push(redirect); + }, + "Error submitting ingestion workflow", // errorMessage + )); export const submitAnalysisWorkflowRun = (serviceBaseUrl, workflow, inputs, redirect, hist) => (dispatch) => From 5af17599d3ec956917f42c744a0ae5df59b3880e Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 14 Nov 2023 11:35:32 -0500 Subject: [PATCH 12/26] chore: implement drop-box ingestion for new workflow system --- .../explorer/IndividualExperiments.js | 2 +- src/components/explorer/IndividualTracks.js | 2 +- .../manager/DatasetSelectionModal.js | 47 -------- .../manager/ManagerDropBoxContent.js | 112 ++++++++---------- src/components/manager/RunSetupInputForm.js | 5 +- src/components/manager/WorkflowListItem.js | 12 +- src/components/manager/WorkflowSelection.js | 12 +- src/modules/drs/actions.js | 2 +- src/utils/files.js | 37 ++++++ src/utils/guessFileType.js | 15 --- 10 files changed, 111 insertions(+), 135 deletions(-) delete mode 100644 src/components/manager/DatasetSelectionModal.js create mode 100644 src/utils/files.js delete mode 100644 src/utils/guessFileType.js diff --git a/src/components/explorer/IndividualExperiments.js b/src/components/explorer/IndividualExperiments.js index 5e7871eda..2abd1695f 100644 --- a/src/components/explorer/IndividualExperiments.js +++ b/src/components/explorer/IndividualExperiments.js @@ -7,7 +7,7 @@ import { Button, Descriptions, Icon, Popover, Table, Tooltip, Typography } from import { experimentPropTypesShape, experimentResultPropTypesShape, individualPropTypesShape } from "../../propTypes"; import { getFileDownloadUrlsFromDrs } from "../../modules/drs/actions"; -import { guessFileType } from "../../utils/guessFileType"; +import { guessFileType } from "../../utils/files"; import { useDeduplicatedIndividualBiosamples, useIndividualResources } from "./utils"; import { VIEWABLE_FILE_EXTENSIONS } from "../display/FileDisplay"; diff --git a/src/components/explorer/IndividualTracks.js b/src/components/explorer/IndividualTracks.js index af5ee81eb..f9883f155 100644 --- a/src/components/explorer/IndividualTracks.js +++ b/src/components/explorer/IndividualTracks.js @@ -9,7 +9,7 @@ import { BENTO_PUBLIC_URL, BENTO_URL } from "../../config"; import { individualPropTypesShape } from "../../propTypes"; import { getIgvUrlsFromDrs } from "../../modules/drs/actions"; import { setIgvPosition } from "../../modules/explorer/actions"; -import { guessFileType } from "../../utils/guessFileType"; +import { guessFileType } from "../../utils/files"; import {useDeduplicatedIndividualBiosamples} from "./utils"; const SQUISHED_CALL_HEIGHT = 10; diff --git a/src/components/manager/DatasetSelectionModal.js b/src/components/manager/DatasetSelectionModal.js deleted file mode 100644 index a57932b38..000000000 --- a/src/components/manager/DatasetSelectionModal.js +++ /dev/null @@ -1,47 +0,0 @@ -import React, {useCallback, useState} from "react"; -import PropTypes from "prop-types"; - -import {Form, Modal} from "antd"; - -import DatasetTreeSelect from "./DatasetTreeSelect"; - -import {nop} from "../../utils/misc"; - -const WIDTH_100 = {width: "100%"}; - -const DatasetSelectionModal = ({dataType, title, visible, onCancel, onOk}) => { - - const [selectedProject, setSelectedProject] = useState(undefined); - const [selectedDataset, setSelectedDataset] = useState(undefined); - - const onChangeInner = useCallback((project, dataset) => { - setSelectedProject(project); - setSelectedDataset(dataset); - }, []); - - const onOkInner = useCallback( - () => (onOk || nop)(selectedProject, selectedDataset, dataType), - [onOk, selectedProject, selectedDataset, dataType], - ); - - return -
- - - - - ; -}; - -DatasetSelectionModal.propTypes = { - dataType: PropTypes.string, - title: PropTypes.string, - visible: PropTypes.bool, - onCancel: PropTypes.func, - onOk: PropTypes.func, -}; - -export default DatasetSelectionModal; diff --git a/src/components/manager/ManagerDropBoxContent.js b/src/components/manager/ManagerDropBoxContent.js index ef79787fe..b302b7b69 100644 --- a/src/components/manager/ManagerDropBoxContent.js +++ b/src/components/manager/ManagerDropBoxContent.js @@ -1,10 +1,10 @@ -import React, {useCallback, useEffect, useMemo, useRef, useState} from "react"; -import {useDispatch, useSelector} from "react-redux"; -import {useHistory} from "react-router-dom"; +import React, { useCallback, useEffect, useMemo, useRef, useState } from "react"; +import { useDispatch, useSelector } from "react-redux"; +import { useHistory } from "react-router-dom"; import PropTypes from "prop-types"; -import {filesize} from "filesize"; +import { filesize } from "filesize"; import { Alert, @@ -26,17 +26,17 @@ import { message, } from "antd"; -import {LAYOUT_CONTENT_STYLE} from "../../styles/layoutContent"; +import { LAYOUT_CONTENT_STYLE } from "../../styles/layoutContent"; import DownloadButton from "../DownloadButton"; import DropBoxTreeSelect from "./DropBoxTreeSelect"; -import DatasetSelectionModal from "./DatasetSelectionModal"; import FileModal from "../display/FileModal"; -import {BENTO_DROP_BOX_FS_BASE_PATH} from "../../config"; -import {STEP_INPUT} from "./workflowCommon"; +import { BENTO_DROP_BOX_FS_BASE_PATH } from "../../config"; +import { STEP_INPUT } from "./workflowCommon"; import { workflowsStateToPropsMixin } from "../../propTypes"; -import {useResourcePermissions} from "../../lib/auth/utils"; -import {getFalse} from "../../utils/misc"; +import { useResourcePermissions } from "../../lib/auth/utils"; +import { testFileAgainstPattern } from "../../utils/files"; +import { getFalse } from "../../utils/misc"; import { beginDropBoxPuttingObjects, endDropBoxPuttingObjects, @@ -44,8 +44,8 @@ import { putDropBoxObject, deleteDropBoxObject, } from "../../modules/manager/actions"; -import {RESOURCE_EVERYTHING} from "../../lib/auth/resources"; -import {deleteDropBox, ingestDropBox} from "../../lib/auth/permissions"; +import { RESOURCE_EVERYTHING } from "../../lib/auth/resources"; +import { deleteDropBox, ingestDropBox } from "../../lib/auth/permissions"; import { VIEWABLE_FILE_EXTENSIONS } from "../display/FileDisplay"; @@ -290,6 +290,9 @@ const ManagerDropBoxContent = () => { const dropBoxService = useSelector(state => state.services.dropBoxService); const {tree, isFetching: treeLoading, isDeleting} = useSelector(state => state.dropBox); const ingestionWorkflows = useSelector(state => workflowsStateToPropsMixin(state).workflows.ingestion); + const ingestionWorkflowsByID = useMemo( + () => Object.fromEntries(ingestionWorkflows.map((iw) => [iw.id, iw])), + [ingestionWorkflows]); const filesByPath = useMemo(() => Object.fromEntries( recursivelyFlattenFileTree([], tree).map(f => [f.relativePath, f])), [tree]); @@ -311,9 +314,6 @@ const ManagerDropBoxContent = () => { const [fileDeleteModal, setFileDeleteModal] = useState(false); const [fileDeleteModalTitle, setFileDeleteModalTitle] = useState(""); // cache to allow close animation - const [selectedWorkflow, setSelectedWorkflow] = useState(null); - const [datasetSelectionModal, setDatasetSelectionModal] = useState(false); - const showUploadModal = useCallback(() => setUploadModal(true), []); const hideUploadModal = useCallback(() => setUploadModal(false), []); const showFileInfoModal = useCallback(() => setFileInfoModal(true), []); @@ -321,39 +321,37 @@ const ManagerDropBoxContent = () => { const showFileContentsModal = useCallback(() => setFileContentsModal(true), []); const hideFileContentsModal = useCallback(() => setFileContentsModal(false), []); - const showDatasetSelectionModal = useCallback(workflow => { - setSelectedWorkflow(workflow); - setDatasetSelectionModal(true); - }, []); - const hideDatasetSelectionModal = useCallback(() => setDatasetSelectionModal(false), []); - const getWorkflowFit = useCallback(w => { let workflowSupported = true; - let filesLeft = [...selectedEntries]; + let entriesLeft = [...selectedEntries]; + const inputs = {}; - for (const i of w.inputs.filter(i => i.type.startsWith("file"))) { - const isFileArray = i.type.endsWith("[]"); + for (const i of w.inputs) { + const isArray = i.type.endsWith("[]"); + const isFileType = i.type.startsWith("file"); + const isDirType = i.type.startsWith("directory"); - // Find datasets that support the data type - // TODO + if (!isFileType && !isDirType) { + continue; // Nothing for us to do with non-file/directory inputs + } - // Find files where 1+ of the valid extensions (e.g. jpeg or jpg) match. - const compatibleFiles = filesLeft.filter(f => !!i.extensions.find(e => f.toLowerCase().endsWith(e))); - if (compatibleFiles.length === 0) { + // Find compatible entries which match the specified pattern if one is given. + const compatEntries = entriesLeft + .filter(e => (isFileType ? !e.contents : e.contents !== undefined) + && testFileAgainstPattern(e, i.pattern)); + if (compatEntries.length === 0) { workflowSupported = false; break; } - // Steal the first compatible file, or all if it's an array - const filesToTake = filesLeft.filter(f => - isFileArray ? compatibleFiles.includes(f) : f === compatibleFiles[0]); - - inputs[i.id] = BENTO_DROP_BOX_FS_BASE_PATH + (isFileArray ? filesToTake : filesToTake[0]); - filesLeft = filesLeft.filter(f => !filesToTake.includes(f)); + // Steal the first compatible entry, or all if it's an array + const entriesToTake = entriesLeft.filter(e => isArray ? compatEntries.includes(e) : e === compatEntries[0]); + inputs[i.id] = BENTO_DROP_BOX_FS_BASE_PATH + (isArray ? entriesToTake : entriesToTake[0]); + entriesLeft = entriesLeft.filter(f => !entriesToTake.includes(f)); } - if (filesLeft.length > 0) { + if (entriesLeft.length > 0) { // If there are unclaimed files remaining at the end, the workflow is not compatible with the // total selection of files. workflowSupported = false; @@ -362,33 +360,33 @@ const ManagerDropBoxContent = () => { return [workflowSupported, inputs]; }, [selectedEntries]); - const ingestIntoDataset = useCallback((project, dataset, dataType) => { + const continueToIngestion = useCallback((selectedWorkflow, initialInputValues) => { history.push("/admin/data/manager/ingestion", { step: STEP_INPUT, - workflowSelectionValues: { - selectedProject: project, - selectedDataset: dataset, - selectedDataType: dataType, + initialWorkflowFilterValues: { + text: "", + tags: [...selectedWorkflow.tags], }, selectedWorkflow, - initialInputValues: getWorkflowFit(selectedWorkflow)[1], + initialInputValues, }); - }, [history, selectedWorkflow]); + }, [history]); const handleViewFile = useCallback(() => { showFileContentsModal(); }, []); const workflowsSupported = useMemo( - () => ingestionWorkflows.filter(w => getWorkflowFit(w)[0]), - [ingestionWorkflows]); + () => Object.fromEntries(ingestionWorkflows.map(w => [w.id, getWorkflowFit(w)])), + [ingestionWorkflows, getWorkflowFit]); + const workflowMenuItemClick = useCallback( + (i) => continueToIngestion(ingestionWorkflowsByID[i.key], workflowsSupported[i.key][1]), + [ingestionWorkflowsByID, continueToIngestion, workflowsSupported]); const workflowMenu = ( {ingestionWorkflows.map(w => ( - w2.id === w.id) === -1} - onClick={() => showDatasetSelectionModal(w)}> + Ingest with Workflow “{w.name}” ))} @@ -396,9 +394,11 @@ const ManagerDropBoxContent = () => { ); const handleIngest = useCallback(() => { - if (workflowsSupported.length !== 1) return; - showDatasetSelectionModal(workflowsSupported[0]); - }, [workflowsSupported]); + const wfs = Object.entries(workflowsSupported).filter(([_, ws]) => ws[0]); + if (wfs.length !== 1) return; + const [wfID, wfSupportedTuple] = wfs[0]; + continueToIngestion(ingestionWorkflowsByID[wfID], wfSupportedTuple[1]); + }, [ingestionWorkflowsByID, workflowsSupported, continueToIngestion]); const hasUploadPermission = permissions.includes(ingestDropBox); const hasDeletePermission = permissions.includes(deleteDropBox); @@ -475,7 +475,7 @@ const ManagerDropBoxContent = () => { const uploadDisabled = !selectedFolder || !hasUploadPermission; // TODO: at least one ingest:data on all datasets vvv const ingestIntoDatasetDisabled = !dropBoxService || - selectedEntries.length === 0 || workflowsSupported.length === 0; + selectedEntries.length === 0 || Object.values(workflowsSupported).filter((w) => w[0]).length === 0; const handleUpload = useCallback(() => { if (selectedFolder) setInitialUploadFolder(selectedEntries[0]); @@ -503,14 +503,6 @@ const ManagerDropBoxContent = () => { onCancel={hideUploadModal} /> - - { @@ -64,7 +65,7 @@ EnumSelect.propTypes = { const getInputComponentAndOptions = ({ id, type, pattern, values, required, repeatable }) => { const dropBoxTreeNodeEnabled = ({ name, contents }) => - contents !== undefined || !pattern || (new RegExp(pattern)).test(name); + contents !== undefined || testFileAgainstPattern(name, pattern); const options = { // Default to requiring the field unless the "required" property is set on the input @@ -183,7 +184,7 @@ RunSetupInputForm.propTypes = { export default Form.create({ name: "run_setup_input_form", mapPropsToFields: ({workflow, formValues}) => - Object.fromEntries(workflow.inputs.map(i => [i.id, Form.createFormField({...formValues[i.id]})])), + Object.fromEntries((workflow?.inputs ?? []).map(i => [i.id, Form.createFormField({...formValues[i.id]})])), onFieldsChange: ({onChange}, _, allFields) => { onChange({...allFields}); }, diff --git a/src/components/manager/WorkflowListItem.js b/src/components/manager/WorkflowListItem.js index 4efe0f717..63888e741 100644 --- a/src/components/manager/WorkflowListItem.js +++ b/src/components/manager/WorkflowListItem.js @@ -44,7 +44,10 @@ const ioTagWithType = (id, ioType, typeContent = "") => ( ); -const WorkflowListItem = ({onClick, workflow}) => { +const FLEX_1 = { flex: 1 }; +const MARGIN_LEFT_1EM = { marginLeft: "1em" }; + +const WorkflowListItem = ({ onClick, workflow, rightAlignedTags }) => { const {inputs, name, description, data_type: dt} = workflow; const typeTag = dt ? {dt} : null; @@ -62,16 +65,18 @@ const WorkflowListItem = ({onClick, workflow}) => { const selectable = !!onClick; // Can be selected if a click handler exists + const workflowNameStyle = rightAlignedTags ? FLEX_1 : MARGIN_LEFT_1EM; + return (onClick || nop)()} style={{ display: "flex" }}> - + {name} {typeTag} : - {name} + {name} {typeTag} } description={description || ""} @@ -94,6 +99,7 @@ WorkflowListItem.propTypes = { workflow: workflowPropTypesShape, selectable: PropTypes.bool, onClick: PropTypes.func, + rightAlignedTags: PropTypes.bool, }; export default WorkflowListItem; diff --git a/src/components/manager/WorkflowSelection.js b/src/components/manager/WorkflowSelection.js index fe2952058..2aff109fa 100644 --- a/src/components/manager/WorkflowSelection.js +++ b/src/components/manager/WorkflowSelection.js @@ -49,6 +49,11 @@ WorkflowFilter.propTypes = { onChange: PropTypes.func, }; +const INITIAL_FILTER_STATE = { + text: "", + tags: [], +}; + const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowClick }) => { const { workflows, workflowsLoading } = useSelector(workflowsStateToPropsMixin); @@ -58,14 +63,11 @@ const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowCl [workflowsOfType], ); - const [filterValues, setFilterValues] = useState({ - text: "", - tags: [], - }); + const [filterValues, setFilterValues] = useState(INITIAL_FILTER_STATE); useEffect(() => { if (filterValues.text === "" && !filterValues.tags.length && initialFilterValues) { - setFilterValues(initialFilterValues); + setFilterValues({...INITIAL_FILTER_STATE, filterValues}); } }, [initialFilterValues]); diff --git a/src/modules/drs/actions.js b/src/modules/drs/actions.js index 7a5cffc64..b6cef78e5 100644 --- a/src/modules/drs/actions.js +++ b/src/modules/drs/actions.js @@ -2,7 +2,7 @@ import { createNetworkActionTypes, networkAction, } from "../../utils/actions"; -import { guessFileType } from "../../utils/guessFileType"; +import { guessFileType } from "../../utils/files"; import {message} from "antd"; export const PERFORM_SEARCH_BY_FUZZYNAME = createNetworkActionTypes("PERFORM_SEARCH_BY_FUZZYNAME"); diff --git a/src/utils/files.js b/src/utils/files.js new file mode 100644 index 000000000..45d6ebf03 --- /dev/null +++ b/src/utils/files.js @@ -0,0 +1,37 @@ +// file type guesses for igv files, for cases where this information is missing +export const guessFileType = (filename) => { + if (filename.toLowerCase().endsWith(".vcf.gz")) { + return ("vcf"); + } + if (filename.toLowerCase().endsWith(".cram")) { + return ("cram"); + } + if (filename.toLowerCase().endsWith(".bw") || filename.toLowerCase().endsWith(".bigwig")) { + return "bigwig"; + } + + // expand here accordingly + return null; +}; + +/** @type {Object.} */ +const FILE_TEST_REGEX_CACHE = {}; + +const _getFileRegExp = (pattern) => { + if (pattern in FILE_TEST_REGEX_CACHE) { + return FILE_TEST_REGEX_CACHE[pattern]; + } + + const r = new RegExp(pattern, "i"); + FILE_TEST_REGEX_CACHE[pattern] = r; + return r; +}; + +export const testFileAgainstPattern = (fileName, pattern) => { + if (!pattern) { + // No pattern => everything matches + return true; + } + const r = _getFileRegExp(pattern); + return r.test(fileName); +}; diff --git a/src/utils/guessFileType.js b/src/utils/guessFileType.js deleted file mode 100644 index f71182013..000000000 --- a/src/utils/guessFileType.js +++ /dev/null @@ -1,15 +0,0 @@ -// file type guesses for igv files, for cases where this information is missing -export const guessFileType = (filename) => { - if (filename.toLowerCase().endsWith(".vcf.gz")) { - return ("vcf"); - } - if (filename.toLowerCase().endsWith(".cram")) { - return ("cram"); - } - if (filename.toLowerCase().endsWith(".bw") || filename.toLowerCase().endsWith(".bigwig")) { - return "bigwig"; - } - - // expand here accordingly - return null; -}; From 95e401e0baa4d791e8dcd38b5722fb4a0acb6100 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 14 Nov 2023 14:59:11 -0500 Subject: [PATCH 13/26] style: fix styling for workflow tags --- src/components/manager/ManagerWorkflowsContent.js | 5 ++++- src/components/manager/WorkflowListItem.js | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/components/manager/ManagerWorkflowsContent.js b/src/components/manager/ManagerWorkflowsContent.js index d6885380b..ec71484ec 100644 --- a/src/components/manager/ManagerWorkflowsContent.js +++ b/src/components/manager/ManagerWorkflowsContent.js @@ -32,7 +32,10 @@ const ManagerWorkflowsContent = () => { {workflowsLoading ? : - {items.map(w => )}} + {items.map(w => ( + + ))} + } ))} diff --git a/src/components/manager/WorkflowListItem.js b/src/components/manager/WorkflowListItem.js index 63888e741..3ad5e56c7 100644 --- a/src/components/manager/WorkflowListItem.js +++ b/src/components/manager/WorkflowListItem.js @@ -45,7 +45,7 @@ const ioTagWithType = (id, ioType, typeContent = "") => ( ); const FLEX_1 = { flex: 1 }; -const MARGIN_LEFT_1EM = { marginLeft: "1em" }; +const MARGIN_RIGHT_1EM = { marginRight: "1em" }; const WorkflowListItem = ({ onClick, workflow, rightAlignedTags }) => { const {inputs, name, description, data_type: dt} = workflow; @@ -65,7 +65,7 @@ const WorkflowListItem = ({ onClick, workflow, rightAlignedTags }) => { const selectable = !!onClick; // Can be selected if a click handler exists - const workflowNameStyle = rightAlignedTags ? FLEX_1 : MARGIN_LEFT_1EM; + const workflowNameStyle = rightAlignedTags ? FLEX_1 : MARGIN_RIGHT_1EM; return Date: Tue, 14 Nov 2023 17:27:34 -0500 Subject: [PATCH 14/26] chore: support new ingest flow in dataset data type list --- src/components/datasets/Dataset.js | 4 +- src/components/datasets/DatasetDataTypes.js | 234 ++++++++++-------- .../manager/ManagerDropBoxContent.js | 24 +- src/components/manager/projects/Project.js | 3 - .../manager/projects/RoutedProject.js | 17 +- src/components/manager/workflowCommon.js | 18 ++ 6 files changed, 154 insertions(+), 146 deletions(-) diff --git a/src/components/datasets/Dataset.js b/src/components/datasets/Dataset.js index 127de6ae3..1a69eda45 100644 --- a/src/components/datasets/Dataset.js +++ b/src/components/datasets/Dataset.js @@ -138,8 +138,7 @@ class Dataset extends Component { isPrivate={isPrivate} />, data_types: , + isPrivate={isPrivate} />, linked_field_sets: ( <> @@ -295,7 +294,6 @@ Dataset.propTypes = { value: datasetPropTypesShape, onEdit: PropTypes.func, - onDatasetIngest: PropTypes.func, addLinkedFieldSet: PropTypes.func, deleteProjectDataset: PropTypes.func, diff --git a/src/components/datasets/DatasetDataTypes.js b/src/components/datasets/DatasetDataTypes.js index 319bd00ce..6fc84f1b7 100644 --- a/src/components/datasets/DatasetDataTypes.js +++ b/src/components/datasets/DatasetDataTypes.js @@ -1,130 +1,152 @@ import React, {useCallback, useMemo, useState} from "react"; import { useSelector, useDispatch } from "react-redux"; -import { Button, Col, Row, Table, Typography } from "antd"; - import PropTypes from "prop-types"; -import { datasetPropTypesShape, projectPropTypesShape } from "../../propTypes"; + +import { Button, Col, Dropdown, Icon, Menu, Row, Table, Typography } from "antd"; + +import { useStartIngestionFlow } from "../manager/workflowCommon"; +import { datasetPropTypesShape, projectPropTypesShape, workflowsStateToPropsMixin } from "../../propTypes"; import { clearDatasetDataType } from "../../modules/metadata/actions"; import { fetchDatasetDataTypesSummariesIfPossible } from "../../modules/datasets/actions"; import genericConfirm from "../ConfirmationModal"; import DataTypeSummaryModal from "./datatype/DataTypeSummaryModal"; -import { nop } from "../../utils/misc"; const NA_TEXT = N/A; -const DatasetDataTypes = React.memo( - ({isPrivate, project, dataset, onDatasetIngest}) => { - const dispatch = useDispatch(); - const datasetDataTypes = useSelector((state) => Object.values( - state.datasetDataTypes.itemsByID[dataset.identifier]?.itemsByID ?? {})); - const datasetSummaries = useSelector((state) => state.datasetSummaries.itemsByID[dataset.identifier]); - const isFetchingDataset = useSelector( - (state) => state.datasetDataTypes.itemsByID[dataset.identifier]?.isFetching); - - const [datatypeSummaryVisible, setDatatypeSummaryVisible] = useState(false); - const [selectedDataType, setSelectedDataType] = useState(null); - - const selectedSummary = datasetSummaries?.data?.[selectedDataType?.id] ?? {}; - - const handleClearDataType = useCallback((dataType) => { - genericConfirm({ - title: `Are you sure you want to delete the "${dataType.label || dataType.id}" data type?`, - content: "Deleting this means all instances of this data type contained in the dataset " + - "will be deleted permanently, and will no longer be available for exploration.", - onOk: async () => { - await dispatch(clearDatasetDataType(dataset.identifier, dataType.id)); - await dispatch(fetchDatasetDataTypesSummariesIfPossible(dataset.identifier)); - }, - }); - }, [dispatch, dataset]); +const DatasetDataTypes = React.memo(({isPrivate, project, dataset}) => { + const dispatch = useDispatch(); + const datasetDataTypes = useSelector((state) => Object.values( + state.datasetDataTypes.itemsByID[dataset.identifier]?.itemsByID ?? {})); + const datasetSummaries = useSelector((state) => state.datasetSummaries.itemsByID[dataset.identifier]); + const isFetchingDataset = useSelector( + (state) => state.datasetDataTypes.itemsByID[dataset.identifier]?.isFetching); - const showDataTypeSummary = useCallback((dataType) => { - setSelectedDataType(dataType); - setDatatypeSummaryVisible(true); - }, []); + const ingestionWorkflows = useSelector(state => workflowsStateToPropsMixin(state).workflows.ingestion); - const dataTypesColumns = useMemo(() => [ - { - title: "Name", - key: "label", - render: (dt) => - isPrivate ? ( - showDataTypeSummary(dt)}> - {dt.label ?? NA_TEXT} - - ) : dt.label ?? NA_TEXT, - defaultSortOrder: "ascend", - sorter: (a, b) => a.label.localeCompare(b.label), + const [datatypeSummaryVisible, setDatatypeSummaryVisible] = useState(false); + const [selectedDataType, setSelectedDataType] = useState(null); + + const selectedSummary = datasetSummaries?.data?.[selectedDataType?.id] ?? {}; + + const handleClearDataType = useCallback((dataType) => { + genericConfirm({ + title: `Are you sure you want to delete the "${dataType.label || dataType.id}" data type?`, + content: "Deleting this means all instances of this data type contained in the dataset " + + "will be deleted permanently, and will no longer be available for exploration.", + onOk: async () => { + await dispatch(clearDatasetDataType(dataset.identifier, dataType.id)); + await dispatch(fetchDatasetDataTypesSummariesIfPossible(dataset.identifier)); }, + }); + }, [dispatch, dataset]); + + const showDataTypeSummary = useCallback((dataType) => { + setSelectedDataType(dataType); + setDatatypeSummaryVisible(true); + }, []); + + const startIngestionFlow = useStartIngestionFlow(); + + const dataTypesColumns = useMemo(() => [ + { + title: "Name", + key: "label", + render: (dt) => + isPrivate ? ( + showDataTypeSummary(dt)}> + {dt.label ?? NA_TEXT} + + ) : dt.label ?? NA_TEXT, + defaultSortOrder: "ascend", + sorter: (a, b) => a.label.localeCompare(b.label), + }, + { + title: "Count", + dataIndex: "count", + render: (c) => (c ?? NA_TEXT), + }, + ...(isPrivate ? [ { - title: "Count", - dataIndex: "count", - render: (c) => (c ?? NA_TEXT), - }, - ...(isPrivate ? [ - { - title: "Actions", - key: "actions", - width: 230, - render: (dt) => ( - -
- - - - - - - ), + + ); + + return ( + + + {ingestDropdown} + + + + + + ); }, - ] : null), - ], [isPrivate, project, dataset, onDatasetIngest]); - - const onDataTypeSummaryModalCancel = useCallback(() => setDatatypeSummaryVisible(false), []); - - return ( - <> - - - - Data Types - - -
- - ); - }); + }, + ] : null), + ], [isPrivate, project, dataset, ingestionWorkflows, startIngestionFlow]); + + const onDataTypeSummaryModalCancel = useCallback(() => setDatatypeSummaryVisible(false), []); + + return ( + <> + + + + Data Types + + +
+ + ); +}); DatasetDataTypes.propTypes = { isPrivate: PropTypes.bool, project: projectPropTypesShape, dataset: datasetPropTypesShape, - onDatasetIngest: PropTypes.func, }; export default DatasetDataTypes; diff --git a/src/components/manager/ManagerDropBoxContent.js b/src/components/manager/ManagerDropBoxContent.js index b302b7b69..a59ab7367 100644 --- a/src/components/manager/ManagerDropBoxContent.js +++ b/src/components/manager/ManagerDropBoxContent.js @@ -1,6 +1,5 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useDispatch, useSelector } from "react-redux"; -import { useHistory } from "react-router-dom"; import PropTypes from "prop-types"; @@ -32,9 +31,9 @@ import DropBoxTreeSelect from "./DropBoxTreeSelect"; import FileModal from "../display/FileModal"; import { BENTO_DROP_BOX_FS_BASE_PATH } from "../../config"; -import { STEP_INPUT } from "./workflowCommon"; import { workflowsStateToPropsMixin } from "../../propTypes"; import { useResourcePermissions } from "../../lib/auth/utils"; +import { useStartIngestionFlow } from "./workflowCommon"; import { testFileAgainstPattern } from "../../utils/files"; import { getFalse } from "../../utils/misc"; import { @@ -283,7 +282,6 @@ const DROP_BOX_ROOT_KEY = "/"; const ManagerDropBoxContent = () => { const dispatch = useDispatch(); - const history = useHistory(); const {permissions, hasAttempted} = useResourcePermissions(RESOURCE_EVERYTHING) ?? {}; @@ -360,17 +358,7 @@ const ManagerDropBoxContent = () => { return [workflowSupported, inputs]; }, [selectedEntries]); - const continueToIngestion = useCallback((selectedWorkflow, initialInputValues) => { - history.push("/admin/data/manager/ingestion", { - step: STEP_INPUT, - initialWorkflowFilterValues: { - text: "", - tags: [...selectedWorkflow.tags], - }, - selectedWorkflow, - initialInputValues, - }); - }, [history]); + const startIngestionFlow = useStartIngestionFlow(); const handleViewFile = useCallback(() => { showFileContentsModal(); @@ -381,8 +369,8 @@ const ManagerDropBoxContent = () => { [ingestionWorkflows, getWorkflowFit]); const workflowMenuItemClick = useCallback( - (i) => continueToIngestion(ingestionWorkflowsByID[i.key], workflowsSupported[i.key][1]), - [ingestionWorkflowsByID, continueToIngestion, workflowsSupported]); + (i) => startIngestionFlow(ingestionWorkflowsByID[i.key], workflowsSupported[i.key][1]), + [ingestionWorkflowsByID, startIngestionFlow, workflowsSupported]); const workflowMenu = ( {ingestionWorkflows.map(w => ( @@ -397,8 +385,8 @@ const ManagerDropBoxContent = () => { const wfs = Object.entries(workflowsSupported).filter(([_, ws]) => ws[0]); if (wfs.length !== 1) return; const [wfID, wfSupportedTuple] = wfs[0]; - continueToIngestion(ingestionWorkflowsByID[wfID], wfSupportedTuple[1]); - }, [ingestionWorkflowsByID, workflowsSupported, continueToIngestion]); + startIngestionFlow(ingestionWorkflowsByID[wfID], wfSupportedTuple[1]); + }, [ingestionWorkflowsByID, workflowsSupported, startIngestionFlow]); const hasUploadPermission = permissions.includes(ingestDropBox); const hasDeletePermission = permissions.includes(deleteDropBox); diff --git a/src/components/manager/projects/Project.js b/src/components/manager/projects/Project.js index ed1cfcddb..896e499ad 100644 --- a/src/components/manager/projects/Project.js +++ b/src/components/manager/projects/Project.js @@ -141,7 +141,6 @@ class Project extends Component { project={this.props.value} value={d} onEdit={() => (this.props.onEditDataset || nop)(d)} - onDatasetIngest={this.props.onDatasetIngest || nop} /> , @@ -201,8 +200,6 @@ Project.propTypes = { onAddDataset: PropTypes.func, onEditDataset: PropTypes.func, onAddJsonSchema: PropTypes.func, - - onDatasetIngest: PropTypes.func, }; export default Project; diff --git a/src/components/manager/projects/RoutedProject.js b/src/components/manager/projects/RoutedProject.js index 73c686ca7..8fc0f926c 100644 --- a/src/components/manager/projects/RoutedProject.js +++ b/src/components/manager/projects/RoutedProject.js @@ -29,7 +29,6 @@ class RoutedProject extends Component { this.showDatasetAdditionModal = this.showDatasetAdditionModal.bind(this); this.hideDatasetAdditionModal = this.hideDatasetAdditionModal.bind(this); this.hideDatasetEditModal = this.hideDatasetEditModal.bind(this); - this.ingestIntoDataset = this.ingestIntoDataset.bind(this); this.handleDeleteProject = this.handleDeleteProject.bind(this); } @@ -40,19 +39,6 @@ class RoutedProject extends Component { } } - ingestIntoDataset(p, d, dt) { - this.props.history.push( - "/admin/data/manager/ingestion", - { - workflowSelectionValues: { - selectedProject: p.identifier, - selectedDataset: d.identifier, - selectedDataType: dt.id, - }, - }, - ); - } - handleProjectSave(project) { // TODO: Form validation for project this.props.saveProjectIfPossible(project); @@ -135,8 +121,7 @@ class RoutedProject extends Component { selectedDataset: dataset, datasetEditModal: true, })} - onAddJsonSchema={() => this.setJsonSchemaModalVisible(true)} - onDatasetIngest={(p, d, dt) => this.ingestIntoDataset(p, d, dt)}/> + onAddJsonSchema={() => this.setJsonSchemaModalVisible(true)} /> ; } } diff --git a/src/components/manager/workflowCommon.js b/src/components/manager/workflowCommon.js index 6f43409c2..33add55dd 100644 --- a/src/components/manager/workflowCommon.js +++ b/src/components/manager/workflowCommon.js @@ -1,3 +1,6 @@ +import { useHistory } from "react-router-dom"; +import { useCallback } from "react"; + export const FORM_LABEL_COL = {md: {span: 24}, lg: {span: 4}, xl: {span: 6}}; export const FORM_WRAPPER_COL = {md: {span: 24}, lg: {span: 16}, xl: {span: 12}}; export const FORM_BUTTON_COL = { @@ -9,3 +12,18 @@ export const FORM_BUTTON_COL = { export const STEP_WORKFLOW_SELECTION = 0; export const STEP_INPUT = 1; export const STEP_CONFIRM = 2; + +export const useStartIngestionFlow = () => { + const history = useHistory(); + return useCallback((selectedWorkflow, initialInputValues) => { + history.push("/admin/data/manager/ingestion", { + step: STEP_INPUT, + initialWorkflowFilterValues: { + text: "", + tags: [...selectedWorkflow.tags], + }, + selectedWorkflow, + initialInputValues, + }); + }, [history]); +}; From 4b54d1e4b72b6fd86a95589e7ab0c6fa7897f788 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 15 Nov 2023 10:09:00 -0500 Subject: [PATCH 15/26] fix: typo in routed project --- src/components/manager/projects/RoutedProject.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/manager/projects/RoutedProject.js b/src/components/manager/projects/RoutedProject.js index 8fc0f926c..b5cd8bfc9 100644 --- a/src/components/manager/projects/RoutedProject.js +++ b/src/components/manager/projects/RoutedProject.js @@ -160,7 +160,7 @@ const mapStateToProps = state => ({ projects: state.projects.items, projectsByID: state.projects.itemsByID, - loadingProjects: state.projects.isAdding || state.projects.isFetching, + loadingProjects: state.projects.isCreating || state.projects.isFetching, isDeletingProject: state.projects.isDeleting, }); From d0e70b2172c36943492108e28f2f77d3254d9055 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Wed, 15 Nov 2023 10:12:50 -0500 Subject: [PATCH 16/26] fix(manager): runsetupinputstable usememo with wrong deps --- src/components/manager/RunSetupInputsTable.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/components/manager/RunSetupInputsTable.js b/src/components/manager/RunSetupInputsTable.js index 976f8210c..b36e59633 100644 --- a/src/components/manager/RunSetupInputsTable.js +++ b/src/components/manager/RunSetupInputsTable.js @@ -22,6 +22,7 @@ const RunSetupInputsTable = ({ selectedWorkflow, inputs }) => { return EM_DASH; } + // TODO: link these to new tab: manager page on project/dataset (when we can route datasets) if (input.inputConfig.type === "project:dataset") { const [projectID, datasetID] = value.split(":"); return
@@ -39,7 +40,7 @@ const RunSetupInputsTable = ({ selectedWorkflow, inputs }) => { return value.toString(); }, }, - ], []); + ], [projectsByID, datasetsByID]); const dataSource = useMemo( () => selectedWorkflow.inputs From c1dc766b7c87297ffcd30fea5c0002b3c940d24f Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Thu, 16 Nov 2023 14:54:53 -0500 Subject: [PATCH 17/26] style: tweak style for dataset data type table --- src/components/datasets/DatasetDataTypes.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/components/datasets/DatasetDataTypes.js b/src/components/datasets/DatasetDataTypes.js index 6fc84f1b7..3dbbbe331 100644 --- a/src/components/datasets/DatasetDataTypes.js +++ b/src/components/datasets/DatasetDataTypes.js @@ -69,7 +69,7 @@ const DatasetDataTypes = React.memo(({isPrivate, project, dataset}) => { { title: "Actions", key: "actions", - width: 250, + width: 240, render: (dt) => { const dtIngestionWorkflows = ingestionWorkflows .filter((wf) => wf.data_type === dt.id || (wf.tags ?? []).includes(dt.id)); @@ -87,7 +87,7 @@ const DatasetDataTypes = React.memo(({isPrivate, project, dataset}) => { ); const ingestDropdown = ( - + @@ -134,6 +134,8 @@ const DatasetDataTypes = React.memo(({isPrivate, project, dataset}) => {
Date: Thu, 16 Nov 2023 14:57:30 -0500 Subject: [PATCH 18/26] fix(manager): correctly reset workflow inputs when changing workflow --- src/components/manager/RunSetupWizard.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/components/manager/RunSetupWizard.js b/src/components/manager/RunSetupWizard.js index 2c6509c93..b14e97079 100644 --- a/src/components/manager/RunSetupWizard.js +++ b/src/components/manager/RunSetupWizard.js @@ -55,9 +55,16 @@ const RunSetupWizard = ({ }, [location]); const handleWorkflowClick = useCallback((workflow) => { - setSelectedWorkflow(workflow); + if (workflow.id !== selectedWorkflow?.id) { + // If we had pre-defined initial values / form values, but we change the workflow, reset these inputs. + setInitialInputValues({}); + setInputFormFields({}); + + // Change to the new selected workflow + setSelectedWorkflow(workflow); + } setStep(STEP_INPUT); - }, []); + }, [selectedWorkflow]); const handleInputSubmit = useCallback(inputs => { setInputs(inputs); From 395022af56184c5d2e77b852e69d3d5e2cccb975 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Fri, 17 Nov 2023 13:43:35 -0500 Subject: [PATCH 19/26] fix(manager): properly incorporate all workflow tags into tag-set --- src/components/manager/WorkflowSelection.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/components/manager/WorkflowSelection.js b/src/components/manager/WorkflowSelection.js index 2aff109fa..01c7a7892 100644 --- a/src/components/manager/WorkflowSelection.js +++ b/src/components/manager/WorkflowSelection.js @@ -59,7 +59,7 @@ const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowCl const workflowsOfType = workflows[workflowType] ?? []; const tags = useMemo( - () => Array.from(new Set(workflowsOfType.map(w => w.data_type))), + () => Array.from(new Set(workflowsOfType.flatMap(w => [w.data_type, ...(w.tags ?? [])]))), [workflowsOfType], ); @@ -78,14 +78,19 @@ const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowCl const ftTags = filterValues.tags; return workflowsOfType - .filter(w => - ( + .filter(w => { + const wTags = new Set(w.tags ?? []); + return ( !ftLower || w.name.toLowerCase().includes(ftLower) || w.description.toLowerCase().includes(ftLower) || - w.data_type.includes(ftLower) - ) && (ftTags.length === 0 || ftTags.includes(w.data_type)), - ) // TODO: tags too, properly + w.data_type.includes(ftLower) || + wTags.has(ftLower) + ) && ( + ftTags.length === 0 || + ftTags.reduce((acc, t) => acc && wTags.has(t), true) + ); + }) .map(w => Date: Tue, 21 Nov 2023 12:52:54 -0500 Subject: [PATCH 20/26] fix(manager): add missing deps to useCallbacks in WorkflowFilter --- src/components/manager/WorkflowSelection.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/manager/WorkflowSelection.js b/src/components/manager/WorkflowSelection.js index 01c7a7892..c54464085 100644 --- a/src/components/manager/WorkflowSelection.js +++ b/src/components/manager/WorkflowSelection.js @@ -14,8 +14,8 @@ const filterValuesPropType = PropTypes.shape({ }); const WorkflowFilter = ({ loading, tags, value, onChange }) => { - const onChangeText = useCallback(e => onChange({ ...value, text: e.target.value }), [onChange]); - const onChangeTags = useCallback(tags => onChange({ ...value, tags }), [onChange]); + const onChangeText = useCallback(e => onChange({ ...value, text: e.target.value }), [value, onChange]); + const onChangeTags = useCallback(tags => onChange({ ...value, tags }), [value, onChange]); return From 9a43e0146fea266ca42a89f6f671a9a8c3c94077 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 21 Nov 2023 12:58:25 -0500 Subject: [PATCH 21/26] refact: turn workflow mixin into a hook w/ byID access too lint: clean up prop types file --- src/components/datasets/DatasetDataTypes.js | 9 ++- .../manager/ManagerDropBoxContent.js | 10 +-- .../manager/ManagerIngestionContent.js | 2 - .../manager/ManagerWorkflowsContent.js | 17 ++--- src/components/manager/WorkflowSelection.js | 10 +-- src/hooks.js | 32 +++++++++ src/propTypes.js | 71 ------------------- 7 files changed, 56 insertions(+), 95 deletions(-) create mode 100644 src/hooks.js diff --git a/src/components/datasets/DatasetDataTypes.js b/src/components/datasets/DatasetDataTypes.js index 3dbbbe331..3456aaddf 100644 --- a/src/components/datasets/DatasetDataTypes.js +++ b/src/components/datasets/DatasetDataTypes.js @@ -4,16 +4,18 @@ import PropTypes from "prop-types"; import { Button, Col, Dropdown, Icon, Menu, Row, Table, Typography } from "antd"; +import { useWorkflows } from "../../hooks"; import { useStartIngestionFlow } from "../manager/workflowCommon"; -import { datasetPropTypesShape, projectPropTypesShape, workflowsStateToPropsMixin } from "../../propTypes"; +import { datasetPropTypesShape, projectPropTypesShape } from "../../propTypes"; import { clearDatasetDataType } from "../../modules/metadata/actions"; import { fetchDatasetDataTypesSummariesIfPossible } from "../../modules/datasets/actions"; + import genericConfirm from "../ConfirmationModal"; import DataTypeSummaryModal from "./datatype/DataTypeSummaryModal"; const NA_TEXT = N/A; -const DatasetDataTypes = React.memo(({isPrivate, project, dataset}) => { +const DatasetDataTypes = React.memo(({ isPrivate, project, dataset }) => { const dispatch = useDispatch(); const datasetDataTypes = useSelector((state) => Object.values( state.datasetDataTypes.itemsByID[dataset.identifier]?.itemsByID ?? {})); @@ -21,7 +23,8 @@ const DatasetDataTypes = React.memo(({isPrivate, project, dataset}) => { const isFetchingDataset = useSelector( (state) => state.datasetDataTypes.itemsByID[dataset.identifier]?.isFetching); - const ingestionWorkflows = useSelector(state => workflowsStateToPropsMixin(state).workflows.ingestion); + const { workflowsByType } = useWorkflows(); + const ingestionWorkflows = workflowsByType.ingestion.items; const [datatypeSummaryVisible, setDatatypeSummaryVisible] = useState(false); const [selectedDataType, setSelectedDataType] = useState(null); diff --git a/src/components/manager/ManagerDropBoxContent.js b/src/components/manager/ManagerDropBoxContent.js index a59ab7367..eda0ec78c 100644 --- a/src/components/manager/ManagerDropBoxContent.js +++ b/src/components/manager/ManagerDropBoxContent.js @@ -31,7 +31,6 @@ import DropBoxTreeSelect from "./DropBoxTreeSelect"; import FileModal from "../display/FileModal"; import { BENTO_DROP_BOX_FS_BASE_PATH } from "../../config"; -import { workflowsStateToPropsMixin } from "../../propTypes"; import { useResourcePermissions } from "../../lib/auth/utils"; import { useStartIngestionFlow } from "./workflowCommon"; import { testFileAgainstPattern } from "../../utils/files"; @@ -47,6 +46,7 @@ import { RESOURCE_EVERYTHING } from "../../lib/auth/resources"; import { deleteDropBox, ingestDropBox } from "../../lib/auth/permissions"; import { VIEWABLE_FILE_EXTENSIONS } from "../display/FileDisplay"; +import { useWorkflows } from "../../hooks"; const DROP_BOX_CONTENT_CONTAINER_STYLE = { display: "flex", flexDirection: "column", gap: 8 }; const DROP_BOX_ACTION_CONTAINER_STYLE = { @@ -287,10 +287,10 @@ const ManagerDropBoxContent = () => { const dropBoxService = useSelector(state => state.services.dropBoxService); const {tree, isFetching: treeLoading, isDeleting} = useSelector(state => state.dropBox); - const ingestionWorkflows = useSelector(state => workflowsStateToPropsMixin(state).workflows.ingestion); - const ingestionWorkflowsByID = useMemo( - () => Object.fromEntries(ingestionWorkflows.map((iw) => [iw.id, iw])), - [ingestionWorkflows]); + + const { workflowsByType } = useWorkflows(); + const ingestionWorkflows = workflowsByType.ingestion.items; + const ingestionWorkflowsByID = workflowsByType.ingestion.itemsByID; const filesByPath = useMemo(() => Object.fromEntries( recursivelyFlattenFileTree([], tree).map(f => [f.relativePath, f])), [tree]); diff --git a/src/components/manager/ManagerIngestionContent.js b/src/components/manager/ManagerIngestionContent.js index cf1050439..e35fa5f9b 100644 --- a/src/components/manager/ManagerIngestionContent.js +++ b/src/components/manager/ManagerIngestionContent.js @@ -15,7 +15,6 @@ import { FORM_BUTTON_COL, } from "./workflowCommon"; -import { workflowTarget } from "../../propTypes"; import RunSetupWizard from "./RunSetupWizard"; import RunSetupInputsTable from "./RunSetupInputsTable"; @@ -53,7 +52,6 @@ const IngestConfirmDisplay = ({selectedWorkflow, inputs, handleRunWorkflow}) => ); }; IngestConfirmDisplay.propTypes = { - target: workflowTarget, selectedWorkflow: PropTypes.object, inputs: PropTypes.object, handleRunWorkflow: PropTypes.func, diff --git a/src/components/manager/ManagerWorkflowsContent.js b/src/components/manager/ManagerWorkflowsContent.js index ec71484ec..457c7bd23 100644 --- a/src/components/manager/ManagerWorkflowsContent.js +++ b/src/components/manager/ManagerWorkflowsContent.js @@ -1,13 +1,11 @@ import React from "react"; -import {useSelector} from "react-redux"; import {Layout, List, Skeleton, Spin, Tabs, Typography} from "antd"; - import WorkflowListItem from "./WorkflowListItem"; -import {LAYOUT_CONTENT_STYLE} from "../../styles/layoutContent"; -import {workflowsStateToPropsMixin} from "../../propTypes"; +import { useWorkflows } from "../../hooks"; +import { LAYOUT_CONTENT_STYLE } from "../../styles/layoutContent"; const workflowTypesToTitles = { ingestion: "Ingestion", @@ -16,16 +14,15 @@ const workflowTypesToTitles = { }; const ManagerWorkflowsContent = () => { - // TODO: real key - - const {workflows, workflowsLoading} = useSelector(state => workflowsStateToPropsMixin(state)); + const { workflowsByType, workflowsLoading } = useWorkflows(); + // noinspection JSValidateTypes return - {Object.entries(workflows) + {Object.entries(workflowsByType) .filter(([wt, _]) => wt in workflowTypesToTitles) - .map(([wt, items]) => ( + .map(([wt, { items }]) => ( {workflowTypesToTitles[wt]} Workflows @@ -33,7 +30,7 @@ const ManagerWorkflowsContent = () => { ? : {items.map(w => ( - + ))} } diff --git a/src/components/manager/WorkflowSelection.js b/src/components/manager/WorkflowSelection.js index c54464085..b3ed34e60 100644 --- a/src/components/manager/WorkflowSelection.js +++ b/src/components/manager/WorkflowSelection.js @@ -5,8 +5,9 @@ import PropTypes from "prop-types"; import { Col, Form, Input, List, Row, Select, Skeleton, Spin } from "antd"; import WorkflowListItem from "./WorkflowListItem"; -import { workflowsStateToPropsMixin, workflowTypePropType } from "../../propTypes"; +import { workflowTypePropType } from "../../propTypes"; import { FORM_LABEL_COL, FORM_WRAPPER_COL } from "./workflowCommon"; +import { useWorkflows } from "../../hooks"; const filterValuesPropType = PropTypes.shape({ text: PropTypes.string, @@ -55,11 +56,11 @@ const INITIAL_FILTER_STATE = { }; const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowClick }) => { - const { workflows, workflowsLoading } = useSelector(workflowsStateToPropsMixin); + const { workflowsByType, workflowsLoading } = useWorkflows(); - const workflowsOfType = workflows[workflowType] ?? []; + const workflowsOfType = workflowsByType[workflowType] ?? []; const tags = useMemo( - () => Array.from(new Set(workflowsOfType.flatMap(w => [w.data_type, ...(w.tags ?? [])]))), + () => Array.from(new Set(workflowsOfType.items.flatMap(w => [w.data_type, ...(w.tags ?? [])]))), [workflowsOfType], ); @@ -78,6 +79,7 @@ const WorkflowSelection = ({ workflowType, initialFilterValues, handleWorkflowCl const ftTags = filterValues.tags; return workflowsOfType + .items .filter(w => { const wTags = new Set(w.tags ?? []); return ( diff --git a/src/hooks.js b/src/hooks.js new file mode 100644 index 000000000..2108ba5de --- /dev/null +++ b/src/hooks.js @@ -0,0 +1,32 @@ +import { useMemo } from "react"; +import { useSelector } from "react-redux"; + +export const useWorkflows = () => { + const isFetchingAllServices = useSelector((state) => state.services.isFetchingAll); + const isFetchingServiceWorkflows = useSelector((state) => state.serviceWorkflows.isFetching); + + const workflowsLoading = isFetchingAllServices || isFetchingServiceWorkflows; + + const serviceWorkflows = useSelector((state) => state.serviceWorkflows.items); + + return useMemo(() => { + const workflowsByType = { + ingestion: { items: [], itemsByID: {} }, + analysis: { items: [], itemsByID: {} }, + export: { items: [], itemsByID: {} }, + }; + + Object.entries(serviceWorkflows).forEach(([workflowType, workflowTypeWorkflows]) => { + if (!(workflowType in workflowsByType)) return; + + // noinspection JSCheckFunctionSignatures + Object.entries(workflowTypeWorkflows).forEach(([k, v]) => { + const wf = { ...v, id: k }; + workflowsByType[workflowType].items.push(wf); + workflowsByType[workflowType].itemsByID[k] = wf; + }); + }); + + return { workflowsByType, workflowsLoading }; + }, [serviceWorkflows, workflowsLoading]) +}; diff --git a/src/propTypes.js b/src/propTypes.js index 123f01ab5..8e0fccd9f 100644 --- a/src/propTypes.js +++ b/src/propTypes.js @@ -49,20 +49,6 @@ export const dropBoxTreeStateToPropsMixin = state => ({ treeLoading: state.dropBox.isFetching, }); -// Any components which include dropBoxTreeStateToPropsMixin should include this as well in their prop types. -export const dropBoxTreeStateToPropsMixinPropTypes = { - tree: PropTypes.arrayOf(PropTypes.shape({ - name: PropTypes.string.isRequired, - filePath: PropTypes.string.isRequired, - relativePath: PropTypes.string.isRequired, - uri: PropTypes.string, - lastModified: PropTypes.number, - lastMetadataChange: PropTypes.number, - contents: PropTypes.arrayOf(PropTypes.object), - })), // TODO: This is going to change - treeLoading: PropTypes.bool, -}; - export const linkedFieldSetPropTypesShape = PropTypes.shape({ name: PropTypes.string, fields: PropTypes.objectOf(PropTypes.arrayOf(PropTypes.string)), // TODO: Properties pattern? @@ -146,39 +132,6 @@ export const runPropTypesShape = PropTypes.shape({ // Prop types object shape for a single table summary object. export const summaryPropTypesShape = PropTypes.object; -// Prop types object shape describing the target of a workflow (project, dataset and data-type) -export const workflowTarget = PropTypes.shape({ - selectedProject: PropTypes.string, - selectedDataset: PropTypes.string, - selectedDataType: PropTypes.string, -}); - -// Gives components which include this in their state to props connection access to workflows and loading status. -export const workflowsStateToPropsMixin = state => { - const workflowsByType = { - ingestion: [], - analysis: [], - export: [], - }; - - Object.entries(state.serviceWorkflows.items).forEach(([workflowType, workflowTypeWorkflows]) => { - if (!(workflowType in workflowsByType)) return; - - // noinspection JSCheckFunctionSignatures - workflowsByType[workflowType].push( - ...Object.entries(workflowTypeWorkflows).map(([k, v]) => ({ - ...v, - id: k, - })), - ); - }); - - return { - workflows: workflowsByType, - workflowsLoading: state.services.isFetchingAll || state.serviceWorkflows.isFetching, - }; -}; - // Prop types object shape for a single workflow object. export const workflowPropTypesShape = PropTypes.shape({ id: PropTypes.string, @@ -200,16 +153,6 @@ export const workflowPropTypesShape = PropTypes.shape({ export const workflowTypePropType = PropTypes.oneOf(["ingestion", "analysis", "export"]); -// Any components which include workflowStateToPropsMixin should include this as well in their prop types. -export const workflowsStateToPropsMixinPropTypes = { - workflows: PropTypes.shape({ - ingestion: PropTypes.arrayOf(workflowPropTypesShape), - analysis: PropTypes.arrayOf(workflowPropTypesShape), - export: PropTypes.arrayOf(workflowPropTypesShape), - }), - workflowsLoading: PropTypes.bool, -}; - // Shape of a phenopackets ontology object export const ontologyShape = PropTypes.shape({ id: PropTypes.string, // CURIE ID @@ -370,20 +313,6 @@ export const overviewSummaryPropTypesShape = PropTypes.shape({ }), }); -export const searchAllRecordsPropTypesShape = PropTypes.shape({ - data: PropTypes.shape({ - // TODO: more precision - phenopackets: PropTypes.number, - data_type_specific: PropTypes.shape({ - biosamples: PropTypes.object, - diseases: PropTypes.object, - individuals: PropTypes.object, - phenotypic_features: PropTypes.object, - }), - }), -}); - - // Explorer search results format export const explorerSearchResultsPropTypesShape = PropTypes.shape({ From 419d392a161dd4d9752460359861311fef953fd4 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 21 Nov 2023 13:01:48 -0500 Subject: [PATCH 22/26] chore(manager): document where props come from for getInputComponentAndOptions --- src/components/manager/RunSetupInputForm.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/components/manager/RunSetupInputForm.js b/src/components/manager/RunSetupInputForm.js index ced50c035..bedff66fb 100644 --- a/src/components/manager/RunSetupInputForm.js +++ b/src/components/manager/RunSetupInputForm.js @@ -63,6 +63,10 @@ EnumSelect.propTypes = { }; +// These properties come from the inputs as listed in the WorkflowDefinition in the workflow-providing service. +// For all possible workflow input types, see: +// https://github.com/bento-platform/bento_lib/blob/master/bento_lib/workflows/models.py +// This component is responsible for transforming these workflow input definitions into form elements. const getInputComponentAndOptions = ({ id, type, pattern, values, required, repeatable }) => { const dropBoxTreeNodeEnabled = ({ name, contents }) => contents !== undefined || testFileAgainstPattern(name, pattern); From d32da420115f9ab6ae20c0eb63e0b3f0a8900048 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 21 Nov 2023 13:11:08 -0500 Subject: [PATCH 23/26] refact(manager): factor workflow input tag into component --- src/components/manager/WorkflowListItem.js | 49 +++++++++++++--------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/src/components/manager/WorkflowListItem.js b/src/components/manager/WorkflowListItem.js index 3ad5e56c7..65c2e37eb 100644 --- a/src/components/manager/WorkflowListItem.js +++ b/src/components/manager/WorkflowListItem.js @@ -1,4 +1,4 @@ -import React from "react"; +import React, { useMemo } from "react"; import PropTypes from "prop-types"; import {Icon, List, Tag} from "antd"; @@ -37,31 +37,40 @@ const TYPE_TAG_DISPLAY = { }, }; -const ioTagWithType = (id, ioType, typeContent = "") => ( - -   - {id} ({typeContent || ioType}{ioType.endsWith("[]") ? " array" : ""}) - -); +const WorkflowInputTag = ({ id, type, children }) => { + const display = useMemo(() => TYPE_TAG_DISPLAY[type.replace("[]", "")], [type]); + return ( + +   + {id} ({children || type}{type.endsWith("[]") ? " array" : ""}) + + ); +}; +WorkflowInputTag.propTypes = { + id: PropTypes.string, + type: PropTypes.string, + children: PropTypes.node, +}; const FLEX_1 = { flex: 1 }; const MARGIN_RIGHT_1EM = { marginRight: "1em" }; const WorkflowListItem = ({ onClick, workflow, rightAlignedTags }) => { - const {inputs, name, description, data_type: dt} = workflow; + const { inputs, name, description, data_type: dt } = workflow; const typeTag = dt ? {dt} : null; - const inputTags = inputs - .filter(i => !i.hidden && !i.injected) // Filter out hidden (often injected/FROM_CONFIG) inputs - .map(i => - ioTagWithType( - i.id, - i.type, - i.type.startsWith("file") - ? i.extensions ? (i.extensions.join(" / ")) : (i.pattern ?? "") - : "", - )); + const inputTags = useMemo( + () => + inputs + .filter(i => !i.hidden && !i.injected) // Filter out hidden/injected inputs + .map(({ id, type, pattern }) => ( + + {type.startsWith("file") ? pattern ?? "" : ""} + + )), + [inputs], + ); const selectable = !!onClick; // Can be selected if a click handler exists @@ -82,8 +91,8 @@ const WorkflowListItem = ({ onClick, workflow, rightAlignedTags }) => { description={description || ""} /> -
- Inputs: +
+ Inputs: {inputTags}
From 0c69b6e61765b21bfa84c8033c2ae23df213c9b8 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 21 Nov 2023 13:12:37 -0500 Subject: [PATCH 24/26] lint --- src/components/manager/WorkflowListItem.js | 12 ++++++------ src/components/manager/WorkflowSelection.js | 1 - src/hooks.js | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/components/manager/WorkflowListItem.js b/src/components/manager/WorkflowListItem.js index 65c2e37eb..5aca60738 100644 --- a/src/components/manager/WorkflowListItem.js +++ b/src/components/manager/WorkflowListItem.js @@ -63,12 +63,12 @@ const WorkflowListItem = ({ onClick, workflow, rightAlignedTags }) => { const inputTags = useMemo( () => inputs - .filter(i => !i.hidden && !i.injected) // Filter out hidden/injected inputs - .map(({ id, type, pattern }) => ( - - {type.startsWith("file") ? pattern ?? "" : ""} - - )), + .filter(i => !i.hidden && !i.injected) // Filter out hidden/injected inputs + .map(({ id, type, pattern }) => ( + + {type.startsWith("file") ? pattern ?? "" : ""} + + )), [inputs], ); diff --git a/src/components/manager/WorkflowSelection.js b/src/components/manager/WorkflowSelection.js index b3ed34e60..31c4d05e5 100644 --- a/src/components/manager/WorkflowSelection.js +++ b/src/components/manager/WorkflowSelection.js @@ -1,5 +1,4 @@ import React, { useCallback, useEffect, useMemo, useState } from "react"; -import { useSelector } from "react-redux"; import PropTypes from "prop-types"; import { Col, Form, Input, List, Row, Select, Skeleton, Spin } from "antd"; diff --git a/src/hooks.js b/src/hooks.js index 2108ba5de..137ae113a 100644 --- a/src/hooks.js +++ b/src/hooks.js @@ -28,5 +28,5 @@ export const useWorkflows = () => { }); return { workflowsByType, workflowsLoading }; - }, [serviceWorkflows, workflowsLoading]) + }, [serviceWorkflows, workflowsLoading]); }; From 11f237e90f51ba2506d50b0deb08d35c705ba828 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 21 Nov 2023 13:15:31 -0500 Subject: [PATCH 25/26] refact(manager): rm submit-workflow action params + make tags opt. --- src/modules/wes/actions.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/modules/wes/actions.js b/src/modules/wes/actions.js index 2be14fb8c..6a9c23d3d 100644 --- a/src/modules/wes/actions.js +++ b/src/modules/wes/actions.js @@ -99,11 +99,11 @@ export const fetchRunLogStreamsIfPossibleAndNeeded = runID => (dispatch, getStat export const submitWorkflowRun = networkAction( - (types, serviceBaseUrl, workflow, params, inputs, tags, onSuccess, errorMessage) => (dispatch, getState) => { + (types, serviceBaseUrl, workflow, inputs, onSuccess, errorMessage, tags) => (dispatch, getState) => { const serviceUrlRStrip = serviceBaseUrl.replace(/\/$/, ""); const runRequest = { - workflow_params: Object.fromEntries(Object.entries(inputs) + workflow_params: Object.fromEntries(Object.entries(inputs ?? {}) .map(([k, v]) => [`${workflow.id}.${k}`, v])), workflow_type: "WDL", // TODO: Should eventually not be hard-coded workflow_type_version: "1.0", // TODO: " @@ -112,13 +112,13 @@ export const submitWorkflowRun = networkAction( tags: { workflow_id: workflow.id, workflow_metadata: workflow, - ...tags, + ...(tags ?? {}), }, }; return { types, - params: {request: runRequest, ...params}, + params: { request: runRequest }, url: `${getState().services.wesService.url}/runs`, req: { method: "POST", @@ -135,9 +135,7 @@ export const submitIngestionWorkflowRun = (serviceBaseUrl, workflow, inputs, red SUBMIT_INGESTION_RUN, serviceBaseUrl, workflow, - {}, // params inputs, - {}, // tags run => { // onSuccess message.success(`Ingestion with run ID "${run.run_id}" submitted!`); if (redirect) hist.push(redirect); @@ -151,9 +149,7 @@ export const submitAnalysisWorkflowRun = (serviceBaseUrl, workflow, inputs, redi SUBMIT_ANALYSIS_RUN, serviceBaseUrl, workflow, - {}, // params inputs, - {}, // tags run => { // onSuccess message.success(`Analysis with run ID "${run.run_id}" submitted!`); if (redirect) hist.push(redirect); From 7aed7c4b78bfb3132f6eb1cc912e112056b49364 Mon Sep 17 00:00:00 2001 From: David Lougheed Date: Tue, 21 Nov 2023 16:27:33 -0500 Subject: [PATCH 26/26] fix(manager): make latest ingestion page work again --- src/components/manager/runs/RunLastContent.js | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/src/components/manager/runs/RunLastContent.js b/src/components/manager/runs/RunLastContent.js index 6ffcbd50c..01a167462 100644 --- a/src/components/manager/runs/RunLastContent.js +++ b/src/components/manager/runs/RunLastContent.js @@ -102,34 +102,49 @@ const buildKeyFromRecord = (record) => `${record.dataType}-${record.datasetId}`; const fileNameFromPath = (path) => path.split("/").at(-1); +const namespacedInput = (workflowId, input) => `${workflowId}.${input.id}`; + +const getFirstProjectDatasetInputFromWorkflow = (workflowId, {inputs}) => + inputs + .filter(input => input.type === "project:dataset") + .map(input => namespacedInput(workflowId, input))[0]; + const getFileInputsFromWorkflow = (workflowId, {inputs}) => inputs .filter(input => ["file", "file[]"].includes(input.type)) - .map(input => `${workflowId}.${input.id}`); + .map(input => namespacedInput(workflowId, input)); const processIngestions = (data, currentDatasets) => { const currentDatasetIds = new Set((currentDatasets || []).map((ds) => ds.identifier)); - const ingestionsByDataType = data.reduce((ingestions, run) => { + const ingestions = {}; + + data.forEach((run) => { if (run.state !== "COMPLETE") { - return ingestions; + return; } - const { - workflow_id: workflowId, - workflow_metadata: workflowMetadata, - dataset_id: datasetId, - } = run.details.request.tags; + const workflowParams = run.details.request.workflow_params; + const { workflow_id: workflowId, workflow_metadata: workflowMetadata } = run.details.request.tags; + const projectDatasetKey = getFirstProjectDatasetInputFromWorkflow(workflowId, workflowMetadata); + if (!projectDatasetKey) { + return; + } + const datasetId = workflowParams[projectDatasetKey].split(":")[1]; if (datasetId === undefined || !currentDatasetIds.has(datasetId)) { - return ingestions; + return; + } + + if (!workflowMetadata.data_type) { + return; } const fileNames = getFileInputsFromWorkflow(workflowId ?? workflowMetadata.id, workflowMetadata) .flatMap(key => { - const paramValue = run.details.request.workflow_params[key]; + const paramValue = workflowParams[key]; if (!paramValue) { // Key isn't in workflow params or is null // - possibly optional field or something else going wrong @@ -153,10 +168,9 @@ const processIngestions = (data, currentDatasets) => { } else { ingestions[dataTypeAndDatasetId] = currentIngestion; } - return ingestions; }, {}); - return Object.values(ingestionsByDataType).sort((a, b) => Date.parse(b.date) - Date.parse(a.date)); + return Object.values(ingestions).sort((a, b) => Date.parse(b.date) - Date.parse(a.date)); }; const LastIngestionTable = () => {