diff --git a/src/components/discovery/DataTypeExplorationModal.js b/src/components/discovery/DataTypeExplorationModal.js index ff2b6dac4..dfd439905 100644 --- a/src/components/discovery/DataTypeExplorationModal.js +++ b/src/components/discovery/DataTypeExplorationModal.js @@ -46,6 +46,8 @@ class DataTypeExplorationModal extends Component { } render() { + const filteredDataTypes = this.props.dataTypes || []; + return Table Detail View - {Object.values(this.props.dataTypes).flatMap(ds => (ds.items ?? []) - .filter(dataType => (dataType.queryable ?? true) && dataType.count > 0) - .map(dataType => - - {this.state.view === "tree" ? ( - - ) : ( - <> - this.onFilterChange(e.target.value)} - placeholder="Search for a field..." - style={{marginBottom: "16px"}} - /> - - - )} - , - ))} - + {filteredDataTypes.map(dataType => { + return ( + + {this.state.view === "tree" ? ( + + ) : ( + <> + this.onFilterChange(e.target.value)} + placeholder="Search for a field..." + style={{marginBottom: "16px"}} + /> +
+ + )} + + ); + })} + ; } } DataTypeExplorationModal.propTypes = { - dataTypes: PropTypes.object, // TODO: Shape + dataTypes: PropTypes.array, visible: PropTypes.bool, onCancel: PropTypes.func, }; diff --git a/src/components/discovery/DiscoveryQueryBuilder.js b/src/components/discovery/DiscoveryQueryBuilder.js index 8e4085e39..161702724 100644 --- a/src/components/discovery/DiscoveryQueryBuilder.js +++ b/src/components/discovery/DiscoveryQueryBuilder.js @@ -131,19 +131,25 @@ class DiscoveryQueryBuilder extends Component { } render() { - // Filter out services without data types and then flat-map the service's data types to make the dropdown. + const variantDatasetIds = this.props.serviceTables.items + .filter(item => item.data_type === "variant") + .map(item => item.dataset); + + const { activeDataset, dataTypesByDataset } = this.props; + const dataTypesForActiveDataset = dataTypesByDataset.itemsByDatasetID[activeDataset] || []; + + // The second filter condition is a temporary workaround given that + // gohan_does not yet support the 'queryable' flag for data types by dataset. + // TODO: Remove this once gohan_does supports the 'queryable' flag and variant count by dataset. + const filteredDataTypes = dataTypesForActiveDataset + .filter(dt => (dt.queryable ?? true) && dt.count > 0) + .filter(dt => dt.data_type === "variant" || variantDatasetIds.includes(activeDataset)); + const dataTypeMenu = ( - {this.props.servicesInfo - .filter(s => (this.props.dataTypes[s.id]?.items ?? []).length) - .flatMap(s => - this.props.dataTypes[s.id].items - .filter(dt => (dt.queryable ?? true) && dt.count > 0) - .map(dt => - {dt.label ?? dt.id}, - ), - ) - } + {filteredDataTypes.map(dt => ( + {dt.label ?? dt.id} + ))} ); @@ -169,24 +175,29 @@ class DiscoveryQueryBuilder extends Component { }); const addConditionsOnDataType = (buttonProps = {style: {float: "right"}}) => ( - + ); return Advanced Search {addConditionsOnDataType()} - + {this.props.dataTypeForms.length > 0 @@ -218,6 +229,7 @@ class DiscoveryQueryBuilder extends Component { } DiscoveryQueryBuilder.propTypes = { + activeDataset: PropTypes.string, isInternal: PropTypes.bool, requiredDataTypes: PropTypes.arrayOf(PropTypes.string), @@ -225,6 +237,8 @@ DiscoveryQueryBuilder.propTypes = { dataTypes: PropTypes.object, dataTypesByID: PropTypes.object, dataTypesLoading: PropTypes.bool, + dataTypesByDataset: PropTypes.object, + serviceTables: PropTypes.object, searchLoading: PropTypes.bool, formValues: PropTypes.object, @@ -247,6 +261,8 @@ const mapStateToProps = state => ({ servicesInfo: state.services.items, dataTypes: state.serviceDataTypes.dataTypesByServiceID, dataTypesByID: state.serviceDataTypes.itemsByID, + dataTypesByDataset: state.serviceDataTypes, + serviceTables: state.serviceTables, autoQuery: state.explorer.autoQuery, isFetchingTextSearch: state.explorer.fetchingTextSearch || false, diff --git a/src/components/explorer/ExplorerDatasetSearch.js b/src/components/explorer/ExplorerDatasetSearch.js index 81797b0ba..9ac2f9694 100644 --- a/src/components/explorer/ExplorerDatasetSearch.js +++ b/src/components/explorer/ExplorerDatasetSearch.js @@ -83,6 +83,7 @@ const ExplorerDatasetSearch = () => { Explore Dataset {selectedDataset.title} { const cs = bentoServicesByKind[s.bento?.serviceKind ?? s.type.artifact] ?? {}; @@ -136,6 +135,7 @@ class RoutedProject extends Component { console.log("tll", tableList); + // TODO: Inconsistent schemas const strayTables = [ ...this.props.serviceTables.filter(t2 => diff --git a/src/modules/metadata/actions.js b/src/modules/metadata/actions.js index acfd45674..1001627d8 100644 --- a/src/modules/metadata/actions.js +++ b/src/modules/metadata/actions.js @@ -9,6 +9,8 @@ import { } from "../services/actions"; import {endProjectEditing} from "../manager/actions"; +import {fetchServicesByDataset} from "../services/actions"; + import { createNetworkActionTypes, createFlowActionTypes, @@ -81,6 +83,7 @@ export const fetchProjectsWithDatasetsAndTables = () => async (dispatch, getStat dispatch(beginFlow(FETCHING_PROJECTS_WITH_TABLES)); await dispatch(fetchProjects()); await dispatch(fetchProjectTables(getState().projects.itemsByID)); + await dispatch(fetchServicesByDataset()); dispatch(endFlow(FETCHING_PROJECTS_WITH_TABLES)); }; diff --git a/src/modules/services/actions.js b/src/modules/services/actions.js index b1ddae8dc..2658aa2cd 100644 --- a/src/modules/services/actions.js +++ b/src/modules/services/actions.js @@ -39,6 +39,9 @@ export const DELETING_SERVICE_TABLE = createFlowActionTypes("DELETING_SERVICE_TA export const FETCH_SERVICE_WORKFLOWS = createNetworkActionTypes("FETCH_SERVICE_WORKFLOWS"); export const LOADING_SERVICE_WORKFLOWS = createFlowActionTypes("LOADING_SERVICE_WORKFLOWS"); +export const FETCH_SERVICE_DATA_TYPES_BY_DATASET = createNetworkActionTypes("FETCH_SERVICE_DATA_TYPES_BY_DATASET"); +export const LOADING_SERVICE_DATA_TYPES_BY_DATASET = createFlowActionTypes("LOADING_SERVICE_DATA_TYPES_BY_DATASET"); + export const endAddingServiceTable = (serviceInfo, dataTypeID, table) => ({ type: ADDING_SERVICE_TABLE.END, @@ -75,6 +78,13 @@ export const fetchDataServiceDataTypes = networkAction((serviceInfo) => ({ err: `Error fetching data types from service '${serviceInfo.name}'`, })); +export const fetchDataServiceDataTypesById = networkAction((serviceInfo, datasetID) => ({ + types: FETCH_SERVICE_DATA_TYPES_BY_DATASET, + params: {serviceInfo, datasetID}, + url: `${serviceInfo.url}/data-types?dataset=${encodeURIComponent(datasetID)}`, + err: `Error fetching data types from service '${serviceInfo.name}'`, +})); + export const fetchDataServiceDataTypeTables = networkAction((serviceInfo, dataType) => ({ types: FETCH_SERVICE_TABLES, params: {serviceInfo, dataTypeID: dataType.id}, @@ -144,6 +154,29 @@ export const fetchServicesWithMetadataAndDataTypesAndTables = (onServiceFetchFin dispatch(endFlow(LOADING_ALL_SERVICE_DATA)); }; +export const fetchServicesByDataset = () => async (dispatch, getState) => { + dispatch(beginFlow(LOADING_SERVICE_DATA_TYPES_BY_DATASET)); + const datasetIdentifiers = Object.values(getState().projects.itemsByID).flatMap( + project => project.datasets?.map(d => d.identifier) || []); + + const dataServicesInfo = getState().services.items.filter(s => s?.type && s?.id).map(s => { + const serviceKind = s.bento?.serviceKind ?? s.type.artifact; + return { + ...s, + bentoService: getState().bentoServices.itemsByKind[serviceKind] ?? null, + }; + }).filter(s => s.bentoService?.data_service ?? false); + + if (datasetIdentifiers.length > 0 && dataServicesInfo.length > 0) { + await Promise.all(dataServicesInfo.flatMap(s => + datasetIdentifiers.map(id => dispatch(fetchDataServiceDataTypesById(s, id))), + )); + } + + dispatch(endFlow(LOADING_SERVICE_DATA_TYPES_BY_DATASET)); +}; + + export const fetchServicesWithMetadataAndDataTypesAndTablesIfNeeded = (onServiceFetchFinish) => (dispatch, getState) => { const state = getState(); diff --git a/src/modules/services/reducers.js b/src/modules/services/reducers.js index 7198ce73c..78ad674a6 100644 --- a/src/modules/services/reducers.js +++ b/src/modules/services/reducers.js @@ -9,6 +9,9 @@ import { FETCH_SERVICE_DATA_TYPES, LOADING_SERVICE_DATA_TYPES, + FETCH_SERVICE_DATA_TYPES_BY_DATASET, + LOADING_SERVICE_DATA_TYPES_BY_DATASET, + FETCH_SERVICE_TABLES, LOADING_SERVICE_TABLES, @@ -134,10 +137,12 @@ export const services = ( export const serviceDataTypes = ( state = { isFetchingAll: false, + isFetchingItemsByDatasetID: false, itemsByID: {}, dataTypesByServiceID: {}, dataTypesByServiceArtifact: {}, dataTypesByServiceKind: {}, + itemsByDatasetID: {}, }, action, ) => { @@ -236,11 +241,43 @@ export const serviceDataTypes = ( }; } + // Handle itemsByDatasetID + case LOADING_SERVICE_DATA_TYPES_BY_DATASET.BEGIN: + return {...state, isFetchingItemsByDatasetID: true}; + + case LOADING_SERVICE_DATA_TYPES_BY_DATASET.END: + case LOADING_SERVICE_DATA_TYPES_BY_DATASET.TERMINATE: + return {...state, isFetchingItemsByDatasetID: false}; + + case FETCH_SERVICE_DATA_TYPES_BY_DATASET.RECEIVE: { + const { datasetID, data } = action; + const existingItems = state.itemsByDatasetID[datasetID] || []; + return { + ...state, + itemsByDatasetID: { + ...state.itemsByDatasetID, + [datasetID]: [...existingItems, ...data], + }, + }; + } + + case FETCH_SERVICE_DATA_TYPES_BY_DATASET.ERROR: { + const { datasetID } = action; + return { + ...state, + itemsByDatasetID: { + ...state.itemsByDatasetID, + [datasetID]: state.itemsByDatasetID[datasetID] || [], + }, + }; + } + default: return state; } }; + export const serviceTables = ( state = { isFetchingAll: false,