feat: dataset-level disabling of data type querying by count

src/components/discovery/DataTypeExplorationModal.js

@@ -76,36 +76,41 @@ class DataTypeExplorationModal extends Component {
                     <Radio.Button value="table"><Icon type="table" /> Table Detail View</Radio.Button>
                 </Radio.Group>
                 <Tabs>
-                    {Object.values(this.props.dataTypes).flatMap(ds => (ds.items ?? [])
-                        .filter(dataType => (dataType.queryable ?? true) && dataType.count > 0)
-                        .map(dataType =>
-                            <Tabs.TabPane tab={dataType.label ?? dataType.id} key={dataType.id}>
-                                {this.state.view === "tree" ? (
-                                    <SchemaTree schema={dataType.schema} />
-                                ) : (
-                                    <>
-                                        <Input.Search
-                                            allowClear={true}
-                                            onChange={e => this.onFilterChange(e.target.value)}
-                                            placeholder="Search for a field..."
-                                            style={{marginBottom: "16px"}}
-                                        />
-                                        <Table
-                                            bordered={true}
-                                            columns={FIELD_COLUMNS}
-                                            dataSource={this.getTableData(dataType)} />
-                                    </>
-                                )}
-                            </Tabs.TabPane>,
-                        ))}
-                </Tabs>
+                {this.props.dataTypesByDataset.itemsByDatasetID
+                    .filter(ds => ds.items?.length > 0 && ds.datasetIdentifier === this.props.activeDataset)
+                    .flatMap(ds =>
+                        ds.items
+                            .filter(dataType => (dataType.queryable ?? true) && dataType.count > 0)
+                            .map(dataType =>
+                                <Tabs.TabPane tab={dataType.label ?? dataType.id} key={dataType.id}>
+                                    {this.state.view === "tree" ? (
+                                        <SchemaTree schema={dataType.schema} />
+                                    ) : (
+                                        <>
+                                            <Input.Search
+                                                allowClear={true}
+                                                onChange={e => this.onFilterChange(e.target.value)}
+                                                placeholder="Search for a field..."
+                                                style={{marginBottom: "16px"}}
+                                            />
+                                            <Table
+                                                bordered={true}
+                                                columns={FIELD_COLUMNS}
+                                                dataSource={this.getTableData(dataType)} />
+                                        </>
+                                    )}
+                                </Tabs.TabPane>,
+                            ))}
+            </Tabs>
             </div>
         </Modal>;
     }
 }
 
 DataTypeExplorationModal.propTypes = {
     dataTypes: PropTypes.object,  // TODO: Shape
+    activeDataset: PropTypes.string,
+    dataTypesByDataset: PropTypes.object,  // TODO: Shape
     visible: PropTypes.bool,
     onCancel: PropTypes.func,
 };

src/components/discovery/DiscoveryQueryBuilder.js

@@ -134,18 +134,18 @@ class DiscoveryQueryBuilder extends Component {
         // Filter out services without data types and then flat-map the service's data types to make the dropdown.
         const dataTypeMenu = (
             <Menu onClick={this.handleAddDataTypeQueryForm}>
-                {this.props.servicesInfo
-                    .filter(s => (this.props.dataTypes[s.id]?.items ?? []).length)
+                {this.props.dataTypesByDataset.itemsByDatasetID
+                    .filter(s => s.items?.length > 0 && s.datasetIdentifier === this.props.activeDataset)
                     .flatMap(s =>
-                        this.props.dataTypes[s.id].items
+                        s.items
                             .filter(dt => (dt.queryable ?? true) && dt.count > 0)
                             .map(dt =>
-                                <Menu.Item key={`${s.id}:${dt.id}`}>{dt.label ?? dt.id}</Menu.Item>,
+                                <Menu.Item key={`${s.datasetIdentifier}:${dt.id}`}>{dt.label ?? dt.id}</Menu.Item>,
                             ),
                     )
                 }
-            </Menu>
-        );
+            </Menu>);
+
 
         const dataTypeTabPanes = this.props.dataTypeForms.map(({dataType, formValues}) => {
             // Use data type label for tab name, unless it isn't specified - then fall back to ID.
@@ -178,8 +178,10 @@ class DiscoveryQueryBuilder extends Component {
         return <Card style={{marginBottom: "1.5em"}}>
             <DataTypeExplorationModal
                 dataTypes={this.props.dataTypes}
+                dataTypesByDataset={this.props.dataTypesByDataset}
                 visible={this.state.schemasModalShown}
                 onCancel={this.handleHelpAndSchemasToggle}
+                activeDataset={this.props.activeDataset}
             />
 
             <Typography.Title level={3} style={{marginBottom: "1.5rem"}}>
@@ -218,13 +220,15 @@ class DiscoveryQueryBuilder extends Component {
 }
 
 DiscoveryQueryBuilder.propTypes = {
+    activeDataset: PropTypes.string,
     isInternal: PropTypes.bool,
     requiredDataTypes: PropTypes.arrayOf(PropTypes.string),
 
     servicesInfo: PropTypes.arrayOf(PropTypes.object),
     dataTypes: PropTypes.object,
     dataTypesByID: PropTypes.object,
     dataTypesLoading: PropTypes.bool,
+    dataTypesByDataset: PropTypes.object,
 
     searchLoading: PropTypes.bool,
     formValues: PropTypes.object,
@@ -247,6 +251,7 @@ const mapStateToProps = state => ({
     servicesInfo: state.services.items,
     dataTypes: state.serviceDataTypes.dataTypesByServiceID,
     dataTypesByID: state.serviceDataTypes.itemsByID,
+    dataTypesByDataset: state.serviceDataTypesByDataset,
 
     autoQuery: state.explorer.autoQuery,
     isFetchingTextSearch: state.explorer.fetchingTextSearch || false,

src/components/explorer/ExplorerDatasetSearch.js

@@ -83,6 +83,7 @@ const ExplorerDatasetSearch = () => {
             <Typography.Title level={4}>Explore Dataset {selectedDataset.title}</Typography.Title>
             <SearchAllRecords datasetID={dataset} />
             <DiscoveryQueryBuilder
+                activeDataset={dataset}
                 isInternal={true}
                 dataTypeForms={dataTypeForms}
                 onSubmit={performSearch}

src/components/manager/projects/RoutedProject.js

@@ -111,7 +111,7 @@ class RoutedProject extends Component {
 
         const bentoServicesByKind = this.props.bentoServicesByKind;
         const serviceDataTypesByServiceID = this.props.serviceDataTypesByServiceID;
-
+        console.log("setvicesjul", this.props.services);
         const manageableDataTypes = this.props.services
             .filter(s => {
                 const cs = bentoServicesByKind[s.bento?.serviceKind ?? s.type.artifact] ?? {};
@@ -136,6 +136,8 @@ class RoutedProject extends Component {
 
         console.log("tll", tableList);
 
+        console.log("julithis.props.servicesByID", this.props.servicesByID);
+
         // TODO: Inconsistent schemas
         const strayTables = [
             ...this.props.serviceTables.filter(t2 =>

src/modules/metadata/actions.js

@@ -9,6 +9,8 @@ import {
 } from "../services/actions";
 import {endProjectEditing} from "../manager/actions";
 
+import {fetchServicesByDataset} from "../services/actions";
+
 import {
     createNetworkActionTypes,
     createFlowActionTypes,
@@ -80,6 +82,7 @@ export const fetchProjectsWithDatasetsAndTables = () => async (dispatch, getStat
     dispatch(beginFlow(FETCHING_PROJECTS_WITH_TABLES));
     await dispatch(fetchProjects());
     await dispatch(fetchProjectTables(getState().projects.itemsByID));
+    await dispatch(fetchServicesByDataset());
     dispatch(endFlow(FETCHING_PROJECTS_WITH_TABLES));
 };
 

src/modules/services/actions.js

@@ -39,6 +39,9 @@ export const DELETING_SERVICE_TABLE = createFlowActionTypes("DELETING_SERVICE_TA
 export const FETCH_SERVICE_WORKFLOWS = createNetworkActionTypes("FETCH_SERVICE_WORKFLOWS");
 export const LOADING_SERVICE_WORKFLOWS = createFlowActionTypes("LOADING_SERVICE_WORKFLOWS");
 
+export const FETCH_SERVICE_DATA_TYPES_BY_DATASET = createNetworkActionTypes("FETCH_SERVICE_DATA_TYPES_BY_DATASET");
+export const LOADING_SERVICE_DATA_TYPES_BY_DATASET = createFlowActionTypes("LOADING_SERVICE_DATA_TYPES_BY_DATASET");
+
 
 export const endAddingServiceTable = (serviceInfo, dataTypeID, table) => ({
     type: ADDING_SERVICE_TABLE.END,
@@ -75,6 +78,13 @@ export const fetchDataServiceDataTypes = networkAction((serviceInfo) => ({
     err: `Error fetching data types from service '${serviceInfo.name}'`,
 }));
 
+export const fetchDataServiceDataTypesById = networkAction((serviceInfo, datasetID) => ({
+    types:  FETCH_SERVICE_DATA_TYPES_BY_DATASET,
+    params: {serviceInfo, datasetID},
+    url: `${serviceInfo.url}/data-types?dataset=${encodeURIComponent(datasetID)}`,
+    err: `Error fetching data types from service '${serviceInfo.name}'`,
+}));
+
 export const fetchDataServiceDataTypeTables = networkAction((serviceInfo, dataType) => ({
     types: FETCH_SERVICE_TABLES,
     params: {serviceInfo, dataTypeID: dataType.id},
@@ -144,6 +154,29 @@ export const fetchServicesWithMetadataAndDataTypesAndTables = (onServiceFetchFin
     dispatch(endFlow(LOADING_ALL_SERVICE_DATA));
 };
 
+export const fetchServicesByDataset = () => async (dispatch, getState) => {
+    dispatch(beginFlow(LOADING_SERVICE_DATA_TYPES_BY_DATASET));
+    const datasetIdentifiers = Object.values(getState().projects.itemsByID).flatMap(
+        project => project.datasets?.map(d => d.identifier) || []);
+
+    const dataServicesInfo = getState().services.items.filter(s => s?.type && s?.id).map(s => {
+        const serviceKind = s.bento?.serviceKind ?? s.type.artifact;
+        return {
+            ...s,
+            bentoService: getState().bentoServices.itemsByKind[serviceKind] ?? null,
+        };
+    }).filter(s => s.bentoService?.data_service ?? false);
+
+    if (datasetIdentifiers.length > 0 && dataServicesInfo.length > 0) {
+        await Promise.all(dataServicesInfo.flatMap(s =>
+            datasetIdentifiers.map(id => dispatch(fetchDataServiceDataTypesById(s, id))),
+        ));
+    }
+
+    dispatch(endFlow(LOADING_SERVICE_DATA_TYPES_BY_DATASET));
+};
+
+
 export const fetchServicesWithMetadataAndDataTypesAndTablesIfNeeded = (onServiceFetchFinish) =>
     (dispatch, getState) => {
         const state = getState();

src/modules/services/reducers.js

@@ -9,6 +9,9 @@ import {
     FETCH_SERVICE_DATA_TYPES,
     LOADING_SERVICE_DATA_TYPES,
 
+    FETCH_SERVICE_DATA_TYPES_BY_DATASET,
+    LOADING_SERVICE_DATA_TYPES_BY_DATASET,
+
     FETCH_SERVICE_TABLES,
     LOADING_SERVICE_TABLES,
 
@@ -241,6 +244,62 @@ export const serviceDataTypes = (
     }
 };
 
+export const serviceDataTypesByDataset = (
+    state = {
+        isFetchingAll: false,
+        itemsByDatasetID: [],
+    },
+    action,
+) => {
+    switch (action.type) {
+        case LOADING_SERVICE_DATA_TYPES_BY_DATASET.BEGIN:
+            return {...state, isFetchingAll: true};
+
+        case LOADING_SERVICE_DATA_TYPES_BY_DATASET.END:
+        case LOADING_SERVICE_DATA_TYPES_BY_DATASET.TERMINATE:
+            return {...state, isFetchingAll: false};
+
+        case FETCH_SERVICE_DATA_TYPES_BY_DATASET.RECEIVE: {
+            const { datasetID, data } = action;
+
+            const existingDatasetInfo = state.itemsByDatasetID.find(
+                datasetInfo => datasetInfo.datasetIdentifier === datasetID);
+
+            const mergedData = existingDatasetInfo ? [...existingDatasetInfo.items, ...data] : data;
+
+            const newDatasetInfo = { datasetIdentifier: datasetID, items: mergedData };
+
+            const itemsByDatasetID = existingDatasetInfo
+                ? state.itemsByDatasetID.map(
+                    datasetInfo => datasetInfo.datasetIdentifier === datasetID ? newDatasetInfo : datasetInfo)
+                : [...state.itemsByDatasetID, newDatasetInfo];
+
+            return {
+                ...state,
+                itemsByDatasetID,
+                lastUpdated: action.receivedAt,
+            };
+        }
+
+        case FETCH_SERVICE_DATA_TYPES_BY_DATASET.ERROR: {
+            const {datasetID} = action;
+            return {
+                ...state,
+                itemsByDatasetID: {
+                    ...state.itemsByDatasetID,
+                    [datasetID]: {
+                        ...(state.itemsByDatasetID[datasetID] ?? {items: null, itemsByID: null}),
+                        isFetching: false,
+                    },
+                },
+            };
+        }
+
+        default:
+            return state;
+    }
+};
+
 export const serviceTables = (
     state = {
         isFetchingAll: false,

src/reducers.js

@@ -19,6 +19,7 @@ import {
     bentoServices,
     services,
     serviceDataTypes,
+    serviceDataTypesByDataset,
     serviceTables,
     serviceWorkflows,
 } from "./modules/services/reducers";
@@ -58,6 +59,7 @@ const rootReducer = combineReducers({
     bentoServices,
     services,
     serviceDataTypes,
+    serviceDataTypesByDataset,
     serviceTables,
     serviceWorkflows,
 

src/utils/actions.js

@@ -80,7 +80,6 @@ const _networkAction = (fn, ...args) => async (dispatch, getState) => {
 
     let {parse} = fnResult;
     if (!parse) parse = r => r.json();
-
     dispatch({type: types.REQUEST, ...params});
     try {
         const data = await (paginated ? _paginatedNetworkFetch : _unpaginatedNetworkFetch)(