feat: last ingestion time from data services

main.go

@@ -23,7 +23,7 @@ const ConfigLogTemplate = `Config --
 	Static Files: %s
 	Client Name: %s
 	Katsu URL: %v
-	WES URL: %v
+	Gohan URL: %v
 	Bento Portal Url: %s
 	Port: %d
 	Translated: %t
@@ -37,7 +37,7 @@ type BentoConfig struct {
 	StaticFilesPath string `envconfig:"BENTO_PUBLIC_STATIC_FILES_PATH" default:"./www"`
 	ClientName      string `envconfig:"BENTO_PUBLIC_CLIENT_NAME"`
 	KatsuUrl        string `envconfig:"BENTO_PUBLIC_KATSU_URL"`
-	WesUrl          string `envconfig:"BENTO_PUBLIC_WES_URL"`
+	GohanUrl        string `envconfig:"BENTO_PUBLIC_GOHAN_URL"`
 	BentoPortalUrl  string `envconfig:"BENTO_PUBLIC_PORTAL_URL"`
 	Port            int    `envconfig:"INTERNAL_PORT" default:"8090"`
 	Translated      bool   `envconfig:"BENTO_PUBLIC_TRANSLATED" default:"true"`
@@ -114,7 +114,7 @@ func main() {
 		cfg.StaticFilesPath,
 		cfg.ClientName,
 		cfg.KatsuUrl,
-		cfg.WesUrl,
+		cfg.GohanUrl,
 		cfg.BentoPortalUrl,
 		cfg.Port,
 		cfg.Translated,
@@ -176,14 +176,6 @@ func main() {
 		return c.JSON(http.StatusOK, result)
 	}
 
-	wesRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatterFunc) error {
-		result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.WesUrl, path), qs, c, rf)
-		if err != nil {
-			return err
-		}
-		return c.JSON(http.StatusOK, result)
-	}
-
 	katsuRequestBasic := func(path string, c echo.Context) error {
 		return katsuRequest(path, nil, c, jsonDeserialize)
 	}
@@ -201,11 +193,28 @@ func main() {
 		return jsonFormattedData, nil
 	}
 
-	wesRequestWithDetailsAndPublic := func(c echo.Context) error {
-		qs := url.Values{}
-		qs.Add("with_details", "true")
-		qs.Add("public", "true")
-		return wesRequest("/runs", qs, c, jsonDeserialize)
+	dataTypesEndpointHandler := func(baseUrl string) echo.HandlerFunc {
+		return func(c echo.Context) error {
+			fullPath := fmt.Sprintf("%s/data-types", baseUrl)
+			result, err := genericRequestJsonOnly(fullPath, nil, c, jsonDeserialize)
+			if err != nil {
+				return err
+			}
+
+			resultSlice, ok := result.([]JsonLike)
+			if !ok {
+				return fmt.Errorf("result is not of type []JsonLike")
+			}
+
+			var modifiedResult []JsonLike
+			// Update the "count" value
+			for _, item := range resultSlice {
+				item["count"] = nil
+				modifiedResult = append(modifiedResult, item)
+			}
+
+			return c.JSON(http.StatusOK, modifiedResult)
+		}
 	}
 
 	fetchAndSetKatsuPublic := func(c echo.Context, katsuCache *cache.Cache) (JsonLike, error) {
@@ -359,8 +368,6 @@ func main() {
 		return katsuRequestBasic("/api/public_search_fields", c)
 	})
 
-	e.GET("/wes-runs", wesRequestWithDetailsAndPublic)
-
 	e.GET("/provenance", func(c echo.Context) error {
 		// Query Katsu for datasets provenance
 		return katsuRequestBasic("/api/public_dataset", c)
@@ -382,6 +389,10 @@ func main() {
 		return c.String(http.StatusOK, string(data))
 	})
 
+	e.GET("/gohan/data-types", dataTypesEndpointHandler(cfg.GohanUrl))
+
+	e.GET("/katsu/data-types", dataTypesEndpointHandler(cfg.KatsuUrl))
+
 	// Run
 	e.Logger.Fatal(e.Start(fmt.Sprintf(":%d", cfg.Port)))
 }

src/js/components/Overview/LastIngestion.tsx

@@ -6,10 +6,14 @@ import { useTranslation } from 'react-i18next';
 import { DEFAULT_TRANSLATION } from '@/constants/configConstants';
 import { useAppSelector } from '@/hooks';
 import { getDataTypeLabel } from '@/types/dataTypes';
+import { LastIngestionDataTypeResponse, DataResponseArray } from '@/types/lastIngestionDataTypeResponse';
 
 const LastIngestionInfo: React.FC = () => {
   const { t, i18n } = useTranslation(DEFAULT_TRANSLATION);
-  const lastEndTimesByDataType = useAppSelector((state) => state.ingestionData?.lastEndTimesByDataType) || {};
+
+  const lastEndTimesByDataType: DataResponseArray = useAppSelector((state) => state.lastIngestionData?.dataTypes) || [];
+
+  const queryableDataTypes = lastEndTimesByDataType.filter((item: LastIngestionDataTypeResponse) => item.queryable);
 
   const formatDate = useCallback(
     (dateString: string) => {
@@ -28,19 +32,19 @@ const LastIngestionInfo: React.FC = () => {
     [i18n.language]
   );
 
-  const hasData = Object.keys(lastEndTimesByDataType).length > 0;
+  const hasData = queryableDataTypes.length > 0;
 
   return (
     <Space direction="vertical" size={0}>
       <Typography.Title level={3}>{t('Latest Data Ingestion')}</Typography.Title>
       <Space direction="horizontal">
         {hasData ? (
-          Object.entries(lastEndTimesByDataType).map(([dataType, endTime]) => (
-            <Card key={dataType}>
+          queryableDataTypes.map((dataType: LastIngestionDataTypeResponse) => (
+            <Card key={dataType.id}>
               <Space direction="vertical">
-                <Typography.Text style={{ color: 'rgba(0,0,0,0.45)' }}>{t(getDataTypeLabel(dataType))}</Typography.Text>
+                <Typography.Text style={{ color: 'rgba(0,0,0,0.45)' }}>{getDataTypeLabel(dataType.id)}</Typography.Text>
                 <Typography.Text>
-                  <CalendarOutlined /> {formatDate(endTime)}
+                  <CalendarOutlined /> {dataType.last_ingested ? formatDate(dataType.last_ingested) : 'Not Available'}
                 </Typography.Text>
               </Space>
             </Card>

src/js/components/TabbedDashboard.tsx

@@ -10,8 +10,9 @@ import { makeGetAboutRequest } from '@/features/content/content.store';
 import { makeGetDataRequestThunk } from '@/features/data/data.store';
 import { makeGetSearchFields } from '@/features/search/query.store';
 import { makeGetProvenanceRequest } from '@/features/provenance/provenance.store';
-import { makeGetIngestionDataRequest } from '@/features/ingestion/ingestion.store';
 import { getBeaconConfig } from '@/features/beacon/beaconConfig.store';
+import { makeGetDataTypes } from '@/features/dataTypes/dataTypes.store';
+import { fetchGohanData, fetchKatsuData } from '@/features/ingestion/lastIngestion.store';
 
 import Loader from './Loader';
 import PublicOverview from './Overview/PublicOverview';
@@ -35,7 +36,9 @@ const TabbedDashboard = () => {
     dispatch(makeGetDataRequestThunk());
     dispatch(makeGetSearchFields());
     dispatch(makeGetProvenanceRequest());
-    dispatch(makeGetIngestionDataRequest());
+    dispatch(fetchKatsuData());
+    dispatch(fetchGohanData());
+    dispatch(makeGetDataTypes());
   }, []);
 
   const isFetchingOverviewData = useAppSelector((state) => state.data.isFetchingData);

src/js/constants/configConstants.ts

@@ -5,7 +5,8 @@ export const publicOverviewUrl = '/overview';
 export const searchFieldsUrl = '/fields';
 export const katsuUrl = '/katsu';
 export const provenanceUrl = '/provenance';
-export const lastIngestionsUrl = '/wes-runs';
+export const katsuLastIngestionsUrl = '/katsu/data-types';
+export const gohanLastIngestionsUrl = '/gohan/data-types';
 
 export const DEFAULT_TRANSLATION = 'default_translation';
 export const NON_DEFAULT_TRANSLATION = 'translation';

src/js/features/ingestion/lastIngestion.store.ts

@@ -0,0 +1,73 @@
+import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit';
+import axios from 'axios';
+import { katsuLastIngestionsUrl, gohanLastIngestionsUrl } from '@/constants/configConstants';
+import { printAPIError } from '@/utils/error.util';
+
+export interface DataTypeResponse {
+  count: number | null;
+  id: string;
+  label: string;
+  last_ingested: string | null;
+  queryable: boolean;
+}
+
+export type DataResponseArray = DataTypeResponse[];
+
+// Async thunks to fetch data from the two endpoints
+export const fetchKatsuData = createAsyncThunk('dataTypes/fetchKatsuData', (_, { rejectWithValue }) =>
+  axios
+    .get(katsuLastIngestionsUrl)
+    .then((res) => res.data)
+    .catch(printAPIError(rejectWithValue))
+);
+
+export const fetchGohanData = createAsyncThunk('dataTypes/fetchGohanData', (_, { rejectWithValue }) =>
+  axios
+    .get(gohanLastIngestionsUrl)
+    .then((res) => res.data)
+    .catch(printAPIError(rejectWithValue))
+);
+
+// Define the state structure
+export interface DataTypeState {
+  isFetchingData: boolean;
+  dataTypes: DataResponseArray;
+}
+
+// Initialize the state
+const initialDataTypeState: DataTypeState = {
+  isFetchingData: false,
+  dataTypes: [],
+};
+
+const reduceServiceDataTypes = (state: DataTypeState, { payload }: PayloadAction<DataResponseArray>) => {
+  const uniqueIds = new Set(state.dataTypes.map((data: DataTypeResponse) => data.id));
+  const newData = payload.filter((data: DataTypeResponse) => !uniqueIds.has(data.id));
+  state.dataTypes = [...state.dataTypes, ...newData];
+  state.isFetchingData = false;
+};
+
+// Create a slice to manage the state
+const DataTypeStore = createSlice({
+  name: 'dataTypes',
+  initialState: initialDataTypeState,
+  reducers: {},
+  extraReducers: (builder) => {
+    builder.addCase(fetchKatsuData.pending, (state) => {
+      state.isFetchingData = true;
+    });
+    builder.addCase(fetchGohanData.pending, (state) => {
+      state.isFetchingData = true;
+    });
+    builder.addCase(fetchKatsuData.fulfilled, reduceServiceDataTypes);
+    builder.addCase(fetchGohanData.fulfilled, reduceServiceDataTypes);
+    builder.addCase(fetchKatsuData.rejected, (state) => {
+      state.isFetchingData = false;
+    });
+    builder.addCase(fetchGohanData.rejected, (state) => {
+      state.isFetchingData = false;
+    });
+  },
+});
+
+export default DataTypeStore.reducer;

src/js/store.ts

@@ -4,7 +4,7 @@ import configReducer from '@/features/config/config.store';
 import contentReducer from '@/features/content/content.store';
 import dataReducer from '@/features/data/data.store';
 import queryReducer from '@/features/search/query.store';
-import ingestionDataReducer from '@/features/ingestion/ingestion.store';
+import lastIngestionDataReducer from '@/features/ingestion/lastIngestion.store';
 import provenanceReducer from '@/features/provenance/provenance.store';
 import beaconConfigReducer from './features/beacon/beaconConfig.store';
 import beaconQueryReducer from './features/beacon/beaconQuery.store';
@@ -16,7 +16,7 @@ export const store = configureStore({
     data: dataReducer,
     query: queryReducer,
     provenance: provenanceReducer,
-    ingestionData: ingestionDataReducer,
+    lastIngestionData: lastIngestionDataReducer,
     beaconConfig: beaconConfigReducer,
     beaconQuery: beaconQueryReducer,
   },

src/js/types/lastIngestionDataTypeResponse.ts

@@ -0,0 +1,9 @@
+export interface LastIngestionDataTypeResponse {
+  count: number | null;
+  id: string;
+  label: string;
+  last_ingested: string | null;
+  queryable: boolean;
+}
+
+export type DataResponseArray = LastIngestionDataTypeResponse[];