diff --git a/main.go b/main.go index 85c30d7f..f3dc055a 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,7 @@ const ConfigLogTemplate = `Config -- Static Files: %s Client Name: %s Katsu URL: %v - WES URL: %v + Gohan URL: %v Bento Portal Url: %s Port: %d Translated: %t @@ -37,7 +37,7 @@ type BentoConfig struct { StaticFilesPath string `envconfig:"BENTO_PUBLIC_STATIC_FILES_PATH" default:"./www"` ClientName string `envconfig:"BENTO_PUBLIC_CLIENT_NAME"` KatsuUrl string `envconfig:"BENTO_PUBLIC_KATSU_URL"` - WesUrl string `envconfig:"BENTO_PUBLIC_WES_URL"` + GohanUrl string `envconfig:"BENTO_PUBLIC_GOHAN_URL"` BentoPortalUrl string `envconfig:"BENTO_PUBLIC_PORTAL_URL"` Port int `envconfig:"INTERNAL_PORT" default:"8090"` Translated bool `envconfig:"BENTO_PUBLIC_TRANSLATED" default:"true"` @@ -114,7 +114,7 @@ func main() { cfg.StaticFilesPath, cfg.ClientName, cfg.KatsuUrl, - cfg.WesUrl, + cfg.GohanUrl, cfg.BentoPortalUrl, cfg.Port, cfg.Translated, @@ -176,14 +176,6 @@ func main() { return c.JSON(http.StatusOK, result) } - wesRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatterFunc) error { - result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.WesUrl, path), qs, c, rf) - if err != nil { - return err - } - return c.JSON(http.StatusOK, result) - } - katsuRequestBasic := func(path string, c echo.Context) error { return katsuRequest(path, nil, c, jsonDeserialize) } @@ -201,11 +193,28 @@ func main() { return jsonFormattedData, nil } - wesRequestWithDetailsAndPublic := func(c echo.Context) error { - qs := url.Values{} - qs.Add("with_details", "true") - qs.Add("public", "true") - return wesRequest("/runs", qs, c, jsonDeserialize) + dataTypesEndpointHandler := func(baseUrl string) echo.HandlerFunc { + return func(c echo.Context) error { + fullPath := fmt.Sprintf("%s/data-types", baseUrl) + result, err := genericRequestJsonOnly(fullPath, nil, c, jsonDeserialize) + if err != nil { + return err + } + + resultSlice, ok := result.([]JsonLike) + if !ok { + return fmt.Errorf("result is not of type []JsonLike") + } + + var modifiedResult []JsonLike + // Update the "count" value + for _, item := range resultSlice { + item["count"] = nil + modifiedResult = append(modifiedResult, item) + } + + return c.JSON(http.StatusOK, modifiedResult) + } } fetchAndSetKatsuPublic := func(c echo.Context, katsuCache *cache.Cache) (JsonLike, error) { @@ -359,8 +368,6 @@ func main() { return katsuRequestBasic("/api/public_search_fields", c) }) - e.GET("/wes-runs", wesRequestWithDetailsAndPublic) - e.GET("/provenance", func(c echo.Context) error { // Query Katsu for datasets provenance return katsuRequestBasic("/api/public_dataset", c) @@ -382,6 +389,10 @@ func main() { return c.String(http.StatusOK, string(data)) }) + e.GET("/gohan/data-types", dataTypesEndpointHandler(cfg.GohanUrl)) + + e.GET("/katsu/data-types", dataTypesEndpointHandler(cfg.KatsuUrl)) + // Run e.Logger.Fatal(e.Start(fmt.Sprintf(":%d", cfg.Port))) } diff --git a/src/js/components/Overview/LastIngestion.tsx b/src/js/components/Overview/LastIngestion.tsx index ccb01c37..d486153b 100644 --- a/src/js/components/Overview/LastIngestion.tsx +++ b/src/js/components/Overview/LastIngestion.tsx @@ -7,9 +7,17 @@ import { DEFAULT_TRANSLATION } from '@/constants/configConstants'; import { useAppSelector } from '@/hooks'; import { getDataTypeLabel } from '@/types/dataTypes'; +import { LastIngestionDataTypeResponse } from '@/types/lastIngestionDataTypeResponse'; + const LastIngestionInfo: React.FC = () => { const { t, i18n } = useTranslation(DEFAULT_TRANSLATION); - const lastEndTimesByDataType = useAppSelector((state) => state.ingestionData?.lastEndTimesByDataType) || {}; + + const dataTypesObject = useAppSelector((state) => state.lastIngestionData?.dataTypes) || {}; + + const sortedDataTypes = Object.values(dataTypesObject).sort((a, b) => a.label.localeCompare(b.label)); + + // Filter out the queryable data types + const queryableDataTypes = sortedDataTypes.filter((dataType: LastIngestionDataTypeResponse) => dataType.queryable); const formatDate = useCallback( (dateString: string) => { @@ -28,19 +36,21 @@ const LastIngestionInfo: React.FC = () => { [i18n.language] ); - const hasData = Object.keys(lastEndTimesByDataType).length > 0; + const hasData = queryableDataTypes.length > 0; return ( {t('Latest Data Ingestion')} {hasData ? ( - Object.entries(lastEndTimesByDataType).map(([dataType, endTime]) => ( - + queryableDataTypes.map((dataType: LastIngestionDataTypeResponse) => ( + - {t(getDataTypeLabel(dataType))} + + {t(getDataTypeLabel(dataType.id))} + - {formatDate(endTime)} + {dataType.last_ingested ? formatDate(dataType.last_ingested) : 'Not Available'} diff --git a/src/js/components/TabbedDashboard.tsx b/src/js/components/TabbedDashboard.tsx index 3a1f8a6e..0b2c3b64 100644 --- a/src/js/components/TabbedDashboard.tsx +++ b/src/js/components/TabbedDashboard.tsx @@ -10,8 +10,8 @@ import { makeGetAboutRequest } from '@/features/content/content.store'; import { makeGetDataRequestThunk } from '@/features/data/data.store'; import { makeGetSearchFields } from '@/features/search/query.store'; import { makeGetProvenanceRequest } from '@/features/provenance/provenance.store'; -import { makeGetIngestionDataRequest } from '@/features/ingestion/ingestion.store'; import { getBeaconConfig } from '@/features/beacon/beaconConfig.store'; +import { fetchGohanData, fetchKatsuData } from '@/features/ingestion/lastIngestion.store'; import Loader from './Loader'; import PublicOverview from './Overview/PublicOverview'; @@ -35,7 +35,9 @@ const TabbedDashboard = () => { dispatch(makeGetDataRequestThunk()); dispatch(makeGetSearchFields()); dispatch(makeGetProvenanceRequest()); - dispatch(makeGetIngestionDataRequest()); + dispatch(fetchKatsuData()); + dispatch(fetchGohanData()); + //TODO: Dispatch makeGetDataTypes to get the data types from service-registry }, []); const isFetchingOverviewData = useAppSelector((state) => state.data.isFetchingData); diff --git a/src/js/constants/configConstants.ts b/src/js/constants/configConstants.ts index b59e6650..02425459 100644 --- a/src/js/constants/configConstants.ts +++ b/src/js/constants/configConstants.ts @@ -5,7 +5,8 @@ export const publicOverviewUrl = '/overview'; export const searchFieldsUrl = '/fields'; export const katsuUrl = '/katsu'; export const provenanceUrl = '/provenance'; -export const lastIngestionsUrl = '/wes-runs'; +export const katsuLastIngestionsUrl = '/katsu/data-types'; +export const gohanLastIngestionsUrl = '/gohan/data-types'; export const DEFAULT_TRANSLATION = 'default_translation'; export const NON_DEFAULT_TRANSLATION = 'translation'; diff --git a/src/js/features/ingestion/ingestion.store.ts b/src/js/features/ingestion/ingestion.store.ts deleted file mode 100644 index 2ce4d686..00000000 --- a/src/js/features/ingestion/ingestion.store.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit'; -import axios from 'axios'; -import { lastIngestionsUrl } from '@/constants/configConstants'; -import { printAPIError } from '@/utils/error.util'; -import { IngestionData, LastIngestionResponse } from '@/types/lastIngestionResponse'; - -export const makeGetIngestionDataRequest = createAsyncThunk( - 'ingestionData/getIngestionData', - (_, { rejectWithValue }) => - axios - .get(lastIngestionsUrl) - .then((res) => res.data) - .catch(printAPIError(rejectWithValue)) -); - -export interface IngestionDataState { - isFetchingIngestionData: boolean; - ingestionData: IngestionData[]; - lastEndTimesByDataType: { [dataType: string]: string }; -} - -const initialState: IngestionDataState = { - isFetchingIngestionData: false, - ingestionData: [], - lastEndTimesByDataType: {}, -}; - -const IngestionDataStore = createSlice({ - name: 'ingestionData', - initialState, - reducers: {}, - extraReducers: (builder) => { - builder.addCase(makeGetIngestionDataRequest.pending, (state) => { - state.isFetchingIngestionData = true; - }); - builder.addCase( - makeGetIngestionDataRequest.fulfilled, - (state, { payload }: PayloadAction) => { - state.ingestionData = payload; - payload.forEach((ingestion) => { - const dataType = ingestion.details.request.tags.workflow_metadata.data_type; - const endTime = ingestion.details.run_log.end_time; - const previousEndTime = state.lastEndTimesByDataType[dataType]; - if (!previousEndTime || new Date(endTime) > new Date(previousEndTime)) { - state.lastEndTimesByDataType[dataType] = endTime; - } - }); - state.isFetchingIngestionData = false; - } - ); - builder.addCase(makeGetIngestionDataRequest.rejected, (state) => { - state.isFetchingIngestionData = false; - }); - }, -}); - -export default IngestionDataStore.reducer; diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts new file mode 100644 index 00000000..09146e2e --- /dev/null +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -0,0 +1,73 @@ +import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit'; +import axios from 'axios'; +import { katsuLastIngestionsUrl, gohanLastIngestionsUrl } from '@/constants/configConstants'; +import { printAPIError } from '@/utils/error.util'; + +import { LastIngestionDataTypeResponse, DataTypeMap } from '@/types/lastIngestionDataTypeResponse'; + +// Async thunks to fetch data from the two endpoints +export const fetchKatsuData = createAsyncThunk('dataTypes/fetchKatsuData', (_, { rejectWithValue }) => + axios + .get(katsuLastIngestionsUrl) + .then((res) => res.data) + .catch(printAPIError(rejectWithValue)) +); + +export const fetchGohanData = createAsyncThunk('dataTypes/fetchGohanData', (_, { rejectWithValue }) => + axios + .get(gohanLastIngestionsUrl) + .then((res) => res.data) + .catch(printAPIError(rejectWithValue)) +); + +// Define the state structure +export interface DataTypeState { + isFetchingKatsuData: boolean; + isFetchingGohanData: boolean; + dataTypes: DataTypeMap; +} + +// Initialize the state +const initialDataTypeState: DataTypeState = { + isFetchingKatsuData: false, + isFetchingGohanData: false, + dataTypes: {}, +}; + +const reduceServiceDataTypes = (state: DataTypeState, { payload }: PayloadAction) => { + state.dataTypes = { + ...state.dataTypes, + ...Object.fromEntries(payload.map((data) => [data.id, data])), + }; +}; + +// Create a slice to manage the state +const DataTypeStore = createSlice({ + name: 'dataTypes', + initialState: initialDataTypeState, + reducers: {}, + extraReducers: (builder) => { + builder.addCase(fetchKatsuData.pending, (state) => { + state.isFetchingKatsuData = true; + }); + builder.addCase(fetchGohanData.pending, (state) => { + state.isFetchingGohanData = true; + }); + builder.addCase(fetchKatsuData.fulfilled, (state, action) => { + reduceServiceDataTypes(state, action); + state.isFetchingKatsuData = false; + }); + builder.addCase(fetchGohanData.fulfilled, (state, action) => { + reduceServiceDataTypes(state, action); + state.isFetchingGohanData = false; + }); + builder.addCase(fetchKatsuData.rejected, (state) => { + state.isFetchingKatsuData = false; + }); + builder.addCase(fetchGohanData.rejected, (state) => { + state.isFetchingGohanData = false; + }); + }, +}); + +export default DataTypeStore.reducer; diff --git a/src/js/store.ts b/src/js/store.ts index 313180b1..921821a8 100644 --- a/src/js/store.ts +++ b/src/js/store.ts @@ -4,7 +4,7 @@ import configReducer from '@/features/config/config.store'; import contentReducer from '@/features/content/content.store'; import dataReducer from '@/features/data/data.store'; import queryReducer from '@/features/search/query.store'; -import ingestionDataReducer from '@/features/ingestion/ingestion.store'; +import lastIngestionDataReducer from '@/features/ingestion/lastIngestion.store'; import provenanceReducer from '@/features/provenance/provenance.store'; import beaconConfigReducer from './features/beacon/beaconConfig.store'; import beaconQueryReducer from './features/beacon/beaconQuery.store'; @@ -16,7 +16,7 @@ export const store = configureStore({ data: dataReducer, query: queryReducer, provenance: provenanceReducer, - ingestionData: ingestionDataReducer, + lastIngestionData: lastIngestionDataReducer, beaconConfig: beaconConfigReducer, beaconQuery: beaconQueryReducer, }, diff --git a/src/js/types/lastIngestionDataTypeResponse.ts b/src/js/types/lastIngestionDataTypeResponse.ts new file mode 100644 index 00000000..617bd148 --- /dev/null +++ b/src/js/types/lastIngestionDataTypeResponse.ts @@ -0,0 +1,9 @@ +export interface LastIngestionDataTypeResponse { + count: number | null; + id: string; + label: string; + last_ingested: string | null; + queryable: boolean; +} + +export type DataTypeMap = { [id: string]: LastIngestionDataTypeResponse }; diff --git a/src/js/types/lastIngestionResponse.ts b/src/js/types/lastIngestionResponse.ts deleted file mode 100644 index 22357371..00000000 --- a/src/js/types/lastIngestionResponse.ts +++ /dev/null @@ -1,36 +0,0 @@ -interface Tags { - table_id: string; - workflow_id: string; - workflow_metadata: { - data_type: string; - id: string; - }; -} - -interface Request { - tags: Tags; - workflow_type: string; -} - -interface RunLog { - end_time: string; - id: string; - start_time: string; -} - -export interface IngestionDetails { - end_time: string; - request: Request; - run_id: string; - run_log: RunLog; - state: string; - task_logs: null; -} - -export interface IngestionData { - details: IngestionDetails; - run_id: string; - state: string; -} - -export type LastIngestionResponse = IngestionData[]; diff --git a/src/public/locales/en/default_translation_en.json b/src/public/locales/en/default_translation_en.json index b3587487..74c37cf8 100644 --- a/src/public/locales/en/default_translation_en.json +++ b/src/public/locales/en/default_translation_en.json @@ -74,5 +74,6 @@ "e.g.": "e.g.", "or": "or", "Ingestion History Is Empty": "Ingestion History Is Empty", - "Latest Data Ingestion": "Latest Data Ingestion" + "Latest Data Ingestion": "Latest Data Ingestion", + "Clinical Data": "Clinical Data" } \ No newline at end of file diff --git a/src/public/locales/fr/default_translation_fr.json b/src/public/locales/fr/default_translation_fr.json index b5e11373..1b2194ce 100644 --- a/src/public/locales/fr/default_translation_fr.json +++ b/src/public/locales/fr/default_translation_fr.json @@ -74,5 +74,6 @@ "e.g.": "par exemple", "or": "ou", "Ingestion History Is Empty": "L'historique d'ingestion est vide", - "Latest Data Ingestion": "Dernière ingestion de données" + "Latest Data Ingestion": "Dernière ingestion de données", + "Clinical Data": "Données cliniques" } \ No newline at end of file