From d2afa2005616d8076b074b044b0e74de7642382d Mon Sep 17 00:00:00 2001 From: Julian Date: Tue, 10 Oct 2023 08:37:56 -0400 Subject: [PATCH 01/32] temporal changes --- main.go | 83 ++++++++++++++++++++ src/js/components/Overview/LastIngestion.tsx | 2 +- src/js/components/TabbedDashboard.tsx | 4 + src/js/constants/configConstants.ts | 2 + src/js/features/datasets/datasets.store.ts | 52 ++++++++++++ src/js/features/ingestion/ingestion.store.ts | 2 + src/js/store.ts | 2 + 7 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 src/js/features/datasets/datasets.store.ts diff --git a/main.go b/main.go index 85c30d7f..8392816e 100644 --- a/main.go +++ b/main.go @@ -24,6 +24,7 @@ const ConfigLogTemplate = `Config -- Client Name: %s Katsu URL: %v WES URL: %v + Gohan URL: %v Bento Portal Url: %s Port: %d Translated: %t @@ -38,6 +39,7 @@ type BentoConfig struct { ClientName string `envconfig:"BENTO_PUBLIC_CLIENT_NAME"` KatsuUrl string `envconfig:"BENTO_PUBLIC_KATSU_URL"` WesUrl string `envconfig:"BENTO_PUBLIC_WES_URL"` + GohanUrl string `envconfig:"BENTO_PUBLIC_GOHAN_URL"` BentoPortalUrl string `envconfig:"BENTO_PUBLIC_PORTAL_URL"` Port int `envconfig:"INTERNAL_PORT" default:"8090"` Translated bool `envconfig:"BENTO_PUBLIC_TRANSLATED" default:"true"` @@ -115,6 +117,7 @@ func main() { cfg.ClientName, cfg.KatsuUrl, cfg.WesUrl, + cfg.GohanUrl, cfg.BentoPortalUrl, cfg.Port, cfg.Translated, @@ -131,6 +134,8 @@ func main() { var req *http.Request var err error + fmt.Println("urlXXXXX: ", url) + if qs != nil { req, err = http.NewRequest("GET", fmt.Sprintf("%s?%s", url, qs.Encode()), nil) if err != nil { @@ -184,6 +189,14 @@ func main() { return c.JSON(http.StatusOK, result) } + gohanRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatterFunc) error { + result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.GohanUrl, path), qs, c, rf) + if err != nil { + return err + } + return c.JSON(http.StatusOK, result) + } + katsuRequestBasic := func(path string, c echo.Context) error { return katsuRequest(path, nil, c, jsonDeserialize) } @@ -208,6 +221,28 @@ func main() { return wesRequest("/runs", qs, c, jsonDeserialize) } + gohanRequestPublic := func(c echo.Context) error { + + // Construct the path using the extracted dataset ID. + path := fmt.Sprintf("/data-types") + + // Debug print statements to verify the constructed path and the GohanUrl. + fmt.Fprintf(os.Stderr, "pathCFCFCFCF: %s\n", path) + fmt.Println("DEBUG: cfg.GohanUrl =", cfg.GohanUrl) + + // Make a request to the Gohan service and deserialize the response. + err := gohanRequest(path, nil, c, jsonDeserialize) + if err != nil { + // Return the error if there is one. + return err + } + + // If everything is successful, return a success status to the caller. + return c.JSON(http.StatusOK, map[string]string{ + "message": "Success", + }) + } + fetchAndSetKatsuPublic := func(c echo.Context, katsuCache *cache.Cache) (JsonLike, error) { fmt.Println("'publicOverview' not found or expired in 'katsuCache' - fetching") publicOverviewInterface, err := genericRequestJsonOnly( @@ -361,6 +396,8 @@ func main() { e.GET("/wes-runs", wesRequestWithDetailsAndPublic) + e.GET("/gohan/data-types", gohanRequestPublic) + e.GET("/provenance", func(c echo.Context) error { // Query Katsu for datasets provenance return katsuRequestBasic("/api/public_dataset", c) @@ -382,6 +419,52 @@ func main() { return c.String(http.StatusOK, string(data)) }) + e.GET("/datasets", func(c echo.Context) error { + data, err := katsuRequestFormattedData("/api/datasets", c) + if err != nil { + return err + } + + fmt.Println("FFFFF") + fmt.Println("data: ", string(data)) + fmt.Println("FFFFF") + + return c.String(http.StatusOK, string(data)) + + }) + + e.GET("/datasets/:id/data-types/:data-type/public", func(c echo.Context) error { + id := c.Param("id") + dataType := c.Param("data-type") + relativeUrl := fmt.Sprintf("/datasets/%s/data-types/%s/public", id, dataType) + + data, err := katsuRequestFormattedData(relativeUrl, c) + if err != nil { + return err + } + + fmt.Println("XXXXXXX") + + fmt.Println("data: ", string(data)) + + fmt.Println("XXXXXXX") + + return c.String(http.StatusOK, string(data)) + }) + + e.GET("/katsu-data-types", func(c echo.Context) error { + data, err := katsuRequestFormattedData("/data-types", c) + if err != nil { + return err + } + + fmt.Println("XXXXXXX") + fmt.Println("data: ", string(data)) + fmt.Println("XXXXXXX") + + return c.String(http.StatusOK, string(data)) + }) + // Run e.Logger.Fatal(e.Start(fmt.Sprintf(":%d", cfg.Port))) } diff --git a/src/js/components/Overview/LastIngestion.tsx b/src/js/components/Overview/LastIngestion.tsx index ccb01c37..46ed6b03 100644 --- a/src/js/components/Overview/LastIngestion.tsx +++ b/src/js/components/Overview/LastIngestion.tsx @@ -32,7 +32,7 @@ const LastIngestionInfo: React.FC = () => { return ( - {t('Latest Data Ingestion')} + {t('Latest Data IngestionX')} {hasData ? ( Object.entries(lastEndTimesByDataType).map(([dataType, endTime]) => ( diff --git a/src/js/components/TabbedDashboard.tsx b/src/js/components/TabbedDashboard.tsx index 3a1f8a6e..a23bd2b7 100644 --- a/src/js/components/TabbedDashboard.tsx +++ b/src/js/components/TabbedDashboard.tsx @@ -10,8 +10,10 @@ import { makeGetAboutRequest } from '@/features/content/content.store'; import { makeGetDataRequestThunk } from '@/features/data/data.store'; import { makeGetSearchFields } from '@/features/search/query.store'; import { makeGetProvenanceRequest } from '@/features/provenance/provenance.store'; +import {fetchDatasets } from '@/features/datasets/datasets.store'; import { makeGetIngestionDataRequest } from '@/features/ingestion/ingestion.store'; import { getBeaconConfig } from '@/features/beacon/beaconConfig.store'; +import {makeGetDataTypes } from '@/features/dataTypes/dataTypes.store'; import Loader from './Loader'; import PublicOverview from './Overview/PublicOverview'; @@ -36,6 +38,8 @@ const TabbedDashboard = () => { dispatch(makeGetSearchFields()); dispatch(makeGetProvenanceRequest()); dispatch(makeGetIngestionDataRequest()); + dispatch(fetchDatasets()); + dispatch(makeGetDataTypes()); }, []); const isFetchingOverviewData = useAppSelector((state) => state.data.isFetchingData); diff --git a/src/js/constants/configConstants.ts b/src/js/constants/configConstants.ts index b59e6650..f3eb98af 100644 --- a/src/js/constants/configConstants.ts +++ b/src/js/constants/configConstants.ts @@ -6,6 +6,8 @@ export const searchFieldsUrl = '/fields'; export const katsuUrl = '/katsu'; export const provenanceUrl = '/provenance'; export const lastIngestionsUrl = '/wes-runs'; +export const datasetsUrl = '/datasets'; +export const datasetDataTypePublicUrl = '/datasets/:id/data-types/:data-type/public'; export const DEFAULT_TRANSLATION = 'default_translation'; export const NON_DEFAULT_TRANSLATION = 'translation'; diff --git a/src/js/features/datasets/datasets.store.ts b/src/js/features/datasets/datasets.store.ts new file mode 100644 index 00000000..5ccfb31a --- /dev/null +++ b/src/js/features/datasets/datasets.store.ts @@ -0,0 +1,52 @@ +import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit'; +import axios from 'axios'; +import { datasetsUrl } from '@/constants/configConstants'; +import { printAPIError } from '@/utils/error.util'; + +// Type definition for Dataset response. You need to define this according to the response structure +interface DatasetResponse { + // Define the structure based on the response of the /datasets endpoint. + // Here's a placeholder type; replace with actual fields + id: string; + name: string; + description: string; +} + +export const fetchDatasets = createAsyncThunk( + 'datasets/fetchDatasets', + (_, { rejectWithValue }) => + axios + .get(datasetsUrl) + .then((res) => res.data) + .catch(printAPIError(rejectWithValue)) +); + +export interface DatasetsState { + isFetchingDatasets: boolean; + datasets: DatasetResponse[]; +} + +const initialState: DatasetsState = { + isFetchingDatasets: false, + datasets: [], +}; + +const DatasetsStore = createSlice({ + name: 'datasets', + initialState, + reducers: {}, + extraReducers: (builder) => { + builder.addCase(fetchDatasets.pending, (state) => { + state.isFetchingDatasets = true; + }); + builder.addCase(fetchDatasets.fulfilled, (state, { payload }: PayloadAction) => { + state.datasets = payload; + state.isFetchingDatasets = false; + }); + builder.addCase(fetchDatasets.rejected, (state) => { + state.isFetchingDatasets = false; + }); + }, +}); + +export default DatasetsStore.reducer; diff --git a/src/js/features/ingestion/ingestion.store.ts b/src/js/features/ingestion/ingestion.store.ts index 2ce4d686..0fb8964f 100644 --- a/src/js/features/ingestion/ingestion.store.ts +++ b/src/js/features/ingestion/ingestion.store.ts @@ -39,6 +39,8 @@ const IngestionDataStore = createSlice({ state.ingestionData = payload; payload.forEach((ingestion) => { const dataType = ingestion.details.request.tags.workflow_metadata.data_type; + console.log("losDATATipes",dataType); + //// here use dataType const endTime = ingestion.details.run_log.end_time; const previousEndTime = state.lastEndTimesByDataType[dataType]; if (!previousEndTime || new Date(endTime) > new Date(previousEndTime)) { diff --git a/src/js/store.ts b/src/js/store.ts index 313180b1..6ef8adbb 100644 --- a/src/js/store.ts +++ b/src/js/store.ts @@ -4,6 +4,7 @@ import configReducer from '@/features/config/config.store'; import contentReducer from '@/features/content/content.store'; import dataReducer from '@/features/data/data.store'; import queryReducer from '@/features/search/query.store'; +import datasetsReducer from '@/features/datasets/datasets.store'; import ingestionDataReducer from '@/features/ingestion/ingestion.store'; import provenanceReducer from '@/features/provenance/provenance.store'; import beaconConfigReducer from './features/beacon/beaconConfig.store'; @@ -16,6 +17,7 @@ export const store = configureStore({ data: dataReducer, query: queryReducer, provenance: provenanceReducer, + datasets: datasetsReducer, ingestionData: ingestionDataReducer, beaconConfig: beaconConfigReducer, beaconQuery: beaconQueryReducer, From 89ee9bf0430ea1ebdd7b142295bc6e41b0431857 Mon Sep 17 00:00:00 2001 From: Julian Date: Tue, 10 Oct 2023 14:39:29 -0400 Subject: [PATCH 02/32] remove logs --- main.go | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/main.go b/main.go index 8392816e..b5b5ea3b 100644 --- a/main.go +++ b/main.go @@ -134,8 +134,6 @@ func main() { var req *http.Request var err error - fmt.Println("urlXXXXX: ", url) - if qs != nil { req, err = http.NewRequest("GET", fmt.Sprintf("%s?%s", url, qs.Encode()), nil) if err != nil { @@ -224,11 +222,7 @@ func main() { gohanRequestPublic := func(c echo.Context) error { // Construct the path using the extracted dataset ID. - path := fmt.Sprintf("/data-types") - - // Debug print statements to verify the constructed path and the GohanUrl. - fmt.Fprintf(os.Stderr, "pathCFCFCFCF: %s\n", path) - fmt.Println("DEBUG: cfg.GohanUrl =", cfg.GohanUrl) + path := fmt.Sprintf("/public/data-types") // Make a request to the Gohan service and deserialize the response. err := gohanRequest(path, nil, c, jsonDeserialize) @@ -452,16 +446,12 @@ func main() { return c.String(http.StatusOK, string(data)) }) - e.GET("/katsu-data-types", func(c echo.Context) error { - data, err := katsuRequestFormattedData("/data-types", c) + e.GET("/katsu/data-types", func(c echo.Context) error { + data, err := katsuRequestFormattedData("/public/data-types", c) if err != nil { return err } - fmt.Println("XXXXXXX") - fmt.Println("data: ", string(data)) - fmt.Println("XXXXXXX") - return c.String(http.StatusOK, string(data)) }) From 2c0cff86bf1db30151e96197913f5e054144e67a Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:28:22 -0400 Subject: [PATCH 03/32] types for last ingestion by datatypes --- src/js/types/lastIngestionDataTypeResponse.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/js/types/lastIngestionDataTypeResponse.ts diff --git a/src/js/types/lastIngestionDataTypeResponse.ts b/src/js/types/lastIngestionDataTypeResponse.ts new file mode 100644 index 00000000..9e14a98d --- /dev/null +++ b/src/js/types/lastIngestionDataTypeResponse.ts @@ -0,0 +1,10 @@ +export interface LastIngestionDataTypeResponse { + count: number | null; + id: string; + label: string; + last_ingested: string | null; + queryable: boolean; + } + + export type DataResponseArray = LastIngestionDataTypeResponse[]; + \ No newline at end of file From 485fa67d5cad8845b00b116018960b07928864f4 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:30:23 -0400 Subject: [PATCH 04/32] Remove unnecessary files for wes request --- src/js/features/datasets/datasets.store.ts | 52 ----------------- src/js/features/ingestion/ingestion.store.ts | 59 -------------------- src/js/types/lastIngestionResponse.ts | 36 ------------ 3 files changed, 147 deletions(-) delete mode 100644 src/js/features/datasets/datasets.store.ts delete mode 100644 src/js/features/ingestion/ingestion.store.ts delete mode 100644 src/js/types/lastIngestionResponse.ts diff --git a/src/js/features/datasets/datasets.store.ts b/src/js/features/datasets/datasets.store.ts deleted file mode 100644 index 5ccfb31a..00000000 --- a/src/js/features/datasets/datasets.store.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit'; -import axios from 'axios'; -import { datasetsUrl } from '@/constants/configConstants'; -import { printAPIError } from '@/utils/error.util'; - -// Type definition for Dataset response. You need to define this according to the response structure -interface DatasetResponse { - // Define the structure based on the response of the /datasets endpoint. - // Here's a placeholder type; replace with actual fields - id: string; - name: string; - description: string; -} - -export const fetchDatasets = createAsyncThunk( - 'datasets/fetchDatasets', - (_, { rejectWithValue }) => - axios - .get(datasetsUrl) - .then((res) => res.data) - .catch(printAPIError(rejectWithValue)) -); - -export interface DatasetsState { - isFetchingDatasets: boolean; - datasets: DatasetResponse[]; -} - -const initialState: DatasetsState = { - isFetchingDatasets: false, - datasets: [], -}; - -const DatasetsStore = createSlice({ - name: 'datasets', - initialState, - reducers: {}, - extraReducers: (builder) => { - builder.addCase(fetchDatasets.pending, (state) => { - state.isFetchingDatasets = true; - }); - builder.addCase(fetchDatasets.fulfilled, (state, { payload }: PayloadAction) => { - state.datasets = payload; - state.isFetchingDatasets = false; - }); - builder.addCase(fetchDatasets.rejected, (state) => { - state.isFetchingDatasets = false; - }); - }, -}); - -export default DatasetsStore.reducer; diff --git a/src/js/features/ingestion/ingestion.store.ts b/src/js/features/ingestion/ingestion.store.ts deleted file mode 100644 index 0fb8964f..00000000 --- a/src/js/features/ingestion/ingestion.store.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit'; -import axios from 'axios'; -import { lastIngestionsUrl } from '@/constants/configConstants'; -import { printAPIError } from '@/utils/error.util'; -import { IngestionData, LastIngestionResponse } from '@/types/lastIngestionResponse'; - -export const makeGetIngestionDataRequest = createAsyncThunk( - 'ingestionData/getIngestionData', - (_, { rejectWithValue }) => - axios - .get(lastIngestionsUrl) - .then((res) => res.data) - .catch(printAPIError(rejectWithValue)) -); - -export interface IngestionDataState { - isFetchingIngestionData: boolean; - ingestionData: IngestionData[]; - lastEndTimesByDataType: { [dataType: string]: string }; -} - -const initialState: IngestionDataState = { - isFetchingIngestionData: false, - ingestionData: [], - lastEndTimesByDataType: {}, -}; - -const IngestionDataStore = createSlice({ - name: 'ingestionData', - initialState, - reducers: {}, - extraReducers: (builder) => { - builder.addCase(makeGetIngestionDataRequest.pending, (state) => { - state.isFetchingIngestionData = true; - }); - builder.addCase( - makeGetIngestionDataRequest.fulfilled, - (state, { payload }: PayloadAction) => { - state.ingestionData = payload; - payload.forEach((ingestion) => { - const dataType = ingestion.details.request.tags.workflow_metadata.data_type; - console.log("losDATATipes",dataType); - //// here use dataType - const endTime = ingestion.details.run_log.end_time; - const previousEndTime = state.lastEndTimesByDataType[dataType]; - if (!previousEndTime || new Date(endTime) > new Date(previousEndTime)) { - state.lastEndTimesByDataType[dataType] = endTime; - } - }); - state.isFetchingIngestionData = false; - } - ); - builder.addCase(makeGetIngestionDataRequest.rejected, (state) => { - state.isFetchingIngestionData = false; - }); - }, -}); - -export default IngestionDataStore.reducer; diff --git a/src/js/types/lastIngestionResponse.ts b/src/js/types/lastIngestionResponse.ts deleted file mode 100644 index 22357371..00000000 --- a/src/js/types/lastIngestionResponse.ts +++ /dev/null @@ -1,36 +0,0 @@ -interface Tags { - table_id: string; - workflow_id: string; - workflow_metadata: { - data_type: string; - id: string; - }; -} - -interface Request { - tags: Tags; - workflow_type: string; -} - -interface RunLog { - end_time: string; - id: string; - start_time: string; -} - -export interface IngestionDetails { - end_time: string; - request: Request; - run_id: string; - run_log: RunLog; - state: string; - task_logs: null; -} - -export interface IngestionData { - details: IngestionDetails; - run_id: string; - state: string; -} - -export type LastIngestionResponse = IngestionData[]; From 471083aae844d1313adb47ffe00272f81221c151 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:31:18 -0400 Subject: [PATCH 05/32] add store for last ingestion in datatypes --- .../features/ingestion/lastIngestion.store.ts | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 src/js/features/ingestion/lastIngestion.store.ts diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts new file mode 100644 index 00000000..af916a0d --- /dev/null +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -0,0 +1,79 @@ +import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit'; +import axios from 'axios'; +import { katsuLastIngestionsUrl, gohanLastIngestionsUrl } from '@/constants/configConstants'; +import { printAPIError } from '@/utils/error.util'; + +export interface DataTypeResponse { + count: number | null; + id: string; + label: string; + last_ingested: string | null; + queryable: boolean; +} + +export type DataResponseArray = DataTypeResponse[]; + +// Async thunks to fetch data from the two endpoints +export const fetchKatsuData = createAsyncThunk( + 'dataTypes/fetchKatsuData', + (_, { rejectWithValue }) => + axios.get(katsuLastIngestionsUrl).then((res) => res.data).catch(printAPIError(rejectWithValue)) +); + +export const fetchGohanData = createAsyncThunk( + 'dataTypes/fetchGohanData', + (_, { rejectWithValue }) => + axios.get(gohanLastIngestionsUrl).then((res) => res.data).catch(printAPIError(rejectWithValue)) +); + +// Define the state structure +export interface DataTypeState { + isFetchingData: boolean; + dataTypes: DataResponseArray; +} + +// Initialize the state +const initialDataTypeState: DataTypeState = { + isFetchingData: false, + dataTypes: [], +}; + +// Create a slice to manage the state +const DataTypeStore = createSlice({ + name: 'dataTypes', + initialState: initialDataTypeState, + reducers: {}, + extraReducers: (builder) => { + builder.addCase(fetchKatsuData.pending, (state) => { + state.isFetchingData = true; + }); + builder.addCase(fetchGohanData.pending, (state) => { + state.isFetchingData = true; + }); + builder.addCase( + fetchKatsuData.fulfilled, + (state, { payload }: PayloadAction) => { + const uniqueIds = new Set(state.dataTypes.map(data => data.id)); + const newData = payload.filter(data => !uniqueIds.has(data.id)); + state.dataTypes = [...state.dataTypes, ...newData]; + } + ); + builder.addCase( + fetchGohanData.fulfilled, + (state, { payload }: PayloadAction) => { + const uniqueIds = new Set(state.dataTypes.map(data => data.id)); + const newData = payload.filter(data => !uniqueIds.has(data.id)); + state.dataTypes = [...state.dataTypes, ...newData]; + state.isFetchingData = false; + } + ); + builder.addCase(fetchKatsuData.rejected, (state) => { + state.isFetchingData = false; + }); + builder.addCase(fetchGohanData.rejected, (state) => { + state.isFetchingData = false; + }); + }, +}); + +export default DataTypeStore.reducer; From bc2c8d9b068727c1e29cd4d839f7224af6da955a Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:32:14 -0400 Subject: [PATCH 06/32] refactor gohan request --- main.go | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/main.go b/main.go index b5b5ea3b..4f3197bc 100644 --- a/main.go +++ b/main.go @@ -220,21 +220,13 @@ func main() { } gohanRequestPublic := func(c echo.Context) error { - - // Construct the path using the extracted dataset ID. - path := fmt.Sprintf("/public/data-types") - - // Make a request to the Gohan service and deserialize the response. - err := gohanRequest(path, nil, c, jsonDeserialize) + path := fmt.Sprintf("/data-types") + result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.GohanUrl, path), nil, c, jsonDeserialize) if err != nil { - // Return the error if there is one. return err } - // If everything is successful, return a success status to the caller. - return c.JSON(http.StatusOK, map[string]string{ - "message": "Success", - }) + return c.JSON(http.StatusOK, result) } fetchAndSetKatsuPublic := func(c echo.Context, katsuCache *cache.Cache) (JsonLike, error) { From 3d9e9ba50bed47be2a71098aaaaa325072bf1af9 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:34:57 -0400 Subject: [PATCH 07/32] add gohan/data-types endpoint --- main.go | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/main.go b/main.go index 4f3197bc..75dae30e 100644 --- a/main.go +++ b/main.go @@ -405,38 +405,7 @@ func main() { return c.String(http.StatusOK, string(data)) }) - e.GET("/datasets", func(c echo.Context) error { - data, err := katsuRequestFormattedData("/api/datasets", c) - if err != nil { - return err - } - - fmt.Println("FFFFF") - fmt.Println("data: ", string(data)) - fmt.Println("FFFFF") - - return c.String(http.StatusOK, string(data)) - - }) - - e.GET("/datasets/:id/data-types/:data-type/public", func(c echo.Context) error { - id := c.Param("id") - dataType := c.Param("data-type") - relativeUrl := fmt.Sprintf("/datasets/%s/data-types/%s/public", id, dataType) - - data, err := katsuRequestFormattedData(relativeUrl, c) - if err != nil { - return err - } - - fmt.Println("XXXXXXX") - - fmt.Println("data: ", string(data)) - - fmt.Println("XXXXXXX") - - return c.String(http.StatusOK, string(data)) - }) + e.GET("/gohan/data-types", gohanRequestPublic) e.GET("/katsu/data-types", func(c echo.Context) error { data, err := katsuRequestFormattedData("/public/data-types", c) From 9e55a48471d51e9c0849a9b6e3c8dafe39373a32 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:35:46 -0400 Subject: [PATCH 08/32] refactor katsu datatypes retrieval --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 75dae30e..361e6f91 100644 --- a/main.go +++ b/main.go @@ -408,7 +408,7 @@ func main() { e.GET("/gohan/data-types", gohanRequestPublic) e.GET("/katsu/data-types", func(c echo.Context) error { - data, err := katsuRequestFormattedData("/public/data-types", c) + data, err := katsuRequestFormattedData("/data-types", c) if err != nil { return err } From 01de0f6d617e62ff89c9def39fd3c6b92632cda1 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:36:12 -0400 Subject: [PATCH 09/32] remove wes request --- main.go | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/main.go b/main.go index 361e6f91..70075b81 100644 --- a/main.go +++ b/main.go @@ -179,22 +179,6 @@ func main() { return c.JSON(http.StatusOK, result) } - wesRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatterFunc) error { - result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.WesUrl, path), qs, c, rf) - if err != nil { - return err - } - return c.JSON(http.StatusOK, result) - } - - gohanRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatterFunc) error { - result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.GohanUrl, path), qs, c, rf) - if err != nil { - return err - } - return c.JSON(http.StatusOK, result) - } - katsuRequestBasic := func(path string, c echo.Context) error { return katsuRequest(path, nil, c, jsonDeserialize) } @@ -212,13 +196,6 @@ func main() { return jsonFormattedData, nil } - wesRequestWithDetailsAndPublic := func(c echo.Context) error { - qs := url.Values{} - qs.Add("with_details", "true") - qs.Add("public", "true") - return wesRequest("/runs", qs, c, jsonDeserialize) - } - gohanRequestPublic := func(c echo.Context) error { path := fmt.Sprintf("/data-types") result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.GohanUrl, path), nil, c, jsonDeserialize) @@ -380,10 +357,6 @@ func main() { return katsuRequestBasic("/api/public_search_fields", c) }) - e.GET("/wes-runs", wesRequestWithDetailsAndPublic) - - e.GET("/gohan/data-types", gohanRequestPublic) - e.GET("/provenance", func(c echo.Context) error { // Query Katsu for datasets provenance return katsuRequestBasic("/api/public_dataset", c) From 88d359b5e1c6125efd4e7e81e9a5fe403469f68a Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:36:50 -0400 Subject: [PATCH 10/32] configure stire for lastIngestionData --- src/js/store.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/js/store.ts b/src/js/store.ts index 6ef8adbb..3477ad2e 100644 --- a/src/js/store.ts +++ b/src/js/store.ts @@ -4,8 +4,7 @@ import configReducer from '@/features/config/config.store'; import contentReducer from '@/features/content/content.store'; import dataReducer from '@/features/data/data.store'; import queryReducer from '@/features/search/query.store'; -import datasetsReducer from '@/features/datasets/datasets.store'; -import ingestionDataReducer from '@/features/ingestion/ingestion.store'; +import lastIngestionDataReducer from '@/features/ingestion/lastIngestion.store'; import provenanceReducer from '@/features/provenance/provenance.store'; import beaconConfigReducer from './features/beacon/beaconConfig.store'; import beaconQueryReducer from './features/beacon/beaconQuery.store'; @@ -17,8 +16,7 @@ export const store = configureStore({ data: dataReducer, query: queryReducer, provenance: provenanceReducer, - datasets: datasetsReducer, - ingestionData: ingestionDataReducer, + lastIngestionData: lastIngestionDataReducer, beaconConfig: beaconConfigReducer, beaconQuery: beaconQueryReducer, }, From 2c436eff69d5021f0bcfa8f9eb360c9f0db3ec9e Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:38:23 -0400 Subject: [PATCH 11/32] dispatch acction for retrieve data-types with last ingestion data --- src/js/components/TabbedDashboard.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/js/components/TabbedDashboard.tsx b/src/js/components/TabbedDashboard.tsx index a23bd2b7..0c61e0b8 100644 --- a/src/js/components/TabbedDashboard.tsx +++ b/src/js/components/TabbedDashboard.tsx @@ -14,6 +14,7 @@ import {fetchDatasets } from '@/features/datasets/datasets.store'; import { makeGetIngestionDataRequest } from '@/features/ingestion/ingestion.store'; import { getBeaconConfig } from '@/features/beacon/beaconConfig.store'; import {makeGetDataTypes } from '@/features/dataTypes/dataTypes.store'; +import { fetchGohanData, fetchKatsuData } from '@/features/ingestion/lastIngestion.store'; import Loader from './Loader'; import PublicOverview from './Overview/PublicOverview'; @@ -37,8 +38,8 @@ const TabbedDashboard = () => { dispatch(makeGetDataRequestThunk()); dispatch(makeGetSearchFields()); dispatch(makeGetProvenanceRequest()); - dispatch(makeGetIngestionDataRequest()); - dispatch(fetchDatasets()); + dispatch(fetchKatsuData()); + dispatch(fetchGohanData()); dispatch(makeGetDataTypes()); }, []); From 1efabb18c21f1e532e87c3240e2ff622d2a97a49 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:39:12 -0400 Subject: [PATCH 12/32] remove unnecessary dispatchs --- src/js/components/TabbedDashboard.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/js/components/TabbedDashboard.tsx b/src/js/components/TabbedDashboard.tsx index 0c61e0b8..46d39016 100644 --- a/src/js/components/TabbedDashboard.tsx +++ b/src/js/components/TabbedDashboard.tsx @@ -10,8 +10,6 @@ import { makeGetAboutRequest } from '@/features/content/content.store'; import { makeGetDataRequestThunk } from '@/features/data/data.store'; import { makeGetSearchFields } from '@/features/search/query.store'; import { makeGetProvenanceRequest } from '@/features/provenance/provenance.store'; -import {fetchDatasets } from '@/features/datasets/datasets.store'; -import { makeGetIngestionDataRequest } from '@/features/ingestion/ingestion.store'; import { getBeaconConfig } from '@/features/beacon/beaconConfig.store'; import {makeGetDataTypes } from '@/features/dataTypes/dataTypes.store'; import { fetchGohanData, fetchKatsuData } from '@/features/ingestion/lastIngestion.store'; From d171b73e9fe8cf0fe0fa8fee986479b15de2a4e7 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:39:46 -0400 Subject: [PATCH 13/32] refactor LastIngestionInfo for new data --- src/js/components/Overview/LastIngestion.tsx | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/js/components/Overview/LastIngestion.tsx b/src/js/components/Overview/LastIngestion.tsx index 46ed6b03..57ff48bb 100644 --- a/src/js/components/Overview/LastIngestion.tsx +++ b/src/js/components/Overview/LastIngestion.tsx @@ -6,10 +6,14 @@ import { useTranslation } from 'react-i18next'; import { DEFAULT_TRANSLATION } from '@/constants/configConstants'; import { useAppSelector } from '@/hooks'; import { getDataTypeLabel } from '@/types/dataTypes'; +import { LastIngestionDataTypeResponse, DataResponseArray } from '@/types/lastIngestionDataTypeResponse'; const LastIngestionInfo: React.FC = () => { const { t, i18n } = useTranslation(DEFAULT_TRANSLATION); - const lastEndTimesByDataType = useAppSelector((state) => state.ingestionData?.lastEndTimesByDataType) || {}; + + const lastEndTimesByDataType: DataResponseArray = useAppSelector((state) => state.lastIngestionData?.dataTypes) || []; + + const queryableDataTypes = lastEndTimesByDataType.filter((item: LastIngestionDataTypeResponse) => item.queryable); const formatDate = useCallback( (dateString: string) => { @@ -28,19 +32,19 @@ const LastIngestionInfo: React.FC = () => { [i18n.language] ); - const hasData = Object.keys(lastEndTimesByDataType).length > 0; + const hasData = queryableDataTypes.length > 0; return ( {t('Latest Data IngestionX')} {hasData ? ( - Object.entries(lastEndTimesByDataType).map(([dataType, endTime]) => ( - + queryableDataTypes.map((dataType: LastIngestionDataTypeResponse) => ( + - {t(getDataTypeLabel(dataType))} + {getDataTypeLabel(dataType.id)} - {formatDate(endTime)} + {dataType.last_ingested ? formatDate(dataType.last_ingested) : 'Not Available'} From 8829bf732abc1a50584d4d24edb8874a7970620a Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:40:34 -0400 Subject: [PATCH 14/32] add partial urls for data-ty retrieval pron katsu and gohan --- src/js/constants/configConstants.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/js/constants/configConstants.ts b/src/js/constants/configConstants.ts index f3eb98af..c3c27e97 100644 --- a/src/js/constants/configConstants.ts +++ b/src/js/constants/configConstants.ts @@ -6,8 +6,8 @@ export const searchFieldsUrl = '/fields'; export const katsuUrl = '/katsu'; export const provenanceUrl = '/provenance'; export const lastIngestionsUrl = '/wes-runs'; -export const datasetsUrl = '/datasets'; -export const datasetDataTypePublicUrl = '/datasets/:id/data-types/:data-type/public'; +export const katsuLastIngestionsUrl = '/katsu/data-types'; +export const gohanLastIngestionsUrl = '/gohan/data-types'; export const DEFAULT_TRANSLATION = 'default_translation'; export const NON_DEFAULT_TRANSLATION = 'translation'; From 8d5411d4714922ef062c2a39b3a4b8e0acd9cfc4 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 11 Oct 2023 14:53:22 -0400 Subject: [PATCH 15/32] prettier --- src/js/components/TabbedDashboard.tsx | 2 +- .../features/ingestion/lastIngestion.store.ts | 46 +++++++++---------- src/js/store.ts | 2 +- src/js/types/lastIngestionDataTypeResponse.ts | 17 ++++--- 4 files changed, 31 insertions(+), 36 deletions(-) diff --git a/src/js/components/TabbedDashboard.tsx b/src/js/components/TabbedDashboard.tsx index 46d39016..cdff2d2b 100644 --- a/src/js/components/TabbedDashboard.tsx +++ b/src/js/components/TabbedDashboard.tsx @@ -11,7 +11,7 @@ import { makeGetDataRequestThunk } from '@/features/data/data.store'; import { makeGetSearchFields } from '@/features/search/query.store'; import { makeGetProvenanceRequest } from '@/features/provenance/provenance.store'; import { getBeaconConfig } from '@/features/beacon/beaconConfig.store'; -import {makeGetDataTypes } from '@/features/dataTypes/dataTypes.store'; +import { makeGetDataTypes } from '@/features/dataTypes/dataTypes.store'; import { fetchGohanData, fetchKatsuData } from '@/features/ingestion/lastIngestion.store'; import Loader from './Loader'; diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts index af916a0d..36554893 100644 --- a/src/js/features/ingestion/lastIngestion.store.ts +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -14,16 +14,18 @@ export interface DataTypeResponse { export type DataResponseArray = DataTypeResponse[]; // Async thunks to fetch data from the two endpoints -export const fetchKatsuData = createAsyncThunk( - 'dataTypes/fetchKatsuData', - (_, { rejectWithValue }) => - axios.get(katsuLastIngestionsUrl).then((res) => res.data).catch(printAPIError(rejectWithValue)) +export const fetchKatsuData = createAsyncThunk('dataTypes/fetchKatsuData', (_, { rejectWithValue }) => + axios + .get(katsuLastIngestionsUrl) + .then((res) => res.data) + .catch(printAPIError(rejectWithValue)) ); -export const fetchGohanData = createAsyncThunk( - 'dataTypes/fetchGohanData', - (_, { rejectWithValue }) => - axios.get(gohanLastIngestionsUrl).then((res) => res.data).catch(printAPIError(rejectWithValue)) +export const fetchGohanData = createAsyncThunk('dataTypes/fetchGohanData', (_, { rejectWithValue }) => + axios + .get(gohanLastIngestionsUrl) + .then((res) => res.data) + .catch(printAPIError(rejectWithValue)) ); // Define the state structure @@ -50,23 +52,17 @@ const DataTypeStore = createSlice({ builder.addCase(fetchGohanData.pending, (state) => { state.isFetchingData = true; }); - builder.addCase( - fetchKatsuData.fulfilled, - (state, { payload }: PayloadAction) => { - const uniqueIds = new Set(state.dataTypes.map(data => data.id)); - const newData = payload.filter(data => !uniqueIds.has(data.id)); - state.dataTypes = [...state.dataTypes, ...newData]; - } - ); - builder.addCase( - fetchGohanData.fulfilled, - (state, { payload }: PayloadAction) => { - const uniqueIds = new Set(state.dataTypes.map(data => data.id)); - const newData = payload.filter(data => !uniqueIds.has(data.id)); - state.dataTypes = [...state.dataTypes, ...newData]; - state.isFetchingData = false; - } - ); + builder.addCase(fetchKatsuData.fulfilled, (state, { payload }: PayloadAction) => { + const uniqueIds = new Set(state.dataTypes.map((data) => data.id)); + const newData = payload.filter((data) => !uniqueIds.has(data.id)); + state.dataTypes = [...state.dataTypes, ...newData]; + }); + builder.addCase(fetchGohanData.fulfilled, (state, { payload }: PayloadAction) => { + const uniqueIds = new Set(state.dataTypes.map((data) => data.id)); + const newData = payload.filter((data) => !uniqueIds.has(data.id)); + state.dataTypes = [...state.dataTypes, ...newData]; + state.isFetchingData = false; + }); builder.addCase(fetchKatsuData.rejected, (state) => { state.isFetchingData = false; }); diff --git a/src/js/store.ts b/src/js/store.ts index 3477ad2e..921821a8 100644 --- a/src/js/store.ts +++ b/src/js/store.ts @@ -16,7 +16,7 @@ export const store = configureStore({ data: dataReducer, query: queryReducer, provenance: provenanceReducer, - lastIngestionData: lastIngestionDataReducer, + lastIngestionData: lastIngestionDataReducer, beaconConfig: beaconConfigReducer, beaconQuery: beaconQueryReducer, }, diff --git a/src/js/types/lastIngestionDataTypeResponse.ts b/src/js/types/lastIngestionDataTypeResponse.ts index 9e14a98d..c3e7eebe 100644 --- a/src/js/types/lastIngestionDataTypeResponse.ts +++ b/src/js/types/lastIngestionDataTypeResponse.ts @@ -1,10 +1,9 @@ export interface LastIngestionDataTypeResponse { - count: number | null; - id: string; - label: string; - last_ingested: string | null; - queryable: boolean; - } - - export type DataResponseArray = LastIngestionDataTypeResponse[]; - \ No newline at end of file + count: number | null; + id: string; + label: string; + last_ingested: string | null; + queryable: boolean; +} + +export type DataResponseArray = LastIngestionDataTypeResponse[]; From c8c53a80687ec0dd0620890596aa8717c132b3a9 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 8 Nov 2023 14:59:26 -0500 Subject: [PATCH 16/32] remove wes last ingestion retrieval --- main.go | 3 --- src/js/constants/configConstants.ts | 1 - 2 files changed, 4 deletions(-) diff --git a/main.go b/main.go index 70075b81..46c42be4 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,6 @@ const ConfigLogTemplate = `Config -- Static Files: %s Client Name: %s Katsu URL: %v - WES URL: %v Gohan URL: %v Bento Portal Url: %s Port: %d @@ -38,7 +37,6 @@ type BentoConfig struct { StaticFilesPath string `envconfig:"BENTO_PUBLIC_STATIC_FILES_PATH" default:"./www"` ClientName string `envconfig:"BENTO_PUBLIC_CLIENT_NAME"` KatsuUrl string `envconfig:"BENTO_PUBLIC_KATSU_URL"` - WesUrl string `envconfig:"BENTO_PUBLIC_WES_URL"` GohanUrl string `envconfig:"BENTO_PUBLIC_GOHAN_URL"` BentoPortalUrl string `envconfig:"BENTO_PUBLIC_PORTAL_URL"` Port int `envconfig:"INTERNAL_PORT" default:"8090"` @@ -116,7 +114,6 @@ func main() { cfg.StaticFilesPath, cfg.ClientName, cfg.KatsuUrl, - cfg.WesUrl, cfg.GohanUrl, cfg.BentoPortalUrl, cfg.Port, diff --git a/src/js/constants/configConstants.ts b/src/js/constants/configConstants.ts index c3c27e97..02425459 100644 --- a/src/js/constants/configConstants.ts +++ b/src/js/constants/configConstants.ts @@ -5,7 +5,6 @@ export const publicOverviewUrl = '/overview'; export const searchFieldsUrl = '/fields'; export const katsuUrl = '/katsu'; export const provenanceUrl = '/provenance'; -export const lastIngestionsUrl = '/wes-runs'; export const katsuLastIngestionsUrl = '/katsu/data-types'; export const gohanLastIngestionsUrl = '/gohan/data-types'; From e91b8182824279e295427a7dde5d3cbbf390e632 Mon Sep 17 00:00:00 2001 From: Julian Date: Fri, 10 Nov 2023 14:51:08 -0500 Subject: [PATCH 17/32] Refactor last ingestion store --- .../features/ingestion/lastIngestion.store.ts | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts index 36554893..4d11bab6 100644 --- a/src/js/features/ingestion/lastIngestion.store.ts +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -40,6 +40,16 @@ const initialDataTypeState: DataTypeState = { dataTypes: [], }; +const reduceServiceDataTypes = ( + state: DataTypeState, + { payload }: PayloadAction +) => { + const uniqueIds = new Set(state.dataTypes.map((data: DataTypeResponse) => data.id)); + const newData = payload.filter((data: DataTypeResponse) => !uniqueIds.has(data.id)); + state.dataTypes = [...state.dataTypes, ...newData]; + state.isFetchingData = false; +}; + // Create a slice to manage the state const DataTypeStore = createSlice({ name: 'dataTypes', @@ -52,17 +62,8 @@ const DataTypeStore = createSlice({ builder.addCase(fetchGohanData.pending, (state) => { state.isFetchingData = true; }); - builder.addCase(fetchKatsuData.fulfilled, (state, { payload }: PayloadAction) => { - const uniqueIds = new Set(state.dataTypes.map((data) => data.id)); - const newData = payload.filter((data) => !uniqueIds.has(data.id)); - state.dataTypes = [...state.dataTypes, ...newData]; - }); - builder.addCase(fetchGohanData.fulfilled, (state, { payload }: PayloadAction) => { - const uniqueIds = new Set(state.dataTypes.map((data) => data.id)); - const newData = payload.filter((data) => !uniqueIds.has(data.id)); - state.dataTypes = [...state.dataTypes, ...newData]; - state.isFetchingData = false; - }); + builder.addCase(fetchKatsuData.fulfilled, reduceServiceDataTypes); + builder.addCase(fetchGohanData.fulfilled, reduceServiceDataTypes); builder.addCase(fetchKatsuData.rejected, (state) => { state.isFetchingData = false; }); From ebee93d2c980ed94d6779f77f6ce971831bb623b Mon Sep 17 00:00:00 2001 From: Julian Date: Fri, 10 Nov 2023 14:54:06 -0500 Subject: [PATCH 18/32] lint --- src/js/features/ingestion/lastIngestion.store.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts index 4d11bab6..b1521894 100644 --- a/src/js/features/ingestion/lastIngestion.store.ts +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -40,10 +40,7 @@ const initialDataTypeState: DataTypeState = { dataTypes: [], }; -const reduceServiceDataTypes = ( - state: DataTypeState, - { payload }: PayloadAction -) => { +const reduceServiceDataTypes = (state: DataTypeState, { payload }: PayloadAction) => { const uniqueIds = new Set(state.dataTypes.map((data: DataTypeResponse) => data.id)); const newData = payload.filter((data: DataTypeResponse) => !uniqueIds.has(data.id)); state.dataTypes = [...state.dataTypes, ...newData]; From bf98a5f23191644fc64e1fd42a1e8bfce75ca4ef Mon Sep 17 00:00:00 2001 From: noctillion Date: Fri, 10 Nov 2023 19:11:19 -0500 Subject: [PATCH 19/32] Update LastIngestion.tsx remove typo --- src/js/components/Overview/LastIngestion.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/components/Overview/LastIngestion.tsx b/src/js/components/Overview/LastIngestion.tsx index 57ff48bb..88ea533c 100644 --- a/src/js/components/Overview/LastIngestion.tsx +++ b/src/js/components/Overview/LastIngestion.tsx @@ -36,7 +36,7 @@ const LastIngestionInfo: React.FC = () => { return ( - {t('Latest Data IngestionX')} + {t('Latest Data Ingestion')} {hasData ? ( queryableDataTypes.map((dataType: LastIngestionDataTypeResponse) => ( From e413073f4a0874139204248008b6bd9236463c39 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 15 Nov 2023 16:42:07 -0500 Subject: [PATCH 20/32] Set count to null from gohan data-types response --- main.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index 46c42be4..f8413d15 100644 --- a/main.go +++ b/main.go @@ -200,7 +200,22 @@ func main() { return err } - return c.JSON(http.StatusOK, result) + resultSlice, ok := result.([]JsonLike) + if !ok { + return fmt.Errorf("result is not of type []JsonLike") + } + + var modifiedResult []JsonLike + + // Remove the "count" key + for _, item := range resultSlice { + item["count"] = nil + modifiedResult = append(modifiedResult, item) + } + + fmt.Printf("modifiedResult: %v\n", modifiedResult) + + return c.JSON(http.StatusOK, modifiedResult) } fetchAndSetKatsuPublic := func(c echo.Context, katsuCache *cache.Cache) (JsonLike, error) { From d48c0a2737fc27bd05daaec6396860b0388a843c Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 15 Nov 2023 16:45:32 -0500 Subject: [PATCH 21/32] correct comment --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index f8413d15..ad747ee2 100644 --- a/main.go +++ b/main.go @@ -207,7 +207,7 @@ func main() { var modifiedResult []JsonLike - // Remove the "count" key + // Update the "count" value for _, item := range resultSlice { item["count"] = nil modifiedResult = append(modifiedResult, item) From e91b57ab7e3dd4db3cdba68ad35fbc7df374ab35 Mon Sep 17 00:00:00 2001 From: Julian Date: Wed, 15 Nov 2023 16:59:01 -0500 Subject: [PATCH 22/32] remove logs --- main.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/main.go b/main.go index ad747ee2..df5f9562 100644 --- a/main.go +++ b/main.go @@ -213,8 +213,6 @@ func main() { modifiedResult = append(modifiedResult, item) } - fmt.Printf("modifiedResult: %v\n", modifiedResult) - return c.JSON(http.StatusOK, modifiedResult) } From 4cee04329635a1103738ca4033c7ad6a7d702888 Mon Sep 17 00:00:00 2001 From: Julian Date: Thu, 16 Nov 2023 17:33:08 -0500 Subject: [PATCH 23/32] Add request handling for data-types endpoint and count value removal --- main.go | 46 ++++++++++++++++++++-------------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/main.go b/main.go index df5f9562..f3dc055a 100644 --- a/main.go +++ b/main.go @@ -193,27 +193,28 @@ func main() { return jsonFormattedData, nil } - gohanRequestPublic := func(c echo.Context) error { - path := fmt.Sprintf("/data-types") - result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.GohanUrl, path), nil, c, jsonDeserialize) - if err != nil { - return err - } + dataTypesEndpointHandler := func(baseUrl string) echo.HandlerFunc { + return func(c echo.Context) error { + fullPath := fmt.Sprintf("%s/data-types", baseUrl) + result, err := genericRequestJsonOnly(fullPath, nil, c, jsonDeserialize) + if err != nil { + return err + } - resultSlice, ok := result.([]JsonLike) - if !ok { - return fmt.Errorf("result is not of type []JsonLike") - } + resultSlice, ok := result.([]JsonLike) + if !ok { + return fmt.Errorf("result is not of type []JsonLike") + } - var modifiedResult []JsonLike + var modifiedResult []JsonLike + // Update the "count" value + for _, item := range resultSlice { + item["count"] = nil + modifiedResult = append(modifiedResult, item) + } - // Update the "count" value - for _, item := range resultSlice { - item["count"] = nil - modifiedResult = append(modifiedResult, item) + return c.JSON(http.StatusOK, modifiedResult) } - - return c.JSON(http.StatusOK, modifiedResult) } fetchAndSetKatsuPublic := func(c echo.Context, katsuCache *cache.Cache) (JsonLike, error) { @@ -388,16 +389,9 @@ func main() { return c.String(http.StatusOK, string(data)) }) - e.GET("/gohan/data-types", gohanRequestPublic) + e.GET("/gohan/data-types", dataTypesEndpointHandler(cfg.GohanUrl)) - e.GET("/katsu/data-types", func(c echo.Context) error { - data, err := katsuRequestFormattedData("/data-types", c) - if err != nil { - return err - } - - return c.String(http.StatusOK, string(data)) - }) + e.GET("/katsu/data-types", dataTypesEndpointHandler(cfg.KatsuUrl)) // Run e.Logger.Fatal(e.Start(fmt.Sprintf(":%d", cfg.Port))) From 8525ad89d556527f06d226037edb81b18266a998 Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 09:50:21 -0500 Subject: [PATCH 24/32] Add clinical data to translation --- src/public/locales/en/default_translation_en.json | 3 ++- src/public/locales/fr/default_translation_fr.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/public/locales/en/default_translation_en.json b/src/public/locales/en/default_translation_en.json index b3587487..74c37cf8 100644 --- a/src/public/locales/en/default_translation_en.json +++ b/src/public/locales/en/default_translation_en.json @@ -74,5 +74,6 @@ "e.g.": "e.g.", "or": "or", "Ingestion History Is Empty": "Ingestion History Is Empty", - "Latest Data Ingestion": "Latest Data Ingestion" + "Latest Data Ingestion": "Latest Data Ingestion", + "Clinical Data": "Clinical Data" } \ No newline at end of file diff --git a/src/public/locales/fr/default_translation_fr.json b/src/public/locales/fr/default_translation_fr.json index b5e11373..1b2194ce 100644 --- a/src/public/locales/fr/default_translation_fr.json +++ b/src/public/locales/fr/default_translation_fr.json @@ -74,5 +74,6 @@ "e.g.": "par exemple", "or": "ou", "Ingestion History Is Empty": "L'historique d'ingestion est vide", - "Latest Data Ingestion": "Dernière ingestion de données" + "Latest Data Ingestion": "Dernière ingestion de données", + "Clinical Data": "Données cliniques" } \ No newline at end of file From 1bfae35a33fbfc1fdf8179f7fd2ecefb8884443d Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 09:54:37 -0500 Subject: [PATCH 25/32] Remove redundant types --- src/js/features/ingestion/lastIngestion.store.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts index b1521894..0010bcb9 100644 --- a/src/js/features/ingestion/lastIngestion.store.ts +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -3,15 +3,7 @@ import axios from 'axios'; import { katsuLastIngestionsUrl, gohanLastIngestionsUrl } from '@/constants/configConstants'; import { printAPIError } from '@/utils/error.util'; -export interface DataTypeResponse { - count: number | null; - id: string; - label: string; - last_ingested: string | null; - queryable: boolean; -} - -export type DataResponseArray = DataTypeResponse[]; +import { LastIngestionDataTypeResponse, DataTypeMap } from '@/types/lastIngestionDataTypeResponse'; // Async thunks to fetch data from the two endpoints export const fetchKatsuData = createAsyncThunk('dataTypes/fetchKatsuData', (_, { rejectWithValue }) => From 012932d0122df23fbb09b17d9131ab1361a3d447 Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 09:55:45 -0500 Subject: [PATCH 26/32] Refactor lastIngestion store to keep rendering order --- .../features/ingestion/lastIngestion.store.ts | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts index 0010bcb9..9206e91c 100644 --- a/src/js/features/ingestion/lastIngestion.store.ts +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -22,21 +22,25 @@ export const fetchGohanData = createAsyncThunk('dataTypes/fetchGohanData', (_, { // Define the state structure export interface DataTypeState { - isFetchingData: boolean; - dataTypes: DataResponseArray; + isFetchingKatsuData: boolean; + isFetchingGohanData: boolean; + dataTypes: DataTypeMap; } // Initialize the state const initialDataTypeState: DataTypeState = { - isFetchingData: false, - dataTypes: [], + isFetchingKatsuData: false, + isFetchingGohanData: false, + dataTypes: {}, }; -const reduceServiceDataTypes = (state: DataTypeState, { payload }: PayloadAction) => { - const uniqueIds = new Set(state.dataTypes.map((data: DataTypeResponse) => data.id)); - const newData = payload.filter((data: DataTypeResponse) => !uniqueIds.has(data.id)); - state.dataTypes = [...state.dataTypes, ...newData]; - state.isFetchingData = false; +const reduceServiceDataTypes = (state: DataTypeState, { payload }: PayloadAction) => { + state.dataTypes = { + ...state.dataTypes, + ...Object.fromEntries(payload.map((data) => [data.id, data])), + }; + state.isFetchingKatsuData = false; + state.isFetchingGohanData = false; }; // Create a slice to manage the state @@ -46,18 +50,18 @@ const DataTypeStore = createSlice({ reducers: {}, extraReducers: (builder) => { builder.addCase(fetchKatsuData.pending, (state) => { - state.isFetchingData = true; + state.isFetchingKatsuData = true; }); builder.addCase(fetchGohanData.pending, (state) => { - state.isFetchingData = true; + state.isFetchingGohanData = true; }); builder.addCase(fetchKatsuData.fulfilled, reduceServiceDataTypes); builder.addCase(fetchGohanData.fulfilled, reduceServiceDataTypes); builder.addCase(fetchKatsuData.rejected, (state) => { - state.isFetchingData = false; + state.isFetchingKatsuData = false; }); builder.addCase(fetchGohanData.rejected, (state) => { - state.isFetchingData = false; + state.isFetchingGohanData = false; }); }, }); From 5025155ea851b6ef69b1143eb4bf63422dc708f7 Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 09:56:16 -0500 Subject: [PATCH 27/32] Refactor LastIngestion types --- src/js/types/lastIngestionDataTypeResponse.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/types/lastIngestionDataTypeResponse.ts b/src/js/types/lastIngestionDataTypeResponse.ts index c3e7eebe..617bd148 100644 --- a/src/js/types/lastIngestionDataTypeResponse.ts +++ b/src/js/types/lastIngestionDataTypeResponse.ts @@ -6,4 +6,4 @@ export interface LastIngestionDataTypeResponse { queryable: boolean; } -export type DataResponseArray = LastIngestionDataTypeResponse[]; +export type DataTypeMap = { [id: string]: LastIngestionDataTypeResponse }; From a6e6fe3a9d2b66b07b0c8a8f9ada3c8691f2c131 Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 09:56:56 -0500 Subject: [PATCH 28/32] Refactor lastIngestion rendering --- src/js/components/Overview/LastIngestion.tsx | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/js/components/Overview/LastIngestion.tsx b/src/js/components/Overview/LastIngestion.tsx index 88ea533c..0ccd0bc4 100644 --- a/src/js/components/Overview/LastIngestion.tsx +++ b/src/js/components/Overview/LastIngestion.tsx @@ -6,14 +6,19 @@ import { useTranslation } from 'react-i18next'; import { DEFAULT_TRANSLATION } from '@/constants/configConstants'; import { useAppSelector } from '@/hooks'; import { getDataTypeLabel } from '@/types/dataTypes'; -import { LastIngestionDataTypeResponse, DataResponseArray } from '@/types/lastIngestionDataTypeResponse'; + +import { LastIngestionDataTypeResponse } from '@/types/lastIngestionDataTypeResponse'; const LastIngestionInfo: React.FC = () => { const { t, i18n } = useTranslation(DEFAULT_TRANSLATION); - const lastEndTimesByDataType: DataResponseArray = useAppSelector((state) => state.lastIngestionData?.dataTypes) || []; + const dataTypesObject = useAppSelector((state) => state.lastIngestionData?.dataTypes) || {}; + const dataTypesArray = Object.values(dataTypesObject); + + const sortedDataTypes = dataTypesArray.sort((a, b) => a.label.localeCompare(b.label)); - const queryableDataTypes = lastEndTimesByDataType.filter((item: LastIngestionDataTypeResponse) => item.queryable); + // Filter out the queryable data types + const queryableDataTypes = sortedDataTypes.filter((dataType: LastIngestionDataTypeResponse) => dataType.queryable); const formatDate = useCallback( (dateString: string) => { @@ -42,7 +47,7 @@ const LastIngestionInfo: React.FC = () => { queryableDataTypes.map((dataType: LastIngestionDataTypeResponse) => ( - {getDataTypeLabel(dataType.id)} + {t(getDataTypeLabel(dataType.id))} {dataType.last_ingested ? formatDate(dataType.last_ingested) : 'Not Available'} From 89eb3d5ac188bf96e91b322b66e17da1a8f5999a Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 09:59:05 -0500 Subject: [PATCH 29/32] lint --- src/js/components/Overview/LastIngestion.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/js/components/Overview/LastIngestion.tsx b/src/js/components/Overview/LastIngestion.tsx index 0ccd0bc4..76fe7739 100644 --- a/src/js/components/Overview/LastIngestion.tsx +++ b/src/js/components/Overview/LastIngestion.tsx @@ -47,7 +47,9 @@ const LastIngestionInfo: React.FC = () => { queryableDataTypes.map((dataType: LastIngestionDataTypeResponse) => ( - {t(getDataTypeLabel(dataType.id))} + + {t(getDataTypeLabel(dataType.id))} + {dataType.last_ingested ? formatDate(dataType.last_ingested) : 'Not Available'} From b39f43ac3663175a8bba41cdb18e98e6ada668a0 Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 10:52:17 -0500 Subject: [PATCH 30/32] Refactor combine array creation and sorting --- src/js/components/Overview/LastIngestion.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/js/components/Overview/LastIngestion.tsx b/src/js/components/Overview/LastIngestion.tsx index 76fe7739..d486153b 100644 --- a/src/js/components/Overview/LastIngestion.tsx +++ b/src/js/components/Overview/LastIngestion.tsx @@ -13,9 +13,8 @@ const LastIngestionInfo: React.FC = () => { const { t, i18n } = useTranslation(DEFAULT_TRANSLATION); const dataTypesObject = useAppSelector((state) => state.lastIngestionData?.dataTypes) || {}; - const dataTypesArray = Object.values(dataTypesObject); - const sortedDataTypes = dataTypesArray.sort((a, b) => a.label.localeCompare(b.label)); + const sortedDataTypes = Object.values(dataTypesObject).sort((a, b) => a.label.localeCompare(b.label)); // Filter out the queryable data types const queryableDataTypes = sortedDataTypes.filter((dataType: LastIngestionDataTypeResponse) => dataType.queryable); From 8a038e5b6babc1e28f19f221ba0e7ab2070f28a0 Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 11:05:01 -0500 Subject: [PATCH 31/32] Update fetching status flags --- src/js/features/ingestion/lastIngestion.store.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/js/features/ingestion/lastIngestion.store.ts b/src/js/features/ingestion/lastIngestion.store.ts index 9206e91c..09146e2e 100644 --- a/src/js/features/ingestion/lastIngestion.store.ts +++ b/src/js/features/ingestion/lastIngestion.store.ts @@ -39,8 +39,6 @@ const reduceServiceDataTypes = (state: DataTypeState, { payload }: PayloadAction ...state.dataTypes, ...Object.fromEntries(payload.map((data) => [data.id, data])), }; - state.isFetchingKatsuData = false; - state.isFetchingGohanData = false; }; // Create a slice to manage the state @@ -55,8 +53,14 @@ const DataTypeStore = createSlice({ builder.addCase(fetchGohanData.pending, (state) => { state.isFetchingGohanData = true; }); - builder.addCase(fetchKatsuData.fulfilled, reduceServiceDataTypes); - builder.addCase(fetchGohanData.fulfilled, reduceServiceDataTypes); + builder.addCase(fetchKatsuData.fulfilled, (state, action) => { + reduceServiceDataTypes(state, action); + state.isFetchingKatsuData = false; + }); + builder.addCase(fetchGohanData.fulfilled, (state, action) => { + reduceServiceDataTypes(state, action); + state.isFetchingGohanData = false; + }); builder.addCase(fetchKatsuData.rejected, (state) => { state.isFetchingKatsuData = false; }); From 4006fdcc7e4eb362675c5f21287c1fd3611595e7 Mon Sep 17 00:00:00 2001 From: Julian Date: Mon, 20 Nov 2023 13:22:55 -0500 Subject: [PATCH 32/32] Disable makeGetDataTypes dispatch --- src/js/components/TabbedDashboard.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/js/components/TabbedDashboard.tsx b/src/js/components/TabbedDashboard.tsx index cdff2d2b..0b2c3b64 100644 --- a/src/js/components/TabbedDashboard.tsx +++ b/src/js/components/TabbedDashboard.tsx @@ -11,7 +11,6 @@ import { makeGetDataRequestThunk } from '@/features/data/data.store'; import { makeGetSearchFields } from '@/features/search/query.store'; import { makeGetProvenanceRequest } from '@/features/provenance/provenance.store'; import { getBeaconConfig } from '@/features/beacon/beaconConfig.store'; -import { makeGetDataTypes } from '@/features/dataTypes/dataTypes.store'; import { fetchGohanData, fetchKatsuData } from '@/features/ingestion/lastIngestion.store'; import Loader from './Loader'; @@ -38,7 +37,7 @@ const TabbedDashboard = () => { dispatch(makeGetProvenanceRequest()); dispatch(fetchKatsuData()); dispatch(fetchGohanData()); - dispatch(makeGetDataTypes()); + //TODO: Dispatch makeGetDataTypes to get the data types from service-registry }, []); const isFetchingOverviewData = useAppSelector((state) => state.data.isFetchingData);