Skip to content

Commit

Permalink
#38 - Use HTTP Headers from published datasets URL to retrieve versio…
Browse files Browse the repository at this point in the history 
…ning of the application
  • Loading branch information
@jrmartin
jrmartin committed Dec 17, 2024
1 parent 4ffaace commit 532bff9
Show file tree
Hide file tree
Showing 6 changed files with 6,597 additions and 6,168 deletions.
7 changes: 5 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@
"private": true,
"homepage": "./",
"dependencies": {
"@craco/craco": "^6.1.2",
"@craco/craco": "^7.1.0",
"@frogcat/ttl2jsonld": "^0.0.7",
"@material-ui/core": "^4.12.1",
"@material-ui/icons": "^4.11.2",
"@material-ui/lab": "^4.0.0-alpha.60",
"@mui/material" : "^6.2.0",
"@metacell/geppetto-meta-client": "^0.0.3",
"@metacell/geppetto-meta-core": "^0.0.3",
"@metacell/geppetto-meta-ui": "^0.0.3",
Expand All @@ -21,6 +22,7 @@
"@types/node": "^15.0.2",
"@types/react": "^17.0.5",
"@types/react-dom": "^17.0.3",
"@types/three" : "^1.4.0",
"axios": "^0.21.1",
"fs": "^0.0.1-security",
"gh-pages": "^3.2.3",
Expand All @@ -35,7 +37,8 @@
"react-dom": "^17.0.2",
"react-hot-loader": "^4.13.0",
"react-redux": "^7.2.4",
"react-scripts": "^4.0.3",
"react-scripts": "^5.0.1",
"three": "^0.171.0",
"ts-jest": "^26.4.4",
"typescript": "^4.2.4",
"uuid": "^8.3.2"
Expand Down
9 changes: 5 additions & 4 deletions src/App.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import Box from '@material-ui/core/Box';
import Splinter from './utils/Splinter';
import MainLayout from './app/mainLayout';
import FileHandler from './utils/filesHandler';
import { fetchHeaders } from './utils/versionHandler';
import { useSelector, useDispatch } from 'react-redux';
import Sidebar from './components/Sidebar/Sidebar';
import EmptyContainer from './components/EmptyContainer';
Expand Down Expand Up @@ -131,7 +132,7 @@ const App = () => {
const splinter = new DatasetsListSplinter(undefined, file.data);
let graph = await splinter.getGraph();
let datasets = graph.nodes.filter((node) => node?.attributes?.hasDoi);
let version = config.version
let version = await fetchHeaders(config.repository_url + config.available_datasets);
const match = datasets.find( node => node.attributes?.hasDoi?.[0]?.includes(doi));
if ( match ) {
const datasetID = match.name;
Expand All @@ -156,14 +157,14 @@ const App = () => {
}
};

useEffect(() => {
useEffect(async () => {
if (datasetID && datasetID !== "" ) {
loadFiles(datasetID);
}

if (doi && doi !== "" ) {
if ( doiMatch ){
let version = config.version;
let version = await fetchHeaders(config.repository_url + config.available_datasets);
const storage = JSON.parse(localStorage.getItem(config.datasetsStorage));
const storageVersion = storage?.version
if ( storageVersion === version ) {
Expand All @@ -179,7 +180,7 @@ const App = () => {
} else {
const fileHandler = new FileHandler();
const summaryURL = config.repository_url + config.available_datasets;
fileHandler.get_remote_file(summaryURL, loadDatsetFromDOI);
await fileHandler.get_remote_file(summaryURL, loadDatsetFromDOI);
}
}
}
Expand Down
45 changes: 26 additions & 19 deletions src/components/DatasetsListViewer/DatasetsListDialog.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ import DatasetsListSplinter from "./DatasetsListSplinter";
import { WidgetStatus } from "@metacell/geppetto-meta-client/common/layout/model";
import FileHandler from "../../utils/filesHandler";
import UploadSubmit from "./../FileUploader/UploadSubmit";
import { fetchHeaders } from "../../utils/versionHandler";
import CLOSE from '../../images/close.svg';

import config from "./../../config/app.json";

const DatasetsListDialog = (props) => {
Expand All @@ -33,6 +33,7 @@ const DatasetsListDialog = (props) => {
const [selectedIndex, setSelectedIndex] = React.useState(undefined);
const datasets = useSelector((state) => state.sdsState.available_datasets);
const [filteredDatasets, setFilteredDatasets] = React.useState(datasets);
const [versionID, setVersionID] = React.useState("")
const PUBLISHED = "PUBLISHED";

let turtle_url = "";
Expand Down Expand Up @@ -92,7 +93,7 @@ const DatasetsListDialog = (props) => {
});
};

const loadDatasets = () => {
const loadDatasets = async (versionID) => {
const fileHandler = new FileHandler();
const callback = async (url, fileData) => {
let file = {
Expand All @@ -106,25 +107,24 @@ const DatasetsListDialog = (props) => {
let datasets = graph.nodes.filter((node) => node?.attributes?.hasUriApi);
datasets.forEach( node => node.attributes ? node.attributes.lowerCaseLabel = node.attributes?.label?.[0]?.toLowerCase() : null );
datasets = datasets.filter( node => node?.attributes?.statusOnPlatform?.[0]?.includes(PUBLISHED) );

let version = config.version;

let datasetStorage = {};
let parsedDatasets = []
datasets?.forEach( node => {
parsedDatasets.push({ name : node.name , doi : node.attributes?.hasDoi?.[0], label : node.attributes ? node.attributes?.lowerCaseLabel : null});
});
datasetStorage = {
version : version,
version : versionID,
datasets : parsedDatasets
}

localStorage.setItem(config.datasetsStorage, JSON.stringify(datasetStorage));
dispatch(setDatasetsList(datasetStorage.datasets));
setFilteredDatasets(datasetStorage.datasets);
};
const summaryURL = config.repository_url + config.available_datasets;
fileHandler.get_remote_file(summaryURL, callback);
};
};

const handleChange = (event) => {
const lowerCaseSearch = event.target.value.toLowerCase();
Expand Down Expand Up @@ -152,19 +152,26 @@ const DatasetsListDialog = (props) => {
}

useEffect(() => {
if ( open && datasets.length === 0 ) {
let version = config.version;
const storage = JSON.parse(localStorage.getItem(config.datasetsStorage));
const storageVersion = storage?.version
if ( localStorage.getItem(config.datasetsStorage) && version === storageVersion ) {
let storedDatasetsInfo = JSON.parse(localStorage.getItem(config.datasetsStorage));
dispatch(setDatasetsList(storedDatasetsInfo.datasets));
setFilteredDatasets(storedDatasetsInfo.datasets);
} else {
loadDatasets();
const fetchData = async () => {
if (open && datasets.length === 0) {
try {
const versionID = await fetchHeaders(config.repository_url + config.available_datasets);
const storage = JSON.parse(localStorage.getItem(config.datasetsStorage));
const storageVersion = storage?.version;

if (storage && versionID === storageVersion) {
dispatch(setDatasetsList(storage.datasets));
setFilteredDatasets(storage.datasets);
} else {
await loadDatasets(versionID);
}
} catch (error) {
console.error("Error in useEffect:", error);
}
}
}
});
};
fetchData();
}, [open]);

return (
<Dialog
Expand Down
39 changes: 20 additions & 19 deletions src/utils/filesHandler.js
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,18 @@
const axios = require('axios');
import axios from 'axios';

export default class FileHandler {

get_local_file(path, callback, progressCallback) {
if (path === undefined)
return undefined;
if (path === undefined) return undefined;

const reader = new FileReader();

reader.onload = function(response) {
callback(response.target.result)
reader.onload = function (response) {
callback(response.target.result);
};

reader.onprogress = function (data) {
if (data.lengthComputable) {
var progress = parseInt((data.loaded / data.total) * 100, 10);
const progress = parseInt((data.loaded / data.total) * 100, 10);
progressCallback(progress);
}
};
Expand All @@ -23,17 +21,20 @@ export default class FileHandler {
}

get_remote_file(url, callback, errorCallback) {
axios.get(url, {
crossDomain: true,
responseType: "stream",
onDownloadProgress: (progressEvent) => {
// Progress callback to be used for the loader.
}
}).then(response => {
callback(url, response.data);
}).catch(error => {
console.log(error);
errorCallback && errorCallback(undefined);
});
axios
.get(url, {
crossDomain: true,
responseType: "text", // Updated for browser compatibility
onDownloadProgress: (progressEvent) => {
// Optionally handle download progress
},
})
.then((response) => {
callback(url, response.data);
})
.catch((error) => {
console.error("Error fetching remote file:", error);
if (errorCallback) errorCallback(error);
});
}
}
20 changes: 20 additions & 0 deletions src/utils/versionHandler.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
export const fetchHeaders = async (url) => {
try {
const response = await fetch(url, {
method: 'HEAD', // Use HEAD method to fetch headers only
});

if (!response.ok) {
throw new Error(`HTTP error! status: ${response.status}`);
}

const headersObj = {};
response.headers.forEach((value, key) => {
headersObj[key] = value;
});

return headersObj?.["last-modified"];
} catch (err) {
console.log("Error retrieving headers " , err)
}
};
Loading

0 comments on commit 532bff9

Please sign in to comment.