Skip to content

Commit

Permalink
fix*data-table): Fix UI freeze when export huge features
Browse files Browse the repository at this point in the history 
Closes Canadian-Geospatial-Platform#2580
  • Loading branch information
@jolevesq
jolevesq committed Nov 5, 2024
1 parent 13a555b commit 2ab433d
Show file tree
Hide file tree
Showing 9 changed files with 355 additions and 87 deletions.
6 changes: 6 additions & 0 deletions ...ges/geoview-core/src/api/event-processors/event-processor-children/app-event-processor.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,9 @@ import { TypeHTMLElement } from '@/core/types/global-types';
import { createGuideObject } from '@/core/utils/utilities';
import { MapViewer } from '@/geo/map/map-viewer';
import { MapEventProcessor } from './map-event-processor';
import { SnackbarType } from '@/core/utils/notifications';
import { logger } from '@/core/utils/logger';
import { api } from '@/app';

// GV Important: See notes in header of MapEventProcessor file for information on the paradigm to apply when working with UIEventProcessor vs UIState

Expand Down Expand Up @@ -56,6 +58,10 @@ export class AppEventProcessor extends AbstractEventProcessor {
return this.getAppState(mapId).displayTheme;
}

static addMessage(mapId: string, type: SnackbarType, message: string): void {
api.maps[mapId].notifications.showMessage(message, undefined, false);
}

static async addNotification(mapId: string, notif: NotificationDetailsType): Promise<void> {
// because notification is called before map is created, we use the async
// version of getAppStateAsync
Expand Down
280 changes: 204 additions & 76 deletions packages/geoview-core/src/core/components/data-table/json-export-button.tsx
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
import { useCallback } from 'react';
/* eslint-disable no-await-in-loop */
import { useCallback, useState } from 'react';
import { useTranslation } from 'react-i18next';

import { Geometry, Point, Polygon, LineString, MultiPoint } from 'ol/geom';
// import { Geometry, Point, Polygon, LineString, MultiPoint } from 'ol/geom';
// import { Geometry } from 'ol/geom';

import { MenuItem } from '@/ui';
import { useMapStoreActions } from '@/core/stores/store-interface-and-intial-values/map-state';
// import { useMapStoreActions } from '@/core/stores/store-interface-and-intial-values/map-state';
import { TypeFeatureInfoEntry } from '@/geo/map/map-schema-types';
import { TypeJsonObject } from '@/core/types/global-types';
// import { TypeJsonObject } from '@/core/types/global-types';
import { useLayerStoreActions } from '@/core/stores/store-interface-and-intial-values/layer-state';
import { useAppStoreActions } from '@/core/stores/store-interface-and-intial-values/app-state';
// import { logger } from '@/core/utils/logger';

interface JSONExportButtonProps {
rows: unknown[];
Expand All @@ -26,85 +30,60 @@ function JSONExportButton({ rows, features, layerPath }: JSONExportButtonProps):
const { t } = useTranslation<string>();

// get store value - projection config to transfer lat long and layer
const { transformPoints } = useMapStoreActions();
// const { transformPoints } = useMapStoreActions();
const { getLayer } = useLayerStoreActions();

/**
* Creates a geometry json
* @param {Geometry} geometry - The geometry
* @returns {TypeJsonObject} The geometry json
*/
const buildGeometry = useCallback(
(geometry: Geometry): TypeJsonObject => {
let builtGeometry = {};

if (geometry instanceof Polygon) {
builtGeometry = {
type: 'Polygon',
coordinates: geometry.getCoordinates().map((coords) => {
return coords.map((coord) => transformPoints([coord], 4326)[0]);
}),
};
} else if (geometry instanceof LineString) {
builtGeometry = { type: 'LineString', coordinates: geometry.getCoordinates().map((coord) => transformPoints([coord], 4326)[0]) };
} else if (geometry instanceof Point) {
builtGeometry = { type: 'Point', coordinates: transformPoints([geometry.getCoordinates()], 4326)[0] };
} else if (geometry instanceof MultiPoint) {
builtGeometry = { type: 'MultiPoint', coordinates: geometry.getCoordinates().map((coord) => transformPoints([coord], 4326)[0]) };
}
// const buildGeometry = useCallback(
// (geometry: Geometry): TypeJsonObject => {
// let builtGeometry = {};

return builtGeometry;
},
[transformPoints]
);
// if (geometry instanceof Polygon) {
// builtGeometry = {
// type: 'Polygon',
// coordinates: geometry.getCoordinates().map((coords) => {
// return coords.map((coord) => transformPoints([coord], 4326)[0]);
// }),
// };
// } else if (geometry instanceof LineString) {
// builtGeometry = { type: 'LineString', coordinates: geometry.getCoordinates().map((coord) => transformPoints([coord], 4326)[0]) };
// } else if (geometry instanceof Point) {
// builtGeometry = { type: 'Point', coordinates: transformPoints([geometry.getCoordinates()], 4326)[0] };
// } else if (geometry instanceof MultiPoint) {
// builtGeometry = { type: 'MultiPoint', coordinates: geometry.getCoordinates().map((coord) => transformPoints([coord], 4326)[0]) };
// }

// return builtGeometry;
// },
// [transformPoints]
// );

/**
* Builds the JSON file
* @returns {string} Json file content as string
*/
const getJson = useCallback((): string => {
// Filter features from filtered rows
const rowsID = rows.map((row) => {
if (
typeof row === 'object' &&
row !== null &&
'geoviewID' in row &&
typeof row.geoviewID === 'object' &&
row.geoviewID !== null &&
'value' in row.geoviewID
) {
return row.geoviewID.value;
}
return '';
});

const filteredFeatures = features.filter((feature) => rowsID.includes(feature.fieldInfo.geoviewID!.value));

// create GeoJSON feature
const geoData = filteredFeatures.map((feature) => {
const { geometry, fieldInfo } = feature;

// Format the feature info to extract only value and remove the geoviewID field
const formattedInfo: Record<string, unknown>[] = [];
Object.keys(fieldInfo).forEach((key) => {
if (key !== 'geoviewID') {
const tmpObj: Record<string, unknown> = {};
tmpObj[key] = fieldInfo[key]!.value;
formattedInfo.push(tmpObj);
}
});

// TODO: fix issue with geometry not available for esriDynamic: https://github.com/Canadian-Geospatial-Platform/geoview/issues/2545
return {
type: 'Feature',
geometry: buildGeometry(geometry?.getGeometry() as Geometry),
properties: formattedInfo,
};
});
// // Create a Web Worker for buildGeometry
// const geometryWorker = new Worker(new URL('./geometryWorker.ts', import.meta.url));
// // Add error handling to the worker
// geometryWorker.onerror = (error) => {
// console.error('Web Worker error:', error);
// };

// Stringify with some indentation
return JSON.stringify({ type: 'FeatureCollection', features: geoData }, null, 2);
}, [buildGeometry, features, rows]);
// Function to use Web Worker
// eslint-disable-next-line @typescript-eslint/no-explicit-any
// const buildGeometryAsync = (geometry: Geometry): Promise<any> => {
// return new Promise((resolve) => {
// geometryWorker.postMessage({ test: geometry });
// geometryWorker.onmessage = (event) => {
// resolve(event.data);
// };
// });
// };

/**
* Exports the blob to a file
Expand All @@ -126,16 +105,165 @@ function JSONExportButton({ rows, features, layerPath }: JSONExportButtonProps):
/**
* Exports data table in csv format.
*/
const handleExportData = useCallback((): void => {
const jsonString = getJson();
const blob = new Blob([jsonString], {
type: 'text/json',
});

exportBlob(blob, `table-${getLayer(layerPath)?.layerName.replaceAll(' ', '-')}.json`);
}, [exportBlob, getJson, getLayer, layerPath]);
// const getJson = useCallback((): string => {
// // eslint-disable-next-line @typescript-eslint/no-explicit-any
// const rowsIDSet = new Set(rows.map((row: any) => row?.geoviewID?.value).filter(Boolean));

// const geoData = features
// .filter((feature) => rowsIDSet.has(feature.fieldInfo.geoviewID?.value))
// .map(({ geometry, fieldInfo }) => ({
// type: 'Feature',
// geometry: buildGeometry(geometry?.getGeometry() as Geometry),
// properties: Object.fromEntries(
// Object.entries(fieldInfo)
// .filter(([key]) => key !== 'geoviewID')
// .map(([key, value]) => [key, value?.value])
// ),
// }));

// return JSON.stringify({ type: 'FeatureCollection', features: geoData }, null, 2);
// }, [features, rows, buildGeometry]);

// Helper function to serialize geometry
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const serializeGeometry = (geometry: any) => {
if (!geometry) return null;
return {
type: geometry.getType(),
coordinates: geometry.getCoordinates(),
// Add any other properties that might be needed
};
};

const getJson = useCallback(
async function* (): AsyncGenerator<string> {
const worker = new Worker(new URL('./worker.ts', import.meta.url));
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const rowsIDSet = new Set(rows.map((row: any) => row?.geoviewID?.value).filter(Boolean));
const chunkSize = 1000; // Adjust based on performance testing

// Get projection information
const projectionInfo = {
sourceCRS: 'EPSG:3978', // Projection.getSourceCRS(), // Replace with actual method to get source CRS
targetCRS: 'EPSG:4326', // Replace with actual method to get target CRS
// Add any other necessary projection parameters
};

// Initialize worker with projection info
worker.postMessage({ type: 'init', projectionInfo });

const processChunk = async (chunk: any[], isFirst: boolean): Promise<string> => {
return new Promise((resolve) => {
worker.onmessage = (event) => resolve(event.data);
worker.postMessage({ type: 'process', chunk, isFirst });
});
};

try {
for (let i = 0; i < features.length; i += chunkSize) {
const chunk = features.slice(i, i + chunkSize);
const serializedChunk = chunk
.filter((feature) => rowsIDSet.has(feature.fieldInfo.geoviewID?.value))
.map((feature) => ({
geometry: serializeGeometry(feature.geometry?.getGeometry().clone()),
properties: Object.fromEntries(
Object.entries(feature.fieldInfo)
.filter(([key]) => key !== 'geoviewID')
.map(([key, value]) => [key, value?.value])
),
}));

if (serializedChunk.length > 0) {
const result = await processChunk(serializedChunk, i === 0);
yield result;
}

return <MenuItem onClick={handleExportData}>{t('dataTable.jsonExportBtn')}</MenuItem>;
// Allow UI to update
await new Promise((resolve) => setTimeout(resolve, 0));
}

yield ']}';
} finally {
worker.terminate();
}
},
[features, rows]
);

const [isExporting, setIsExporting] = useState(false);
const { addMessage } = useAppStoreActions();

const handleExportData = useCallback(async () => {
setIsExporting(true);
try {
const jsonGenerator = getJson();
const chunks = [];
let i = 0;

for await (const chunk of jsonGenerator) {
chunks.push(chunk);
i++;

// Optionally update progress here
addMessage('info', `Processing ${i * 1000} ... of ${rows.length}`);
}

const fullJson = chunks.join('');
const blob = new Blob([fullJson], { type: 'application/json' });
exportBlob(blob, `table-${getLayer(layerPath)?.layerName.replaceAll(' ', '-')}.json`);
// Do something with the full JSON, e.g., save to file
} catch (error) {
console.error('Export failed:', error);
} finally {
setIsExporting(false);
}
}, [getJson]);

// const handleExportData = useCallback(async (): Promise<void> => {
// try {
// const jsonGenerator = getJson();

// const stream = new ReadableStream({
// async start(controller) {
// try {
// for await (const chunk of jsonGenerator) {
// controller.enqueue(new TextEncoder().encode(chunk));
// }
// } catch (error) {
// console.error('Error in stream processing:', error);
// controller.error(error);
// } finally {
// controller.close();
// }
// },
// });

// const response = new Response(stream);
// const blob = await response.blob();

// exportBlob(blob, `table-${getLayer(layerPath)?.layerName.replaceAll(' ', '-')}.json`);
// } catch (error) {
// console.error('Export failed:', error);
// // Optionally, display an error message to the user
// }
// }, [exportBlob, getJson, getLayer, layerPath]);

// const handleExportData = useCallback((): void => {
// try {
// const jsonString = getJson();
// const blob = new Blob([jsonString], { type: 'application/json' });
// exportBlob(blob, `table-${getLayer(layerPath)?.layerName.replaceAll(' ', '-')}.json`);
// } catch (error) {
// logger.logError('Table download as GeoJSON failed:', error);
// }
// }, [exportBlob, getJson, getLayer, layerPath]);

return (
<MenuItem onClick={handleExportData} disabled={isExporting}>
{t('dataTable.jsonExportBtn')}
</MenuItem>
);
}

export default JSONExportButton;
Loading

0 comments on commit 2ab433d

Please sign in to comment.