From efb99ff00dda45a5ac8094085e4989851a1f662b Mon Sep 17 00:00:00 2001 From: Blayne Chard Date: Wed, 15 Nov 2023 13:11:46 +1300 Subject: [PATCH] fix(config): allow config to be loaded where tiffs are not in Z-X-Y.tiff format --- packages/config/src/json/json.config.ts | 94 ++++++++++++++++--------- packages/config/src/json/tiff.config.ts | 29 ++++---- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/packages/config/src/json/json.config.ts b/packages/config/src/json/json.config.ts index a8c2307545..b5c193007e 100644 --- a/packages/config/src/json/json.config.ts +++ b/packages/config/src/json/json.config.ts @@ -9,8 +9,9 @@ import { VectorFormat, } from '@basemaps/geo'; import { fsa } from '@chunkd/fs'; -import { Cotar } from '@cotar/core'; import { CogTiff } from '@cogeotiff/core'; +import { Cotar } from '@cotar/core'; +import PLimit from 'p-limit'; import { basename } from 'path'; import ulid from 'ulid'; import { ConfigId } from '../base.config.js'; @@ -27,7 +28,7 @@ import { LogType } from './log.js'; import { zProviderConfig } from './parse.provider.js'; import { zStyleJson } from './parse.style.js'; import { TileSetConfigSchemaLayer, zTileSetConfig } from './parse.tile.set.js'; -import PLimit from 'p-limit'; +import { loadTiffsFromPaths } from './tiff.config.js'; const Q = PLimit(10); @@ -251,38 +252,67 @@ export class ConfigJson { const fileList = await fsa.toArray(fsa.details(uri)); const tiffFiles = fileList.filter((f) => f.path.endsWith('.tiff') || f.path.endsWith('.tif')); + const tiffTiles = []; + // Files can be stored as `{z}-{x}-{y}.tiff` + // Check to see if all tiffs match the z-x-y format + for (const tiff of tiffFiles) { + const tileName = basename(tiff.path).replace('.tiff', ''); + const [z, x, y] = tileName.split('-').map((f) => Number(f)); + if (isNaN(x) || isNaN(y) || isNaN(z)) break; + + tiffTiles.push({ tiff, tile: { z, x, y } }); + } + let bounds: Bounds | null = null; - // Files are stored as `{z}-{x}-{y}.tiff` - // TODO the files could actually be smaller than the tile size, - // we should really load the tiff at some point to validate the size - const imageList = await Promise.all( - tiffFiles.map(async (c) => { - const tileName = basename(c.path).replace('.tiff', ''); - const [z, x, y] = tileName.split('-').map((f) => Number(f)); - if (isNaN(z) || isNaN(y) || isNaN(z)) throw new Error('Failed to parse XYZ from: ' + c); - - // This tiff is really small, validate that the tiff actually has data - if (c.size != null && c.size < SmallTiffSizeBytes) { - const isEmpty = await isEmptyTiff(c.path); - if (isEmpty) { - this.logger.warn({ uri: c.path, imageId: id, size: c.size }, 'Imagery:Empty'); - return null; - } else { - this.logger.trace({ uri: c.path, imageId: id, size: c.size }, 'Imagery:CheckSmall:NotEmpty'); - } - } - const tile = tileMatrix.tileToSourceBounds({ z, x, y }); - // Expand the total bounds to cover this tile - if (bounds == null) bounds = Bounds.fromJson(tile); - else bounds = bounds.union(Bounds.fromJson(tile)); - return { ...tile, name: tileName }; - }), - ); + const imageList: NamedBounds[] = []; + if (tiffTiles.length !== tiffFiles.length) { + // some of the tiffs are not named `{z}-{x}-{y}.tiff` so extract bounds from the tiff + const tiffs = await loadTiffsFromPaths( + tiffFiles.map((m) => m.path), + Q, + ); + + for (const tiff of tiffs) { + const gsd = tiff.images[0].resolution[0]; + + const gsdRound = Math.floor(gsd * 100) / 10000; + const bbox = tiff.images[0].bbox.map((f) => Math.floor(f / gsdRound) * gsdRound); + const imgBounds = Bounds.fromBbox(bbox); + + if (bounds == null) bounds = imgBounds; + else bounds = bounds.union(imgBounds); + + const tileName = basename(tiff.source.uri); + imageList.push({ ...imgBounds, name: tileName }); + } + } else { + await Promise.all( + tiffTiles.map((t) => { + return Q(async () => { + if (t.tiff.size != null && t.tiff.size < SmallTiffSizeBytes) { + const isEmpty = await isEmptyTiff(t.tiff.path); + if (isEmpty) { + this.logger.warn({ uri: t.tiff.path, imageId: id, size: t.tiff.size }, 'Imagery:Empty'); + return; + } else { + this.logger.trace({ uri: t.tiff.path, imageId: id, size: t.tiff.size }, 'Imagery:CheckSmall:NotEmpty'); + } + } + const tileName = basename(t.tiff.path); + + const tile = tileMatrix.tileToSourceBounds(t.tile); + // Expand the total bounds to cover this tile + if (bounds == null) bounds = Bounds.fromJson(tile); + else bounds = bounds.union(Bounds.fromJson(tile)); + imageList.push({ ...tile, name: tileName }); + }); + }), + ); + } - const files = imageList.filter((f) => f != null) as NamedBounds[]; // Sort the files by Z, X, Y - files.sort((a, b): number => { + imageList.sort((a, b): number => { const widthSize = a.width - b.width; if (widthSize !== 0) return widthSize; @@ -298,7 +328,7 @@ export class ConfigJson { return bXyz[2] - aXyz[2]; }); - this.logger.debug({ uri, imageId, files: files.length }, 'FetchImagery:Done'); + this.logger.debug({ uri, imageId, files: imageList.length }, 'FetchImagery:Done'); if (bounds == null) throw new Error('Failed to get bounds from URI: ' + uri); const now = Date.now(); @@ -311,7 +341,7 @@ export class ConfigJson { tileMatrix: tileMatrix.identifier, uri, bounds, - files, + files: imageList, }; output.overviews = await ConfigJson.findImageryOverviews(output); diff --git a/packages/config/src/json/tiff.config.ts b/packages/config/src/json/tiff.config.ts index aa5c9ae8b3..23d0c14901 100644 --- a/packages/config/src/json/tiff.config.ts +++ b/packages/config/src/json/tiff.config.ts @@ -197,6 +197,21 @@ export function getImageryName(target: URL): string { return basename(target.pathname); } +export async function loadTiffsFromPaths(sourceFiles: string[], Q: LimitFunction): Promise { + // Load metadata about all the tiffs ignoring any empty sparse tiffs + return ( + await Promise.all( + sourceFiles.filter(isTiff).map((c) => + Q(async () => { + const tiff = await new CogTiff(fsa.source(c)).init(true); + if (await isEmptyTiff(tiff)) return null; + return tiff; + }), + ), + ) + ).filter((f) => f != null) as CogTiff[]; +} + /** * Attempt to load all imagery inside of a path and create a configuration from it * @@ -212,19 +227,7 @@ export async function initImageryFromTiffUrl( ): Promise { const targetPath = urlToString(target); const sourceFiles = await fsa.toArray(fsa.list(targetPath)); - - // Load metadata about all the tiffs ignoring any empty sparse tiffs - const tiffs = ( - await Promise.all( - sourceFiles.filter(isTiff).map((c) => - Q(async () => { - const tiff = await new CogTiff(fsa.source(c)).init(true); - if (await isEmptyTiff(tiff)) return null; - return tiff; - }), - ), - ) - ).filter((f) => f != null) as CogTiff[]; + const tiffs = await loadTiffsFromPaths(sourceFiles, Q); try { const stac = await loadStacFromURL(target);