diff --git a/packages/cli/package.json b/packages/cli/package.json index 6c1e5f029..338dd5d40 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -41,6 +41,7 @@ "dependencies": { "@basemaps/config": "^6.46.0", "@basemaps/geo": "^6.44.0", + "@basemaps/server": "^6.46.0", "@basemaps/shared": "^6.46.0", "@basemaps/tiler": "^6.44.0", "@basemaps/tiler-sharp": "^6.45.0", diff --git a/packages/cli/src/cli/server/action.serve.ts b/packages/cli/src/cli/server/action.serve.ts new file mode 100644 index 000000000..e28ab50cf --- /dev/null +++ b/packages/cli/src/cli/server/action.serve.ts @@ -0,0 +1,60 @@ +import { createServer } from '@basemaps/server'; +import { Const, Env, LogConfig } from '@basemaps/shared'; +import { + CommandLineAction, + CommandLineFlagParameter, + CommandLineIntegerParameter, + CommandLineStringParameter, +} from '@rushstack/ts-command-line'; + +const DefaultPort = 5000; + +export class CommandServe extends CommandLineAction { + config: CommandLineStringParameter; + assets: CommandLineStringParameter; + port: CommandLineIntegerParameter; + noConfig: CommandLineFlagParameter; + + public constructor() { + super({ + actionName: 'serve', + summary: 'Cli tool to create sprite sheet', + documentation: 'Create a sprite sheet from a folder of sprites', + }); + } + + protected onDefineParameters(): void { + this.config = this.defineStringParameter({ + argumentName: 'CONFIG', + parameterLongName: '--config', + description: 'Configuration source to use', + }); + this.assets = this.defineStringParameter({ + argumentName: 'ASSETS', + parameterLongName: '--assets', + description: 'Where the assets (sprites, fonts) are located', + }); + this.port = this.defineIntegerParameter({ + argumentName: 'PORT', + parameterLongName: '--port', + description: 'port to use', + defaultValue: DefaultPort, + }); + } + + protected async onExecute(): Promise { + const logger = LogConfig.get(); + + const config = this.config.value ?? 'dynamodb://' + Const.TileMetadata.TableName; + const port = this.port.value; + const assets = this.assets.value; + + // Force a default url base so WMTS requests know their relative url + const ServerUrl = Env.get(Env.PublicUrlBase) ?? `http://localhost:${port}`; + process.env[Env.PublicUrlBase] = ServerUrl; + + const server = await createServer({ config, assets }, logger); + await server.listen({ port: port ?? DefaultPort, host: '0.0.0.0' }); + logger.info({ url: ServerUrl }, 'ServerStarted'); + } +} diff --git a/packages/cli/src/cog/__tests__/builder.test.ts b/packages/cli/src/cog/__tests__/builder.test.ts new file mode 100644 index 000000000..2576e8da5 --- /dev/null +++ b/packages/cli/src/cog/__tests__/builder.test.ts @@ -0,0 +1,98 @@ +import { Bounds, Epsg, EpsgCode, GoogleTms } from '@basemaps/geo'; +import { LogConfig } from '@basemaps/shared'; +import { fsa } from '@chunkd/fs'; +import { CogTiff } from '@cogeotiff/core'; + +import o from 'ospec'; +import { CogBuilder, guessProjection } from '../builder.js'; + +o.spec('Builder', () => { + o('should guess WKT', () => { + o( + guessProjection( + 'PCS Name = NZGD_2000_New_Zealand_Transverse_Mercator|GCS Name = GCS_NZGD_2000|Ellipsoid = GRS_1980|Primem = Greenwich||', + ), + ).equals(Epsg.Nztm2000); + + o( + guessProjection('NZGD2000_New_Zealand_Transverse_Mercator_2000|GCS Name = GCS_NZGD_2000|Primem = Greenwich||'), + ).equals(Epsg.Nztm2000); + }); + + o('should not guess unknown wkt', () => { + o(guessProjection('')).equals(null); + o(guessProjection('NZTM')).equals(null); + o(guessProjection('NZGD2000')).equals(null); + }); + + o.spec('tiff', () => { + const googleBuilder = new CogBuilder(GoogleTms, 1, LogConfig.get()); + const origInit = CogTiff.prototype.init; + const origGetImage = CogTiff.prototype.getImage; + + o.after(() => { + CogTiff.prototype.init = origInit; + CogTiff.prototype.getImage = origGetImage; + }); + + o('bounds', async () => { + const localTiff = fsa.source('/local/file.tiff')!; + const s3Tiff = fsa.source('s3://bucket/file.tiff')!; + + const imageLocal = { + resolution: [0.1], + value: (): any => [1], + valueGeo: (): any => EpsgCode.Nztm2000, + bbox: Bounds.fromJson({ + x: 1492000, + y: 6198000, + width: 24000, + height: 36000, + }).toBbox(), + }; + + const imageS3 = { + resolution: [0.1], + value: (): any => [1], + valueGeo: (): any => EpsgCode.Nztm2000, + bbox: Bounds.fromJson({ + x: 1492000 + 24000, + y: 6198000, + width: 24000, + height: 36000, + }).toBbox(), + }; + + CogTiff.prototype.init = o.spy() as any; + CogTiff.prototype.getImage = function (): any { + return this.source === localTiff ? imageLocal : imageS3; + }; + + const ans = await googleBuilder.bounds([localTiff, s3Tiff]); + + o(ans).deepEquals({ + projection: 2193, + nodata: 1, + bands: 1, + bounds: [ + { + x: 1492000, + y: 6198000, + width: 24000, + height: 36000, + name: '/local/file.tiff', + }, + { + x: 1516000, + y: 6198000, + width: 24000, + height: 36000, + name: 's3://bucket/file.tiff', + }, + ], + pixelScale: 0.1, + resZoom: 21, + }); + }); + }); +}); diff --git a/packages/cli/src/cog/__tests__/cog.stac.job.test.ts b/packages/cli/src/cog/__tests__/cog.stac.job.test.ts new file mode 100644 index 000000000..b9f166c47 --- /dev/null +++ b/packages/cli/src/cog/__tests__/cog.stac.job.test.ts @@ -0,0 +1,350 @@ +import { Bounds, Epsg, EpsgCode, GoogleTms, Nztm2000Tms, Stac, StacCollection } from '@basemaps/geo'; +import { Projection } from '@basemaps/geo'; +import { mockFileOperator } from '@basemaps/shared/build/file/__tests__/file.operator.test.helper.js'; +import { round } from '@basemaps/test/build/rounding.js'; +import { Ring } from '@linzjs/geojson'; +import o from 'ospec'; +import { CogStacJob, JobCreationContext } from '../cog.stac.job.js'; +import { CogBuilderMetadata, CogJobJson } from '../types.js'; + +type RecursivePartial = { + [P in keyof T]?: RecursivePartial; +}; +o.spec('CogJob', () => { + o.spec('build', () => { + const id = 'jobid1'; + const imageryName = 'auckland_rural_2010-2012_0-50m'; + + const jobPath = 's3://target-bucket/path/3857/auckland_rural_2010-2012_0-50m/jobid1'; + + const ring1 = [ + [170, -41], + [-175, -40], + [-177, -42], + [175, -42], + [170, -41], + ].map(Projection.get(Epsg.Nztm2000).fromWgs84) as Ring; + const ring2 = [ + [-150, -40], + [-140, -41], + [-150, -41], + [-150, -40], + ].map(Projection.get(Epsg.Nztm2000).fromWgs84) as Ring; + + const srcPoly = [[ring1], [ring2]]; + const bounds = srcPoly.map((poly, i) => ({ ...Bounds.fromMultiPolygon([poly]), name: 'ring' + i })); + + const metadata: CogBuilderMetadata = { + bands: 3, + bounds, + projection: EpsgCode.Nztm2000, + pixelScale: 0.075, + resZoom: 22, + nodata: 0, + files: [{ name: '0-0-0', x: 1, y: 2, width: 2, height: 3 }], + targetBounds: { x: 1, y: 2, width: 2, height: 3 }, + }; + const addAlpha = true; + const ctx = { + tileMatrix: GoogleTms, + sourceLocation: { type: 's3', path: 's3://source-bucket/path' }, + outputLocation: { type: 's3', path: 's3://target-bucket/path' }, + cutline: { blend: 20, href: 's3://curline-bucket/path' }, + oneCogCovering: false, + } as JobCreationContext; + + const mockFs = mockFileOperator(); + + o.beforeEach(mockFs.setup); + + o.afterEach(mockFs.teardown); + + o('with job.json', async () => { + mockFs.jsStore['s3://source-bucket/path/collection.json'] = { + title: 'The Title', + description: 'The Description', + license: 'The License', + keywords: ['keywords'], + extent: { + spatial: { bbox: [9, 8, 7, 6] }, + temporal: { interval: [['2020-01-01T00:00:00.000', '2020-08-08T19:18:23.456Z']] }, + }, + + providers: [ + { name: 'provider name', roles: ['licensor'], url: 'https://provider.com' }, + { name: 'unknown url', roles: ['processor'], url: 'unknown' }, + ], + }; + const job = await CogStacJob.create({ id, imageryName, metadata, ctx, addAlpha, cutlinePoly: [] }); + o(job.getJobPath('job.json')).equals(jobPath + '/job.json'); + + o(job.title).equals('The Title'); + o(job.description).equals('The Description'); + + const stac = round(mockFs.jsStore[jobPath + '/collection.json'], 4); + + o(stac.title).equals('The Title'); + o(stac.description).equals('The Description'); + o(stac.license).equals('The License'); + o(stac.keywords).deepEquals(['keywords']); + o(stac.providers).deepEquals([ + { name: 'provider name', roles: ['licensor'], url: 'https://provider.com' }, + { name: 'unknown url', roles: ['processor'], url: undefined }, + ]); + o(stac.links[2]).deepEquals({ + href: 's3://source-bucket/path/collection.json', + rel: 'sourceImagery', + type: 'application/json', + }); + o(round(stac.extent.spatial.bbox, 4)).deepEquals([[169.3347, -51.868, -145.9143, -32.8496]]); + o(stac.extent.temporal).deepEquals({ interval: [['2020-01-01T00:00:00.000', '2020-08-08T19:18:23.456Z']] }); + }); + + o('no source collection.json', async () => { + const startNow = Date.now(); + const job = await CogStacJob.create({ + id, + imageryName, + metadata, + ctx: { ...ctx, oneCogCovering: true }, + addAlpha, + cutlinePoly: [], + }); + const afterNow = Date.now(); + const jobJson = round(job.json, 4) as CogJobJson; + + o(jobJson).deepEquals(round(mockFs.jsStore[jobPath + '/job.json'], 4)); + o(jobJson.id).equals('jobid1'); + o(jobJson.name).equals('auckland_rural_2010-2012_0-50m'); + o(jobJson.title).equals('Auckland rural 2010-2012 0.50m'); + o(jobJson.description).equals('No description'); + o(jobJson.source.gsd).equals(0.075); + o(jobJson.source.epsg).equals(2193); + o(jobJson.source.files[0]).deepEquals({ + x: 1347679.1521, + y: 5301577.5262, + width: 1277913.1293, + height: 201072.641, + name: 'ring0', + }); + o(jobJson.source.files[1]).deepEquals({ + x: 4728648.2332, + y: 4247288.7253, + width: 837784.3351, + height: 609622.9622, + name: 'ring1', + }); + o(jobJson.source.location).deepEquals({ type: 's3', path: 's3://source-bucket/path' }); + o(jobJson.output.gsd).equals(0.0373); + o(jobJson.output.epsg).equals(3857); + o(jobJson.output.tileMatrix).equals('WebMercatorQuad'); + o(jobJson.output.files).deepEquals([{ name: '0-0-0', x: 1, y: 2, width: 2, height: 3 }]); + o(jobJson.output.location).deepEquals({ type: 's3', path: 's3://target-bucket/path' }); + o(jobJson.output.resampling).deepEquals({ warp: 'bilinear', overview: 'lanczos' }); + o(jobJson.output.quality).equals(90); + o(jobJson.output.cutline).deepEquals({ blend: 20, href: 's3://curline-bucket/path' }); + o(jobJson.output.addAlpha).equals(true); + o(jobJson.output.nodata).equals(0); + o(jobJson.output.bounds).deepEquals({ x: 1, y: 2, width: 2, height: 3 }); + + o(job.id).equals(id); + o(job.source.gsd).equals(0.075); + o(round(job.output.gsd, 4)).equals(0.0373); + o(job.output.oneCogCovering).equals(true); + + const stac = round(mockFs.jsStore[jobPath + '/collection.json'], 4) as StacCollection; + + const stacMeta = stac.summaries as any; + + const generated = stacMeta['linz:generated'][0]; + + const jobNow = +Date.parse(generated.datetime); + o(startNow <= jobNow && jobNow <= afterNow).equals(true); + + // split so that scripts/detect.unlinked.dep.js does not think it is an import + o(generated.package).equals('@' + 'basemaps/cli'); + o(stac.id).equals(job.id); + o(stac.extent.spatial.bbox).deepEquals([[169.3347, -51.868, -145.9143, -32.8496]]); + o(stac.extent.temporal).deepEquals({ interval: [['2010-01-01T00:00:00Z', '2011-01-01T00:00:00Z']] }); + o(stac.stac_version).equals(Stac.Version); + o(stac.stac_extensions).deepEquals([Stac.BaseMapsExtension]); + o(stac.license).deepEquals(Stac.License); + o(stac.keywords).deepEquals(['Imagery', 'New Zealand']); + o(stac.summaries).deepEquals({ + gsd: [0.075], + 'proj:epsg': [3857], + 'linz:output': [ + { + resampling: { warp: 'bilinear', overview: 'lanczos' }, + quality: 90, + cutlineBlend: 20, + addAlpha: true, + nodata: 0, + }, + ], + 'linz:generated': [generated], + }); + o(stac.links).deepEquals([ + { href: 's3://target-bucket/path/collection.json', type: 'application/json', rel: 'self' }, + { href: 'job.json', type: 'application/json', rel: 'linz.basemaps.job' }, + { href: 'cutline.geojson.gz', type: 'application/geo+json+gzip', rel: 'linz.basemaps.cutline' }, + { href: 'covering.geojson', type: 'application/geo+json', rel: 'linz.basemaps.covering' }, + { href: 'source.geojson', type: 'application/geo+json', rel: 'linz.basemaps.source' }, + { href: '0-0-0.json', type: 'application/geo+json', rel: 'item' }, + ]); + + o(round(mockFs.jsStore[jobPath + '/cutline.geojson.gz'], 4)).deepEquals({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + geometry: { type: 'MultiPolygon', coordinates: [] }, + properties: {}, + }, + ], + crs: { + type: 'name', + properties: { name: 'urn:ogc:def:crs:EPSG::3857' }, + }, + }); + + o(round(mockFs.jsStore[jobPath + '/0-0-0.json'], 4)).deepEquals({ + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [0, 0], + [0, 0], + [0, 0], + [0, 0], + ], + ], + }, + properties: { + datetime: new Date(jobNow).toISOString(), + name: '0-0-0', + gsd: 0.0373, + 'proj:epsg': 3857, + }, + bbox: [0, 0, 0, 0], + id: 'jobid1/0-0-0', + collection: 'jobid1', + stac_version: Stac.Version, + stac_extensions: ['projection'], + links: [ + { href: jobPath + '/0-0-0.json', rel: 'self' }, + { href: 'collection.json', rel: 'collection' }, + ], + assets: { + cog: { + href: '0-0-0.tiff', + type: 'image/tiff; application=geotiff; profile=cloud-optimized', + roles: ['data'], + }, + }, + }); + o(round(mockFs.jsStore[jobPath + '/covering.geojson'], 4)).deepEquals({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [0, 0], + [0, 0], + [0, 0], + [0, 0], + [0, 0], + ], + ], + }, + properties: { name: '0-0-0' }, + bbox: [0, 0, 0, 0], + }, + ], + }); + + const sourceGeoJson = mockFs.jsStore[jobPath + '/source.geojson']; + o(sourceGeoJson.type).equals('FeatureCollection'); + o(sourceGeoJson.features.length).equals(2); + o(round(sourceGeoJson.features[0], 4)).deepEquals({ + type: 'Feature', + geometry: { + type: 'MultiPolygon', + coordinates: [ + [ + [ + [169.9343, -42.3971], + [180, -42.2199], + [180, -40.4109], + [170.0185, -40.5885], + [169.9343, -42.3971], + ], + ], + [ + [ + [-180, -42.2199], + [-174.6708, -41.7706], + [-175, -40], + [-180, -40.4109], + [-180, -42.2199], + ], + ], + ], + }, + properties: { name: 'ring0' }, + bbox: [169.9343, -42.3971, -175, -40], + }); + + o(round(sourceGeoJson.features[1], 4)).deepEquals({ + type: 'Feature', + geometry: { + type: 'Polygon', + coordinates: [ + [ + [-147.4403, -44.5167], + [-150.5162, -40.1908], + [-143.1693, -37.1443], + [-140, -41], + [-147.4403, -44.5167], + ], + ], + }, + properties: { name: 'ring1' }, + bbox: [-147.4403, -44.5167, -143.1693, -37.1443], + }); + }); + + o('should create with no tileMatrix', () => { + const cfg: RecursivePartial = { output: { epsg: 2193 } }; + const job = new CogStacJob(cfg as CogJobJson); + o(job.tileMatrix.identifier).equals(Nztm2000Tms.identifier); + }); + + o('should error with invalid tileMatrix', () => { + const cfg: RecursivePartial = { output: { tileMatrix: 'None' } }; + const job = new CogStacJob(cfg as CogJobJson); + o(() => job.tileMatrix).throws('Failed to find TileMatrixSet "None"'); + }); + + o('no source collection.json and not nice name', async () => { + try { + await CogStacJob.create({ + id, + imageryName: 'yucky-name', + metadata, + ctx: { ...ctx, oneCogCovering: true }, + addAlpha, + cutlinePoly: [], + }); + o('').equals('create should not have exceeded'); + } catch (err: any) { + o(err.message).equals('Missing date in imagery name: yucky-name'); + } + }); + }); +}); diff --git a/packages/cli/src/cog/__tests__/cutline.test.ts b/packages/cli/src/cog/__tests__/cutline.test.ts new file mode 100644 index 000000000..21719812c --- /dev/null +++ b/packages/cli/src/cog/__tests__/cutline.test.ts @@ -0,0 +1,353 @@ +import { Bounds, EpsgCode, GoogleTms, Nztm2000Tms } from '@basemaps/geo'; +import { qkToName } from '@basemaps/geo/build/proj/__tests__/test.util.js'; +import { round } from '@basemaps/test/build/rounding.js'; +import { MultiPolygon } from '@linzjs/geojson'; +import o from 'ospec'; +import path from 'path'; +import url from 'url'; +import { AlignedLevel } from '../constants.js'; +import { Cutline, polyContainsBounds } from '../cutline.js'; +import { SourceMetadata } from '../types.js'; +import { SourceTiffTestHelper } from './source.tiff.testhelper.js'; +const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); + +o.spec('cutline', () => { + o.specTimeout(1_000); + const testDir = `${__dirname}/../../../__test.assets__`; + + o.spec('filterSourcesForName', () => { + o('fully within same projection', async () => { + // convert poly to GoogleTms + const cutpoly = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson')).toGeoJson(); + + const cutline = new Cutline(GoogleTms, cutpoly, -100); + + const name = qkToName('311333222321113310'); + + const job = SourceTiffTestHelper.makeCogJob(); + job.output.tileMatrix = GoogleTms.identifier; + job.source.epsg = EpsgCode.Nztm2000; + const sourceBounds = SourceTiffTestHelper.tiffNztmBounds(testDir); + const [tif1, tif2] = sourceBounds; + job.source.files = [tif1, tif2]; + + o(cutline.filterSourcesForName(name, job)).deepEquals([tif2.name]); + }); + }); + + o('polyContainsBounds', () => { + const polys: MultiPolygon = [ + [ + [ + [-4, 2], + [-2, 1], + [-4, -1], + [1, -5], + [2, -5], + [2, -2], + [4, -5], + [6, -2], + [2, 0], + [6, 3], + [2, 7], + [0, 3], + [-4, 6], + [-4, 10], + [10, 10], + [10, -10], + [-10, -10], + [-4, 2], + ], + ], + [ + [ + [2, 3], + [3, 3], + [3, 5], + [2, 5], + [2, 3], + ], + ], + ]; + + o(polyContainsBounds(polys, Bounds.fromBbox([-6, -6, -3, -3]))).equals(true); + + o(polyContainsBounds(polys, Bounds.fromBbox([-3, -4, 4, 4]))).equals(false); + o(polyContainsBounds(polys, Bounds.fromBbox([6, -8, 5, 5]))).equals(false); + }); + + o('loadCutline', async () => { + const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); + const geojson = round(cutline.toGeoJson()); + + o(geojson).deepEquals({ + type: 'FeatureCollection', + features: [ + { + type: 'Feature', + geometry: { + type: 'MultiPolygon', + coordinates: [ + [ + [ + [19456570.81047118, -5023799.46086743], + [19457346.41188008, -5024885.32147075], + [19455543.1261867, -5026720.78623442], + [19455446.57500747, -5025689.98761053], + [19456325.95579277, -5024324.4264959], + [19456431.41331051, -5023815.14487631], + [19456497.91735541, -5023822.4695137], + [19456513.95059036, -5023820.4859541], + [19456528.99998363, -5023810.66149213], + [19456528.41956381, -5023802.3991344], + [19456521.62606925, -5023792.60909871], + [19456521.12156931, -5023788.06857522], + [19456524.31632738, -5023783.03836819], + [19456531.227264, -5023778.34574544], + [19456540.68563587, -5023777.32428773], + [19456551.09434221, -5023782.4891701], + [19456560.03196148, -5023796.37226942], + [19456570.81047118, -5023799.46086743], + ], + ], + [ + [ + [19430214.06028324, -5008476.23288268], + [19429858.86076585, -5008966.74650194], + [19430484.68848696, -5009778.63111311], + [19430907.54505528, -5009085.14634107], + [19430214.06028324, -5008476.23288268], + ], + ], + ], + }, + properties: {}, + }, + ], + crs: { + type: 'name', + properties: { name: 'urn:ogc:def:crs:EPSG::3857' }, + }, + }); + }); + + o('findCovering', async () => { + const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); + const bounds = SourceTiffTestHelper.tiffNztmBounds(); + + o(cutline.clipPoly.length).equals(2); + + (cutline as any).findCovering({ projection: EpsgCode.Nztm2000, bounds, resZoom: 15 }); + + o((cutline as any).srcPoly.length).equals(1); + }); + + o.spec('optimize', async () => { + const bounds = SourceTiffTestHelper.tiffNztmBounds(); + + o('full-extent 3857', () => { + const cutline = new Cutline(GoogleTms); + const covering = cutline.optimizeCovering( + { + projection: EpsgCode.Google, + bounds: [{ ...GoogleTms.extent, name: 'gebco' }], + resZoom: 5, + } as SourceMetadata, + 4, + ); + + o(round(covering, 4)).deepEquals([ + { + name: '1-0-0', + x: -20037508.3428, + y: 0, + width: 20037508.3428, + height: 20037508.3428, + }, + { + name: '1-0-1', + x: -20037508.3428, + y: -20037508.3428, + width: 20037508.3428, + height: 20037508.3428, + }, + { + name: '1-1-0', + x: -0, + y: 0, + width: 20037508.3428, + height: 20037508.3428, + }, + { + name: '1-1-1', + x: -0, + y: -20037508.3428, + width: 20037508.3428, + height: 20037508.3428, + }, + ]); + }); + + o('nztm', () => { + const cutline = new Cutline(Nztm2000Tms); + + const covering = cutline.optimizeCovering({ + projection: EpsgCode.Nztm2000, + bounds, + resZoom: 7 + AlignedLevel, + } as SourceMetadata); + + o(covering[4]).deepEquals({ + name: '7-153-253', + x: 1741760, + y: 5448320, + width: 17920, + height: 17920, + }); + o(covering.map((c) => c.name)).deepEquals([ + '7-152-252', + '7-152-253', + '7-152-254', + '7-153-252', + '7-153-253', + '7-153-254', + '7-154-252', + '7-154-253', + '7-154-254', + '7-155-252', + '7-155-253', + '8-387-635', + ]); + }); + + o('source polygon is smoothed', async () => { + const tiff1 = { + name: '/path/to/tiff1.tiff', + x: 1759315.9336568, + y: 5476120.089572778, + width: 2577.636033663526, + height: 2275.3233966259286, + }; + + const tiff2 = { + name: '/path/to/tiff2.tiff', + x: 1759268.0010635862, + y: 5473899.771969615, + width: 2576.894309356343, + height: 2275.3359155077487, + }; + + const bounds = [tiff1, tiff2]; + const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 500); + + const covering = cutline.optimizeCovering( + { + projection: EpsgCode.Nztm2000, + bounds, + resZoom: 22, + } as SourceMetadata, + 8, + ); + + o(round(cutline.clipPoly, 4)[0][0]).deepEquals([ + [19468576.3516, -4993284.6176], + [19471093.8032, -4993284.6176], + [19471932.5568, -4992600.0637], + [19472095.7462, -4992346.9043], + [19472095.7462, -4987199.9524], + [19471985.5211, -4987199.9524], + [19470978.4379, -4989417.4454], + [19468800.9775, -4991497.5405], + [19468576.3516, -4991797.1824], + [19468576.3516, -4993284.6176], + ]); + + o(covering.map((c) => c.name)).deepEquals([ + '14-16151-10232', + '14-16151-10233', + '14-16152-10231', + '14-16152-10232', + '14-16152-10233', + '18-258444-163695', + ]); + }); + + o('boundary part inland, part coastal', async () => { + const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 500); + const bounds = [ + { + name: 'tiff1', + x: 1759315.9336568, + y: 5476120.089572778, + width: 2577.636033663526, + height: 2275.3233966259286, + }, + ]; + + const covering = cutline.optimizeCovering( + { + projection: EpsgCode.Nztm2000, + bounds, + resZoom: 22, + } as SourceMetadata, + 8, + ); + + o(round(cutline.clipPoly, 4)[0][0]).deepEquals([ + [19470011.8243, -4990340.8368], + [19472095.7007, -4990340.8368], + [19472095.7007, -4987199.9524], + [19471985.5211, -4987199.9524], + [19470978.4379, -4989417.4454], + [19470011.8243, -4990340.8368], + ]); + + o(covering.map((c) => c.name)).deepEquals([ + '14-16152-10231', + '15-32304-20464', + '15-32305-20464', + '18-258444-163695', + ]); + }); + + o('low res', () => { + const cutline = new Cutline(GoogleTms); + + const covering = cutline.optimizeCovering( + { + projection: EpsgCode.Nztm2000, + bounds, + resZoom: 13, + } as SourceMetadata, + 9, + ); + + o(covering.length).equals(covering.length); + o(covering.map((c) => c.name)).deepEquals(['8-252-159', '8-252-160']); + }); + + o('hi res', () => { + const covering2 = new Cutline(GoogleTms).optimizeCovering( + { + projection: EpsgCode.Nztm2000, + bounds, + resZoom: 18, + } as SourceMetadata, + 8, + ); + + o(covering2.length).equals(covering2.length); + + o(covering2.map((c) => c.name)).deepEquals([ + '10-1008-639', + '10-1008-640', + '10-1009-639', + '10-1009-640', + '11-2020-1279', + '11-2020-1280', + '12-4040-2557', + '12-4040-2562', + ]); + }); + }); +}); diff --git a/packages/cli/src/cog/__tests__/projection.loader.test.ts b/packages/cli/src/cog/__tests__/projection.loader.test.ts new file mode 100644 index 000000000..aac709ec3 --- /dev/null +++ b/packages/cli/src/cog/__tests__/projection.loader.test.ts @@ -0,0 +1,30 @@ +import o from 'ospec'; +import { ProjectionLoader } from '../projection.loader.js'; +import sinon from 'sinon'; +import { Nztm2000 } from '@basemaps/geo/build/proj/nztm2000.js'; +import { Epsg } from '@basemaps/geo'; +import { Projection } from '@basemaps/geo'; + +o.spec('ProjectionLoader', () => { + const sandbox = sinon.createSandbox(); + o.afterEach(() => sandbox.restore()); + o('should not load if already loaded', async () => { + const fetchStub = sandbox.stub(ProjectionLoader, '_fetch'); + const res = await ProjectionLoader.load(2193); + o(fetchStub.callCount).equals(0); + o(res.code).equals(2193); + }); + + o('should fetch data from the internet', async () => { + const fetchStub = sandbox + .stub(ProjectionLoader, '_fetch') + .resolves({ ok: true, text: () => Promise.resolve(Nztm2000) } as any); + const res = await ProjectionLoader.load(3790); + + o(fetchStub.callCount).equals(1); + + o(res.code).equals(3790); + o(Epsg.get(3790)).equals(res); + o(Projection.get(3790).epsg).equals(res); + }); +}); diff --git a/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts b/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts new file mode 100644 index 000000000..371a8ed53 --- /dev/null +++ b/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts @@ -0,0 +1,44 @@ +import { EpsgCode, NamedBounds, GoogleTms } from '@basemaps/geo'; +import { CogStacJob } from '../cog.stac.job.js'; +import { CogJobJson } from '../types.js'; + +export const SourceTiffTestHelper = { + makeCogJob(): CogStacJob { + return new CogStacJob({ + source: { + files: [] as NamedBounds[], + epsg: EpsgCode.Nztm2000, + gsd: 0.8, + }, + output: { + tileMatrix: GoogleTms.identifier, + gsd: 0.75, + addAlpha: true, + oneCogCovering: false, + }, + } as CogJobJson); + }, + + tiffNztmBounds(path = '/path/to'): NamedBounds[] { + return [ + { + name: path + '/tiff1.tiff', + x: 1732000, + y: 5442000, + width: 24000, + height: 36000, + }, + { + name: path + '/tiff2.tiff', + x: 1756000, + y: 5442000, + width: 24000, + height: 36000, + }, + ]; + }, + + namedBoundsToPaths(bounds: NamedBounds[]): string[] { + return bounds.map(({ name }): string => `/path/to/tiff${name}.tiff`); + }, +}; diff --git a/packages/cli/src/cog/cog.stac.job.ts b/packages/cli/src/cog/cog.stac.job.ts new file mode 100644 index 000000000..94c5b65f0 --- /dev/null +++ b/packages/cli/src/cog/cog.stac.job.ts @@ -0,0 +1,407 @@ +import { + Bounds, + Epsg, + Projection, + Stac, + StacCollection, + StacLink, + StacProvider, + TileMatrixSet, + TileMatrixSets, +} from '@basemaps/geo'; +import { + CompositeError, + extractYearRangeFromName, + FileConfig, + FileConfigPath, + fsa, + titleizeImageryName, +} from '@basemaps/shared'; +import { CliInfo } from '@basemaps/shared/build/cli/info.js'; +import { MultiPolygon, toFeatureCollection, toFeatureMultiPolygon } from '@linzjs/geojson'; +import { GdalCogBuilderDefaults, GdalCogBuilderResampling } from '../gdal/gdal.config.js'; +import { ProjectionLoader } from './projection.loader.js'; +import { CogStac, CogStacItem, CogStacItemExtensions, CogStacKeywords } from './stac.js'; +import { + CogBuilderMetadata, + CogJob, + CogJobJson, + CogOutputProperties, + CogSourceProperties, + FeatureCollectionWithCrs, +} from './types.js'; + +export const MaxConcurrencyDefault = 50; + +export interface JobCreationContext { + /** Custom Imagery Name if defined */ + imageryName?: string; + + /** Source config */ + sourceLocation: FileConfig | FileConfigPath; + + /** Output config */ + outputLocation: FileConfig; + + /** Should the imagery be cut to a cutline */ + cutline?: { + href: string; + blend: number; + }; + + tileMatrix: TileMatrixSet; + + override?: { + /** Override job id */ + id?: string; + + /** + * Image quality + * @default GdalCogBuilderDefaults.quality + */ + quality?: number; + + /** + * Number of threads to use for fetches + * @default MaxConcurrencyDefault + */ + concurrency?: number; + + /** + * Override the source projection + */ + projection?: Epsg; + + /** + * Override Default aligned zoom level + */ + alignedLevel?: number; + + /** + * Resampling method + * @Default GdalCogBuilderDefaults.resampling + */ + resampling?: GdalCogBuilderResampling; + }; + + /** + * Should this job be submitted to batch now? + * @default false + */ + batch: boolean; + + /** Should this job ignore source coverage and just produce one big COG for EPSG extent */ + oneCogCovering: boolean; +} + +export interface CogJobCreateParams { + /** unique id for imagery set */ + id: string; + /** name of imagery set */ + imageryName: string; + /** information about the source */ + metadata: CogBuilderMetadata; + /** information about the output */ + ctx: JobCreationContext; + /** the polygon to use to clip the source imagery */ + cutlinePoly: MultiPolygon; + /** do we need an alpha band added */ + addAlpha: boolean; +} + +/** + * Information needed to create cogs + */ +export class CogStacJob implements CogJob { + json: CogJobJson; + private cacheTargetZoom?: { + gsd: number; + zoom: number; + }; + + /** + * Load the job.json + + * @param jobpath where to load the job.json from + */ + static async load(jobpath: string): Promise { + const job = new CogStacJob(await fsa.readJson(jobpath)); + await ProjectionLoader.load(job.source.epsg); + return job; + } + + /** + * Create job.json, collection.json, source.geojson, covering.geojson, cutlint.geojson.gz and + * stac descriptions of the target COGs + */ + static async create({ + id, + imageryName, + metadata, + ctx, + cutlinePoly, + addAlpha, + }: CogJobCreateParams): Promise { + let description: string | undefined; + const providers: StacProvider[] = []; + const links: StacLink[] = [ + { + href: fsa.join(ctx.outputLocation.path, 'collection.json'), + type: 'application/json', + rel: 'self', + }, + { + href: 'job.json', + type: 'application/json', + rel: 'linz.basemaps.job', + }, + ]; + let sourceStac = {} as StacCollection; + + const interval: [string, string][] = []; + try { + const sourceCollectionPath = fsa.join(ctx.sourceLocation.path, 'collection.json'); + sourceStac = await fsa.readJson(sourceCollectionPath); + description = sourceStac.description; + interval.push(...(sourceStac.extent?.temporal?.interval ?? [])); + links.push({ href: sourceCollectionPath, rel: 'sourceImagery', type: 'application/json' }); + if (sourceStac.providers != null) { + for (const p of sourceStac.providers) { + if (p.roles.indexOf('host') === -1) { + if (p.url === 'unknown') { + // LINZ LDS has put unknown in some urls + p.url = undefined; + } + providers.push(p); + } + } + } + } catch (err) { + if (!CompositeError.isCompositeError(err) || err.code !== 404) { + throw err; + } + } + const keywords = sourceStac.keywords ?? CogStacKeywords.slice(); + const license = sourceStac.license ?? Stac.License; + const title = sourceStac.title ?? titleizeImageryName(imageryName); + + if (description == null) { + description = 'No description'; + } + + await ProjectionLoader.load(metadata.projection); + const job = new CogStacJob({ + id, + name: imageryName, + title, + description, + source: { + gsd: metadata.pixelScale, + epsg: metadata.projection, + files: metadata.bounds, + location: ctx.sourceLocation, + }, + output: { + gsd: ctx.tileMatrix.pixelScale(metadata.resZoom), + tileMatrix: ctx.tileMatrix.identifier, + epsg: ctx.tileMatrix.projection.code, + files: metadata.files, + location: ctx.outputLocation, + resampling: ctx.override?.resampling ?? GdalCogBuilderDefaults.resampling, + quality: ctx.override?.quality ?? GdalCogBuilderDefaults.quality, + cutline: ctx.cutline, + addAlpha, + nodata: metadata.nodata, + bounds: metadata.targetBounds, + oneCogCovering: ctx.oneCogCovering, + }, + }); + + const nowStr = new Date().toISOString(); + + const sourceProj = Projection.get(job.source.epsg); + + const bbox = [ + sourceProj.boundsToWgs84BoundingBox( + metadata.bounds.map((a) => Bounds.fromJson(a)).reduce((sum, a) => sum.union(a)), + ), + ]; + + if (interval.length === 0) { + const years = extractYearRangeFromName(imageryName); + if (years == null) throw new Error('Missing date in imagery name: ' + imageryName); + interval.push(years.map((y) => `${y}-01-01T00:00:00Z`) as [string, string]); + } + + if (ctx.cutline) { + links.push({ href: 'cutline.geojson.gz', type: 'application/geo+json+gzip', rel: 'linz.basemaps.cutline' }); + } + + links.push({ href: 'covering.geojson', type: 'application/geo+json', rel: 'linz.basemaps.covering' }); + links.push({ href: 'source.geojson', type: 'application/geo+json', rel: 'linz.basemaps.source' }); + + const temporal = { interval }; + + const jobFile = job.getJobPath(`job.json`); + + const stac: CogStac = { + id, + title, + description, + stac_version: Stac.Version, + stac_extensions: [Stac.BaseMapsExtension], + + extent: { + spatial: { bbox }, + temporal, + }, + + license, + keywords, + + providers, + + summaries: { + gsd: [metadata.pixelScale], + 'proj:epsg': [ctx.tileMatrix.projection.code], + 'linz:output': [ + { + resampling: ctx.override?.resampling ?? GdalCogBuilderDefaults.resampling, + quality: ctx.override?.quality ?? GdalCogBuilderDefaults.quality, + cutlineBlend: ctx.cutline?.blend, + addAlpha, + nodata: metadata.nodata, + }, + ], + 'linz:generated': [ + { + ...CliInfo, + datetime: nowStr, + }, + ], + }, + + links, + }; + + await fsa.writeJson(jobFile, job.json); + + const covering = Projection.get(job.tileMatrix).toGeoJson(metadata.files); + + const roles = ['data']; + const collectionLink = { href: 'collection.json', rel: 'collection' }; + + for (const f of covering.features) { + const { name } = f.properties as { name: string }; + const href = name + '.json'; + links.push({ href, type: 'application/geo+json', rel: 'item' }); + const item: CogStacItem = { + ...f, + id: job.id + '/' + name, + collection: job.id, + stac_version: Stac.Version, + stac_extensions: CogStacItemExtensions, + properties: { + ...f.properties, + datetime: nowStr, + gsd: job.output.gsd, + 'proj:epsg': job.tileMatrix.projection.code, + }, + links: [{ href: job.getJobPath(href), rel: 'self' }, collectionLink], + assets: { + cog: { + href: name + '.tiff', + type: 'image/tiff; application=geotiff; profile=cloud-optimized', + roles, + }, + }, + }; + await fsa.writeJson(job.getJobPath(href), item); + } + + if (ctx.cutline != null) { + const geoJsonCutlineOutput = job.getJobPath(`cutline.geojson.gz`); + await fsa.writeJson(geoJsonCutlineOutput, this.toGeoJson(cutlinePoly, ctx.tileMatrix.projection)); + } + + const geoJsonSourceOutput = job.getJobPath(`source.geojson`); + await fsa.writeJson(geoJsonSourceOutput, Projection.get(job.source.epsg).toGeoJson(metadata.bounds)); + + const geoJsonCoveringOutput = job.getJobPath(`covering.geojson`); + await fsa.writeJson(geoJsonCoveringOutput, covering); + + await fsa.writeJson(job.getJobPath(`collection.json`), stac); + + return job; + } + + /** + * build a FeatureCollection from MultiPolygon + */ + static toGeoJson(poly: MultiPolygon, epsg: Epsg): FeatureCollectionWithCrs { + const feature = toFeatureCollection([toFeatureMultiPolygon(poly)]) as FeatureCollectionWithCrs; + feature.crs = { + type: 'name', + properties: { name: epsg.toUrn() }, + }; + return feature; + } + + constructor(json: CogJobJson) { + this.json = json; + } + + get id(): string { + return this.json.id; + } + + get name(): string { + return this.json.name; + } + + get title(): string { + return this.json.title; + } + + get description(): string | undefined { + return this.json.description; + } + + get source(): CogSourceProperties { + return this.json.source; + } + + get output(): CogOutputProperties { + return this.json.output; + } + + get tileMatrix(): TileMatrixSet { + if (this.json.output.tileMatrix) { + const tileMatrix = TileMatrixSets.find(this.json.output.tileMatrix); + if (tileMatrix == null) throw new Error(`Failed to find TileMatrixSet "${this.json.output.tileMatrix}"`); + return tileMatrix; + } + return TileMatrixSets.get(this.json.output.epsg); + } + + get targetZoom(): number { + const { gsd } = this.source; + if (this.cacheTargetZoom?.gsd !== gsd) { + this.cacheTargetZoom = { gsd, zoom: Projection.getTiffResZoom(this.tileMatrix, gsd) }; + } + return this.cacheTargetZoom.zoom; + } + + /** + * Get a nicely formatted folder path based on the job + * + * @param key optional file key inside of the job folder + */ + getJobPath(key?: string): string { + const parts = [this.tileMatrix.projection.code, this.name, this.id]; + if (key != null) { + parts.push(key); + } + return fsa.join(this.output.location.path, parts.join('/')); + } +} diff --git a/packages/cli/src/cog/stac.ts b/packages/cli/src/cog/stac.ts new file mode 100644 index 000000000..9f1fcc8b6 --- /dev/null +++ b/packages/cli/src/cog/stac.ts @@ -0,0 +1,41 @@ +import { EpsgCode, StacCollection, StacItem } from '@basemaps/geo'; +import { CogGdalSettings } from './types.js'; + +export interface CogGenerated { + /** ISO date string */ + datetime: string; + /** Package name of the generator */ + package: string; + /* Version of the generator */ + version: string; + /** Commit hash of the generator */ + hash: string | undefined; +} + +export interface CogOutputSummery extends CogGdalSettings { + cutlineBlend?: number; +} + +export interface CogSummaries { + 'proj:epsg': EpsgCode[]; + /** Ground Sampling Distance in meters per pixel of the generated imagery */ + gsd: number[]; + 'linz:output': CogOutputSummery[]; + 'linz:generated': CogGenerated[]; + + /** How and when this job file was generated */ +} + +export const CogStacKeywords = ['Imagery', 'New Zealand']; +export const CogStacItemExtensions = ['projection']; + +/** STAC compliant structure for storing Job instructions */ +export type CogStac = StacCollection; + +export interface CogStacItemProperties { + datetime: string; + gsd: number; + 'proj:epsg': EpsgCode; +} + +export type CogStacItem = StacItem; diff --git a/packages/cli/src/gdal/__tests__/gdal.progress.test.ts b/packages/cli/src/gdal/__tests__/gdal.progress.test.ts new file mode 100644 index 000000000..34e4b52a9 --- /dev/null +++ b/packages/cli/src/gdal/__tests__/gdal.progress.test.ts @@ -0,0 +1,25 @@ +import o from 'ospec'; +import { GdalProgressParser } from '../gdal.progress.js'; + +o.spec('GdalProgressParser', () => { + o('should emit on progress', () => { + const prog = new GdalProgressParser(); + o(prog.progress).equals(0); + + prog.data(Buffer.from('\n.')); + o(prog.progress.toFixed(2)).equals('3.23'); + }); + + o('should take 31 dots to finish', () => { + const prog = new GdalProgressParser(); + let processCount = 0; + prog.data(Buffer.from('\n')); + prog.on('progress', () => processCount++); + + for (let i = 0; i < 31; i++) { + prog.data(Buffer.from('.')); + o(processCount).equals(i + 1); + } + o(prog.progress.toFixed(2)).equals('100.00'); + }); +});