diff --git a/packages/bathymetry/src/bathy.maker.ts b/packages/bathymetry/src/bathy.maker.ts index ea6da17ee..810106655 100644 --- a/packages/bathymetry/src/bathy.maker.ts +++ b/packages/bathymetry/src/bathy.maker.ts @@ -1,5 +1,3 @@ -import { Gdal } from '@basemaps/cli/build/gdal/gdal.js'; -import { GdalCommand } from '@basemaps/cli/build/gdal/gdal.command.js'; import { Bounds, Epsg, Tile, TileMatrixSet } from '@basemaps/geo'; import { fsa, LogType, s3ToVsis3 } from '@basemaps/shared'; import * as os from 'os'; @@ -11,6 +9,8 @@ import { FilePath, FileType } from './file.js'; import { Hash } from './hash.js'; import { MapnikRender } from './mapnik.js'; import { Stac } from './stac.js'; +import { GdalCommand } from './gdal/gdal.command.js'; +import { Gdal } from './gdal/gdal.js'; interface BathyMakerContext { /** unique id for this build */ @@ -108,7 +108,7 @@ export class BathyMaker { const gdalVersion = await this.gdalVersion; logger.info({ gdalVersion }, 'GdalVersion'); - const promises = []; + const promises: Promise[] = []; let extent: Bounds | null = null; for (let x = 0; x < tmsZoom.matrixWidth; x++) { for (let y = 0; y < tmsZoom.matrixHeight; y++) { diff --git a/packages/cli/src/gdal/__tests__/gdal.progress.test.ts b/packages/bathymetry/src/gdal/__tests__/gdal.progress.test.ts similarity index 100% rename from packages/cli/src/gdal/__tests__/gdal.progress.test.ts rename to packages/bathymetry/src/gdal/__tests__/gdal.progress.test.ts diff --git a/packages/cli/src/gdal/gdal.command.ts b/packages/bathymetry/src/gdal/gdal.command.ts similarity index 100% rename from packages/cli/src/gdal/gdal.command.ts rename to packages/bathymetry/src/gdal/gdal.command.ts diff --git a/packages/cli/src/gdal/gdal.docker.ts b/packages/bathymetry/src/gdal/gdal.docker.ts similarity index 100% rename from packages/cli/src/gdal/gdal.docker.ts rename to packages/bathymetry/src/gdal/gdal.docker.ts diff --git a/packages/cli/src/gdal/gdal.local.ts b/packages/bathymetry/src/gdal/gdal.local.ts similarity index 100% rename from packages/cli/src/gdal/gdal.local.ts rename to packages/bathymetry/src/gdal/gdal.local.ts diff --git a/packages/cli/src/gdal/gdal.progress.ts b/packages/bathymetry/src/gdal/gdal.progress.ts similarity index 100% rename from packages/cli/src/gdal/gdal.progress.ts rename to packages/bathymetry/src/gdal/gdal.progress.ts diff --git a/packages/cli/src/gdal/gdal.ts b/packages/bathymetry/src/gdal/gdal.ts similarity index 100% rename from packages/cli/src/gdal/gdal.ts rename to packages/bathymetry/src/gdal/gdal.ts diff --git a/packages/cli/Dockerfile b/packages/cli/Dockerfile index 5513245a2..d0928a667 100644 --- a/packages/cli/Dockerfile +++ b/packages/cli/Dockerfile @@ -12,7 +12,7 @@ ENV NODE_MAJOR=18 RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list RUN apt-get update -RUN apt-get install -y nodejs git +RUN apt-get install -y nodejs # Install sharp TODO update this when we change sharp versions RUN npm install sharp@0.32.6 diff --git a/packages/cli/src/cli/config/action.import.ts b/packages/cli/src/cli/config/action.import.ts index 0f530c08c..22a4a4b8a 100644 --- a/packages/cli/src/cli/config/action.import.ts +++ b/packages/cli/src/cli/config/action.import.ts @@ -17,7 +17,7 @@ import fetch from 'node-fetch'; import { invalidateCache } from '../util.js'; import { Q, Updater } from './config.update.js'; import { FeatureCollection } from 'geojson'; -import { CogJobJson } from '../../cog/types.js'; +import { CogJobJson } from '@basemaps/shared'; const PublicUrlBase = Env.isProduction() ? 'https://basemaps.linz.govt.nz/' : 'https://dev.basemaps.linz.govt.nz/'; diff --git a/packages/cli/src/cog/__tests__/builder.test.ts b/packages/cli/src/cog/__tests__/builder.test.ts deleted file mode 100644 index 2576e8da5..000000000 --- a/packages/cli/src/cog/__tests__/builder.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { Bounds, Epsg, EpsgCode, GoogleTms } from '@basemaps/geo'; -import { LogConfig } from '@basemaps/shared'; -import { fsa } from '@chunkd/fs'; -import { CogTiff } from '@cogeotiff/core'; - -import o from 'ospec'; -import { CogBuilder, guessProjection } from '../builder.js'; - -o.spec('Builder', () => { - o('should guess WKT', () => { - o( - guessProjection( - 'PCS Name = NZGD_2000_New_Zealand_Transverse_Mercator|GCS Name = GCS_NZGD_2000|Ellipsoid = GRS_1980|Primem = Greenwich||', - ), - ).equals(Epsg.Nztm2000); - - o( - guessProjection('NZGD2000_New_Zealand_Transverse_Mercator_2000|GCS Name = GCS_NZGD_2000|Primem = Greenwich||'), - ).equals(Epsg.Nztm2000); - }); - - o('should not guess unknown wkt', () => { - o(guessProjection('')).equals(null); - o(guessProjection('NZTM')).equals(null); - o(guessProjection('NZGD2000')).equals(null); - }); - - o.spec('tiff', () => { - const googleBuilder = new CogBuilder(GoogleTms, 1, LogConfig.get()); - const origInit = CogTiff.prototype.init; - const origGetImage = CogTiff.prototype.getImage; - - o.after(() => { - CogTiff.prototype.init = origInit; - CogTiff.prototype.getImage = origGetImage; - }); - - o('bounds', async () => { - const localTiff = fsa.source('/local/file.tiff')!; - const s3Tiff = fsa.source('s3://bucket/file.tiff')!; - - const imageLocal = { - resolution: [0.1], - value: (): any => [1], - valueGeo: (): any => EpsgCode.Nztm2000, - bbox: Bounds.fromJson({ - x: 1492000, - y: 6198000, - width: 24000, - height: 36000, - }).toBbox(), - }; - - const imageS3 = { - resolution: [0.1], - value: (): any => [1], - valueGeo: (): any => EpsgCode.Nztm2000, - bbox: Bounds.fromJson({ - x: 1492000 + 24000, - y: 6198000, - width: 24000, - height: 36000, - }).toBbox(), - }; - - CogTiff.prototype.init = o.spy() as any; - CogTiff.prototype.getImage = function (): any { - return this.source === localTiff ? imageLocal : imageS3; - }; - - const ans = await googleBuilder.bounds([localTiff, s3Tiff]); - - o(ans).deepEquals({ - projection: 2193, - nodata: 1, - bands: 1, - bounds: [ - { - x: 1492000, - y: 6198000, - width: 24000, - height: 36000, - name: '/local/file.tiff', - }, - { - x: 1516000, - y: 6198000, - width: 24000, - height: 36000, - name: 's3://bucket/file.tiff', - }, - ], - pixelScale: 0.1, - resZoom: 21, - }); - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/cog.stac.job.test.ts b/packages/cli/src/cog/__tests__/cog.stac.job.test.ts deleted file mode 100644 index b9f166c47..000000000 --- a/packages/cli/src/cog/__tests__/cog.stac.job.test.ts +++ /dev/null @@ -1,350 +0,0 @@ -import { Bounds, Epsg, EpsgCode, GoogleTms, Nztm2000Tms, Stac, StacCollection } from '@basemaps/geo'; -import { Projection } from '@basemaps/geo'; -import { mockFileOperator } from '@basemaps/shared/build/file/__tests__/file.operator.test.helper.js'; -import { round } from '@basemaps/test/build/rounding.js'; -import { Ring } from '@linzjs/geojson'; -import o from 'ospec'; -import { CogStacJob, JobCreationContext } from '../cog.stac.job.js'; -import { CogBuilderMetadata, CogJobJson } from '../types.js'; - -type RecursivePartial = { - [P in keyof T]?: RecursivePartial; -}; -o.spec('CogJob', () => { - o.spec('build', () => { - const id = 'jobid1'; - const imageryName = 'auckland_rural_2010-2012_0-50m'; - - const jobPath = 's3://target-bucket/path/3857/auckland_rural_2010-2012_0-50m/jobid1'; - - const ring1 = [ - [170, -41], - [-175, -40], - [-177, -42], - [175, -42], - [170, -41], - ].map(Projection.get(Epsg.Nztm2000).fromWgs84) as Ring; - const ring2 = [ - [-150, -40], - [-140, -41], - [-150, -41], - [-150, -40], - ].map(Projection.get(Epsg.Nztm2000).fromWgs84) as Ring; - - const srcPoly = [[ring1], [ring2]]; - const bounds = srcPoly.map((poly, i) => ({ ...Bounds.fromMultiPolygon([poly]), name: 'ring' + i })); - - const metadata: CogBuilderMetadata = { - bands: 3, - bounds, - projection: EpsgCode.Nztm2000, - pixelScale: 0.075, - resZoom: 22, - nodata: 0, - files: [{ name: '0-0-0', x: 1, y: 2, width: 2, height: 3 }], - targetBounds: { x: 1, y: 2, width: 2, height: 3 }, - }; - const addAlpha = true; - const ctx = { - tileMatrix: GoogleTms, - sourceLocation: { type: 's3', path: 's3://source-bucket/path' }, - outputLocation: { type: 's3', path: 's3://target-bucket/path' }, - cutline: { blend: 20, href: 's3://curline-bucket/path' }, - oneCogCovering: false, - } as JobCreationContext; - - const mockFs = mockFileOperator(); - - o.beforeEach(mockFs.setup); - - o.afterEach(mockFs.teardown); - - o('with job.json', async () => { - mockFs.jsStore['s3://source-bucket/path/collection.json'] = { - title: 'The Title', - description: 'The Description', - license: 'The License', - keywords: ['keywords'], - extent: { - spatial: { bbox: [9, 8, 7, 6] }, - temporal: { interval: [['2020-01-01T00:00:00.000', '2020-08-08T19:18:23.456Z']] }, - }, - - providers: [ - { name: 'provider name', roles: ['licensor'], url: 'https://provider.com' }, - { name: 'unknown url', roles: ['processor'], url: 'unknown' }, - ], - }; - const job = await CogStacJob.create({ id, imageryName, metadata, ctx, addAlpha, cutlinePoly: [] }); - o(job.getJobPath('job.json')).equals(jobPath + '/job.json'); - - o(job.title).equals('The Title'); - o(job.description).equals('The Description'); - - const stac = round(mockFs.jsStore[jobPath + '/collection.json'], 4); - - o(stac.title).equals('The Title'); - o(stac.description).equals('The Description'); - o(stac.license).equals('The License'); - o(stac.keywords).deepEquals(['keywords']); - o(stac.providers).deepEquals([ - { name: 'provider name', roles: ['licensor'], url: 'https://provider.com' }, - { name: 'unknown url', roles: ['processor'], url: undefined }, - ]); - o(stac.links[2]).deepEquals({ - href: 's3://source-bucket/path/collection.json', - rel: 'sourceImagery', - type: 'application/json', - }); - o(round(stac.extent.spatial.bbox, 4)).deepEquals([[169.3347, -51.868, -145.9143, -32.8496]]); - o(stac.extent.temporal).deepEquals({ interval: [['2020-01-01T00:00:00.000', '2020-08-08T19:18:23.456Z']] }); - }); - - o('no source collection.json', async () => { - const startNow = Date.now(); - const job = await CogStacJob.create({ - id, - imageryName, - metadata, - ctx: { ...ctx, oneCogCovering: true }, - addAlpha, - cutlinePoly: [], - }); - const afterNow = Date.now(); - const jobJson = round(job.json, 4) as CogJobJson; - - o(jobJson).deepEquals(round(mockFs.jsStore[jobPath + '/job.json'], 4)); - o(jobJson.id).equals('jobid1'); - o(jobJson.name).equals('auckland_rural_2010-2012_0-50m'); - o(jobJson.title).equals('Auckland rural 2010-2012 0.50m'); - o(jobJson.description).equals('No description'); - o(jobJson.source.gsd).equals(0.075); - o(jobJson.source.epsg).equals(2193); - o(jobJson.source.files[0]).deepEquals({ - x: 1347679.1521, - y: 5301577.5262, - width: 1277913.1293, - height: 201072.641, - name: 'ring0', - }); - o(jobJson.source.files[1]).deepEquals({ - x: 4728648.2332, - y: 4247288.7253, - width: 837784.3351, - height: 609622.9622, - name: 'ring1', - }); - o(jobJson.source.location).deepEquals({ type: 's3', path: 's3://source-bucket/path' }); - o(jobJson.output.gsd).equals(0.0373); - o(jobJson.output.epsg).equals(3857); - o(jobJson.output.tileMatrix).equals('WebMercatorQuad'); - o(jobJson.output.files).deepEquals([{ name: '0-0-0', x: 1, y: 2, width: 2, height: 3 }]); - o(jobJson.output.location).deepEquals({ type: 's3', path: 's3://target-bucket/path' }); - o(jobJson.output.resampling).deepEquals({ warp: 'bilinear', overview: 'lanczos' }); - o(jobJson.output.quality).equals(90); - o(jobJson.output.cutline).deepEquals({ blend: 20, href: 's3://curline-bucket/path' }); - o(jobJson.output.addAlpha).equals(true); - o(jobJson.output.nodata).equals(0); - o(jobJson.output.bounds).deepEquals({ x: 1, y: 2, width: 2, height: 3 }); - - o(job.id).equals(id); - o(job.source.gsd).equals(0.075); - o(round(job.output.gsd, 4)).equals(0.0373); - o(job.output.oneCogCovering).equals(true); - - const stac = round(mockFs.jsStore[jobPath + '/collection.json'], 4) as StacCollection; - - const stacMeta = stac.summaries as any; - - const generated = stacMeta['linz:generated'][0]; - - const jobNow = +Date.parse(generated.datetime); - o(startNow <= jobNow && jobNow <= afterNow).equals(true); - - // split so that scripts/detect.unlinked.dep.js does not think it is an import - o(generated.package).equals('@' + 'basemaps/cli'); - o(stac.id).equals(job.id); - o(stac.extent.spatial.bbox).deepEquals([[169.3347, -51.868, -145.9143, -32.8496]]); - o(stac.extent.temporal).deepEquals({ interval: [['2010-01-01T00:00:00Z', '2011-01-01T00:00:00Z']] }); - o(stac.stac_version).equals(Stac.Version); - o(stac.stac_extensions).deepEquals([Stac.BaseMapsExtension]); - o(stac.license).deepEquals(Stac.License); - o(stac.keywords).deepEquals(['Imagery', 'New Zealand']); - o(stac.summaries).deepEquals({ - gsd: [0.075], - 'proj:epsg': [3857], - 'linz:output': [ - { - resampling: { warp: 'bilinear', overview: 'lanczos' }, - quality: 90, - cutlineBlend: 20, - addAlpha: true, - nodata: 0, - }, - ], - 'linz:generated': [generated], - }); - o(stac.links).deepEquals([ - { href: 's3://target-bucket/path/collection.json', type: 'application/json', rel: 'self' }, - { href: 'job.json', type: 'application/json', rel: 'linz.basemaps.job' }, - { href: 'cutline.geojson.gz', type: 'application/geo+json+gzip', rel: 'linz.basemaps.cutline' }, - { href: 'covering.geojson', type: 'application/geo+json', rel: 'linz.basemaps.covering' }, - { href: 'source.geojson', type: 'application/geo+json', rel: 'linz.basemaps.source' }, - { href: '0-0-0.json', type: 'application/geo+json', rel: 'item' }, - ]); - - o(round(mockFs.jsStore[jobPath + '/cutline.geojson.gz'], 4)).deepEquals({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { type: 'MultiPolygon', coordinates: [] }, - properties: {}, - }, - ], - crs: { - type: 'name', - properties: { name: 'urn:ogc:def:crs:EPSG::3857' }, - }, - }); - - o(round(mockFs.jsStore[jobPath + '/0-0-0.json'], 4)).deepEquals({ - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [ - [ - [0, 0], - [0, 0], - [0, 0], - [0, 0], - [0, 0], - ], - ], - }, - properties: { - datetime: new Date(jobNow).toISOString(), - name: '0-0-0', - gsd: 0.0373, - 'proj:epsg': 3857, - }, - bbox: [0, 0, 0, 0], - id: 'jobid1/0-0-0', - collection: 'jobid1', - stac_version: Stac.Version, - stac_extensions: ['projection'], - links: [ - { href: jobPath + '/0-0-0.json', rel: 'self' }, - { href: 'collection.json', rel: 'collection' }, - ], - assets: { - cog: { - href: '0-0-0.tiff', - type: 'image/tiff; application=geotiff; profile=cloud-optimized', - roles: ['data'], - }, - }, - }); - o(round(mockFs.jsStore[jobPath + '/covering.geojson'], 4)).deepEquals({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [ - [ - [0, 0], - [0, 0], - [0, 0], - [0, 0], - [0, 0], - ], - ], - }, - properties: { name: '0-0-0' }, - bbox: [0, 0, 0, 0], - }, - ], - }); - - const sourceGeoJson = mockFs.jsStore[jobPath + '/source.geojson']; - o(sourceGeoJson.type).equals('FeatureCollection'); - o(sourceGeoJson.features.length).equals(2); - o(round(sourceGeoJson.features[0], 4)).deepEquals({ - type: 'Feature', - geometry: { - type: 'MultiPolygon', - coordinates: [ - [ - [ - [169.9343, -42.3971], - [180, -42.2199], - [180, -40.4109], - [170.0185, -40.5885], - [169.9343, -42.3971], - ], - ], - [ - [ - [-180, -42.2199], - [-174.6708, -41.7706], - [-175, -40], - [-180, -40.4109], - [-180, -42.2199], - ], - ], - ], - }, - properties: { name: 'ring0' }, - bbox: [169.9343, -42.3971, -175, -40], - }); - - o(round(sourceGeoJson.features[1], 4)).deepEquals({ - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [ - [ - [-147.4403, -44.5167], - [-150.5162, -40.1908], - [-143.1693, -37.1443], - [-140, -41], - [-147.4403, -44.5167], - ], - ], - }, - properties: { name: 'ring1' }, - bbox: [-147.4403, -44.5167, -143.1693, -37.1443], - }); - }); - - o('should create with no tileMatrix', () => { - const cfg: RecursivePartial = { output: { epsg: 2193 } }; - const job = new CogStacJob(cfg as CogJobJson); - o(job.tileMatrix.identifier).equals(Nztm2000Tms.identifier); - }); - - o('should error with invalid tileMatrix', () => { - const cfg: RecursivePartial = { output: { tileMatrix: 'None' } }; - const job = new CogStacJob(cfg as CogJobJson); - o(() => job.tileMatrix).throws('Failed to find TileMatrixSet "None"'); - }); - - o('no source collection.json and not nice name', async () => { - try { - await CogStacJob.create({ - id, - imageryName: 'yucky-name', - metadata, - ctx: { ...ctx, oneCogCovering: true }, - addAlpha, - cutlinePoly: [], - }); - o('').equals('create should not have exceeded'); - } catch (err: any) { - o(err.message).equals('Missing date in imagery name: yucky-name'); - } - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/cutline.test.ts b/packages/cli/src/cog/__tests__/cutline.test.ts deleted file mode 100644 index 21719812c..000000000 --- a/packages/cli/src/cog/__tests__/cutline.test.ts +++ /dev/null @@ -1,353 +0,0 @@ -import { Bounds, EpsgCode, GoogleTms, Nztm2000Tms } from '@basemaps/geo'; -import { qkToName } from '@basemaps/geo/build/proj/__tests__/test.util.js'; -import { round } from '@basemaps/test/build/rounding.js'; -import { MultiPolygon } from '@linzjs/geojson'; -import o from 'ospec'; -import path from 'path'; -import url from 'url'; -import { AlignedLevel } from '../constants.js'; -import { Cutline, polyContainsBounds } from '../cutline.js'; -import { SourceMetadata } from '../types.js'; -import { SourceTiffTestHelper } from './source.tiff.testhelper.js'; -const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); - -o.spec('cutline', () => { - o.specTimeout(1_000); - const testDir = `${__dirname}/../../../__test.assets__`; - - o.spec('filterSourcesForName', () => { - o('fully within same projection', async () => { - // convert poly to GoogleTms - const cutpoly = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson')).toGeoJson(); - - const cutline = new Cutline(GoogleTms, cutpoly, -100); - - const name = qkToName('311333222321113310'); - - const job = SourceTiffTestHelper.makeCogJob(); - job.output.tileMatrix = GoogleTms.identifier; - job.source.epsg = EpsgCode.Nztm2000; - const sourceBounds = SourceTiffTestHelper.tiffNztmBounds(testDir); - const [tif1, tif2] = sourceBounds; - job.source.files = [tif1, tif2]; - - o(cutline.filterSourcesForName(name, job)).deepEquals([tif2.name]); - }); - }); - - o('polyContainsBounds', () => { - const polys: MultiPolygon = [ - [ - [ - [-4, 2], - [-2, 1], - [-4, -1], - [1, -5], - [2, -5], - [2, -2], - [4, -5], - [6, -2], - [2, 0], - [6, 3], - [2, 7], - [0, 3], - [-4, 6], - [-4, 10], - [10, 10], - [10, -10], - [-10, -10], - [-4, 2], - ], - ], - [ - [ - [2, 3], - [3, 3], - [3, 5], - [2, 5], - [2, 3], - ], - ], - ]; - - o(polyContainsBounds(polys, Bounds.fromBbox([-6, -6, -3, -3]))).equals(true); - - o(polyContainsBounds(polys, Bounds.fromBbox([-3, -4, 4, 4]))).equals(false); - o(polyContainsBounds(polys, Bounds.fromBbox([6, -8, 5, 5]))).equals(false); - }); - - o('loadCutline', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - const geojson = round(cutline.toGeoJson()); - - o(geojson).deepEquals({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - type: 'MultiPolygon', - coordinates: [ - [ - [ - [19456570.81047118, -5023799.46086743], - [19457346.41188008, -5024885.32147075], - [19455543.1261867, -5026720.78623442], - [19455446.57500747, -5025689.98761053], - [19456325.95579277, -5024324.4264959], - [19456431.41331051, -5023815.14487631], - [19456497.91735541, -5023822.4695137], - [19456513.95059036, -5023820.4859541], - [19456528.99998363, -5023810.66149213], - [19456528.41956381, -5023802.3991344], - [19456521.62606925, -5023792.60909871], - [19456521.12156931, -5023788.06857522], - [19456524.31632738, -5023783.03836819], - [19456531.227264, -5023778.34574544], - [19456540.68563587, -5023777.32428773], - [19456551.09434221, -5023782.4891701], - [19456560.03196148, -5023796.37226942], - [19456570.81047118, -5023799.46086743], - ], - ], - [ - [ - [19430214.06028324, -5008476.23288268], - [19429858.86076585, -5008966.74650194], - [19430484.68848696, -5009778.63111311], - [19430907.54505528, -5009085.14634107], - [19430214.06028324, -5008476.23288268], - ], - ], - ], - }, - properties: {}, - }, - ], - crs: { - type: 'name', - properties: { name: 'urn:ogc:def:crs:EPSG::3857' }, - }, - }); - }); - - o('findCovering', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - const bounds = SourceTiffTestHelper.tiffNztmBounds(); - - o(cutline.clipPoly.length).equals(2); - - (cutline as any).findCovering({ projection: EpsgCode.Nztm2000, bounds, resZoom: 15 }); - - o((cutline as any).srcPoly.length).equals(1); - }); - - o.spec('optimize', async () => { - const bounds = SourceTiffTestHelper.tiffNztmBounds(); - - o('full-extent 3857', () => { - const cutline = new Cutline(GoogleTms); - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Google, - bounds: [{ ...GoogleTms.extent, name: 'gebco' }], - resZoom: 5, - } as SourceMetadata, - 4, - ); - - o(round(covering, 4)).deepEquals([ - { - name: '1-0-0', - x: -20037508.3428, - y: 0, - width: 20037508.3428, - height: 20037508.3428, - }, - { - name: '1-0-1', - x: -20037508.3428, - y: -20037508.3428, - width: 20037508.3428, - height: 20037508.3428, - }, - { - name: '1-1-0', - x: -0, - y: 0, - width: 20037508.3428, - height: 20037508.3428, - }, - { - name: '1-1-1', - x: -0, - y: -20037508.3428, - width: 20037508.3428, - height: 20037508.3428, - }, - ]); - }); - - o('nztm', () => { - const cutline = new Cutline(Nztm2000Tms); - - const covering = cutline.optimizeCovering({ - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 7 + AlignedLevel, - } as SourceMetadata); - - o(covering[4]).deepEquals({ - name: '7-153-253', - x: 1741760, - y: 5448320, - width: 17920, - height: 17920, - }); - o(covering.map((c) => c.name)).deepEquals([ - '7-152-252', - '7-152-253', - '7-152-254', - '7-153-252', - '7-153-253', - '7-153-254', - '7-154-252', - '7-154-253', - '7-154-254', - '7-155-252', - '7-155-253', - '8-387-635', - ]); - }); - - o('source polygon is smoothed', async () => { - const tiff1 = { - name: '/path/to/tiff1.tiff', - x: 1759315.9336568, - y: 5476120.089572778, - width: 2577.636033663526, - height: 2275.3233966259286, - }; - - const tiff2 = { - name: '/path/to/tiff2.tiff', - x: 1759268.0010635862, - y: 5473899.771969615, - width: 2576.894309356343, - height: 2275.3359155077487, - }; - - const bounds = [tiff1, tiff2]; - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 500); - - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 22, - } as SourceMetadata, - 8, - ); - - o(round(cutline.clipPoly, 4)[0][0]).deepEquals([ - [19468576.3516, -4993284.6176], - [19471093.8032, -4993284.6176], - [19471932.5568, -4992600.0637], - [19472095.7462, -4992346.9043], - [19472095.7462, -4987199.9524], - [19471985.5211, -4987199.9524], - [19470978.4379, -4989417.4454], - [19468800.9775, -4991497.5405], - [19468576.3516, -4991797.1824], - [19468576.3516, -4993284.6176], - ]); - - o(covering.map((c) => c.name)).deepEquals([ - '14-16151-10232', - '14-16151-10233', - '14-16152-10231', - '14-16152-10232', - '14-16152-10233', - '18-258444-163695', - ]); - }); - - o('boundary part inland, part coastal', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 500); - const bounds = [ - { - name: 'tiff1', - x: 1759315.9336568, - y: 5476120.089572778, - width: 2577.636033663526, - height: 2275.3233966259286, - }, - ]; - - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 22, - } as SourceMetadata, - 8, - ); - - o(round(cutline.clipPoly, 4)[0][0]).deepEquals([ - [19470011.8243, -4990340.8368], - [19472095.7007, -4990340.8368], - [19472095.7007, -4987199.9524], - [19471985.5211, -4987199.9524], - [19470978.4379, -4989417.4454], - [19470011.8243, -4990340.8368], - ]); - - o(covering.map((c) => c.name)).deepEquals([ - '14-16152-10231', - '15-32304-20464', - '15-32305-20464', - '18-258444-163695', - ]); - }); - - o('low res', () => { - const cutline = new Cutline(GoogleTms); - - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 13, - } as SourceMetadata, - 9, - ); - - o(covering.length).equals(covering.length); - o(covering.map((c) => c.name)).deepEquals(['8-252-159', '8-252-160']); - }); - - o('hi res', () => { - const covering2 = new Cutline(GoogleTms).optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 18, - } as SourceMetadata, - 8, - ); - - o(covering2.length).equals(covering2.length); - - o(covering2.map((c) => c.name)).deepEquals([ - '10-1008-639', - '10-1008-640', - '10-1009-639', - '10-1009-640', - '11-2020-1279', - '11-2020-1280', - '12-4040-2557', - '12-4040-2562', - ]); - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/projection.loader.test.ts b/packages/cli/src/cog/__tests__/projection.loader.test.ts deleted file mode 100644 index aac709ec3..000000000 --- a/packages/cli/src/cog/__tests__/projection.loader.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import o from 'ospec'; -import { ProjectionLoader } from '../projection.loader.js'; -import sinon from 'sinon'; -import { Nztm2000 } from '@basemaps/geo/build/proj/nztm2000.js'; -import { Epsg } from '@basemaps/geo'; -import { Projection } from '@basemaps/geo'; - -o.spec('ProjectionLoader', () => { - const sandbox = sinon.createSandbox(); - o.afterEach(() => sandbox.restore()); - o('should not load if already loaded', async () => { - const fetchStub = sandbox.stub(ProjectionLoader, '_fetch'); - const res = await ProjectionLoader.load(2193); - o(fetchStub.callCount).equals(0); - o(res.code).equals(2193); - }); - - o('should fetch data from the internet', async () => { - const fetchStub = sandbox - .stub(ProjectionLoader, '_fetch') - .resolves({ ok: true, text: () => Promise.resolve(Nztm2000) } as any); - const res = await ProjectionLoader.load(3790); - - o(fetchStub.callCount).equals(1); - - o(res.code).equals(3790); - o(Epsg.get(3790)).equals(res); - o(Projection.get(3790).epsg).equals(res); - }); -}); diff --git a/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts b/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts deleted file mode 100644 index 371a8ed53..000000000 --- a/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { EpsgCode, NamedBounds, GoogleTms } from '@basemaps/geo'; -import { CogStacJob } from '../cog.stac.job.js'; -import { CogJobJson } from '../types.js'; - -export const SourceTiffTestHelper = { - makeCogJob(): CogStacJob { - return new CogStacJob({ - source: { - files: [] as NamedBounds[], - epsg: EpsgCode.Nztm2000, - gsd: 0.8, - }, - output: { - tileMatrix: GoogleTms.identifier, - gsd: 0.75, - addAlpha: true, - oneCogCovering: false, - }, - } as CogJobJson); - }, - - tiffNztmBounds(path = '/path/to'): NamedBounds[] { - return [ - { - name: path + '/tiff1.tiff', - x: 1732000, - y: 5442000, - width: 24000, - height: 36000, - }, - { - name: path + '/tiff2.tiff', - x: 1756000, - y: 5442000, - width: 24000, - height: 36000, - }, - ]; - }, - - namedBoundsToPaths(bounds: NamedBounds[]): string[] { - return bounds.map(({ name }): string => `/path/to/tiff${name}.tiff`); - }, -}; diff --git a/packages/cli/src/cog/builder.ts b/packages/cli/src/cog/builder.ts index 59a55186d..e69cb7d34 100644 --- a/packages/cli/src/cog/builder.ts +++ b/packages/cli/src/cog/builder.ts @@ -5,8 +5,8 @@ import { CogTiff, TiffTag, TiffTagGeo } from '@cogeotiff/core'; import pLimit, { LimitFunction } from 'p-limit'; import { basename } from 'path'; import { Cutline } from './cutline.js'; // -import { ProjectionLoader } from './projection.loader.js'; -import { CogBuilderMetadata, SourceMetadata } from './types.js'; +import { ProjectionLoader } from '@basemaps/geo'; +import { CogBuilderMetadata, SourceMetadata } from '@basemaps/shared'; export const InvalidProjectionCode = 32767; diff --git a/packages/cli/src/cog/cog.stac.job.ts b/packages/cli/src/cog/cog.stac.job.ts deleted file mode 100644 index 94c5b65f0..000000000 --- a/packages/cli/src/cog/cog.stac.job.ts +++ /dev/null @@ -1,407 +0,0 @@ -import { - Bounds, - Epsg, - Projection, - Stac, - StacCollection, - StacLink, - StacProvider, - TileMatrixSet, - TileMatrixSets, -} from '@basemaps/geo'; -import { - CompositeError, - extractYearRangeFromName, - FileConfig, - FileConfigPath, - fsa, - titleizeImageryName, -} from '@basemaps/shared'; -import { CliInfo } from '@basemaps/shared/build/cli/info.js'; -import { MultiPolygon, toFeatureCollection, toFeatureMultiPolygon } from '@linzjs/geojson'; -import { GdalCogBuilderDefaults, GdalCogBuilderResampling } from '../gdal/gdal.config.js'; -import { ProjectionLoader } from './projection.loader.js'; -import { CogStac, CogStacItem, CogStacItemExtensions, CogStacKeywords } from './stac.js'; -import { - CogBuilderMetadata, - CogJob, - CogJobJson, - CogOutputProperties, - CogSourceProperties, - FeatureCollectionWithCrs, -} from './types.js'; - -export const MaxConcurrencyDefault = 50; - -export interface JobCreationContext { - /** Custom Imagery Name if defined */ - imageryName?: string; - - /** Source config */ - sourceLocation: FileConfig | FileConfigPath; - - /** Output config */ - outputLocation: FileConfig; - - /** Should the imagery be cut to a cutline */ - cutline?: { - href: string; - blend: number; - }; - - tileMatrix: TileMatrixSet; - - override?: { - /** Override job id */ - id?: string; - - /** - * Image quality - * @default GdalCogBuilderDefaults.quality - */ - quality?: number; - - /** - * Number of threads to use for fetches - * @default MaxConcurrencyDefault - */ - concurrency?: number; - - /** - * Override the source projection - */ - projection?: Epsg; - - /** - * Override Default aligned zoom level - */ - alignedLevel?: number; - - /** - * Resampling method - * @Default GdalCogBuilderDefaults.resampling - */ - resampling?: GdalCogBuilderResampling; - }; - - /** - * Should this job be submitted to batch now? - * @default false - */ - batch: boolean; - - /** Should this job ignore source coverage and just produce one big COG for EPSG extent */ - oneCogCovering: boolean; -} - -export interface CogJobCreateParams { - /** unique id for imagery set */ - id: string; - /** name of imagery set */ - imageryName: string; - /** information about the source */ - metadata: CogBuilderMetadata; - /** information about the output */ - ctx: JobCreationContext; - /** the polygon to use to clip the source imagery */ - cutlinePoly: MultiPolygon; - /** do we need an alpha band added */ - addAlpha: boolean; -} - -/** - * Information needed to create cogs - */ -export class CogStacJob implements CogJob { - json: CogJobJson; - private cacheTargetZoom?: { - gsd: number; - zoom: number; - }; - - /** - * Load the job.json - - * @param jobpath where to load the job.json from - */ - static async load(jobpath: string): Promise { - const job = new CogStacJob(await fsa.readJson(jobpath)); - await ProjectionLoader.load(job.source.epsg); - return job; - } - - /** - * Create job.json, collection.json, source.geojson, covering.geojson, cutlint.geojson.gz and - * stac descriptions of the target COGs - */ - static async create({ - id, - imageryName, - metadata, - ctx, - cutlinePoly, - addAlpha, - }: CogJobCreateParams): Promise { - let description: string | undefined; - const providers: StacProvider[] = []; - const links: StacLink[] = [ - { - href: fsa.join(ctx.outputLocation.path, 'collection.json'), - type: 'application/json', - rel: 'self', - }, - { - href: 'job.json', - type: 'application/json', - rel: 'linz.basemaps.job', - }, - ]; - let sourceStac = {} as StacCollection; - - const interval: [string, string][] = []; - try { - const sourceCollectionPath = fsa.join(ctx.sourceLocation.path, 'collection.json'); - sourceStac = await fsa.readJson(sourceCollectionPath); - description = sourceStac.description; - interval.push(...(sourceStac.extent?.temporal?.interval ?? [])); - links.push({ href: sourceCollectionPath, rel: 'sourceImagery', type: 'application/json' }); - if (sourceStac.providers != null) { - for (const p of sourceStac.providers) { - if (p.roles.indexOf('host') === -1) { - if (p.url === 'unknown') { - // LINZ LDS has put unknown in some urls - p.url = undefined; - } - providers.push(p); - } - } - } - } catch (err) { - if (!CompositeError.isCompositeError(err) || err.code !== 404) { - throw err; - } - } - const keywords = sourceStac.keywords ?? CogStacKeywords.slice(); - const license = sourceStac.license ?? Stac.License; - const title = sourceStac.title ?? titleizeImageryName(imageryName); - - if (description == null) { - description = 'No description'; - } - - await ProjectionLoader.load(metadata.projection); - const job = new CogStacJob({ - id, - name: imageryName, - title, - description, - source: { - gsd: metadata.pixelScale, - epsg: metadata.projection, - files: metadata.bounds, - location: ctx.sourceLocation, - }, - output: { - gsd: ctx.tileMatrix.pixelScale(metadata.resZoom), - tileMatrix: ctx.tileMatrix.identifier, - epsg: ctx.tileMatrix.projection.code, - files: metadata.files, - location: ctx.outputLocation, - resampling: ctx.override?.resampling ?? GdalCogBuilderDefaults.resampling, - quality: ctx.override?.quality ?? GdalCogBuilderDefaults.quality, - cutline: ctx.cutline, - addAlpha, - nodata: metadata.nodata, - bounds: metadata.targetBounds, - oneCogCovering: ctx.oneCogCovering, - }, - }); - - const nowStr = new Date().toISOString(); - - const sourceProj = Projection.get(job.source.epsg); - - const bbox = [ - sourceProj.boundsToWgs84BoundingBox( - metadata.bounds.map((a) => Bounds.fromJson(a)).reduce((sum, a) => sum.union(a)), - ), - ]; - - if (interval.length === 0) { - const years = extractYearRangeFromName(imageryName); - if (years == null) throw new Error('Missing date in imagery name: ' + imageryName); - interval.push(years.map((y) => `${y}-01-01T00:00:00Z`) as [string, string]); - } - - if (ctx.cutline) { - links.push({ href: 'cutline.geojson.gz', type: 'application/geo+json+gzip', rel: 'linz.basemaps.cutline' }); - } - - links.push({ href: 'covering.geojson', type: 'application/geo+json', rel: 'linz.basemaps.covering' }); - links.push({ href: 'source.geojson', type: 'application/geo+json', rel: 'linz.basemaps.source' }); - - const temporal = { interval }; - - const jobFile = job.getJobPath(`job.json`); - - const stac: CogStac = { - id, - title, - description, - stac_version: Stac.Version, - stac_extensions: [Stac.BaseMapsExtension], - - extent: { - spatial: { bbox }, - temporal, - }, - - license, - keywords, - - providers, - - summaries: { - gsd: [metadata.pixelScale], - 'proj:epsg': [ctx.tileMatrix.projection.code], - 'linz:output': [ - { - resampling: ctx.override?.resampling ?? GdalCogBuilderDefaults.resampling, - quality: ctx.override?.quality ?? GdalCogBuilderDefaults.quality, - cutlineBlend: ctx.cutline?.blend, - addAlpha, - nodata: metadata.nodata, - }, - ], - 'linz:generated': [ - { - ...CliInfo, - datetime: nowStr, - }, - ], - }, - - links, - }; - - await fsa.writeJson(jobFile, job.json); - - const covering = Projection.get(job.tileMatrix).toGeoJson(metadata.files); - - const roles = ['data']; - const collectionLink = { href: 'collection.json', rel: 'collection' }; - - for (const f of covering.features) { - const { name } = f.properties as { name: string }; - const href = name + '.json'; - links.push({ href, type: 'application/geo+json', rel: 'item' }); - const item: CogStacItem = { - ...f, - id: job.id + '/' + name, - collection: job.id, - stac_version: Stac.Version, - stac_extensions: CogStacItemExtensions, - properties: { - ...f.properties, - datetime: nowStr, - gsd: job.output.gsd, - 'proj:epsg': job.tileMatrix.projection.code, - }, - links: [{ href: job.getJobPath(href), rel: 'self' }, collectionLink], - assets: { - cog: { - href: name + '.tiff', - type: 'image/tiff; application=geotiff; profile=cloud-optimized', - roles, - }, - }, - }; - await fsa.writeJson(job.getJobPath(href), item); - } - - if (ctx.cutline != null) { - const geoJsonCutlineOutput = job.getJobPath(`cutline.geojson.gz`); - await fsa.writeJson(geoJsonCutlineOutput, this.toGeoJson(cutlinePoly, ctx.tileMatrix.projection)); - } - - const geoJsonSourceOutput = job.getJobPath(`source.geojson`); - await fsa.writeJson(geoJsonSourceOutput, Projection.get(job.source.epsg).toGeoJson(metadata.bounds)); - - const geoJsonCoveringOutput = job.getJobPath(`covering.geojson`); - await fsa.writeJson(geoJsonCoveringOutput, covering); - - await fsa.writeJson(job.getJobPath(`collection.json`), stac); - - return job; - } - - /** - * build a FeatureCollection from MultiPolygon - */ - static toGeoJson(poly: MultiPolygon, epsg: Epsg): FeatureCollectionWithCrs { - const feature = toFeatureCollection([toFeatureMultiPolygon(poly)]) as FeatureCollectionWithCrs; - feature.crs = { - type: 'name', - properties: { name: epsg.toUrn() }, - }; - return feature; - } - - constructor(json: CogJobJson) { - this.json = json; - } - - get id(): string { - return this.json.id; - } - - get name(): string { - return this.json.name; - } - - get title(): string { - return this.json.title; - } - - get description(): string | undefined { - return this.json.description; - } - - get source(): CogSourceProperties { - return this.json.source; - } - - get output(): CogOutputProperties { - return this.json.output; - } - - get tileMatrix(): TileMatrixSet { - if (this.json.output.tileMatrix) { - const tileMatrix = TileMatrixSets.find(this.json.output.tileMatrix); - if (tileMatrix == null) throw new Error(`Failed to find TileMatrixSet "${this.json.output.tileMatrix}"`); - return tileMatrix; - } - return TileMatrixSets.get(this.json.output.epsg); - } - - get targetZoom(): number { - const { gsd } = this.source; - if (this.cacheTargetZoom?.gsd !== gsd) { - this.cacheTargetZoom = { gsd, zoom: Projection.getTiffResZoom(this.tileMatrix, gsd) }; - } - return this.cacheTargetZoom.zoom; - } - - /** - * Get a nicely formatted folder path based on the job - * - * @param key optional file key inside of the job folder - */ - getJobPath(key?: string): string { - const parts = [this.tileMatrix.projection.code, this.name, this.id]; - if (key != null) { - parts.push(key); - } - return fsa.join(this.output.location.path, parts.join('/')); - } -} diff --git a/packages/cli/src/cog/cutline.ts b/packages/cli/src/cog/cutline.ts index 9fec5b9b4..30d8322e2 100644 --- a/packages/cli/src/cog/cutline.ts +++ b/packages/cli/src/cog/cutline.ts @@ -10,8 +10,8 @@ import { union, } from '@linzjs/geojson'; import { FeatureCollection } from 'geojson'; -import { AlignedLevel, CoveringFraction } from './constants.js'; -import { CogJob, FeatureCollectionWithCrs, SourceMetadata } from './types.js'; +import { AlignedLevel, CoveringFraction } from '@basemaps/shared'; +import { CogJob, FeatureCollectionWithCrs, SourceMetadata } from '@basemaps/shared'; /** Padding to always apply to image boundies */ const PixelPadding = 200; diff --git a/packages/cli/src/cog/projection.loader.ts b/packages/cli/src/cog/projection.loader.ts deleted file mode 100644 index 72cae9c9f..000000000 --- a/packages/cli/src/cog/projection.loader.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Epsg, Projection } from '@basemaps/geo'; -import fetch from 'node-fetch'; - -export class ProjectionLoader { - // Exposed for testing - static _fetch = fetch; - - /** - * Ensure that a projection EPSG code is avialable for use in Proj4js - * - * If its not already loaded, lookup definition from spatialreference.org - * @param code - */ - static async load(code: number): Promise { - if (Projection.tryGet(code) != null) return Epsg.get(code); - const url = `https://spatialreference.org/ref/epsg/${code}/ogcwkt/`; - - const res = await this._fetch(url); - if (!res.ok) throw new Error('Failed to load projection information for:' + code); - - let epsg = Epsg.tryGet(code); - if (epsg == null) epsg = new Epsg(code); - - const text = await res.text(); - Projection.define(epsg, text); - return epsg; - } -} diff --git a/packages/cli/src/cog/constants.ts b/packages/shared/src/cog/constants.ts similarity index 100% rename from packages/cli/src/cog/constants.ts rename to packages/shared/src/cog/constants.ts diff --git a/packages/cli/src/gdal/gdal.config.ts b/packages/shared/src/cog/gdal.config.ts similarity index 100% rename from packages/cli/src/gdal/gdal.config.ts rename to packages/shared/src/cog/gdal.config.ts diff --git a/packages/cli/src/cog/stac.ts b/packages/shared/src/cog/stac.ts similarity index 100% rename from packages/cli/src/cog/stac.ts rename to packages/shared/src/cog/stac.ts diff --git a/packages/cli/src/cog/types.ts b/packages/shared/src/cog/types.ts similarity index 95% rename from packages/cli/src/cog/types.ts rename to packages/shared/src/cog/types.ts index 35c6778ea..e4703b31d 100644 --- a/packages/cli/src/cog/types.ts +++ b/packages/shared/src/cog/types.ts @@ -1,6 +1,6 @@ import { BoundingBox, EpsgCode, NamedBounds, TileMatrixSet } from '@basemaps/geo'; -import { FileConfig } from '@basemaps/shared'; -import { GdalCogBuilderResampling } from '../gdal/gdal.config.js'; +import { GdalCogBuilderResampling } from './gdal.config.js'; +import { FileConfig } from '../file/file.config.js'; export interface FeatureCollectionWithCrs extends GeoJSON.FeatureCollection { crs: { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 6bf429357..da87b2d3f 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -10,6 +10,8 @@ export { toQueryString } from './url.js'; export { CompositeError } from './composite.error.js'; export { ConfigProviderDynamo } from './dynamo/dynamo.config.js'; export { ConfigDynamoBase } from './dynamo/dynamo.config.base.js'; +export { CogJobJson, CogBuilderMetadata, CogJob, FeatureCollectionWithCrs, SourceMetadata } from './cog/types.js'; +export { AlignedLevel, CoveringFraction } from './cog/constants.js'; export * from './file/index.js'; export * from './util.js';