From 1999e2cb7d278fc190a5c1fde113ff89450acdd4 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Fri, 13 Oct 2023 12:45:48 +1300 Subject: [PATCH] Deprecate all the old cog creation, serve, and sprites cli. --- packages/cli/package.json | 5 - packages/cli/src/cli/base.cli.ts | 0 .../cli/cogify/__tests__/batch.job.test.ts | 105 ----- .../src/cli/cogify/__tests__/semver.test.ts | 69 --- packages/cli/src/cli/cogify/action.cog.ts | 247 ----------- packages/cli/src/cli/cogify/action.job.ts | 223 ---------- .../cli/src/cli/cogify/action.make.cog.pr.ts | 103 ----- .../cli/src/cli/cogify/action.make.cog.ts | 243 ----------- packages/cli/src/cli/cogify/batch.job.ts | 227 ---------- packages/cli/src/cli/cogify/cutline.ts | 34 -- packages/cli/src/cli/cogify/semver.util.ts | 56 --- packages/cli/src/cli/config/action.import.ts | 8 +- packages/cli/src/cli/github/github.ts | 118 ----- packages/cli/src/cli/github/make.cog.pr.ts | 202 --------- packages/cli/src/cli/index.ts | 18 +- .../cli/overview/action.create.overview.ts | 7 +- packages/cli/src/cli/server/action.serve.ts | 60 --- .../cli/src/cli/sprites/action.sprites.ts | 58 --- packages/cli/src/cli/util.ts | 18 +- .../cli/src/cog/__tests__/builder.test.ts | 98 ----- .../src/cog/__tests__/cog.stac.job.test.ts | 350 --------------- packages/cli/src/cog/__tests__/cog.test.ts | 101 ----- .../cli/src/cog/__tests__/cog.vrt.test.ts | 275 ------------ .../cli/src/cog/__tests__/cutline.test.ts | 353 --------------- .../cog/__tests__/projection.loader.test.ts | 30 -- .../cog/__tests__/source.tiff.testhelper.ts | 44 -- packages/cli/src/cog/cog.stac.job.ts | 407 ------------------ packages/cli/src/cog/cog.ts | 101 ----- packages/cli/src/cog/cog.vrt.ts | 144 ------- packages/cli/src/{gdal => cog}/gdal.config.ts | 0 packages/cli/src/cog/job.factory.ts | 120 ------ packages/cli/src/cog/stac.ts | 41 -- packages/cli/src/cog/types.ts | 2 +- .../src/gdal/__tests__/gdal.progress.test.ts | 25 -- packages/cli/src/gdal/__tests__/gdal.test.ts | 64 --- packages/cli/src/gdal/gdal.cog.ts | 143 ------ packages/cli/src/gdal/gdal.command.ts | 116 ----- packages/cli/src/gdal/gdal.docker.ts | 93 ---- packages/cli/src/gdal/gdal.local.ts | 24 -- packages/cli/src/gdal/gdal.progress.ts | 52 --- packages/cli/src/gdal/gdal.ts | 29 -- packages/cli/src/index.ts | 6 - yarn.lock | 78 +--- 43 files changed, 16 insertions(+), 4481 deletions(-) delete mode 100644 packages/cli/src/cli/base.cli.ts delete mode 100644 packages/cli/src/cli/cogify/__tests__/batch.job.test.ts delete mode 100644 packages/cli/src/cli/cogify/__tests__/semver.test.ts delete mode 100644 packages/cli/src/cli/cogify/action.cog.ts delete mode 100644 packages/cli/src/cli/cogify/action.job.ts delete mode 100644 packages/cli/src/cli/cogify/action.make.cog.pr.ts delete mode 100644 packages/cli/src/cli/cogify/action.make.cog.ts delete mode 100644 packages/cli/src/cli/cogify/batch.job.ts delete mode 100644 packages/cli/src/cli/cogify/cutline.ts delete mode 100644 packages/cli/src/cli/cogify/semver.util.ts delete mode 100644 packages/cli/src/cli/github/github.ts delete mode 100644 packages/cli/src/cli/github/make.cog.pr.ts delete mode 100644 packages/cli/src/cli/server/action.serve.ts delete mode 100644 packages/cli/src/cli/sprites/action.sprites.ts delete mode 100644 packages/cli/src/cog/__tests__/builder.test.ts delete mode 100644 packages/cli/src/cog/__tests__/cog.stac.job.test.ts delete mode 100644 packages/cli/src/cog/__tests__/cog.test.ts delete mode 100644 packages/cli/src/cog/__tests__/cog.vrt.test.ts delete mode 100644 packages/cli/src/cog/__tests__/cutline.test.ts delete mode 100644 packages/cli/src/cog/__tests__/projection.loader.test.ts delete mode 100644 packages/cli/src/cog/__tests__/source.tiff.testhelper.ts delete mode 100644 packages/cli/src/cog/cog.stac.job.ts delete mode 100644 packages/cli/src/cog/cog.ts delete mode 100644 packages/cli/src/cog/cog.vrt.ts rename packages/cli/src/{gdal => cog}/gdal.config.ts (100%) delete mode 100644 packages/cli/src/cog/job.factory.ts delete mode 100644 packages/cli/src/cog/stac.ts delete mode 100644 packages/cli/src/gdal/__tests__/gdal.progress.test.ts delete mode 100644 packages/cli/src/gdal/__tests__/gdal.test.ts delete mode 100644 packages/cli/src/gdal/gdal.cog.ts delete mode 100644 packages/cli/src/gdal/gdal.command.ts delete mode 100644 packages/cli/src/gdal/gdal.docker.ts delete mode 100644 packages/cli/src/gdal/gdal.local.ts delete mode 100644 packages/cli/src/gdal/gdal.progress.ts delete mode 100644 packages/cli/src/gdal/gdal.ts delete mode 100644 packages/cli/src/index.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index e9135addd..6c1e5f029 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -41,9 +41,7 @@ "dependencies": { "@basemaps/config": "^6.46.0", "@basemaps/geo": "^6.44.0", - "@basemaps/server": "^6.46.0", "@basemaps/shared": "^6.46.0", - "@basemaps/sprites": "^6.41.0", "@basemaps/tiler": "^6.44.0", "@basemaps/tiler-sharp": "^6.45.0", "@chunkd/fs": "^10.0.2", @@ -52,12 +50,10 @@ "@cotar/core": "^5.4.0", "@cotar/tar": "^5.4.1", "@linzjs/geojson": "^6.43.0", - "@octokit/core": "^4.0.5", "@rushstack/ts-command-line": "^4.3.13", "ansi-colors": "^4.1.1", "deep-diff": "^1.0.2", "flatgeobuf": "^3.23.1", - "get-port": "^6.1.2", "node-fetch": "^3.2.3", "p-limit": "^4.0.0", "pretty-json-log": "^1.0.0", @@ -66,7 +62,6 @@ }, "devDependencies": { "@types/deep-diff": "^1.0.1", - "@types/proj4": "^2.5.2", "@types/sharp": "^0.31.0" }, "publishConfig": { diff --git a/packages/cli/src/cli/base.cli.ts b/packages/cli/src/cli/base.cli.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/cli/src/cli/cogify/__tests__/batch.job.test.ts b/packages/cli/src/cli/cogify/__tests__/batch.job.test.ts deleted file mode 100644 index 347ad4fa4..000000000 --- a/packages/cli/src/cli/cogify/__tests__/batch.job.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { LogConfig } from '@basemaps/shared'; -import o from 'ospec'; -import { CogJob } from '../../../cog/types.js'; -import { BatchJob, extractResolutionFromName } from '../batch.job.js'; - -o.spec('action.batch', () => { - o('extractResolutionFromName', () => { - o(extractResolutionFromName('2013')).equals(-1); - o(extractResolutionFromName('new_zealand_sentinel_2018-19_10m')).equals(10000); - o(extractResolutionFromName('abc2017def_1.00m')).equals(1000); - o(extractResolutionFromName('wellington_urban_2017_0.10m')).equals(100); - o(extractResolutionFromName('wellington_urban_2017_0-10m')).equals(100); - o(extractResolutionFromName('wellington_urban_2017_1.00m')).equals(1000); - o(extractResolutionFromName('wellington_urban_2017_0.025m')).equals(25); - }); - - o('should create valid jobNames', () => { - const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' } as CogJob; - o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-1x-0'); - - fakeJob.name = 'ōtorohanga_urban_2021_0.1m_RGB'; - o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-1x-0'); - - o(BatchJob.id(fakeJob)).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-'); - }); - - o('should truncate job names to 128 characters', () => { - const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' } as CogJob; - - o( - BatchJob.id(fakeJob, [ - 'this is a really long file name', - 'it should over flow 128 characters', - 'so it should be truncated at some point.tiff', - ]), - ).equals( - '01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-3x-it should over flow 128 characters_so it should be truncated at some point.tiff_t', - ); - }); - - o.spec('ChunkJobs', () => { - const fakeGsd = 0.9; - const ChunkJobSmall = 4097; - const ChunkJobMiddle = 8193; - const ChunkJobLarge = 16385; - const fakeFiles = [ - { name: '1-2-0', width: ChunkJobSmall * fakeGsd - 1 }, // Small Job 20 - { name: '1-2-1', width: ChunkJobSmall * fakeGsd - 1 }, // Small Job 40 - { name: '1-2-2', width: ChunkJobLarge * fakeGsd + 1 }, // Single Job - { name: '1-2-3', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 90 - { name: '1-2-4', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 140 - { name: '1-2-5', width: ChunkJobLarge * fakeGsd + 0.1 }, // Single Job - { name: '1-2-6', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 190 - { name: '1-2-7', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 390 - { name: '1-2-8', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 590 - { name: '1-2-9', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 790 - { name: '1-2-10', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 990 - { name: '1-2-11', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 1090 - { name: '1-2-12', width: ChunkJobSmall * fakeGsd - 1 }, // Small Job 20 - { name: '1-2-13', width: ChunkJobLarge * fakeGsd + 1 }, // Single Job - { name: '1-2-14', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 70 - ]; - - const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', output: { files: fakeFiles, gsd: fakeGsd } } as CogJob; - o('should prepare valid chunk jobs', async () => { - o(BatchJob.getJobs(fakeJob, new Set(), LogConfig.get())).deepEquals([ - [fakeFiles[0].name, fakeFiles[1].name, fakeFiles[2].name], - [fakeFiles[3].name, fakeFiles[4].name, fakeFiles[5].name], - [ - fakeFiles[6].name, - fakeFiles[7].name, - fakeFiles[8].name, - fakeFiles[9].name, - fakeFiles[10].name, - fakeFiles[11].name, - fakeFiles[12].name, - fakeFiles[13].name, - ], - [fakeFiles[14].name], - ]); - }); - - o('should skip tiffs that exist', () => { - const existing = new Set([ - `${fakeFiles[0].name}.tiff`, - `${fakeFiles[1].name}.tiff`, - `${fakeFiles[13].name}.tiff`, - ]); - o(BatchJob.getJobs(fakeJob, existing, LogConfig.get())).deepEquals([ - [fakeFiles[2].name], // First single Job - [fakeFiles[3].name, fakeFiles[4].name, fakeFiles[5].name], // Second single Job - [ - fakeFiles[6].name, - fakeFiles[7].name, - fakeFiles[8].name, - fakeFiles[9].name, - fakeFiles[10].name, - fakeFiles[11].name, - fakeFiles[12].name, - fakeFiles[14].name, - ], - ]); - }); - }); -}); diff --git a/packages/cli/src/cli/cogify/__tests__/semver.test.ts b/packages/cli/src/cli/cogify/__tests__/semver.test.ts deleted file mode 100644 index 3670c8d71..000000000 --- a/packages/cli/src/cli/cogify/__tests__/semver.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import o from 'ospec'; -import { SemVer } from '../semver.util.js'; -o.spec('VersionCompare', () => { - o.spec('toNumber', () => { - o('should make correct numbers', () => { - o(SemVer.toNumber('0.0.1')).equals(1); - o(SemVer.toNumber('0.1.0')).equals(1024); - o(SemVer.toNumber('0.1.1')).equals(1025); - o(SemVer.toNumber('0.1.1023')).equals(2047); - o(SemVer.toNumber('1.0.0')).equals(1048576); - }); - - o('should not overflow', () => { - o(SemVer.toNumber('1023.1023.1023') > 0).equals(true); - }); - - o('should support up to three dots', () => { - o(SemVer.toNumber('')).equals(0); - o(SemVer.toNumber('1')).equals(1048576); - o(SemVer.toNumber('1.0')).equals(1048576); - o(SemVer.toNumber('1.0.0')).equals(1048576); - o(SemVer.toNumber('2')).equals(2097152); - o(SemVer.toNumber('')).equals(0); - }); - - o('should be getting bigger', () => { - o(SemVer.toNumber('1.0.0') > SemVer.toNumber('0.0.0')).equals(true); - o(SemVer.toNumber('1.0.1') > SemVer.toNumber('1.0.0')).equals(true); - o(SemVer.toNumber('1.1.1') > SemVer.toNumber('1.1.0')).equals(true); - o(SemVer.toNumber('2.1.1') > SemVer.toNumber('1.1.0')).equals(true); - }); - - o('should support release candidates', () => { - o(SemVer.toNumber('0.0.1-rc1')).equals(1); - o(SemVer.toNumber('1.0.0-rc2')).equals(SemVer.toNumber('1.0.0')); - }); - }); - - o.spec('fromNumber', () => { - o('should create versions', () => { - o(SemVer.fromNumber(1)).equals('0.0.1'); - o(SemVer.fromNumber(1023)).equals('0.0.1023'); - o(SemVer.fromNumber(1024)).equals('0.1.0'); - o(SemVer.fromNumber(1025)).equals('0.1.1'); - o(SemVer.fromNumber(1048576)).equals('1.0.0'); - o(SemVer.fromNumber(2097152)).equals('2.0.0'); - }); - - o('should round trip', () => { - for (const num of ['0.1.0', '1.2.3', '3.2.1']) { - o(SemVer.fromNumber(SemVer.toNumber(num))).equals(num)(`Should roundtrip ${num}`); - } - }); - }); - - o.spec('compare', () => { - o('should compare versions', () => { - o(SemVer.compare('0.0.0', '0.0.1')).equals(-1); - o(SemVer.compare('0.0.1', '0.0.0')).equals(1); - o(SemVer.compare('0.0.0', '0.0.0')).equals(0); - }); - - o('should sort versions', () => { - const versions = ['1.0.0', '2.0.0', '0.0.1', '1.2.3']; - versions.sort(SemVer.compare); - o(versions).deepEquals(['0.0.1', '1.0.0', '1.2.3', '2.0.0']); - }); - }); -}); diff --git a/packages/cli/src/cli/cogify/action.cog.ts b/packages/cli/src/cli/cogify/action.cog.ts deleted file mode 100644 index 54b934229..000000000 --- a/packages/cli/src/cli/cogify/action.cog.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { Env, fsa, LogConfig, LoggerFatalError, LogType } from '@basemaps/shared'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerListParameter, - CommandLineIntegerParameter, - CommandLineStringListParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; -import { createReadStream, promises as fs } from 'fs'; -import { FeatureCollection } from 'geojson'; -import { buildCogForName } from '../../cog/cog.js'; -import { CogStacJob } from '../../cog/cog.stac.job.js'; -import { CogVrt } from '../../cog/cog.vrt.js'; -import { Cutline } from '../../cog/cutline.js'; -import { CogJob } from '../../cog/types.js'; -import { Gdal } from '../../gdal/gdal.js'; -import { makeTempFolder, makeTiffFolder } from '../folder.js'; -import path from 'path'; -import { prepareUrl } from '../util.js'; -import pLimit from 'p-limit'; - -const DefaultConcurrency = 2; - -export class CommandCogCreate extends CommandLineAction { - private job?: CommandLineStringParameter; - private name?: CommandLineStringListParameter; - private commit?: CommandLineFlagParameter; - private cogIndex?: CommandLineIntegerListParameter; - private concurrency?: CommandLineIntegerParameter; - - public constructor() { - super({ - actionName: 'cog', - summary: 'create a COG', - documentation: 'Create a COG for the specified cog name', - }); - } - - parseName(name: string): string[] { - // Name could both be a string or a array type of string - try { - return JSON.parse(name); - } catch (e) { - return [name]; - } - } - - getNames(job: CogJob): Set | null { - const output: Set = new Set(); - const { files } = job.output; - const batchIndex = Env.getNumber(Env.BatchIndex, -1); - if (batchIndex > -1) { - const { name } = files[batchIndex]; - if (name == null) { - throw new LoggerFatalError( - { cogIndex: batchIndex, tileMax: files.length - 1 }, - 'Failed to find cog name from batch index', - ); - } - output.add(name); - return output; - } - - const cogIndex = this.cogIndex?.values; - if (cogIndex != null && cogIndex.length > 0) { - for (const i of cogIndex) { - if (i < 0 || i >= files.length) { - throw new LoggerFatalError({ cogIndex: i, tileMax: files.length - 1 }, 'Failed to find cog name from index'); - } - const { name } = files[i]; - output.add(name); - } - - return output; - } - - const names = this.name?.values; - if (names != null) { - for (const nameStr of names) { - const nameArr = this.parseName(nameStr); - for (const name of nameArr) { - if (!files.find((r) => r.name === name)) - throw new LoggerFatalError( - { name, names: files.map((r) => r.name).join(', ') }, - 'Name does not exist inside job', - ); - output.add(name); - } - } - } - - return output; - } - - async onExecute(): Promise { - const jobLocation = this.job?.value; - if (jobLocation == null) throw new Error('Missing job name'); - - const isCommit = this.commit?.value ?? false; - const job = await CogStacJob.load(jobLocation); - - const logger = LogConfig.get().child({ - correlationId: job.id, - imageryName: job.name, - tileMatrix: job.tileMatrix.identifier, - }); - - LogConfig.set(logger); - logger.info('CogCreate:Start'); - - const gdalVersion = await Gdal.version(logger); - logger.info({ gdalVersion }, 'CogCreate:GdalVersion'); - - const names = this.getNames(job); - if (names == null || names.size === 0) return; - - const tmpFolder = await makeTempFolder(`basemaps-${job.id}-${CliId}`); - - const Q = pLimit(this.concurrency?.value ?? DefaultConcurrency); - - try { - Promise.all( - Array.from(names).map((name) => - Q(async () => { - const tiffJob = await CogStacJob.load(jobLocation); - await this.processTiff(tiffJob, name, tmpFolder, isCommit, logger.child({ tiffName: name })); - }), - ), - ); - } catch (e) { - // Ensure the error is thrown - throw e; - } finally { - // Cleanup! - await fs.rm(tmpFolder, { recursive: true }); - } - } - - async processTiff( - job: CogStacJob, - tiffName: string, - tmpFolder: string, - isCommit: boolean, - logger: LogType, - ): Promise { - const tiffFolder = await makeTiffFolder(tmpFolder, tiffName); - const targetPath = job.getJobPath(`${tiffName}.tiff`); - fsa.configure(job.output.location); - - const outputExists = await fsa.exists(targetPath); - logger.info({ targetPath, outputExists }, 'CogCreate:CheckExists'); - // Output file exists don't try and overwrite it - if (outputExists) { - logger.warn({ targetPath }, 'CogCreate:OutputExists'); - await this.checkJobStatus(job, logger); - return; - } - - let cutlineJson: FeatureCollection | undefined; - if (job.output.cutline != null) { - const cutlinePath = job.getJobPath('cutline.geojson.gz'); - logger.info({ path: cutlinePath }, 'CogCreate:UsingCutLine'); - cutlineJson = await Cutline.loadCutline(cutlinePath); - } else { - logger.warn('CutLine:Skip'); - } - const cutline = new Cutline(job.tileMatrix, cutlineJson, job.output.cutline?.blend); - - const tmpVrtPath = await CogVrt.buildVrt(tiffFolder, job, cutline, tiffName, logger); - - if (tmpVrtPath == null) { - logger.warn('CogCreate:NoMatchingSourceImagery'); - return; - } - - const tmpTiff = fsa.join(tiffFolder, `${tiffName}.tiff`); - - await buildCogForName(job, tiffName, tmpVrtPath, tmpTiff, logger, isCommit); - logger.info({ target: targetPath }, 'CogCreate:StoreTiff'); - if (isCommit) { - await fsa.write(targetPath, createReadStream(tmpTiff)); - await this.checkJobStatus(job, logger); - } else { - logger.warn('DryRun:Done'); - } - } - - /** Check to see how many tiffs are remaining in the job */ - async checkJobStatus(job: CogStacJob, logger: LogType): Promise { - const basePath = job.getJobPath(); - const expectedTiffs = new Set(); - const jobSize = job.output.files.length; - for (const file of job.output.files) expectedTiffs.add(`${file.name}.tiff`); - - for await (const fileName of fsa.list(basePath)) { - const basename = path.basename(fileName); - // Look for tile tiffs only - if (!basename.includes('-') || !basename.endsWith('.tiff')) continue; - expectedTiffs.delete(basename); - } - - if (expectedTiffs.size === 0) { - const url = await prepareUrl(job); - logger.info({ tiffCount: jobSize, tiffTotal: jobSize, url }, 'CogCreate:JobComplete'); - } else { - logger.info({ tiffCount: jobSize, tiffRemaining: expectedTiffs.size }, 'CogCreate:JobProgress'); - } - } - - protected onDefineParameters(): void { - this.job = this.defineStringParameter({ - argumentName: 'JOB', - parameterLongName: '--job', - description: 'Job config source to access', - required: true, - }); - - this.name = this.defineStringListParameter({ - argumentName: 'NAME', - parameterLongName: '--name', - description: 'list of cog names to process', - required: false, - }); - - this.cogIndex = this.defineIntegerListParameter({ - argumentName: 'COG_INDEX', - parameterLongName: '--cog-index', - description: 'list of cog indexes to process', - required: false, - }); - - this.concurrency = this.defineIntegerParameter({ - argumentName: 'CONCURRENCY', - parameterLongName: '--concurrency', - description: 'Number of concurrency create cog jobs', - required: false, - }); - - this.commit = this.defineFlagParameter({ - parameterLongName: '--commit', - description: 'Begin the transformation', - required: false, - }); - } -} diff --git a/packages/cli/src/cli/cogify/action.job.ts b/packages/cli/src/cli/cogify/action.job.ts deleted file mode 100644 index a0cf111a0..000000000 --- a/packages/cli/src/cli/cogify/action.job.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { Epsg, GoogleTms, TileMatrixSets } from '@basemaps/geo'; -import { FileConfig, FileConfigPath, fsa } from '@basemaps/shared'; -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; -import { JobCreationContext } from '../../cog/cog.stac.job.js'; -import { CogJobFactory, MaxConcurrencyDefault } from '../../cog/job.factory.js'; -import { GdalCogBuilderDefaults, GdalCogBuilderResampling, GdalResamplingOptions } from '../../gdal/gdal.config.js'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; - -export class CLiInputData { - path: CommandLineStringParameter; - roleArn: CommandLineStringParameter; - externalId: CommandLineStringParameter; - - constructor(parent: CommandLineAction, prefix: string) { - this.path = parent.defineStringParameter({ - argumentName: prefix.toUpperCase(), - parameterLongName: `--${prefix}`, - description: 'Folder or S3 Bucket location to use', - required: true, - }); - - this.roleArn = parent.defineStringParameter({ - argumentName: prefix.toUpperCase() + '_ARN', - parameterLongName: `--${prefix}-role-arn`, - description: 'Role to be assumed to access the data', - required: false, - }); - - this.externalId = parent.defineStringParameter({ - argumentName: prefix.toUpperCase() + '_EXTERNAL_ID', - parameterLongName: `--${prefix}-role-external-id`, - description: 'Role external id to be assumed to access the data', - required: false, - }); - } -} - -export class CommandJobCreate extends CommandLineAction { - private source: CLiInputData; - private output: CLiInputData; - private maxConcurrency: CommandLineIntegerParameter; - private cutline: CommandLineStringParameter; - private cutlineBlend: CommandLineIntegerParameter; - private overrideId: CommandLineStringParameter; - private overrideWarpReample: CommandLineStringParameter; - private submitBatch: CommandLineFlagParameter; - private quality: CommandLineIntegerParameter; - private sourceProjection: CommandLineIntegerParameter; - private tileMatrix: CommandLineStringParameter; - private oneCog: CommandLineFlagParameter; - private fileList: CommandLineStringParameter; - - public constructor() { - super({ - actionName: 'job', - summary: 'create a list of cogs that need to be processed', - documentation: 'List a folder/bucket full of cogs and determine a optimal processing pipeline', - }); - } - - fsConfig(source: CLiInputData): FileConfig { - if (source.path.value == null) { - throw new Error('Invalid path'); - } - if (!source.path.value.startsWith('s3://')) { - return { type: 'local', path: source.path.value }; - } - if (source.roleArn.value == null) { - return { type: 's3', path: source.path.value }; - } - return { - path: source.path.value, - type: 's3', - roleArn: source.roleArn.value, - externalId: source.externalId.value, - }; - } - - async onExecute(): Promise { - const sourceLocation: FileConfig | FileConfigPath = this.fsConfig(this.source); - const outputLocation = this.fsConfig(this.output); - - fsa.configure(this.fsConfig(this.source)); - fsa.configure(this.fsConfig(this.output)); - - let cutline = undefined; - if (this.cutline?.value) { - cutline = { href: this.cutline.value, blend: this.cutlineBlend.value ?? 20 }; - } - - const tileMatrix = TileMatrixSets.find(this.tileMatrix.value ?? GoogleTms.identifier); - if (tileMatrix == null) throw new Error('Invalid target-projection'); - - let resampling: GdalCogBuilderResampling | undefined; - - if (this.overrideWarpReample?.value != null) { - const warp = GdalResamplingOptions[this.overrideWarpReample?.value]; - if (warp == null) { - throw new Error('Invalid override-warp-resampling'); - } - resampling = { - warp, - overview: 'lanczos', - }; - } - - const fileListPath = this.fileList?.value; - if (fileListPath != null) { - const fileData = await fsa.read(fileListPath); - (sourceLocation as FileConfigPath).files = fileData - .toString() - .trim() - .split('\n') - .map((fn) => sourceLocation.path + '/' + fn); - } - - const ctx: JobCreationContext = { - sourceLocation, - outputLocation, - cutline, - tileMatrix, - override: { - concurrency: this.maxConcurrency?.value ?? MaxConcurrencyDefault, - quality: this.quality?.value ?? GdalCogBuilderDefaults.quality, - id: this.overrideId?.value ?? CliId, - projection: Epsg.tryGet(this.sourceProjection?.value), - resampling, - }, - batch: this.submitBatch?.value ?? false, - oneCogCovering: this.oneCog?.value ?? false, - }; - - await CogJobFactory.create(ctx); - } - - protected onDefineParameters(): void { - this.source = new CLiInputData(this, 'source'); - this.output = new CLiInputData(this, 'output'); - - this.maxConcurrency = this.defineIntegerParameter({ - argumentName: 'MAX_CONCURRENCY', - parameterLongName: '--concurrency', - parameterShortName: '-c', - description: 'Maximum number of requests to use at one time', - defaultValue: MaxConcurrencyDefault, - required: false, - }); - - this.cutline = this.defineStringParameter({ - argumentName: 'CUTLINE', - parameterLongName: '--cutline', - description: 'use a shapefile to crop the COGs', - required: false, - }); - - this.cutlineBlend = this.defineIntegerParameter({ - argumentName: 'CUTLINE_BLEND', - parameterLongName: '--cblend', - description: 'Set a blend distance to use to blend over cutlines (in pixels)', - required: false, - }); - - this.overrideId = this.defineStringParameter({ - argumentName: 'OVERRIDE_ID', - parameterLongName: '--override-id', - description: 'create job with a pre determined job id', - required: false, - }); - - this.overrideWarpReample = this.defineStringParameter({ - argumentName: 'METHOD', - parameterLongName: '--override-warp-resampling', - description: 'Defaults to bilinear. Common options are: nearest, lanczos, cubic', - required: false, - }); - - this.submitBatch = this.defineFlagParameter({ - parameterLongName: `--batch`, - description: 'Submit the job to AWS Batch', - required: false, - }); - - this.quality = this.defineIntegerParameter({ - argumentName: 'QUALITY', - parameterLongName: '--quality', - description: 'Compression quality (0-100)', - required: false, - }); - - this.sourceProjection = this.defineIntegerParameter({ - argumentName: 'SOURCE_PROJECTION', - parameterLongName: '--source-projection', - description: 'The EPSG code of the source imagery', - required: false, - }); - - this.tileMatrix = this.defineStringParameter({ - argumentName: 'TILE_MATRIX', - parameterLongName: '--target-tile-matrix-set', - description: 'The TileMatrixSet to generate the COGS for', - defaultValue: GoogleTms.identifier, - required: false, - }); - - this.oneCog = this.defineFlagParameter({ - parameterLongName: '--one-cog', - description: 'ignore target projection window and just produce one big COG.', - required: false, - }); - - this.fileList = this.defineStringParameter({ - argumentName: 'FILE_LIST', - parameterLongName: '--filelist', - description: 'supply a list of files to use as source imagery', - required: false, - }); - } -} diff --git a/packages/cli/src/cli/cogify/action.make.cog.pr.ts b/packages/cli/src/cli/cogify/action.make.cog.pr.ts deleted file mode 100644 index 78f0be458..000000000 --- a/packages/cli/src/cli/cogify/action.make.cog.pr.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { ConfigLayer, standardizeLayerName } from '@basemaps/config'; -import { LogConfig } from '@basemaps/shared'; -import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line'; -import { MakeCogGithub } from '../github/make.cog.pr.js'; - -export enum Category { - Urban = 'Urban Aerial Photos', - Rural = 'Rural Aerial Photos', - Satellite = 'Satellite Imagery', - Event = 'Event', - Other = 'New Aerial Photos', -} - -export interface CategorySetting { - minZoom?: number; - individual?: boolean; -} - -export const DefaultCategorySetting: Record = { - [Category.Urban]: { minZoom: 14, individual: false }, - [Category.Rural]: { minZoom: 13, individual: false }, - [Category.Satellite]: { minZoom: 5, individual: false }, - [Category.Event]: { individual: true }, - [Category.Other]: { individual: true }, -}; - -export function parseCategory(category: string): Category { - const c = category.toLocaleLowerCase(); - if (c.includes('urban')) return Category.Urban; - else if (c.includes('rural')) return Category.Rural; - else if (c.includes('satellite')) return Category.Satellite; - else return Category.Other; -} - -export class CommandCogPullRequest extends CommandLineAction { - private layer: CommandLineStringParameter; - private category: CommandLineStringParameter; - private repository: CommandLineStringParameter; - private individual: CommandLineFlagParameter; - private vector: CommandLineFlagParameter; - - public constructor() { - super({ - actionName: 'cog-pr', - summary: 'create a github pull request for the import cog workflow', - documentation: 'Given a output of make-cog command and create pull request for the imagery config.', - }); - } - - protected onDefineParameters(): void { - this.layer = this.defineStringParameter({ - argumentName: 'LAYER', - parameterLongName: '--layer', - description: 'Input config layer', - required: false, - }); - this.category = this.defineStringParameter({ - argumentName: 'CATEGORY', - parameterLongName: '--category', - description: 'New Imagery Category, like Rural Aerial Photos, Urban Aerial Photos, Satellite Imagery', - required: false, - }); - this.repository = this.defineStringParameter({ - argumentName: 'REPOSITORY', - parameterLongName: '--repository', - description: 'Github repository reference', - defaultValue: 'linz/basemaps-config', - required: false, - }); - this.individual = this.defineFlagParameter({ - parameterLongName: '--individual', - description: 'Import imagery as individual layer in basemaps.', - required: false, - }); - this.vector = this.defineFlagParameter({ - parameterLongName: '--vector', - description: 'Commit changes for vector map', - required: false, - }); - } - - async onExecute(): Promise { - const logger = LogConfig.get(); - const layerStr = this.layer.value; - const category = this.category.value ? parseCategory(this.category.value) : Category.Other; - const repo = this.repository.value ?? this.repository.defaultValue; - if (layerStr == null) throw new Error('Please provide a valid input layer and urls'); - if (repo == null) throw new Error('Please provide a repository'); - let layer: ConfigLayer; - try { - layer = JSON.parse(layerStr); - } catch { - throw new Error('Please provide a valid input layer'); - } - - //Make sure the imagery name is standardized before update the config - layer.name = standardizeLayerName(layer.name); - - const git = new MakeCogGithub(layer.name, repo, logger); - if (this.vector.value) await git.updateVectorTileSet('topographic', layer); - else await git.updateRasterTileSet('aerial', layer, category, this.individual.value); - } -} diff --git a/packages/cli/src/cli/cogify/action.make.cog.ts b/packages/cli/src/cli/cogify/action.make.cog.ts deleted file mode 100644 index 4b4f1d4a5..000000000 --- a/packages/cli/src/cli/cogify/action.make.cog.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { Epsg, TileMatrixSet, TileMatrixSets } from '@basemaps/geo'; -import { Env, FileConfig, fsa, LogConfig, LogType, titleizeImageryName } from '@basemaps/shared'; -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; -import { CogStacJob, JobCreationContext } from '../../cog/cog.stac.job.js'; -import { ProjectionLoader } from '../../cog/projection.loader.js'; -import * as ulid from 'ulid'; -import { getCutline } from './cutline.js'; -import { CogJobFactory } from '../../cog/job.factory.js'; -import { basename } from 'path'; -import { BatchJob } from './batch.job.js'; -import { CredentialSourceJson } from '@chunkd/source-aws-v2'; -import { ConfigLayer } from '@basemaps/config'; -import { AlignedLevel } from '../../cog/constants.js'; - -interface OutputJobs { - job: string; - names: string[]; -} - -export class CommandMakeCog extends CommandLineAction { - private imagery: CommandLineStringParameter; - private tileMatrix: CommandLineStringParameter; - private name: CommandLineStringParameter; - private target: CommandLineStringParameter; - private cutline: CommandLineStringParameter; - private blend: CommandLineIntegerParameter; - private alignedLevel: CommandLineIntegerParameter; - private maxChunkUnit: CommandLineIntegerParameter; - private output: CommandLineStringParameter; - private aws: CommandLineFlagParameter; - - public constructor() { - super({ - actionName: 'make-cog', - summary: 'Import Basemaps imagery from s3 buckets', - documentation: 'Given a valid path of raw imagery and import into basemaps', - }); - } - - protected onDefineParameters(): void { - this.imagery = this.defineStringParameter({ - argumentName: 'IMAGERY', - parameterShortName: '-i', - parameterLongName: '--imagery', - description: 'Path of source imagery to import.', - required: true, - }); - this.target = this.defineStringParameter({ - argumentName: 'BUCKET', - parameterShortName: '-t', - parameterLongName: '--target', - description: 'Target bucket for job.json', - required: true, - }); - this.name = this.defineStringParameter({ - argumentName: 'NAME', - parameterShortName: '-n', - parameterLongName: '--name', - description: 'Custom imagery name', - required: false, - }); - this.tileMatrix = this.defineStringParameter({ - argumentName: 'TILE_MATRIX', - parameterLongName: '--tile-matrix', - description: 'Target tile matrix', - required: false, - }); - this.cutline = this.defineStringParameter({ - argumentName: 'CUTLINE', - parameterLongName: '--cutline', - description: 'Path of import cutline', - required: false, - }); - this.blend = this.defineIntegerParameter({ - argumentName: 'BLEND', - parameterLongName: '--blend', - description: 'Cutline blend', - required: false, - }); - this.alignedLevel = this.defineIntegerParameter({ - argumentName: 'ALIGNED_LEVEL', - parameterLongName: '--aligned-level', - description: 'Aligned level between resolution and cog', - required: false, - }); - this.maxChunkUnit = this.defineIntegerParameter({ - argumentName: 'MAX_CHUNK_UNIT', - parameterLongName: '--max-chunk-unit', - description: 'Number of small jobs are chunked into one job', - required: false, - }); - this.output = this.defineStringParameter({ - argumentName: 'OUTPUT', - parameterShortName: '-o', - parameterLongName: '--output', - description: 'Output job.json path', - required: false, - }); - this.aws = this.defineFlagParameter({ - parameterLongName: '--aws', - description: 'Running the job on aws', - required: false, - }); - } - - async onExecute(): Promise { - const logger = LogConfig.get(); - const imagery = this.imagery.value; - let name = this.name.value === '' ? undefined : this.name.value; - if (imagery == null) throw new Error('Please provide a valid imagery source'); - await Promise.all([3791, 3790, 3789, 3788].map((code) => ProjectionLoader.load(code))); - - const source = imagery.endsWith('/') ? imagery : imagery + '/'; - logger.info({ imagery: source }, 'FindImagery'); - if (name == null) name = source.split('/').filter(Boolean).pop(); - if (name == null) throw new Error('Failed to find imagery set name'); - - let tileMatrixSets: string[] = []; - const tileMatrix = this.tileMatrix.value; - if (tileMatrix == null) throw new Error('Please provide valid tile set matrix.'); - if (tileMatrix.includes('/')) tileMatrixSets = tileMatrixSets.concat(tileMatrix.split('/')); - else tileMatrixSets.push(tileMatrix); - - const outputJobs: OutputJobs[] = []; - const configLayer: ConfigLayer = { name, title: titleizeImageryName(name) }; - const paths: string[] = []; - for (const identifier of tileMatrixSets) { - const id = ulid.ulid(); - const tileMatrix = TileMatrixSets.find(identifier); - if (tileMatrix == null) throw new Error(`Cannot find tile matrix: ${identifier}`); - logger.info({ id, tileMatrix: tileMatrix.identifier }, 'SetTileMatrix'); - const job = await this.makeCog(id, name, tileMatrix, source); - configLayer.title = job.title; - - const jobLocation = job.getJobPath('job.json'); - // Split the jobs into chunked tasks - const chunkedJobs = await this.splitJob(job, logger); - for (const names of chunkedJobs) { - outputJobs.push({ job: jobLocation, names }); - } - - // Set config layer for output - const path = jobLocation.replace('/job.json', ''); - configLayer[tileMatrix.projection.code] = path; - paths.push(path); - } - - const output = this.output.value; - if (output) { - fsa.write(fsa.join(output, 'jobs.json'), JSON.stringify(outputJobs)); - fsa.write(fsa.join(output, 'layer.json'), JSON.stringify(configLayer)); - fsa.write(fsa.join(output, 'paths.json'), JSON.stringify(paths)); - } - } - - async makeCog(id: string, imageryName: string, tileMatrix: TileMatrixSet, uri: string): Promise { - const bucket = this.target.value; - if (bucket == null && this.aws.value) throw new Error('Please provide a validate bucket for output job.json'); - let resampling; - /** Process Gebco 2193 as one cog of full extent to avoid antimeridian problems */ - if (tileMatrix.projection === Epsg.Nztm2000 && imageryName.includes('gebco')) { - resampling = { - warp: 'nearest', // GDAL doesn't like other warp settings when crossing antimeridian - overview: 'lanczos', - } as const; - } - - if (imageryName.includes('geographx')) { - resampling = { - warp: 'bilinear', - overview: 'bilinear', - } as const; - } - - // Prepare the cutline - let cutline: { href: string; blend: number } | undefined; - const cutlinePath = this.cutline.value === '' ? undefined : this.cutline.value; - const blend = this.blend.value === 0 ? undefined : this.blend.value; - if (cutlinePath && blend) cutline = { href: cutlinePath, blend }; - else if (cutlinePath) new Error('Please provide a blend for the cutline'); - else cutline = getCutline(imageryName); - if (cutline == null) throw new Error(`Cannot found default cutline from imagery name: ${imageryName}`); - - const alignedLevel = this.alignedLevel.value ?? AlignedLevel; - - const ctx: JobCreationContext = { - imageryName, - override: { - id, - projection: Epsg.Nztm2000, - resampling, - alignedLevel, - }, - outputLocation: this.aws.value - ? await this.findLocation(`s3://${bucket}/`) - : { type: 'local' as const, path: '.' }, - sourceLocation: await this.findLocation(uri), - cutline, - batch: false, // Only create the job.json in the make cog cli - tileMatrix, - oneCogCovering: false, - }; - - return await CogJobFactory.create(ctx); - } - - async splitJob(job: CogStacJob, logger: LogType): Promise { - // Get all the existing output tiffs - const existTiffs: Set = new Set(); - for await (const fileName of fsa.list(job.getJobPath())) { - if (fileName.endsWith('.tiff')) existTiffs.add(basename(fileName)); - } - - // Prepare chunk job and individual jobs based on imagery size. - const jobs = await BatchJob.getJobs(job, existTiffs, logger, this.maxChunkUnit.value); - if (jobs.length === 0) { - logger.info('NoJobs'); - return []; - } - logger.info({ jobTotal: job.output.files.length, jobLeft: jobs.length }, 'SplitJob:ChunkedJobs'); - return jobs; - } - - async findLocation(path: string): Promise { - if (path.startsWith('s3://')) { - const configPath = Env.get(Env.AwsRoleConfigPath); - if (configPath == null) throw new Error('No aws config path provided'); - const configs = await fsa.readJson(configPath); - for (const prefix of configs.prefixes) { - if (path.startsWith(prefix.prefix)) return { type: 's3', path, roleArn: prefix.roleArn }; - } - } else { - return { type: 'local', path: path }; - } - - throw new Error(`No valid role to find the path: ${path}`); - } -} diff --git a/packages/cli/src/cli/cogify/batch.job.ts b/packages/cli/src/cli/cogify/batch.job.ts deleted file mode 100644 index 273144820..000000000 --- a/packages/cli/src/cli/cogify/batch.job.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { Projection, TileMatrixSet } from '@basemaps/geo'; -import { Env, fsa, LogConfig, LogType } from '@basemaps/shared'; -import Batch from 'aws-sdk/clients/batch.js'; -import { createHash } from 'crypto'; -import { basename } from 'path'; -import { CogJob } from '../../cog/types.js'; - -const JobQueue = 'CogBatchJobQueue'; -const JobDefinition = 'CogBatchJob'; -const ChunkJobMax = 2000; -const ChunkLargeUnit = 200; // Up to 10 large files in one job -const ChunkMiddleUnit = 50; // Up to 40 middle files in one job -const ChunkSmallUnit = 20; // Up to 100 small files in one job - -/** The base alignment level used by GDAL, Tiffs that are bigger or smaller than this should scale the compute resources */ -const MagicAlignmentLevel = 7; - -const ResolutionRegex = /((?:\d[\.\-])?\d+)m/; -/** - * Attempt to parse a resolution from a imagery name - * @example `wellington_urban_2017_0.10m` -> 100 - * @param name Imagery name to parse - * @returns resolution (millimeters), -1 for failure to parse - */ -export function extractResolutionFromName(name: string): number { - const matches = name.match(ResolutionRegex); - if (matches == null) return -1; - return parseFloat(matches[1].replace('-', '.')) * 1000; -} - -export class BatchJob { - static _batch: Batch; - static get batch(): Batch { - if (this._batch) return this._batch; - const region = Env.get('AWS_REGION') ?? Env.get('AWS_DEFAULT_REGION') ?? 'ap-southeast-2'; - this._batch = new Batch({ region }); - return this._batch; - } - /** - * Create a id for a job - * - * This needs to be within `[a-Z_-]` upto 128 characters log - * @param job job to process - * @param fileNames output filename - * @returns job id - */ - static id(job: CogJob, fileNames?: string[]): string { - // Job names are uncontrolled so hash the name and grab a small slice to use as a identifier - const jobName = createHash('sha256').update(job.name).digest('hex').slice(0, 16); - if (fileNames == null) return `${job.id}-${jobName}-`; - fileNames.sort((a, b) => a.localeCompare(b)); - return `${job.id}-${jobName}-${fileNames.length}x-${fileNames.join('_')}`.slice(0, 128); - } - - static async batchOne( - jobPath: string, - job: CogJob, - names: string[], - isCommit: boolean, - ): Promise<{ jobName: string; jobId: string; memory: number }> { - const jobName = BatchJob.id(job, names); - - let memory = 3900; - if (names.length === 1) { - // Calculate the larger file to provision memory if there is only one imagery in job. - const tile = TileMatrixSet.nameToTile(names[0]); - const alignmentLevels = Projection.findAlignmentLevels(job.tileMatrix, tile, job.source.gsd); - // Give 25% more memory to larger jobs - const resDiff = 1 + Math.max(alignmentLevels - MagicAlignmentLevel, 0) * 0.25; - memory *= resDiff; - } - - if (!isCommit) { - return { jobName, jobId: '', memory }; - } - - let commandStr = ['-V', 'cog', '--job', jobPath, '--commit']; - for (const name of names) commandStr = commandStr.concat(['--name', name]); - - const batchJob = await this.batch - .submitJob({ - jobName, - jobQueue: JobQueue, - jobDefinition: JobDefinition, - containerOverrides: { - memory, - command: commandStr, - }, - retryStrategy: { attempts: 3 }, - }) - .promise(); - return { jobName, jobId: batchJob.jobId, memory }; - } - - /** - * List all the current jobs in batch and their statuses - * @returns a map of JobName to if their status is "ok" (not failed) - */ - static async getCurrentJobList(job: CogJob, logger: LogType): Promise> { - const jobPrefix = BatchJob.id(job); - // For some reason AWS only lets us query one status at a time. - const allStatuses = ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING' /* 'SUCCEEDED' */]; - // Succeeded is not needed as we check to see if the output file exists, if it succeeds and the output file doesn't exist then something has gone wrong - const allJobs = await Promise.all( - allStatuses.map((jobStatus) => this.batch.listJobs({ jobQueue: JobQueue, jobStatus }).promise()), - ); - - const jobIds = new Set(); - - // Find all the relevant jobs that start with our job prefix - for (const status of allJobs) { - for (const job of status.jobSummaryList) { - if (!job.jobName.startsWith(jobPrefix)) continue; - jobIds.add(job.jobId); - } - } - - // Inspect all the jobs for what files are being "processed" - const tiffs = new Set(); - let allJobIds = [...jobIds]; - while (allJobIds.length > 0) { - logger.info({ jobCount: allJobIds.length }, 'JobFetch'); - const jobList = allJobIds.slice(0, 100); - allJobIds = allJobIds.slice(100); - const describedJobs = await this.batch.describeJobs({ jobs: jobList }).promise(); - if (describedJobs.jobs == null) continue; - for (const job of describedJobs.jobs) { - const jobCommand = job.container?.command; - if (jobCommand == null) continue; - - // Extract the tiff names from the job command - for (let i = 0; i < jobCommand.length; i++) { - if (jobCommand[i] === '--name') tiffs.add(jobCommand[i + 1]); - } - } - } - - return tiffs; - } - - static async batchJob(job: CogJob, commit = false, logger: LogType): Promise { - const jobPath = job.getJobPath('job.json'); - if (!jobPath.startsWith('s3://')) { - throw new Error(`AWS Batch collection.json have to be in S3, jobPath:${jobPath}`); - } - LogConfig.set(logger.child({ correlationId: job.id, imageryName: job.name })); - - fsa.configure(job.output.location); - - // Get all the existing output tiffs - const existTiffs: Set = new Set(); - for await (const fileName of fsa.list(job.getJobPath())) { - if (fileName.endsWith('.tiff')) existTiffs.add(basename(fileName)); - } - - const runningJobs = await this.getCurrentJobList(job, logger); - for (const tiffName of runningJobs) existTiffs.add(`${tiffName}.tiff`); - - // Prepare chunk job and individual jobs based on imagery size. - const jobs = await this.getJobs(job, existTiffs, logger); - - if (jobs.length === 0) { - logger.info('NoJobs'); - return; - } - - logger.info( - { - jobTotal: job.output.files.length, - jobLeft: jobs.length, - jobQueue: JobQueue, - jobDefinition: JobDefinition, - }, - 'JobSubmit', - ); - - for (const names of jobs) { - const jobStatus = await BatchJob.batchOne(jobPath, job, names, commit); - logger.info(jobStatus, 'JobSubmitted'); - } - - if (!commit) { - logger.warn('DryRun:Done'); - return; - } - } - - /** - * Prepare the jobs from job files, and chunk the small images into single - * @returns List of jobs including single job and chunk jobs. - */ - static getJobs(job: CogJob, existing: Set, log: LogType, maxChunkUnit = ChunkJobMax): string[][] { - const jobs: string[][] = []; - let chunkJob: string[] = []; - let chunkUnit = 0; // Calculate the chunkUnit based on the size - for (const file of job.output.files) { - const outputFile = `${file.name}.tiff`; - if (existing.has(outputFile)) { - log.debug({ fileName: outputFile }, 'Skip:Exists'); - continue; - } - - const imageSize = file.width / job.output.gsd; - if (imageSize > 16385) { - chunkJob.push(file.name); - chunkUnit += ChunkJobMax; - } else if (imageSize > 8193) { - chunkJob.push(file.name); - chunkUnit += ChunkLargeUnit; - } else if (imageSize > 4097) { - chunkJob.push(file.name); - chunkUnit += ChunkMiddleUnit; - } else { - chunkJob.push(file.name); - chunkUnit += ChunkSmallUnit; - } - if (chunkUnit >= maxChunkUnit) { - jobs.push(chunkJob); - chunkJob = []; - chunkUnit = 0; - } - } - if (chunkJob.length > 0) jobs.push(chunkJob); - - return jobs; - } -} diff --git a/packages/cli/src/cli/cogify/cutline.ts b/packages/cli/src/cli/cogify/cutline.ts deleted file mode 100644 index 28c0f2d4f..000000000 --- a/packages/cli/src/cli/cogify/cutline.ts +++ /dev/null @@ -1,34 +0,0 @@ -const UrbanRuralCutLine = { - href: 's3://linz-basemaps-source/cutline/2020-05-07-cutline-nz-coasts-rural-and-urban.geojson', - blend: 20, -}; - -const DefaultCutlines = [ - { - filter: 'sentinel', - cutline: { href: 's3://linz-basemaps-source/cutline/2020-05-12-cutline-nz-ci.geojson.gz', blend: 5 }, - }, - { - filter: 'chatham', - cutline: { href: 's3://linz-basemaps-source/cutline/2020-05-12-cutline-nz-ci.geojson.gz', blend: 5 }, - }, - { - filter: 'geographx', - cutline: { - href: 's3://linz-basemaps-source/cutline/2021-11-10-cutline-51153-nz-coastlines-and-islands-polygons-topo-150k.geojson.gz', - blend: 0, - }, - }, - { - filter: 'topo50', - cutline: { href: 's3://linz-basemaps-source/cutline/2020-08-10-cutline-topo50.geojson.gz', blend: 0 }, - }, - - { filter: 'satellite', cutline: UrbanRuralCutLine }, - { filter: 'urban', cutline: UrbanRuralCutLine }, - { filter: 'rural', cutline: UrbanRuralCutLine }, -]; - -export function getCutline(imageryName = ''): { href: string; blend: number } | undefined { - return DefaultCutlines.find((f) => imageryName.toLowerCase().includes(f.filter))?.cutline; -} diff --git a/packages/cli/src/cli/cogify/semver.util.ts b/packages/cli/src/cli/cogify/semver.util.ts deleted file mode 100644 index 040807310..000000000 --- a/packages/cli/src/cli/cogify/semver.util.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** major.minor.patch */ -const MaxLength = 3; -/** fits into a int32 with out forcing a uint conversion */ -const EncodeBits = 10; -/** Max size for a single semver number (1024) */ -const MaxEncode = 2 ** EncodeBits - 1; - -export const SemVer = { - /** - * Encode a semver string into a number - * @example - * '0.0.1' => 1 - * 'v0.1.0' => 1024 - * @param v version to encode - */ - toNumber(v: string): number { - if (v.startsWith('v')) v = v.slice(1); - const chunks = v.split('.'); - if (chunks.length > MaxLength) throw new Error(`Failed to parse semver "${v}" too long`); - - let output = 0; - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]; - if (chunk === '') break; - - const num = parseInt(chunk); - if (isNaN(num)) throw new Error(`Failed to parse semver "${v}"`); - if (num > MaxEncode) throw new Error(`Failed to parse semver "${v}" ${chunk} is too large`); - - output |= num << ((MaxLength - i - 1) * EncodeBits); - } - return output; - }, - - /** - * Decoded a number encoded semver - * @example 1 => '0.0.1' - * @param v version to decode - */ - fromNumber(v: number): string { - const major = (v >> 20) & MaxEncode; - const minor = (v >> 10) & MaxEncode; - const patch = v & MaxEncode; - return `${major}.${minor}.${patch}`; - }, - - /** - * Compare two semver's - * @param vA version A - * @param vB version B - * @returns 0 if equal, >0 if vA > vB, <0 otherwise - */ - compare(vA: string, vB: string): number { - return SemVer.toNumber(vA) - SemVer.toNumber(vB); - }, -}; diff --git a/packages/cli/src/cli/config/action.import.ts b/packages/cli/src/cli/config/action.import.ts index a120b4692..0f530c08c 100644 --- a/packages/cli/src/cli/config/action.import.ts +++ b/packages/cli/src/cli/config/action.import.ts @@ -14,10 +14,10 @@ import { GoogleTms, Nztm2000QuadTms, Projection, TileMatrixSet } from '@basemaps import { Env, fsa, getDefaultConfig, LogConfig } from '@basemaps/shared'; import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line'; import fetch from 'node-fetch'; -import { CogStacJob } from '../../cog/cog.stac.job.js'; import { invalidateCache } from '../util.js'; import { Q, Updater } from './config.update.js'; import { FeatureCollection } from 'geojson'; +import { CogJobJson } from '../../cog/types.js'; const PublicUrlBase = Env.isProduction() ? 'https://basemaps.linz.govt.nz/' : 'https://dev.basemaps.linz.govt.nz/'; @@ -316,12 +316,12 @@ export class CommandImport extends CommandLineAction { return; } - _jobs: Map = new Map(); - async _loadJob(path: string): Promise { + _jobs: Map = new Map(); + async _loadJob(path: string): Promise { const existing = this._jobs.get(path); if (existing) return existing; try { - const job = await fsa.readJson(path); + const job = await fsa.readJson(path); this._jobs.set(path, job); return job; } catch { diff --git a/packages/cli/src/cli/github/github.ts b/packages/cli/src/cli/github/github.ts deleted file mode 100644 index 3ab310428..000000000 --- a/packages/cli/src/cli/github/github.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { Env, LogType } from '@basemaps/shared'; -import { execFileSync } from 'child_process'; -import { Octokit } from '@octokit/core'; -import { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js'; -import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'; - -export class Github { - repo: string; - org: string; - logger: LogType; - repoName: string; - octokit: Api; - - constructor(repo: string, logger: LogType) { - this.repo = repo; - this.logger = logger; - const [org, repoName] = repo.split('/'); - if (org == null || repoName == null) throw new Error(`Badly formatted repo name: ${repo}`); - this.org = org; - this.repoName = repoName; - - const token = Env.get(Env.GitHubToken); - if (token == null) throw new Error('Please set up github token environment variable.'); - this.octokit = restEndpointMethods(new Octokit({ auth: token })); - } - - isOk = (s: number): boolean => s >= 200 && s <= 299; - - /** - * Clone the repository - * - */ - clone(): void { - const ssh = `git@github.com:${this.repo}.git`; - this.logger.info({ repository: this.repo }, 'GitHub: Clone'); - execFileSync('git', ['clone', ssh]).toString().trim(); - } - - /** - * Get branch by name if exists, or create a new branch by name. - * - * @returns {branch} github references or the new created branch - */ - getBranch(branch: string): string { - this.logger.info({ branch }, 'GitHub: Get branch'); - try { - execFileSync('git', ['checkout', branch], { cwd: this.repoName }).toString().trim(); - this.logger.info({ branch }, 'GitHub: Branch Checkout'); - return branch; - } catch { - this.logger.info({ branch }, 'GitHub: Create New Branch'); - execFileSync('git', ['checkout', '-b', branch], { cwd: this.repoName }).toString().trim(); - return branch; - } - } - - /** - * Config github user email and user name - * - */ - configUser(): void { - const email = Env.get('GIT_USER_EMAIL') ?? 'basemaps@linz.govt.nz'; - const name = Env.get('GIT_USER_NAME') ?? 'basemaps[bot]'; - this.logger.info({ repository: this.repo }, 'GitHub: Config User Email'); - execFileSync('git', ['config', 'user.email', email], { cwd: this.repoName }).toString().trim(); - this.logger.info({ repository: this.repo }, 'GitHub: Config User Name'); - execFileSync('git', ['config', 'user.name', name], { cwd: this.repoName }).toString().trim(); - } - - /** - * Commit the changes to current branch - * - */ - add(paths: string[]): void { - this.logger.info({ repository: this.repo }, 'GitHub: Add Path'); - for (const path of paths) { - execFileSync('git', ['add', path], { cwd: this.repoName }).toString().trim(); - } - } - - /** - * Commit the changes to current branch - * - */ - commit(message: string): void { - this.logger.info({ repository: this.repo }, 'GitHub: Commit all'); - execFileSync('git', ['commit', '-am', message], { cwd: this.repoName }).toString().trim(); - } - - /** - * Push the local brach - * - */ - push(): void { - this.logger.info({ repository: this.repo }, 'GitHub: Push'); - execFileSync('git', ['push', 'origin', 'HEAD'], { cwd: this.repoName }).toString().trim(); - } - - /** - * Create pull request - * This needs to use github API to create Pull request by access token. - * - */ - async createPullRequests(branch: string, title: string, draft: boolean): Promise { - // Create pull request from the give head - const response = await this.octokit.rest.pulls.create({ - owner: this.org, - repo: this.repoName, - title, - head: branch, - base: 'master', - draft, - }); - if (!this.isOk(response.status)) throw new Error('Failed to create pull request.'); - this.logger.info({ branch, url: response.data.html_url }, 'GitHub: Create Pull Request'); - return response.data.number; - } -} diff --git a/packages/cli/src/cli/github/make.cog.pr.ts b/packages/cli/src/cli/github/make.cog.pr.ts deleted file mode 100644 index fcf662b20..000000000 --- a/packages/cli/src/cli/github/make.cog.pr.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { - ConfigId, - ConfigLayer, - ConfigPrefix, - ConfigTileSetRaster, - ConfigTileSetVector, - TileSetType, -} from '@basemaps/config'; -import { LogType, fsa } from '@basemaps/shared'; -import { Category, DefaultCategorySetting } from '../cogify/action.make.cog.pr.js'; -import { Github } from './github.js'; -import { TileSetConfigSchema } from '@basemaps/config/build/json/parse.tile.set.js'; -import { execFileSync } from 'child_process'; - -export class MakeCogGithub extends Github { - imagery: string; - _formatInstalled = false; - constructor(imagery: string, repo: string, logger: LogType) { - super(repo, logger); - this.imagery = imagery; - } - - /** - * Install the dependencies for the cloned repo - */ - npmInstall(): boolean { - this.logger.info({ repository: this.repo }, 'GitHub: Npm Install'); - execFileSync('npm', ['install', '--include=dev'], { cwd: this.repoName }); - return true; - } - - /** - * Format the config files by prettier - */ - formatConfigFile(path = './config/'): void { - if (!this._formatInstalled) this._formatInstalled = this.npmInstall(); - this.logger.info({ repository: this.repo }, 'GitHub: Prettier'); - execFileSync('npx', ['prettier', '-w', path], { cwd: this.repoName }); - } - - /** - * Prepare and create pull request for the aerial tileset config - */ - async updateRasterTileSet( - filename: string, - layer: ConfigLayer, - category: Category, - individual: boolean, - ): Promise { - const branch = `feat/bot-config-raster-${this.imagery}`; - - // Clone the basemaps-config repo and checkout branch - this.clone(); - this.configUser(); - this.getBranch(branch); - - this.logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file'); - if (individual) { - // Prepare new standalone tileset config - const tileSet: TileSetConfigSchema = { - type: TileSetType.Raster, - id: ConfigId.prefix(ConfigPrefix.TileSet, layer.name), - title: layer.title, - layers: [layer], - }; - const tileSetPath = fsa.joinAll('config', 'tileset', 'individual', `${layer.name}.json`); - const fullPath = fsa.join(this.repoName, tileSetPath); - await fsa.write(fullPath, JSON.stringify(tileSet)); - // Format the config file by prettier - this.formatConfigFile(tileSetPath); - this.add([tileSetPath]); - } else { - // Prepare new aerial tileset config - const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`); - const fullPath = fsa.join(this.repoName, tileSetPath); - const tileSet = await fsa.readJson(fullPath); - const newTileSet = await this.prepareRasterTileSetConfig(layer, tileSet, category); - // skip pull request if not an urban or rural imagery - if (newTileSet == null) return; - await fsa.write(fullPath, JSON.stringify(newTileSet)); - // Format the config file by prettier - this.formatConfigFile(tileSetPath); - this.add([tileSetPath]); - } - - // Commit and push the changes - const message = `config(raster): Add imagery ${this.imagery} to ${filename} config file.`; - this.commit(message); - this.push(); - await this.createPullRequests(branch, message, false); - } - - /** - * Set the default setting for the category - */ - setDefaultConfig(layer: ConfigLayer, category: Category): ConfigLayer { - layer.category = category; - const defaultSetting = DefaultCategorySetting[category]; - if (defaultSetting.minZoom != null && layer.minZoom != null) layer.minZoom = defaultSetting.minZoom; - return layer; - } - - /** - * Add new layer at the bottom of related category - */ - addLayer(layer: ConfigLayer, tileSet: ConfigTileSetRaster, category: Category): ConfigTileSetRaster { - for (let i = tileSet.layers.length - 1; i >= 0; i--) { - // Add new layer at the end of category - if (tileSet.layers[i].category === category) { - // Find first valid category and insert new record above that. - tileSet.layers.splice(i + 1, 0, layer); - break; - } - } - return tileSet; - } - - /** - * Prepare raster tileSet config json - */ - async prepareRasterTileSetConfig( - layer: ConfigLayer, - tileSet: ConfigTileSetRaster, - category: Category, - ): Promise { - // Reprocess existing layer - for (let i = 0; i < tileSet.layers.length; i++) { - if (tileSet.layers[i].name === layer.name) { - tileSet.layers[i] = layer; - return tileSet; - } - } - - // Set default Config if not existing layer - this.setDefaultConfig(layer, category); - - // Set layer zoom level and add to latest order - if (category === Category.Rural) { - for (let i = 0; i < tileSet.layers.length; i++) { - // Add new layer above the first Urban - if (tileSet.layers[i].category === Category.Urban) { - // Find first valid Urban and insert new record above that. - tileSet.layers.splice(i, 0, layer); - break; - } - } - } else if (category === Category.Other) { - // Add new layer at the bottom - tileSet.layers.push(layer); - } else { - this.addLayer(layer, tileSet, category); - } - - return tileSet; - } - - /** - * Prepare and create pull request for the aerial tileset config - */ - async updateVectorTileSet(filename: string, layer: ConfigLayer): Promise { - const branch = `feat/bot-config-vector-${this.imagery}`; - - // Clone the basemaps-config repo and checkout branch - this.clone(); - this.configUser(); - this.getBranch(branch); - - // Prepare new aerial tileset config - this.logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file'); - const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`); - const fullPath = fsa.join(this.repoName, tileSetPath); - const tileSet = await fsa.readJson(fullPath); - const newTileSet = await this.prepareVectorTileSetConfig(layer, tileSet); - - // skip pull request if not an urban or rural imagery - if (newTileSet == null) return; - await fsa.write(fullPath, JSON.stringify(newTileSet)); - // Format the config file by prettier - this.formatConfigFile(tileSetPath); - - // Commit and push the changes - const message = `config(vector): Update the ${this.imagery} to ${filename} config file.`; - this.add([tileSetPath]); - this.commit(message); - this.push(); - await this.createPullRequests(branch, message, false); - } - - /** - * Prepare raster tileSet config json - */ - async prepareVectorTileSetConfig(layer: ConfigLayer, tileSet: ConfigTileSetVector): Promise { - // Reprocess existing layer - for (let i = 0; i < tileSet.layers.length; i++) { - if (tileSet.layers[i].name === layer.name) { - tileSet.layers[i] = layer; - return tileSet; - } - } - return tileSet; - } -} diff --git a/packages/cli/src/cli/index.ts b/packages/cli/src/cli/index.ts index 3b96f81fb..ff1f072bb 100644 --- a/packages/cli/src/cli/index.ts +++ b/packages/cli/src/cli/index.ts @@ -1,18 +1,12 @@ #!/usr/bin/env node import { BaseCommandLine } from '@basemaps/shared/build/cli/base.js'; import 'source-map-support/register.js'; -import { CommandCogCreate } from './cogify/action.cog.js'; import { CommandCreateOverview } from './overview/action.create.overview.js'; -import { CommandCogPullRequest } from './cogify/action.make.cog.pr.js'; -import { CommandJobCreate } from './cogify/action.job.js'; -import { CommandMakeCog } from './cogify/action.make.cog.js'; import { CommandBundleAssets } from './config/action.bundle.assets.js'; import { CommandBundle } from './config/action.bundle.js'; import { CommandCogMapSheet } from './config/action.cog.mapsheet.js'; import { CommandImageryConfig } from './config/action.imagery.config.js'; import { CommandImport } from './config/action.import.js'; -import { CommandServe } from './server/action.serve.js'; -import { CommandSprites } from './sprites/action.sprites.js'; export class BasemapsConfigCommandLine extends BaseCommandLine { constructor() { @@ -21,20 +15,14 @@ export class BasemapsConfigCommandLine extends BaseCommandLine { toolDescription: 'Basemaps config command tools', }); - this.addAction(new CommandCogCreate()); - this.addAction(new CommandJobCreate()); - this.addAction(new CommandMakeCog()); - this.addAction(new CommandCreateOverview()); - this.addAction(new CommandCogPullRequest()); - + // Basemaps Config this.addAction(new CommandBundle()); this.addAction(new CommandBundleAssets()); this.addAction(new CommandImport()); this.addAction(new CommandImageryConfig()); - this.addAction(new CommandSprites()); - this.addAction(new CommandServe()); - + // Argo this.addAction(new CommandCogMapSheet()); + this.addAction(new CommandCreateOverview()); } } diff --git a/packages/cli/src/cli/overview/action.create.overview.ts b/packages/cli/src/cli/overview/action.create.overview.ts index 9205a86f0..573cba5d1 100644 --- a/packages/cli/src/cli/overview/action.create.overview.ts +++ b/packages/cli/src/cli/overview/action.create.overview.ts @@ -12,13 +12,18 @@ import * as path from 'path'; import { resolve } from 'path'; import { CogBuilder } from '../../cog/builder.js'; import { Cutline } from '../../cog/cutline.js'; -import { filterTiff, MaxConcurrencyDefault } from '../../cog/job.factory.js'; import { createOverviewWmtsCapabilities } from './overview.wmts.js'; import { JobTiles, tile } from './tile.generator.js'; import { SimpleTimer } from './timer.js'; const DefaultMaxZoom = 15; // Limitation of maximum overview zoom level to create const MaxNumberTiles = 25000; // Limitation of maximum number of tiles we can create for overview. +export const MaxConcurrencyDefault = 50; + +export function filterTiff(a: string): boolean { + const lowerA = a.toLowerCase(); + return lowerA.endsWith('.tiff') || lowerA.endsWith('.tif'); +} export class CommandCreateOverview extends CommandLineAction { private source: CommandLineStringParameter; diff --git a/packages/cli/src/cli/server/action.serve.ts b/packages/cli/src/cli/server/action.serve.ts deleted file mode 100644 index e28ab50cf..000000000 --- a/packages/cli/src/cli/server/action.serve.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { createServer } from '@basemaps/server'; -import { Const, Env, LogConfig } from '@basemaps/shared'; -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; - -const DefaultPort = 5000; - -export class CommandServe extends CommandLineAction { - config: CommandLineStringParameter; - assets: CommandLineStringParameter; - port: CommandLineIntegerParameter; - noConfig: CommandLineFlagParameter; - - public constructor() { - super({ - actionName: 'serve', - summary: 'Cli tool to create sprite sheet', - documentation: 'Create a sprite sheet from a folder of sprites', - }); - } - - protected onDefineParameters(): void { - this.config = this.defineStringParameter({ - argumentName: 'CONFIG', - parameterLongName: '--config', - description: 'Configuration source to use', - }); - this.assets = this.defineStringParameter({ - argumentName: 'ASSETS', - parameterLongName: '--assets', - description: 'Where the assets (sprites, fonts) are located', - }); - this.port = this.defineIntegerParameter({ - argumentName: 'PORT', - parameterLongName: '--port', - description: 'port to use', - defaultValue: DefaultPort, - }); - } - - protected async onExecute(): Promise { - const logger = LogConfig.get(); - - const config = this.config.value ?? 'dynamodb://' + Const.TileMetadata.TableName; - const port = this.port.value; - const assets = this.assets.value; - - // Force a default url base so WMTS requests know their relative url - const ServerUrl = Env.get(Env.PublicUrlBase) ?? `http://localhost:${port}`; - process.env[Env.PublicUrlBase] = ServerUrl; - - const server = await createServer({ config, assets }, logger); - await server.listen({ port: port ?? DefaultPort, host: '0.0.0.0' }); - logger.info({ url: ServerUrl }, 'ServerStarted'); - } -} diff --git a/packages/cli/src/cli/sprites/action.sprites.ts b/packages/cli/src/cli/sprites/action.sprites.ts deleted file mode 100644 index dce7d08e6..000000000 --- a/packages/cli/src/cli/sprites/action.sprites.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerListParameter, - CommandLineStringListParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; - -import { buildSprites } from '@basemaps/sprites'; - -export class CommandSprites extends CommandLineAction { - paths: CommandLineStringListParameter; - output: CommandLineStringParameter; - ratio: CommandLineIntegerListParameter; - retina: CommandLineFlagParameter; - - public constructor() { - super({ - actionName: 'sprites', - summary: 'Cli tool to create sprite sheet', - documentation: 'Create a sprite sheet from a folder of sprites', - }); - } - - protected onDefineParameters(): void { - this.paths = this.defineStringListParameter({ - argumentName: 'PATH', - parameterLongName: '--path', - description: 'Paths to the sprite files.', - }); - - this.output = this.defineStringParameter({ - argumentName: 'OUTPUT', - parameterLongName: '--output', - description: 'Paths of the output files', - }); - - this.ratio = this.defineIntegerListParameter({ - argumentName: 'RATIO', - parameterLongName: '--ratio', - description: 'Pixel ratio, default: "--ratio 1 --ratio 2"', - }); - - this.retina = this.defineFlagParameter({ - parameterLongName: '--retina', - description: 'Double the pixel ratios, 1x becomes 2x', - }); - } - - protected async onExecute(): Promise { - if (this.paths?.values == null || this.paths.values.length === 0) { - throw new Error('No sprite paths supplied'); - } - const ratio = [...this.ratio.values]; - const paths = [...this.paths.values]; - await buildSprites(ratio, this.retina.value, paths, this.output.value); - } -} diff --git a/packages/cli/src/cli/util.ts b/packages/cli/src/cli/util.ts index 3c03418a0..2865b6432 100644 --- a/packages/cli/src/cli/util.ts +++ b/packages/cli/src/cli/util.ts @@ -1,5 +1,4 @@ -import { Env, fsa, LogConfig } from '@basemaps/shared'; -import { Projection } from '@basemaps/geo'; +import { fsa, LogConfig } from '@basemaps/shared'; import CloudFormation from 'aws-sdk/clients/cloudformation.js'; import CloudFront from 'aws-sdk/clients/cloudfront.js'; import S3 from 'aws-sdk/clients/s3.js'; @@ -8,7 +7,6 @@ import crypto from 'crypto'; import path from 'path'; import { gzip } from 'zlib'; import { promisify } from 'util'; -import { CogStacJob } from '../cog/cog.stac.job.js'; import slugify from 'slugify'; // Cloudfront has to be defined in us-east-1 @@ -130,20 +128,6 @@ export async function uploadStaticFile( return true; } -/** - * Prepare QA urls with center location - */ -export async function prepareUrl(job: CogStacJob): Promise { - const bounds = job.output.bounds; - const center = { x: bounds.x + bounds.width / 2, y: bounds.y + bounds.height / 2 }; - const proj = Projection.get(job.tileMatrix.projection); - const centerLatLon = proj.toWgs84([center.x, center.y]).map((c) => c.toFixed(6)); - const targetZoom = Math.max(job.tileMatrix.findBestZoom(job.output.gsd) - 12, 0); - const base = Env.get(Env.PublicUrlBase); - const url = `${base}/?i=${job.id}&p=${job.tileMatrix.identifier}&debug#@${centerLatLon[1]},${centerLatLon[0]},z${targetZoom}`; - return url; -} - /** * Make a tile imagery title as imagery name * @example diff --git a/packages/cli/src/cog/__tests__/builder.test.ts b/packages/cli/src/cog/__tests__/builder.test.ts deleted file mode 100644 index 2576e8da5..000000000 --- a/packages/cli/src/cog/__tests__/builder.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { Bounds, Epsg, EpsgCode, GoogleTms } from '@basemaps/geo'; -import { LogConfig } from '@basemaps/shared'; -import { fsa } from '@chunkd/fs'; -import { CogTiff } from '@cogeotiff/core'; - -import o from 'ospec'; -import { CogBuilder, guessProjection } from '../builder.js'; - -o.spec('Builder', () => { - o('should guess WKT', () => { - o( - guessProjection( - 'PCS Name = NZGD_2000_New_Zealand_Transverse_Mercator|GCS Name = GCS_NZGD_2000|Ellipsoid = GRS_1980|Primem = Greenwich||', - ), - ).equals(Epsg.Nztm2000); - - o( - guessProjection('NZGD2000_New_Zealand_Transverse_Mercator_2000|GCS Name = GCS_NZGD_2000|Primem = Greenwich||'), - ).equals(Epsg.Nztm2000); - }); - - o('should not guess unknown wkt', () => { - o(guessProjection('')).equals(null); - o(guessProjection('NZTM')).equals(null); - o(guessProjection('NZGD2000')).equals(null); - }); - - o.spec('tiff', () => { - const googleBuilder = new CogBuilder(GoogleTms, 1, LogConfig.get()); - const origInit = CogTiff.prototype.init; - const origGetImage = CogTiff.prototype.getImage; - - o.after(() => { - CogTiff.prototype.init = origInit; - CogTiff.prototype.getImage = origGetImage; - }); - - o('bounds', async () => { - const localTiff = fsa.source('/local/file.tiff')!; - const s3Tiff = fsa.source('s3://bucket/file.tiff')!; - - const imageLocal = { - resolution: [0.1], - value: (): any => [1], - valueGeo: (): any => EpsgCode.Nztm2000, - bbox: Bounds.fromJson({ - x: 1492000, - y: 6198000, - width: 24000, - height: 36000, - }).toBbox(), - }; - - const imageS3 = { - resolution: [0.1], - value: (): any => [1], - valueGeo: (): any => EpsgCode.Nztm2000, - bbox: Bounds.fromJson({ - x: 1492000 + 24000, - y: 6198000, - width: 24000, - height: 36000, - }).toBbox(), - }; - - CogTiff.prototype.init = o.spy() as any; - CogTiff.prototype.getImage = function (): any { - return this.source === localTiff ? imageLocal : imageS3; - }; - - const ans = await googleBuilder.bounds([localTiff, s3Tiff]); - - o(ans).deepEquals({ - projection: 2193, - nodata: 1, - bands: 1, - bounds: [ - { - x: 1492000, - y: 6198000, - width: 24000, - height: 36000, - name: '/local/file.tiff', - }, - { - x: 1516000, - y: 6198000, - width: 24000, - height: 36000, - name: 's3://bucket/file.tiff', - }, - ], - pixelScale: 0.1, - resZoom: 21, - }); - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/cog.stac.job.test.ts b/packages/cli/src/cog/__tests__/cog.stac.job.test.ts deleted file mode 100644 index b9f166c47..000000000 --- a/packages/cli/src/cog/__tests__/cog.stac.job.test.ts +++ /dev/null @@ -1,350 +0,0 @@ -import { Bounds, Epsg, EpsgCode, GoogleTms, Nztm2000Tms, Stac, StacCollection } from '@basemaps/geo'; -import { Projection } from '@basemaps/geo'; -import { mockFileOperator } from '@basemaps/shared/build/file/__tests__/file.operator.test.helper.js'; -import { round } from '@basemaps/test/build/rounding.js'; -import { Ring } from '@linzjs/geojson'; -import o from 'ospec'; -import { CogStacJob, JobCreationContext } from '../cog.stac.job.js'; -import { CogBuilderMetadata, CogJobJson } from '../types.js'; - -type RecursivePartial = { - [P in keyof T]?: RecursivePartial; -}; -o.spec('CogJob', () => { - o.spec('build', () => { - const id = 'jobid1'; - const imageryName = 'auckland_rural_2010-2012_0-50m'; - - const jobPath = 's3://target-bucket/path/3857/auckland_rural_2010-2012_0-50m/jobid1'; - - const ring1 = [ - [170, -41], - [-175, -40], - [-177, -42], - [175, -42], - [170, -41], - ].map(Projection.get(Epsg.Nztm2000).fromWgs84) as Ring; - const ring2 = [ - [-150, -40], - [-140, -41], - [-150, -41], - [-150, -40], - ].map(Projection.get(Epsg.Nztm2000).fromWgs84) as Ring; - - const srcPoly = [[ring1], [ring2]]; - const bounds = srcPoly.map((poly, i) => ({ ...Bounds.fromMultiPolygon([poly]), name: 'ring' + i })); - - const metadata: CogBuilderMetadata = { - bands: 3, - bounds, - projection: EpsgCode.Nztm2000, - pixelScale: 0.075, - resZoom: 22, - nodata: 0, - files: [{ name: '0-0-0', x: 1, y: 2, width: 2, height: 3 }], - targetBounds: { x: 1, y: 2, width: 2, height: 3 }, - }; - const addAlpha = true; - const ctx = { - tileMatrix: GoogleTms, - sourceLocation: { type: 's3', path: 's3://source-bucket/path' }, - outputLocation: { type: 's3', path: 's3://target-bucket/path' }, - cutline: { blend: 20, href: 's3://curline-bucket/path' }, - oneCogCovering: false, - } as JobCreationContext; - - const mockFs = mockFileOperator(); - - o.beforeEach(mockFs.setup); - - o.afterEach(mockFs.teardown); - - o('with job.json', async () => { - mockFs.jsStore['s3://source-bucket/path/collection.json'] = { - title: 'The Title', - description: 'The Description', - license: 'The License', - keywords: ['keywords'], - extent: { - spatial: { bbox: [9, 8, 7, 6] }, - temporal: { interval: [['2020-01-01T00:00:00.000', '2020-08-08T19:18:23.456Z']] }, - }, - - providers: [ - { name: 'provider name', roles: ['licensor'], url: 'https://provider.com' }, - { name: 'unknown url', roles: ['processor'], url: 'unknown' }, - ], - }; - const job = await CogStacJob.create({ id, imageryName, metadata, ctx, addAlpha, cutlinePoly: [] }); - o(job.getJobPath('job.json')).equals(jobPath + '/job.json'); - - o(job.title).equals('The Title'); - o(job.description).equals('The Description'); - - const stac = round(mockFs.jsStore[jobPath + '/collection.json'], 4); - - o(stac.title).equals('The Title'); - o(stac.description).equals('The Description'); - o(stac.license).equals('The License'); - o(stac.keywords).deepEquals(['keywords']); - o(stac.providers).deepEquals([ - { name: 'provider name', roles: ['licensor'], url: 'https://provider.com' }, - { name: 'unknown url', roles: ['processor'], url: undefined }, - ]); - o(stac.links[2]).deepEquals({ - href: 's3://source-bucket/path/collection.json', - rel: 'sourceImagery', - type: 'application/json', - }); - o(round(stac.extent.spatial.bbox, 4)).deepEquals([[169.3347, -51.868, -145.9143, -32.8496]]); - o(stac.extent.temporal).deepEquals({ interval: [['2020-01-01T00:00:00.000', '2020-08-08T19:18:23.456Z']] }); - }); - - o('no source collection.json', async () => { - const startNow = Date.now(); - const job = await CogStacJob.create({ - id, - imageryName, - metadata, - ctx: { ...ctx, oneCogCovering: true }, - addAlpha, - cutlinePoly: [], - }); - const afterNow = Date.now(); - const jobJson = round(job.json, 4) as CogJobJson; - - o(jobJson).deepEquals(round(mockFs.jsStore[jobPath + '/job.json'], 4)); - o(jobJson.id).equals('jobid1'); - o(jobJson.name).equals('auckland_rural_2010-2012_0-50m'); - o(jobJson.title).equals('Auckland rural 2010-2012 0.50m'); - o(jobJson.description).equals('No description'); - o(jobJson.source.gsd).equals(0.075); - o(jobJson.source.epsg).equals(2193); - o(jobJson.source.files[0]).deepEquals({ - x: 1347679.1521, - y: 5301577.5262, - width: 1277913.1293, - height: 201072.641, - name: 'ring0', - }); - o(jobJson.source.files[1]).deepEquals({ - x: 4728648.2332, - y: 4247288.7253, - width: 837784.3351, - height: 609622.9622, - name: 'ring1', - }); - o(jobJson.source.location).deepEquals({ type: 's3', path: 's3://source-bucket/path' }); - o(jobJson.output.gsd).equals(0.0373); - o(jobJson.output.epsg).equals(3857); - o(jobJson.output.tileMatrix).equals('WebMercatorQuad'); - o(jobJson.output.files).deepEquals([{ name: '0-0-0', x: 1, y: 2, width: 2, height: 3 }]); - o(jobJson.output.location).deepEquals({ type: 's3', path: 's3://target-bucket/path' }); - o(jobJson.output.resampling).deepEquals({ warp: 'bilinear', overview: 'lanczos' }); - o(jobJson.output.quality).equals(90); - o(jobJson.output.cutline).deepEquals({ blend: 20, href: 's3://curline-bucket/path' }); - o(jobJson.output.addAlpha).equals(true); - o(jobJson.output.nodata).equals(0); - o(jobJson.output.bounds).deepEquals({ x: 1, y: 2, width: 2, height: 3 }); - - o(job.id).equals(id); - o(job.source.gsd).equals(0.075); - o(round(job.output.gsd, 4)).equals(0.0373); - o(job.output.oneCogCovering).equals(true); - - const stac = round(mockFs.jsStore[jobPath + '/collection.json'], 4) as StacCollection; - - const stacMeta = stac.summaries as any; - - const generated = stacMeta['linz:generated'][0]; - - const jobNow = +Date.parse(generated.datetime); - o(startNow <= jobNow && jobNow <= afterNow).equals(true); - - // split so that scripts/detect.unlinked.dep.js does not think it is an import - o(generated.package).equals('@' + 'basemaps/cli'); - o(stac.id).equals(job.id); - o(stac.extent.spatial.bbox).deepEquals([[169.3347, -51.868, -145.9143, -32.8496]]); - o(stac.extent.temporal).deepEquals({ interval: [['2010-01-01T00:00:00Z', '2011-01-01T00:00:00Z']] }); - o(stac.stac_version).equals(Stac.Version); - o(stac.stac_extensions).deepEquals([Stac.BaseMapsExtension]); - o(stac.license).deepEquals(Stac.License); - o(stac.keywords).deepEquals(['Imagery', 'New Zealand']); - o(stac.summaries).deepEquals({ - gsd: [0.075], - 'proj:epsg': [3857], - 'linz:output': [ - { - resampling: { warp: 'bilinear', overview: 'lanczos' }, - quality: 90, - cutlineBlend: 20, - addAlpha: true, - nodata: 0, - }, - ], - 'linz:generated': [generated], - }); - o(stac.links).deepEquals([ - { href: 's3://target-bucket/path/collection.json', type: 'application/json', rel: 'self' }, - { href: 'job.json', type: 'application/json', rel: 'linz.basemaps.job' }, - { href: 'cutline.geojson.gz', type: 'application/geo+json+gzip', rel: 'linz.basemaps.cutline' }, - { href: 'covering.geojson', type: 'application/geo+json', rel: 'linz.basemaps.covering' }, - { href: 'source.geojson', type: 'application/geo+json', rel: 'linz.basemaps.source' }, - { href: '0-0-0.json', type: 'application/geo+json', rel: 'item' }, - ]); - - o(round(mockFs.jsStore[jobPath + '/cutline.geojson.gz'], 4)).deepEquals({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { type: 'MultiPolygon', coordinates: [] }, - properties: {}, - }, - ], - crs: { - type: 'name', - properties: { name: 'urn:ogc:def:crs:EPSG::3857' }, - }, - }); - - o(round(mockFs.jsStore[jobPath + '/0-0-0.json'], 4)).deepEquals({ - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [ - [ - [0, 0], - [0, 0], - [0, 0], - [0, 0], - [0, 0], - ], - ], - }, - properties: { - datetime: new Date(jobNow).toISOString(), - name: '0-0-0', - gsd: 0.0373, - 'proj:epsg': 3857, - }, - bbox: [0, 0, 0, 0], - id: 'jobid1/0-0-0', - collection: 'jobid1', - stac_version: Stac.Version, - stac_extensions: ['projection'], - links: [ - { href: jobPath + '/0-0-0.json', rel: 'self' }, - { href: 'collection.json', rel: 'collection' }, - ], - assets: { - cog: { - href: '0-0-0.tiff', - type: 'image/tiff; application=geotiff; profile=cloud-optimized', - roles: ['data'], - }, - }, - }); - o(round(mockFs.jsStore[jobPath + '/covering.geojson'], 4)).deepEquals({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [ - [ - [0, 0], - [0, 0], - [0, 0], - [0, 0], - [0, 0], - ], - ], - }, - properties: { name: '0-0-0' }, - bbox: [0, 0, 0, 0], - }, - ], - }); - - const sourceGeoJson = mockFs.jsStore[jobPath + '/source.geojson']; - o(sourceGeoJson.type).equals('FeatureCollection'); - o(sourceGeoJson.features.length).equals(2); - o(round(sourceGeoJson.features[0], 4)).deepEquals({ - type: 'Feature', - geometry: { - type: 'MultiPolygon', - coordinates: [ - [ - [ - [169.9343, -42.3971], - [180, -42.2199], - [180, -40.4109], - [170.0185, -40.5885], - [169.9343, -42.3971], - ], - ], - [ - [ - [-180, -42.2199], - [-174.6708, -41.7706], - [-175, -40], - [-180, -40.4109], - [-180, -42.2199], - ], - ], - ], - }, - properties: { name: 'ring0' }, - bbox: [169.9343, -42.3971, -175, -40], - }); - - o(round(sourceGeoJson.features[1], 4)).deepEquals({ - type: 'Feature', - geometry: { - type: 'Polygon', - coordinates: [ - [ - [-147.4403, -44.5167], - [-150.5162, -40.1908], - [-143.1693, -37.1443], - [-140, -41], - [-147.4403, -44.5167], - ], - ], - }, - properties: { name: 'ring1' }, - bbox: [-147.4403, -44.5167, -143.1693, -37.1443], - }); - }); - - o('should create with no tileMatrix', () => { - const cfg: RecursivePartial = { output: { epsg: 2193 } }; - const job = new CogStacJob(cfg as CogJobJson); - o(job.tileMatrix.identifier).equals(Nztm2000Tms.identifier); - }); - - o('should error with invalid tileMatrix', () => { - const cfg: RecursivePartial = { output: { tileMatrix: 'None' } }; - const job = new CogStacJob(cfg as CogJobJson); - o(() => job.tileMatrix).throws('Failed to find TileMatrixSet "None"'); - }); - - o('no source collection.json and not nice name', async () => { - try { - await CogStacJob.create({ - id, - imageryName: 'yucky-name', - metadata, - ctx: { ...ctx, oneCogCovering: true }, - addAlpha, - cutlinePoly: [], - }); - o('').equals('create should not have exceeded'); - } catch (err: any) { - o(err.message).equals('Missing date in imagery name: yucky-name'); - } - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/cog.test.ts b/packages/cli/src/cog/__tests__/cog.test.ts deleted file mode 100644 index 7b8181489..000000000 --- a/packages/cli/src/cog/__tests__/cog.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { GoogleTms } from '@basemaps/geo'; -import { LogConfig } from '@basemaps/shared'; -import { round } from '@basemaps/test/build/rounding.js'; -import o from 'ospec'; -import { GdalCogBuilder } from '../../gdal/gdal.cog.js'; -import { buildCogForName } from '../cog.js'; -import { SourceTiffTestHelper } from './source.tiff.testhelper.js'; - -LogConfig.disable(); - -o.spec('cog', () => { - o.spec('buildCogForName', () => { - const origConvert = GdalCogBuilder.prototype.convert; - - o.afterEach(() => { - GdalCogBuilder.prototype.convert = origConvert; - }); - - o('gdal_translate args', async () => { - let gdalCogBuilder: GdalCogBuilder | null = null; - let convertArgs: any = null; - const convert = function (this: any, ...args: any[]): void { - gdalCogBuilder = this; // eslint-disable-line @typescript-eslint/no-this-alias - convertArgs = args; - }; - GdalCogBuilder.prototype.convert = convert as any; - - const job = SourceTiffTestHelper.makeCogJob(); - const logger = LogConfig.get(); - - const name = '4-15-10'; - - job.output.files = [{ name, ...job.tileMatrix.tileToSourceBounds({ x: 15, y: 10, z: 4 }) }]; - - await buildCogForName(job, name, '/tmp/test.vrt', '/tmp/out-tiff', logger, true); - o(convertArgs[0].info).equals(logger.info); - - const { config } = gdalCogBuilder!; - config.bbox = round(config.bbox, 4); - config.targetRes = round(config.targetRes, 4); - - o(config).deepEquals({ - bbox: [17532819.7999, -5009377.0857, 20037508.3428, -7514065.6285], - alignmentLevels: 13, - compression: 'webp', - tileMatrix: GoogleTms, - resampling: { warp: 'bilinear', overview: 'lanczos' }, - blockSize: 512, - targetRes: 0.75, - quality: 90, - }); - o(gdalCogBuilder!.source).equals('/tmp/test.vrt'); - o(gdalCogBuilder!.target).equals('/tmp/out-tiff'); - - o(round(gdalCogBuilder!.args, 4)).deepEquals([ - '-of', - 'COG', - '-co', - 'TILING_SCHEME=GoogleMapsCompatible', - '-co', - 'NUM_THREADS=ALL_CPUS', - '--config', - 'GDAL_NUM_THREADS', - 'ALL_CPUS', - '-co', - 'BIGTIFF=YES', - '-co', - 'ADD_ALPHA=YES', - '-co', - 'BLOCKSIZE=512', - '-co', - 'WARP_RESAMPLING=bilinear', - '-co', - 'OVERVIEW_RESAMPLING=lanczos', - '-co', - 'COMPRESS=webp', - '-co', - 'ALIGNED_LEVELS=13', - '-co', - 'QUALITY=90', - '-co', - 'SPARSE_OK=YES', - '--config', - 'GDAL_DISABLE_READDIR_ON_OPEN', - 'EMPTY_DIR', - '-tr', - '0.75', - '0.75', - '-projwin', - '17532819.7999', - '-5009377.0857', - '20037508.3428', - '-7514065.6285', - '-projwin_srs', - 'EPSG:3857', - '/tmp/test.vrt', - '/tmp/out-tiff', - ]); - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/cog.vrt.test.ts b/packages/cli/src/cog/__tests__/cog.vrt.test.ts deleted file mode 100644 index e8314048e..000000000 --- a/packages/cli/src/cog/__tests__/cog.vrt.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { EpsgCode, GoogleTms, Nztm2000Tms } from '@basemaps/geo'; -import { fsa, LogConfig } from '@basemaps/shared'; -import { qkToName } from '@basemaps/geo/build/proj/__tests__/test.util.js'; -import { round } from '@basemaps/test/build/rounding.js'; -import o from 'ospec'; -import path from 'path'; -import url from 'url'; -import { Gdal } from '../../gdal/gdal.js'; -import { CogVrt } from '../cog.vrt.js'; -import { Cutline } from '../cutline.js'; -import { SourceTiffTestHelper } from './source.tiff.testhelper.js'; -const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); - -o.spec('cog.vrt', () => { - const tmpFolder = '/tmp/my-tmp-folder'; - - const job = SourceTiffTestHelper.makeCogJob(); - - const logger = LogConfig.get(); - LogConfig.disable(); - - const testDir = `${__dirname}/../../../__test.assets__`; - - const sourceBounds = SourceTiffTestHelper.tiffNztmBounds(testDir); - const [tif1, tif2] = sourceBounds; - const [tif1Path, tif2Path] = sourceBounds.map(({ name }) => name); - - let cutTiffArgs: Array> = []; - - let runSpy = o.spy(); - - const origFileOperatorWriteJson = fsa.writeJson; - const { create } = Gdal; - - let gdal: any; - - o.after(() => { - fsa.writeJson = origFileOperatorWriteJson; - Gdal.create = create; - }); - - o.beforeEach(() => { - runSpy = o.spy(); - job.output.tileMatrix = GoogleTms.identifier; - job.source.epsg = EpsgCode.Nztm2000; - job.source.gsd = 20; - gdal = { run: runSpy }; - (Gdal as any).create = (): any => gdal; - job.source.files = [tif1, tif2]; - - cutTiffArgs = []; - fsa.writeJson = ((...args: any): any => { - cutTiffArgs.push(args); - }) as any; - - job.output.cutline = undefined; - }); - - o('1 crosses, 1 outside', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson')); - const cl2 = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - cutline.clipPoly.push(...cl2.clipPoly); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, qkToName('31133322'), logger); - - o(job.source.files).deepEquals([tif1, tif2]); - o(cutline.clipPoly.length).equals(2); - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - }); - - o('not within tile', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson')); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, qkToName('3131110001'), logger); - - o(cutTiffArgs.length).equals(0); - o(vrt).equals(null); - o(runSpy.callCount).equals(0); - }); - - o('no cutline same projection', async () => { - const vrt = await CogVrt.buildVrt(tmpFolder, job, new Cutline(GoogleTms), qkToName('31'), logger); - - o(job.source.files).deepEquals([tif1, tif2]); - o(cutTiffArgs.length).equals(0); - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(runSpy.callCount).equals(2); - o(runSpy.args[0]).equals('gdalwarp'); - }); - - o('no cutline diff projection', async () => { - job.output.tileMatrix = Nztm2000Tms.identifier; //EpsgCode.Nztm2000; - const vrt = await CogVrt.buildVrt(tmpFolder, job, new Cutline(GoogleTms), qkToName('31'), logger); - - o(job.source.files).deepEquals([tif1, tif2]); - o(cutTiffArgs.length).equals(0); - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(runSpy.callCount).equals(2); - o(runSpy.args[0]).equals('gdalwarp'); - }); - - o('fully within same projection', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), -100); - - const name = qkToName('311333222321113310'); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, name, logger); - - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(cutTiffArgs.length).equals(0); - o(runSpy.callCount).equals(2); - o(runSpy.args[0]).equals('gdalwarp'); - }); - - o('intersected cutline', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 20); - job.output.cutline = { blend: 20, href: 'cutline.json' }; - - const name = qkToName('311333222321113'); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, name, logger); - - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(cutTiffArgs.length).equals(1); - o(cutTiffArgs[0][1]).deepEquals(cutline.toGeoJson()); - - o(round(cutTiffArgs[0], 6)).deepEquals([ - '/tmp/my-tmp-folder/cutline.geojson', - { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - type: 'MultiPolygon', - coordinates: [ - [ - [ - [19466418.186171, -4995093.077172], - [19467156.308979, -4996108.09485], - [19467854.290313, -4996091.668391], - [19468645.309917, -4995755.689377], - [19469463.193363, -4995206.362781], - [19470215.804402, -4994433.114642], - [19470856.681731, -4993478.14574], - [19471932.55678, -4992600.063654], - [19473538.532795, -4990108.677214], - [19474243.881356, -4988798.711151], - [19474243.881356, -4986828.162353], - [19472131.837905, -4986828.162353], - [19472109.508239, -4986926.945548], - [19470978.43787, -4989417.445436], - [19468800.977534, -4991497.540469], - [19467511.856865, -4993217.174782], - [19466418.186171, -4995093.077172], - ], - ], - ], - }, - properties: {}, - }, - ], - crs: { type: 'name', properties: { name: 'urn:ogc:def:crs:EPSG::3857' } }, - }, - ]); - }); - - o('1 surrounded with s3 files', async () => { - const mount = o.spy(); - const setCredentials = o.spy(); - gdal.mount = mount; - gdal.setCredentials = setCredentials; - - const vtif1 = '/vsis3' + tif1Path; - const vtif2 = '/vsis3' + tif2Path; - - job.source.location = { type: 's3', path: 's3://foo/bar', roleArn: 'a:role:string' }; - - job.source.files = job.source.files.map((o) => { - o = Object.assign({}, o); - o.name = 's3:/' + o.name; - return o; - }); - job.output.cutline = { blend: 10, href: 'cutline.json' }; - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, qkToName('3131110001'), logger); - - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(cutTiffArgs.length).equals(1); - o(cutTiffArgs[0][0]).equals(tmpFolder + '/cutline.geojson'); - - o(cutline.clipPoly.length).equals(1); - - const geo = cutline.toGeoJson(); - - o(geo.type).equals('FeatureCollection'); - - const coordinates = (geo.features[0].geometry as any).coordinates; - if (geo.type === 'FeatureCollection') { - o(geo.features).deepEquals([ - { - type: 'Feature', - properties: {}, - geometry: { - type: 'MultiPolygon', - coordinates, - }, - }, - ]); - } - - o(round(coordinates, 5)).deepEquals([ - [ - [ - [19455446.57501, -5025689.98761], - [19455543.12619, -5026720.78623], - [19457346.41188, -5024885.32147], - [19456570.81047, -5023799.46087], - [19456560.03196, -5023796.37227], - [19456551.09434, -5023782.48917], - [19456540.68564, -5023777.32429], - [19456531.22726, -5023778.34575], - [19456524.31633, -5023783.03837], - [19456521.12157, -5023788.06858], - [19456521.62607, -5023792.6091], - [19456528.41956, -5023802.39913], - [19456528.99998, -5023810.66149], - [19456513.95059, -5023820.48595], - [19456497.91736, -5023822.46951], - [19456431.41331, -5023815.14488], - [19456325.95579, -5024324.4265], - [19455446.57501, -5025689.98761], - ], - ], - ]); - - o(runSpy.callCount).equals(2); - o(mount.calls.map((c: any) => c.args[0])).deepEquals([tmpFolder, ...job.source.files.map((c) => c.name)]); - - o(setCredentials.calls.map((c: any) => c.args[0].service.config.params.RoleArn)).deepEquals(['a:role:string']); - - o(round((runSpy.calls[0] as any).args)).deepEquals([ - 'gdalbuildvrt', - ['-hidenodata', '-allow_projection_difference', '-addalpha', '/tmp/my-tmp-folder/source.vrt', vtif1, vtif2], - logger, - ]); - - o(round(runSpy.args)).deepEquals([ - 'gdalwarp', - [ - '-of', - 'VRT', - '-multi', - '-wo', - 'NUM_THREADS=ALL_CPUS', - '-s_srs', - 'EPSG:2193', - '-t_srs', - 'EPSG:3857', - '-tr', - '0.75', - '0.75', - '-tap', - '-cutline', - '/tmp/my-tmp-folder/cutline.geojson', - '-cblend', - '10', - '/tmp/my-tmp-folder/source.vrt', - '/tmp/my-tmp-folder/cog.vrt', - ], - logger, - ]); - }); -}); diff --git a/packages/cli/src/cog/__tests__/cutline.test.ts b/packages/cli/src/cog/__tests__/cutline.test.ts deleted file mode 100644 index 21719812c..000000000 --- a/packages/cli/src/cog/__tests__/cutline.test.ts +++ /dev/null @@ -1,353 +0,0 @@ -import { Bounds, EpsgCode, GoogleTms, Nztm2000Tms } from '@basemaps/geo'; -import { qkToName } from '@basemaps/geo/build/proj/__tests__/test.util.js'; -import { round } from '@basemaps/test/build/rounding.js'; -import { MultiPolygon } from '@linzjs/geojson'; -import o from 'ospec'; -import path from 'path'; -import url from 'url'; -import { AlignedLevel } from '../constants.js'; -import { Cutline, polyContainsBounds } from '../cutline.js'; -import { SourceMetadata } from '../types.js'; -import { SourceTiffTestHelper } from './source.tiff.testhelper.js'; -const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); - -o.spec('cutline', () => { - o.specTimeout(1_000); - const testDir = `${__dirname}/../../../__test.assets__`; - - o.spec('filterSourcesForName', () => { - o('fully within same projection', async () => { - // convert poly to GoogleTms - const cutpoly = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson')).toGeoJson(); - - const cutline = new Cutline(GoogleTms, cutpoly, -100); - - const name = qkToName('311333222321113310'); - - const job = SourceTiffTestHelper.makeCogJob(); - job.output.tileMatrix = GoogleTms.identifier; - job.source.epsg = EpsgCode.Nztm2000; - const sourceBounds = SourceTiffTestHelper.tiffNztmBounds(testDir); - const [tif1, tif2] = sourceBounds; - job.source.files = [tif1, tif2]; - - o(cutline.filterSourcesForName(name, job)).deepEquals([tif2.name]); - }); - }); - - o('polyContainsBounds', () => { - const polys: MultiPolygon = [ - [ - [ - [-4, 2], - [-2, 1], - [-4, -1], - [1, -5], - [2, -5], - [2, -2], - [4, -5], - [6, -2], - [2, 0], - [6, 3], - [2, 7], - [0, 3], - [-4, 6], - [-4, 10], - [10, 10], - [10, -10], - [-10, -10], - [-4, 2], - ], - ], - [ - [ - [2, 3], - [3, 3], - [3, 5], - [2, 5], - [2, 3], - ], - ], - ]; - - o(polyContainsBounds(polys, Bounds.fromBbox([-6, -6, -3, -3]))).equals(true); - - o(polyContainsBounds(polys, Bounds.fromBbox([-3, -4, 4, 4]))).equals(false); - o(polyContainsBounds(polys, Bounds.fromBbox([6, -8, 5, 5]))).equals(false); - }); - - o('loadCutline', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - const geojson = round(cutline.toGeoJson()); - - o(geojson).deepEquals({ - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - type: 'MultiPolygon', - coordinates: [ - [ - [ - [19456570.81047118, -5023799.46086743], - [19457346.41188008, -5024885.32147075], - [19455543.1261867, -5026720.78623442], - [19455446.57500747, -5025689.98761053], - [19456325.95579277, -5024324.4264959], - [19456431.41331051, -5023815.14487631], - [19456497.91735541, -5023822.4695137], - [19456513.95059036, -5023820.4859541], - [19456528.99998363, -5023810.66149213], - [19456528.41956381, -5023802.3991344], - [19456521.62606925, -5023792.60909871], - [19456521.12156931, -5023788.06857522], - [19456524.31632738, -5023783.03836819], - [19456531.227264, -5023778.34574544], - [19456540.68563587, -5023777.32428773], - [19456551.09434221, -5023782.4891701], - [19456560.03196148, -5023796.37226942], - [19456570.81047118, -5023799.46086743], - ], - ], - [ - [ - [19430214.06028324, -5008476.23288268], - [19429858.86076585, -5008966.74650194], - [19430484.68848696, -5009778.63111311], - [19430907.54505528, -5009085.14634107], - [19430214.06028324, -5008476.23288268], - ], - ], - ], - }, - properties: {}, - }, - ], - crs: { - type: 'name', - properties: { name: 'urn:ogc:def:crs:EPSG::3857' }, - }, - }); - }); - - o('findCovering', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - const bounds = SourceTiffTestHelper.tiffNztmBounds(); - - o(cutline.clipPoly.length).equals(2); - - (cutline as any).findCovering({ projection: EpsgCode.Nztm2000, bounds, resZoom: 15 }); - - o((cutline as any).srcPoly.length).equals(1); - }); - - o.spec('optimize', async () => { - const bounds = SourceTiffTestHelper.tiffNztmBounds(); - - o('full-extent 3857', () => { - const cutline = new Cutline(GoogleTms); - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Google, - bounds: [{ ...GoogleTms.extent, name: 'gebco' }], - resZoom: 5, - } as SourceMetadata, - 4, - ); - - o(round(covering, 4)).deepEquals([ - { - name: '1-0-0', - x: -20037508.3428, - y: 0, - width: 20037508.3428, - height: 20037508.3428, - }, - { - name: '1-0-1', - x: -20037508.3428, - y: -20037508.3428, - width: 20037508.3428, - height: 20037508.3428, - }, - { - name: '1-1-0', - x: -0, - y: 0, - width: 20037508.3428, - height: 20037508.3428, - }, - { - name: '1-1-1', - x: -0, - y: -20037508.3428, - width: 20037508.3428, - height: 20037508.3428, - }, - ]); - }); - - o('nztm', () => { - const cutline = new Cutline(Nztm2000Tms); - - const covering = cutline.optimizeCovering({ - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 7 + AlignedLevel, - } as SourceMetadata); - - o(covering[4]).deepEquals({ - name: '7-153-253', - x: 1741760, - y: 5448320, - width: 17920, - height: 17920, - }); - o(covering.map((c) => c.name)).deepEquals([ - '7-152-252', - '7-152-253', - '7-152-254', - '7-153-252', - '7-153-253', - '7-153-254', - '7-154-252', - '7-154-253', - '7-154-254', - '7-155-252', - '7-155-253', - '8-387-635', - ]); - }); - - o('source polygon is smoothed', async () => { - const tiff1 = { - name: '/path/to/tiff1.tiff', - x: 1759315.9336568, - y: 5476120.089572778, - width: 2577.636033663526, - height: 2275.3233966259286, - }; - - const tiff2 = { - name: '/path/to/tiff2.tiff', - x: 1759268.0010635862, - y: 5473899.771969615, - width: 2576.894309356343, - height: 2275.3359155077487, - }; - - const bounds = [tiff1, tiff2]; - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 500); - - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 22, - } as SourceMetadata, - 8, - ); - - o(round(cutline.clipPoly, 4)[0][0]).deepEquals([ - [19468576.3516, -4993284.6176], - [19471093.8032, -4993284.6176], - [19471932.5568, -4992600.0637], - [19472095.7462, -4992346.9043], - [19472095.7462, -4987199.9524], - [19471985.5211, -4987199.9524], - [19470978.4379, -4989417.4454], - [19468800.9775, -4991497.5405], - [19468576.3516, -4991797.1824], - [19468576.3516, -4993284.6176], - ]); - - o(covering.map((c) => c.name)).deepEquals([ - '14-16151-10232', - '14-16151-10233', - '14-16152-10231', - '14-16152-10232', - '14-16152-10233', - '18-258444-163695', - ]); - }); - - o('boundary part inland, part coastal', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 500); - const bounds = [ - { - name: 'tiff1', - x: 1759315.9336568, - y: 5476120.089572778, - width: 2577.636033663526, - height: 2275.3233966259286, - }, - ]; - - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 22, - } as SourceMetadata, - 8, - ); - - o(round(cutline.clipPoly, 4)[0][0]).deepEquals([ - [19470011.8243, -4990340.8368], - [19472095.7007, -4990340.8368], - [19472095.7007, -4987199.9524], - [19471985.5211, -4987199.9524], - [19470978.4379, -4989417.4454], - [19470011.8243, -4990340.8368], - ]); - - o(covering.map((c) => c.name)).deepEquals([ - '14-16152-10231', - '15-32304-20464', - '15-32305-20464', - '18-258444-163695', - ]); - }); - - o('low res', () => { - const cutline = new Cutline(GoogleTms); - - const covering = cutline.optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 13, - } as SourceMetadata, - 9, - ); - - o(covering.length).equals(covering.length); - o(covering.map((c) => c.name)).deepEquals(['8-252-159', '8-252-160']); - }); - - o('hi res', () => { - const covering2 = new Cutline(GoogleTms).optimizeCovering( - { - projection: EpsgCode.Nztm2000, - bounds, - resZoom: 18, - } as SourceMetadata, - 8, - ); - - o(covering2.length).equals(covering2.length); - - o(covering2.map((c) => c.name)).deepEquals([ - '10-1008-639', - '10-1008-640', - '10-1009-639', - '10-1009-640', - '11-2020-1279', - '11-2020-1280', - '12-4040-2557', - '12-4040-2562', - ]); - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/projection.loader.test.ts b/packages/cli/src/cog/__tests__/projection.loader.test.ts deleted file mode 100644 index aac709ec3..000000000 --- a/packages/cli/src/cog/__tests__/projection.loader.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import o from 'ospec'; -import { ProjectionLoader } from '../projection.loader.js'; -import sinon from 'sinon'; -import { Nztm2000 } from '@basemaps/geo/build/proj/nztm2000.js'; -import { Epsg } from '@basemaps/geo'; -import { Projection } from '@basemaps/geo'; - -o.spec('ProjectionLoader', () => { - const sandbox = sinon.createSandbox(); - o.afterEach(() => sandbox.restore()); - o('should not load if already loaded', async () => { - const fetchStub = sandbox.stub(ProjectionLoader, '_fetch'); - const res = await ProjectionLoader.load(2193); - o(fetchStub.callCount).equals(0); - o(res.code).equals(2193); - }); - - o('should fetch data from the internet', async () => { - const fetchStub = sandbox - .stub(ProjectionLoader, '_fetch') - .resolves({ ok: true, text: () => Promise.resolve(Nztm2000) } as any); - const res = await ProjectionLoader.load(3790); - - o(fetchStub.callCount).equals(1); - - o(res.code).equals(3790); - o(Epsg.get(3790)).equals(res); - o(Projection.get(3790).epsg).equals(res); - }); -}); diff --git a/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts b/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts deleted file mode 100644 index 371a8ed53..000000000 --- a/packages/cli/src/cog/__tests__/source.tiff.testhelper.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { EpsgCode, NamedBounds, GoogleTms } from '@basemaps/geo'; -import { CogStacJob } from '../cog.stac.job.js'; -import { CogJobJson } from '../types.js'; - -export const SourceTiffTestHelper = { - makeCogJob(): CogStacJob { - return new CogStacJob({ - source: { - files: [] as NamedBounds[], - epsg: EpsgCode.Nztm2000, - gsd: 0.8, - }, - output: { - tileMatrix: GoogleTms.identifier, - gsd: 0.75, - addAlpha: true, - oneCogCovering: false, - }, - } as CogJobJson); - }, - - tiffNztmBounds(path = '/path/to'): NamedBounds[] { - return [ - { - name: path + '/tiff1.tiff', - x: 1732000, - y: 5442000, - width: 24000, - height: 36000, - }, - { - name: path + '/tiff2.tiff', - x: 1756000, - y: 5442000, - width: 24000, - height: 36000, - }, - ]; - }, - - namedBoundsToPaths(bounds: NamedBounds[]): string[] { - return bounds.map(({ name }): string => `/path/to/tiff${name}.tiff`); - }, -}; diff --git a/packages/cli/src/cog/cog.stac.job.ts b/packages/cli/src/cog/cog.stac.job.ts deleted file mode 100644 index 94c5b65f0..000000000 --- a/packages/cli/src/cog/cog.stac.job.ts +++ /dev/null @@ -1,407 +0,0 @@ -import { - Bounds, - Epsg, - Projection, - Stac, - StacCollection, - StacLink, - StacProvider, - TileMatrixSet, - TileMatrixSets, -} from '@basemaps/geo'; -import { - CompositeError, - extractYearRangeFromName, - FileConfig, - FileConfigPath, - fsa, - titleizeImageryName, -} from '@basemaps/shared'; -import { CliInfo } from '@basemaps/shared/build/cli/info.js'; -import { MultiPolygon, toFeatureCollection, toFeatureMultiPolygon } from '@linzjs/geojson'; -import { GdalCogBuilderDefaults, GdalCogBuilderResampling } from '../gdal/gdal.config.js'; -import { ProjectionLoader } from './projection.loader.js'; -import { CogStac, CogStacItem, CogStacItemExtensions, CogStacKeywords } from './stac.js'; -import { - CogBuilderMetadata, - CogJob, - CogJobJson, - CogOutputProperties, - CogSourceProperties, - FeatureCollectionWithCrs, -} from './types.js'; - -export const MaxConcurrencyDefault = 50; - -export interface JobCreationContext { - /** Custom Imagery Name if defined */ - imageryName?: string; - - /** Source config */ - sourceLocation: FileConfig | FileConfigPath; - - /** Output config */ - outputLocation: FileConfig; - - /** Should the imagery be cut to a cutline */ - cutline?: { - href: string; - blend: number; - }; - - tileMatrix: TileMatrixSet; - - override?: { - /** Override job id */ - id?: string; - - /** - * Image quality - * @default GdalCogBuilderDefaults.quality - */ - quality?: number; - - /** - * Number of threads to use for fetches - * @default MaxConcurrencyDefault - */ - concurrency?: number; - - /** - * Override the source projection - */ - projection?: Epsg; - - /** - * Override Default aligned zoom level - */ - alignedLevel?: number; - - /** - * Resampling method - * @Default GdalCogBuilderDefaults.resampling - */ - resampling?: GdalCogBuilderResampling; - }; - - /** - * Should this job be submitted to batch now? - * @default false - */ - batch: boolean; - - /** Should this job ignore source coverage and just produce one big COG for EPSG extent */ - oneCogCovering: boolean; -} - -export interface CogJobCreateParams { - /** unique id for imagery set */ - id: string; - /** name of imagery set */ - imageryName: string; - /** information about the source */ - metadata: CogBuilderMetadata; - /** information about the output */ - ctx: JobCreationContext; - /** the polygon to use to clip the source imagery */ - cutlinePoly: MultiPolygon; - /** do we need an alpha band added */ - addAlpha: boolean; -} - -/** - * Information needed to create cogs - */ -export class CogStacJob implements CogJob { - json: CogJobJson; - private cacheTargetZoom?: { - gsd: number; - zoom: number; - }; - - /** - * Load the job.json - - * @param jobpath where to load the job.json from - */ - static async load(jobpath: string): Promise { - const job = new CogStacJob(await fsa.readJson(jobpath)); - await ProjectionLoader.load(job.source.epsg); - return job; - } - - /** - * Create job.json, collection.json, source.geojson, covering.geojson, cutlint.geojson.gz and - * stac descriptions of the target COGs - */ - static async create({ - id, - imageryName, - metadata, - ctx, - cutlinePoly, - addAlpha, - }: CogJobCreateParams): Promise { - let description: string | undefined; - const providers: StacProvider[] = []; - const links: StacLink[] = [ - { - href: fsa.join(ctx.outputLocation.path, 'collection.json'), - type: 'application/json', - rel: 'self', - }, - { - href: 'job.json', - type: 'application/json', - rel: 'linz.basemaps.job', - }, - ]; - let sourceStac = {} as StacCollection; - - const interval: [string, string][] = []; - try { - const sourceCollectionPath = fsa.join(ctx.sourceLocation.path, 'collection.json'); - sourceStac = await fsa.readJson(sourceCollectionPath); - description = sourceStac.description; - interval.push(...(sourceStac.extent?.temporal?.interval ?? [])); - links.push({ href: sourceCollectionPath, rel: 'sourceImagery', type: 'application/json' }); - if (sourceStac.providers != null) { - for (const p of sourceStac.providers) { - if (p.roles.indexOf('host') === -1) { - if (p.url === 'unknown') { - // LINZ LDS has put unknown in some urls - p.url = undefined; - } - providers.push(p); - } - } - } - } catch (err) { - if (!CompositeError.isCompositeError(err) || err.code !== 404) { - throw err; - } - } - const keywords = sourceStac.keywords ?? CogStacKeywords.slice(); - const license = sourceStac.license ?? Stac.License; - const title = sourceStac.title ?? titleizeImageryName(imageryName); - - if (description == null) { - description = 'No description'; - } - - await ProjectionLoader.load(metadata.projection); - const job = new CogStacJob({ - id, - name: imageryName, - title, - description, - source: { - gsd: metadata.pixelScale, - epsg: metadata.projection, - files: metadata.bounds, - location: ctx.sourceLocation, - }, - output: { - gsd: ctx.tileMatrix.pixelScale(metadata.resZoom), - tileMatrix: ctx.tileMatrix.identifier, - epsg: ctx.tileMatrix.projection.code, - files: metadata.files, - location: ctx.outputLocation, - resampling: ctx.override?.resampling ?? GdalCogBuilderDefaults.resampling, - quality: ctx.override?.quality ?? GdalCogBuilderDefaults.quality, - cutline: ctx.cutline, - addAlpha, - nodata: metadata.nodata, - bounds: metadata.targetBounds, - oneCogCovering: ctx.oneCogCovering, - }, - }); - - const nowStr = new Date().toISOString(); - - const sourceProj = Projection.get(job.source.epsg); - - const bbox = [ - sourceProj.boundsToWgs84BoundingBox( - metadata.bounds.map((a) => Bounds.fromJson(a)).reduce((sum, a) => sum.union(a)), - ), - ]; - - if (interval.length === 0) { - const years = extractYearRangeFromName(imageryName); - if (years == null) throw new Error('Missing date in imagery name: ' + imageryName); - interval.push(years.map((y) => `${y}-01-01T00:00:00Z`) as [string, string]); - } - - if (ctx.cutline) { - links.push({ href: 'cutline.geojson.gz', type: 'application/geo+json+gzip', rel: 'linz.basemaps.cutline' }); - } - - links.push({ href: 'covering.geojson', type: 'application/geo+json', rel: 'linz.basemaps.covering' }); - links.push({ href: 'source.geojson', type: 'application/geo+json', rel: 'linz.basemaps.source' }); - - const temporal = { interval }; - - const jobFile = job.getJobPath(`job.json`); - - const stac: CogStac = { - id, - title, - description, - stac_version: Stac.Version, - stac_extensions: [Stac.BaseMapsExtension], - - extent: { - spatial: { bbox }, - temporal, - }, - - license, - keywords, - - providers, - - summaries: { - gsd: [metadata.pixelScale], - 'proj:epsg': [ctx.tileMatrix.projection.code], - 'linz:output': [ - { - resampling: ctx.override?.resampling ?? GdalCogBuilderDefaults.resampling, - quality: ctx.override?.quality ?? GdalCogBuilderDefaults.quality, - cutlineBlend: ctx.cutline?.blend, - addAlpha, - nodata: metadata.nodata, - }, - ], - 'linz:generated': [ - { - ...CliInfo, - datetime: nowStr, - }, - ], - }, - - links, - }; - - await fsa.writeJson(jobFile, job.json); - - const covering = Projection.get(job.tileMatrix).toGeoJson(metadata.files); - - const roles = ['data']; - const collectionLink = { href: 'collection.json', rel: 'collection' }; - - for (const f of covering.features) { - const { name } = f.properties as { name: string }; - const href = name + '.json'; - links.push({ href, type: 'application/geo+json', rel: 'item' }); - const item: CogStacItem = { - ...f, - id: job.id + '/' + name, - collection: job.id, - stac_version: Stac.Version, - stac_extensions: CogStacItemExtensions, - properties: { - ...f.properties, - datetime: nowStr, - gsd: job.output.gsd, - 'proj:epsg': job.tileMatrix.projection.code, - }, - links: [{ href: job.getJobPath(href), rel: 'self' }, collectionLink], - assets: { - cog: { - href: name + '.tiff', - type: 'image/tiff; application=geotiff; profile=cloud-optimized', - roles, - }, - }, - }; - await fsa.writeJson(job.getJobPath(href), item); - } - - if (ctx.cutline != null) { - const geoJsonCutlineOutput = job.getJobPath(`cutline.geojson.gz`); - await fsa.writeJson(geoJsonCutlineOutput, this.toGeoJson(cutlinePoly, ctx.tileMatrix.projection)); - } - - const geoJsonSourceOutput = job.getJobPath(`source.geojson`); - await fsa.writeJson(geoJsonSourceOutput, Projection.get(job.source.epsg).toGeoJson(metadata.bounds)); - - const geoJsonCoveringOutput = job.getJobPath(`covering.geojson`); - await fsa.writeJson(geoJsonCoveringOutput, covering); - - await fsa.writeJson(job.getJobPath(`collection.json`), stac); - - return job; - } - - /** - * build a FeatureCollection from MultiPolygon - */ - static toGeoJson(poly: MultiPolygon, epsg: Epsg): FeatureCollectionWithCrs { - const feature = toFeatureCollection([toFeatureMultiPolygon(poly)]) as FeatureCollectionWithCrs; - feature.crs = { - type: 'name', - properties: { name: epsg.toUrn() }, - }; - return feature; - } - - constructor(json: CogJobJson) { - this.json = json; - } - - get id(): string { - return this.json.id; - } - - get name(): string { - return this.json.name; - } - - get title(): string { - return this.json.title; - } - - get description(): string | undefined { - return this.json.description; - } - - get source(): CogSourceProperties { - return this.json.source; - } - - get output(): CogOutputProperties { - return this.json.output; - } - - get tileMatrix(): TileMatrixSet { - if (this.json.output.tileMatrix) { - const tileMatrix = TileMatrixSets.find(this.json.output.tileMatrix); - if (tileMatrix == null) throw new Error(`Failed to find TileMatrixSet "${this.json.output.tileMatrix}"`); - return tileMatrix; - } - return TileMatrixSets.get(this.json.output.epsg); - } - - get targetZoom(): number { - const { gsd } = this.source; - if (this.cacheTargetZoom?.gsd !== gsd) { - this.cacheTargetZoom = { gsd, zoom: Projection.getTiffResZoom(this.tileMatrix, gsd) }; - } - return this.cacheTargetZoom.zoom; - } - - /** - * Get a nicely formatted folder path based on the job - * - * @param key optional file key inside of the job folder - */ - getJobPath(key?: string): string { - const parts = [this.tileMatrix.projection.code, this.name, this.id]; - if (key != null) { - parts.push(key); - } - return fsa.join(this.output.location.path, parts.join('/')); - } -} diff --git a/packages/cli/src/cog/cog.ts b/packages/cli/src/cog/cog.ts deleted file mode 100644 index 4e72bf3d7..000000000 --- a/packages/cli/src/cog/cog.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Bounds, Projection, TileMatrixSet } from '@basemaps/geo'; -import { Env, isConfigS3Role, LogType } from '@basemaps/shared'; -import { GdalCogBuilder } from '../gdal/gdal.cog.js'; -import { GdalCommand } from '../gdal/gdal.command.js'; -import { GdalProgressParser } from '../gdal/gdal.progress.js'; -import { CogJob } from './types.js'; -import { AwsCredentials } from '@chunkd/source-aws-v2'; - -/** - * Create a onProgress logger - * - * @param keys additional keys to log - * @param logger base logger to use - */ -export function onProgress(gdal: GdalCommand, keys: Record, logger: LogType): void { - let lastTime = Date.now(); - - gdal.parser = new GdalProgressParser(); - gdal.parser.on('progress', (p: number): void => { - logger.trace({ ...keys, progress: parseFloat(p.toFixed(2)), progressTime: Date.now() - lastTime }, 'Progress'); - lastTime = Date.now(); - }); -} - -/** - * Build a COG for a given collection of tiffs - * - * @param job the job to process - * @param name tile name of cog to generate - * @param vrtLocation Location of the source VRT file - * @param outputTiffPath Path to where the output tiff will be stored - * @param logger Logger to use - * @param execute Whether to actually execute the transformation, - */ -export async function buildCogForName( - job: CogJob, - name: string, - vrtLocation: string, - outputTiffPath: string, - logger: LogType, - execute = false, -): Promise { - const startTime = Date.now(); - - const { targetZoom, tileMatrix } = job; - - const nb = job.output.files.find((nb) => nb.name === name); - - if (nb == null) { - throw new Error("Can't find COG named " + name); - } - - const bounds = Bounds.fromJson(nb); - - const tile = TileMatrixSet.nameToTile(name); - - const blockSize = tileMatrix.tileSize * 2; // FIXME is this blockFactor always 2 - const alignmentLevels = Projection.findAlignmentLevels(tileMatrix, tile, job.source.gsd); - - const cogBuild = new GdalCogBuilder(vrtLocation, outputTiffPath, { - bbox: [bounds.x, bounds.bottom, bounds.right, bounds.y], - tileMatrix, - blockSize, - targetRes: job.output.gsd, - alignmentLevels, - resampling: job.output.resampling, - quality: job.output.quality, - }); - - onProgress(cogBuild.gdal, { name, target: 'tiff' }, logger); - - logger.info( - { - imageSize: Projection.getImagePixelWidth(tileMatrix, tile, targetZoom), - name, - tile, - alignmentLevels, - }, - 'CreateCog', - ); - - const sourceLocation = job.source.location; - // If required assume role - if (isConfigS3Role(sourceLocation)) { - const credentials = AwsCredentials.role( - sourceLocation.roleArn, - sourceLocation.externalId, - Env.getNumber(Env.AwsRoleDurationHours, 8) * 60 * 60, - ); - cogBuild.gdal.setCredentials(credentials); - } - - if (cogBuild.gdal.mount != null) { - for (const file of job.source.files) cogBuild.gdal.mount(file.name); - } - - if (execute) { - await cogBuild.convert(logger.child({ name })); - logger.info({ name, duration: Date.now() - startTime }, 'CogCreated'); - } -} diff --git a/packages/cli/src/cog/cog.vrt.ts b/packages/cli/src/cog/cog.vrt.ts deleted file mode 100644 index ab6ad041b..000000000 --- a/packages/cli/src/cog/cog.vrt.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { Epsg } from '@basemaps/geo'; -import { Env, fsa, isConfigS3Role, LogType, s3ToVsis3 } from '@basemaps/shared'; -import { Gdal } from '../gdal/gdal.js'; -import { GdalCommand } from '../gdal/gdal.command.js'; -import { onProgress } from './cog.js'; -import { Cutline } from './cutline.js'; -import { CogJob } from './types.js'; -import { AwsCredentials } from '@chunkd/source-aws-v2'; - -/** - * Build the VRT for the needed source imagery - */ -async function buildPlainVrt( - job: CogJob, - sourceFiles: string[], - vrtPath: string, - gdalCommand: GdalCommand, - logger: LogType, -): Promise { - const buildOpts = ['-hidenodata', '-allow_projection_difference']; - if (job.output.addAlpha) { - buildOpts.push('-addalpha'); - } - - logger.debug({ buildOpts: buildOpts.join(' ') }, 'gdalbuildvrt'); - await gdalCommand.run('gdalbuildvrt', [...buildOpts, vrtPath, ...sourceFiles], logger); -} - -/** - * Warp the source vrt to target projection using an optional cutline - */ -async function buildWarpVrt( - job: CogJob, - sourceVrtPath: string, - gdalCommand: GdalCommand, - cogVrtPath: string, - tr: string, - logger: LogType, - cutlineTarget: string, -): Promise { - const warpOpts = [ - '-of', - 'VRT', - '-multi', - '-wo', - 'NUM_THREADS=ALL_CPUS', - '-s_srs', - Epsg.get(job.source.epsg).toEpsgString(), - '-t_srs', - job.tileMatrix.projection.toEpsgString(), - '-tr', - tr, - tr, - '-tap', - ]; - if (job.output.cutline != null) { - warpOpts.push('-cutline', cutlineTarget); - if (job.output.cutline.blend !== 0) warpOpts.push('-cblend', String(job.output.cutline.blend)); - } - if (job.output.nodata != null) { - warpOpts.push('-srcnodata', String(job.output.nodata), '-dstnodata', String(job.output.nodata)); - } - if (job.output.resampling) { - warpOpts.push('-r', job.output.resampling.warp); - } - - logger.debug({ warpOpts: warpOpts.join(' ') }, 'gdalwarp'); - await gdalCommand.run('gdalwarp', [...warpOpts, cogVrtPath, sourceVrtPath], logger); -} - -export const CogVrt = { - /** - * Build a vrt file for a COG `name` that transforms the source imagery with a cutline - * - * @param tmpFolder temporary `vrt` and `cutline.geojson` will be written here - * @param job - * @param cutline Used to filter the source imagery - * @param name COG tile to reduce vrt and cutline - * @param logger - * - * @return the path to the vrt file - */ - async buildVrt( - tmpFolder: string, - job: CogJob, - cutline: Cutline, - name: string, - logger: LogType, - ): Promise { - logger.info({ name }, 'buildCogVrt'); - - const sourceFiles = cutline.filterSourcesForName(name, job).map(s3ToVsis3); - - if (sourceFiles.length === 0) { - return null; - } - - const sourceVrtPath = fsa.join(tmpFolder, `source.vrt`); - const cogVrtPath = fsa.join(tmpFolder, `cog.vrt`); - - let cutlineTarget = ''; - - if (cutline.clipPoly.length !== 0) { - cutlineTarget = fsa.join(tmpFolder, 'cutline.geojson'); - await fsa.writeJson(cutlineTarget, cutline.toGeoJson()); - } else { - job.output.cutline = undefined; - } - - logger.info( - { - inputTotal: job.source.files.length, - outputTotal: sourceFiles.length, - cutlinePolygons: cutline.clipPoly.length, - }, - 'Tiff count', - ); - - const gdalCommand = Gdal.create(); - - const sourceLocation = job.source.location; - // If required assume role - if (isConfigS3Role(sourceLocation)) { - const credentials = AwsCredentials.role( - sourceLocation.roleArn, - sourceLocation.externalId, - Env.getNumber(Env.AwsRoleDurationHours, 8) * 60 * 60, - ); - gdalCommand.setCredentials(credentials); - } - - if (gdalCommand.mount != null) { - gdalCommand.mount(tmpFolder); - for (const file of job.source.files) gdalCommand.mount(file.name); - } - - const tr = job.output.gsd.toString(); - - onProgress(gdalCommand, { target: `vrt.${job.tileMatrix.projection.code}` }, logger); - await buildPlainVrt(job, sourceFiles, sourceVrtPath, gdalCommand, logger); - await buildWarpVrt(job, cogVrtPath, gdalCommand, sourceVrtPath, tr, logger, cutlineTarget); - return cogVrtPath; - }, -}; diff --git a/packages/cli/src/gdal/gdal.config.ts b/packages/cli/src/cog/gdal.config.ts similarity index 100% rename from packages/cli/src/gdal/gdal.config.ts rename to packages/cli/src/cog/gdal.config.ts diff --git a/packages/cli/src/cog/job.factory.ts b/packages/cli/src/cog/job.factory.ts deleted file mode 100644 index ee31549ae..000000000 --- a/packages/cli/src/cog/job.factory.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { Bounds } from '@basemaps/geo'; -import { fsa, isConfigS3Role, isFileConfigPath, LogConfig } from '@basemaps/shared'; -import { basename } from 'path'; -import * as ulid from 'ulid'; -import { BatchJob } from '../cli/cogify/batch.job.js'; -import { CogBuilder } from '../index.js'; -import { CogStacJob, JobCreationContext } from './cog.stac.job.js'; -import { Cutline } from './cutline.js'; - -export const MaxConcurrencyDefault = 50; - -export function filterTiff(a: string): boolean { - const lowerA = a.toLowerCase(); - return lowerA.endsWith('.tiff') || lowerA.endsWith('.tif'); -} - -/** Group a file list by its zoom */ -function groupFiles(files: { name: string }[]): Record { - const fileCounts: Record = {}; - for (const f of files) { - const zoom = f.name.split('-')[0]; - fileCounts[zoom] = (fileCounts[zoom] ?? 0) + 1; - } - return fileCounts; -} - -export const CogJobFactory = { - /** - * Create a COG Job and potentially submit it to AWS Batch for processing - */ - async create(ctx: JobCreationContext): Promise { - const id = ctx.override?.id ?? ulid.ulid(); - let imageryName = ctx.imageryName; - if (imageryName == null) imageryName = basename(ctx.sourceLocation.path); - - const logger = LogConfig.get().child({ id, imageryName }); - - const { sourceLocation } = ctx; - logger.info( - { source: ctx.sourceLocation.path, sourceRole: isConfigS3Role(sourceLocation) && sourceLocation.roleArn }, - 'ListTiffs', - ); - - fsa.configure(sourceLocation); - - const tiffList = isFileConfigPath(sourceLocation) - ? sourceLocation.files - : (await fsa.toArray(fsa.list(sourceLocation.path))).filter(filterTiff); - - const tiffSource = tiffList.map((path: string) => fsa.source(path)); - - const maxConcurrency = ctx.override?.concurrency ?? MaxConcurrencyDefault; - - logger.info({ source: sourceLocation.path, tiffCount: tiffList.length }, 'LoadingTiffs'); - - const cutline = new Cutline( - ctx.tileMatrix, - ctx.cutline && (await Cutline.loadCutline(ctx.cutline.href)), - ctx.cutline?.blend, - ); - - const builder = new CogBuilder(ctx.tileMatrix, maxConcurrency, logger, ctx.override?.projection); - const metadata = await builder.build(tiffSource, cutline, ctx.override?.alignedLevel); - - if (cutline.clipPoly.length === 0) { - // no cutline needed for this imagery set - ctx.cutline = undefined; - } - - const files = metadata.files.sort(Bounds.compareArea); - if (files.length > 0) { - const bigArea = files[files.length - 1]; - const smallArea = files[0]; - logger.info( - { - tileMatrix: ctx.tileMatrix.identifier, - // Size of the biggest image - big: bigArea.width / cutline.tileMatrix.pixelScale(metadata.resZoom), - // Size of the smallest image - small: smallArea.width / cutline.tileMatrix.pixelScale(metadata.resZoom), - }, - 'Covers', - ); - } - - // Don't log bounds as it is huge - logger.info( - { - ...metadata, - tileMatrix: ctx.tileMatrix.identifier, - bounds: undefined, - files: undefined, - fileCount: files.length, - fileGroups: groupFiles(metadata.files), - }, - 'CoveringGenerated', - ); - - let addAlpha = true; - // -addalpha to vrt adds extra alpha layers even if one already exist - if (metadata.bands > 3) { - logger.info({ bandCount: metadata.bands }, 'Vrt:DetectedAlpha, Disabling -addalpha'); - addAlpha = false; - } - - const job = await CogStacJob.create({ - id, - imageryName, - metadata, - ctx, - addAlpha, - cutlinePoly: cutline.clipPoly, - }); - - if (ctx.batch) await BatchJob.batchJob(job, true, logger); - logger.info({ tileMatrix: ctx.tileMatrix.identifier, job: job.getJobPath() }, 'Done'); - - return job; - }, -}; diff --git a/packages/cli/src/cog/stac.ts b/packages/cli/src/cog/stac.ts deleted file mode 100644 index 9f1fcc8b6..000000000 --- a/packages/cli/src/cog/stac.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { EpsgCode, StacCollection, StacItem } from '@basemaps/geo'; -import { CogGdalSettings } from './types.js'; - -export interface CogGenerated { - /** ISO date string */ - datetime: string; - /** Package name of the generator */ - package: string; - /* Version of the generator */ - version: string; - /** Commit hash of the generator */ - hash: string | undefined; -} - -export interface CogOutputSummery extends CogGdalSettings { - cutlineBlend?: number; -} - -export interface CogSummaries { - 'proj:epsg': EpsgCode[]; - /** Ground Sampling Distance in meters per pixel of the generated imagery */ - gsd: number[]; - 'linz:output': CogOutputSummery[]; - 'linz:generated': CogGenerated[]; - - /** How and when this job file was generated */ -} - -export const CogStacKeywords = ['Imagery', 'New Zealand']; -export const CogStacItemExtensions = ['projection']; - -/** STAC compliant structure for storing Job instructions */ -export type CogStac = StacCollection; - -export interface CogStacItemProperties { - datetime: string; - gsd: number; - 'proj:epsg': EpsgCode; -} - -export type CogStacItem = StacItem; diff --git a/packages/cli/src/cog/types.ts b/packages/cli/src/cog/types.ts index 35c6778ea..525c02a02 100644 --- a/packages/cli/src/cog/types.ts +++ b/packages/cli/src/cog/types.ts @@ -1,6 +1,6 @@ import { BoundingBox, EpsgCode, NamedBounds, TileMatrixSet } from '@basemaps/geo'; import { FileConfig } from '@basemaps/shared'; -import { GdalCogBuilderResampling } from '../gdal/gdal.config.js'; +import { GdalCogBuilderResampling } from './gdal.config.js'; export interface FeatureCollectionWithCrs extends GeoJSON.FeatureCollection { crs: { diff --git a/packages/cli/src/gdal/__tests__/gdal.progress.test.ts b/packages/cli/src/gdal/__tests__/gdal.progress.test.ts deleted file mode 100644 index 34e4b52a9..000000000 --- a/packages/cli/src/gdal/__tests__/gdal.progress.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import o from 'ospec'; -import { GdalProgressParser } from '../gdal.progress.js'; - -o.spec('GdalProgressParser', () => { - o('should emit on progress', () => { - const prog = new GdalProgressParser(); - o(prog.progress).equals(0); - - prog.data(Buffer.from('\n.')); - o(prog.progress.toFixed(2)).equals('3.23'); - }); - - o('should take 31 dots to finish', () => { - const prog = new GdalProgressParser(); - let processCount = 0; - prog.data(Buffer.from('\n')); - prog.on('progress', () => processCount++); - - for (let i = 0; i < 31; i++) { - prog.data(Buffer.from('.')); - o(processCount).equals(i + 1); - } - o(prog.progress.toFixed(2)).equals('100.00'); - }); -}); diff --git a/packages/cli/src/gdal/__tests__/gdal.test.ts b/packages/cli/src/gdal/__tests__/gdal.test.ts deleted file mode 100644 index 3409815aa..000000000 --- a/packages/cli/src/gdal/__tests__/gdal.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import o from 'ospec'; -import { GdalCogBuilder } from '../gdal.cog.js'; -import { normalizeAwsEnv } from '../gdal.command.js'; -import { GdalCogBuilderDefaults } from '../gdal.config.js'; -import { GdalDocker } from '../gdal.docker.js'; - -o.spec('GdalCogBuilder', () => { - o('should default all options', () => { - const builder = new GdalCogBuilder('/foo', 'bar.tiff'); - - o(builder.config.bbox).equals(undefined); - o(builder.config.compression).equals('webp'); - o(builder.config.resampling).deepEquals({ warp: 'bilinear', overview: 'lanczos' }); - o(builder.config.blockSize).equals(512); - o(builder.config.alignmentLevels).equals(1); - - o(builder.config).deepEquals({ ...GdalCogBuilderDefaults, bbox: undefined }); - }); - - o('should create a docker command', () => { - const builder = new GdalCogBuilder('/foo/foo.tiff', '/bar/bar.tiff'); - - const args = builder.args; - - o(args.includes('TILING_SCHEME=GoogleMapsCompatible')).equals(true); - o(args.includes('COMPRESS=webp')).equals(true); - o(builder.args.includes('BLOCKSIZE=512')).equals(true); - - builder.config.compression = 'jpeg'; - o(builder.args.includes('COMPRESS=jpeg')).equals(true); - - builder.config.blockSize = 256; - o(builder.args.includes('BLOCKSIZE=256')).equals(true); - }); - - o('should mount folders', () => { - const gdalDocker = new GdalDocker(); - gdalDocker.mount('/foo/bar'); - o(gdalDocker.mounts).deepEquals(['/foo']); - }); - - o('should not duplicate folders', () => { - const gdalDocker = new GdalDocker(); - gdalDocker.mount('/foo/bar.html'); - gdalDocker.mount('/foo/bar.tiff'); - o(gdalDocker.mounts).deepEquals(['/foo']); - }); - - o('should ignore s3 uris', () => { - const gdalDocker = new GdalDocker(); - gdalDocker.mount('s3://foo/bar.html'); - gdalDocker.mount('s3://foo/bar.tiff'); - o(gdalDocker.mounts).deepEquals([]); - }); - - o('should normalize aws environment vars', () => { - const env = normalizeAwsEnv({ AWS_PROFILE: 'lake' }); - o(env).deepEquals({ AWS_PROFILE: 'lake', AWS_DEFAULT_PROFILE: 'lake' }); - }); - - o('should error if aws config is not sane', () => { - o(() => normalizeAwsEnv({ AWS_PROFILE: 'lake', AWS_DEFAULT_PROFILE: 'other-lake' })).throws(Error); - }); -}); diff --git a/packages/cli/src/gdal/gdal.cog.ts b/packages/cli/src/gdal/gdal.cog.ts deleted file mode 100644 index f20584cb7..000000000 --- a/packages/cli/src/gdal/gdal.cog.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { GoogleTms, Nztm2000Tms } from '@basemaps/geo'; -import { LogType } from '@basemaps/shared'; -import { ChildProcessWithoutNullStreams } from 'child_process'; -import { Gdal } from './gdal.js'; -import { GdalCommand } from './gdal.command.js'; -import { GdalCogBuilderDefaults, GdalCogBuilderOptions } from './gdal.config.js'; - -/** 1% Buffer to the tiff to help prevent gaps between tiles */ -// const TiffBuffer = 1.01; - -/** - * A docker based GDAL Cog Builder - * - * This uses the new 3.1 COG Driver https://gdal.org/drivers/raster/cog.html - * - * When GDAL 3.1 is released docker could be removed from this process. - */ -export class GdalCogBuilder { - config: GdalCogBuilderOptions; - - /** - * Source file generally a .vrt - */ - source: string; - /** - * Output file - */ - target: string; - - /** - * Current running child process - */ - child: ChildProcessWithoutNullStreams | null; - /** - * Promise waiting for child process to finish - */ - promise: Promise | null; - /** When the process started */ - startTime: number; - /** Gdal process */ - gdal: GdalCommand; - - constructor(source: string, target: string, config: Partial = {}) { - this.source = source; - this.target = target; - - this.config = { - bbox: config.bbox, - alignmentLevels: config.alignmentLevels ?? GdalCogBuilderDefaults.alignmentLevels, - compression: config.compression ?? GdalCogBuilderDefaults.compression, - tileMatrix: config.tileMatrix ?? GdalCogBuilderDefaults.tileMatrix, - resampling: config.resampling ?? GdalCogBuilderDefaults.resampling, - blockSize: config.blockSize ?? GdalCogBuilderDefaults.blockSize, - targetRes: config.targetRes ?? GdalCogBuilderDefaults.targetRes, - quality: config.quality ?? GdalCogBuilderDefaults.quality, - }; - this.gdal = Gdal.create(); - this.gdal.mount?.(source); - this.gdal.mount?.(target); - } - - getBounds(): string[] { - if (this.config.bbox == null) { - return []; - } - - // TODO in theory this should be clamped to the lower right of the imagery, as there is no point generating large empty tiffs - const [ulX, ulY, lrX, lrY] = this.config.bbox; - return ['-projwin', ulX, ulY, lrX, lrY, '-projwin_srs', this.config.tileMatrix.projection.toEpsgString()].map( - String, - ); - } - - get tileMatrixFileName(): string { - const tileMatrix = this.config.tileMatrix; - // Gdal built in TileMatrixSets - if (tileMatrix.identifier === GoogleTms.identifier) return 'GoogleMapsCompatible'; - if (tileMatrix.identifier === Nztm2000Tms.identifier) return 'NZTM2000'; - - return 'https://raw.githubusercontent.com/linz/NZTM2000TileMatrixSet/master/raw/NZTM2000Quad.json'; - } - - get args(): string[] { - const tr = this.config.targetRes.toString(); - return [ - // Force output using COG Driver - '-of', - 'COG', - // Force GoogleMaps tiling - '-co', - `TILING_SCHEME=${this.tileMatrixFileName}`, - // Max CPU POWER - '-co', - 'NUM_THREADS=ALL_CPUS', - // in GDAL 3.7.x NUM_THREADS will also set GDAL_NUM_THREADS - '--config', - 'GDAL_NUM_THREADS', - 'ALL_CPUS', - // Force big tiff the extra few bytes savings of using little tiffs does not affect us - '-co', - 'BIGTIFF=YES', - // Force a alpha layer - '-co', - 'ADD_ALPHA=YES', - // User configured output block size - '-co', - `BLOCKSIZE=${this.config.blockSize}`, - // Configured resampling methods - '-co', - `WARP_RESAMPLING=${this.config.resampling.warp}`, - '-co', - `OVERVIEW_RESAMPLING=${this.config.resampling.overview}`, - // User configured compression - '-co', - `COMPRESS=${this.config.compression}`, - // Number of levels to align to web mercator - '-co', - `ALIGNED_LEVELS=${this.config.alignmentLevels}`, - // Default quality of 75 is too low for our needs - '-co', - `QUALITY=${this.config.quality}`, - // most of the imagery contains a lot of empty tiles, no need to output them - '-co', - `SPARSE_OK=YES`, - // Do not attempt to read sidecar files - '--config', - `GDAL_DISABLE_READDIR_ON_OPEN`, - `EMPTY_DIR`, - // Force a target resolution to be better than the imagery not worse - '-tr', - tr, - tr, - ...this.getBounds(), - - this.source, - this.target, - ]; - } - - async convert(log: LogType): Promise { - await this.gdal.run('gdal_translate', this.args, log); - } -} diff --git a/packages/cli/src/gdal/gdal.command.ts b/packages/cli/src/gdal/gdal.command.ts deleted file mode 100644 index 18c30fc7a..000000000 --- a/packages/cli/src/gdal/gdal.command.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { LogType } from '@basemaps/shared'; -import { ChildProcessWithoutNullStreams, spawn } from 'child_process'; -import { GdalProgressParser } from './gdal.progress.js'; - -/** - * GDAL uses AWS_DEFAULT_PROFILE while node uses AWS_PROFILE - * this validates the configuration is sane - * - * @param env environment to normalize - */ -export function normalizeAwsEnv(env: Record): Record { - const awsProfile = env['AWS_PROFILE']; - const awsDefaultProfile = env['AWS_DEFAULT_PROFILE']; - - if (awsProfile == null) return env; - if (awsDefaultProfile == null) { - return { ...env, AWS_DEFAULT_PROFILE: awsProfile }; - } - if (awsDefaultProfile !== awsProfile) { - throw new Error(`$AWS_PROFILE: "${awsProfile}" and $AWS_DEFAULT_PROFILE: "${awsDefaultProfile}" are mismatched`); - } - return env; -} - -export interface GdalCredentials { - needsRefresh(): boolean; - refreshPromise(): Promise; - accessKeyId: string; - secretAccessKey: string; - sessionToken: string; -} - -export abstract class GdalCommand { - parser?: GdalProgressParser; - protected child: ChildProcessWithoutNullStreams; - protected promise?: Promise<{ stdout: string; stderr: string }>; - protected startTime: number; - - /** AWS Access */ - protected credentials?: GdalCredentials; - - mount?(mount: string): void; - env?(): Promise>; - - /** Pass AWS credentials into the container */ - setCredentials(credentials?: GdalCredentials): void { - this.credentials = credentials; - } - - /** - * Run a GDAL command - * @param cmd command to run eg "gdal_translate" - * @param args command arguments - * @param log logger to use - */ - async run(cmd: string, args: string[], log: LogType): Promise<{ stdout: string; stderr: string }> { - if (this.promise != null) throw new Error('Cannot create multiple gdal processes, create a new GdalCommand'); - this.parser?.reset(); - this.startTime = Date.now(); - - const env = normalizeAwsEnv(this.env ? await this.env() : process.env); - - const child = spawn(cmd, args, { env }); - this.child = child; - - const outputBuff: Buffer[] = []; - const errBuff: Buffer[] = []; - child.stderr.on('data', (data: Buffer) => { - const buf = data.toString(); - /** - * Example error line - * `ERROR 1: TIFFReadEncodedTile:Read error at row 4294967295, col 4294967295; got 49005 bytes, expected 49152` - */ - if (buf.includes('ERROR 1')) { - log.error({ data: buf }, 'GdalError'); - } else { - log.warn({ data: buf }, 'GdalWarn'); - } - errBuff.push(data); - }); - - child.stdout.on('data', (data: Buffer) => { - outputBuff.push(data); - this.parser?.data(data); - }); - - this.promise = new Promise((resolve, reject) => { - child.on('exit', (code: number) => { - const stdout = outputBuff.join('').trim(); - const stderr = errBuff.join('').trim(); - const duration = Date.now() - this.startTime; - - if (code !== 0) { - log.error({ code, stdout, stderr, duration }, 'GdalFailed'); - return reject(new Error('Failed to execute GDAL command')); - } - log.trace({ stdout, stderr, duration }, 'GdalDone'); - - this.promise = undefined; - return resolve({ stdout, stderr }); - }); - - child.on('error', (error: Error) => { - const stdout = outputBuff.join('').trim(); - const stderr = errBuff.join('').trim(); - const duration = Date.now() - this.startTime; - - log.error({ stdout, stderr, duration }, 'GdalFailed'); - this.promise = undefined; - reject(error); - }); - }); - - return this.promise; - } -} diff --git a/packages/cli/src/gdal/gdal.docker.ts b/packages/cli/src/gdal/gdal.docker.ts deleted file mode 100644 index 9da2c234d..000000000 --- a/packages/cli/src/gdal/gdal.docker.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { Env, LogType } from '@basemaps/shared'; -import * as os from 'os'; -import * as path from 'path'; -import { GdalCommand } from './gdal.command.js'; - -export class GdalDocker extends GdalCommand { - mounts: string[]; - - constructor() { - super(); - this.mounts = []; - } - - mount(filePath: string): void { - if (filePath.startsWith('s3://')) return; - - const basePath = path.dirname(filePath); - if (this.mounts.includes(basePath)) return; - this.mounts.push(basePath); - } - - private getMounts(): string[] { - if (this.mounts.length === 0) { - return []; - } - const output: string[] = []; - for (const mount of this.mounts) { - output.push('-v'); - output.push(`${mount}:${mount}`); - } - return output; - } - - private async getCredentials(): Promise { - if (this.credentials == null) { - return []; - } - if (this.credentials.needsRefresh()) { - await this.credentials.refreshPromise(); - } - return [ - '--env', - `AWS_ACCESS_KEY_ID=${this.credentials.accessKeyId}`, - '--env', - `AWS_SECRET_ACCESS_KEY=${this.credentials.secretAccessKey}`, - '--env', - `AWS_SESSION_TOKEN=${this.credentials.sessionToken}`, - ]; - } - - /** this could contain sensitive info like AWS access keys */ - private async getDockerArgs(): Promise { - const DOCKER_CONTAINER = Env.get(Env.Gdal.DockerContainer) ?? 'ghcr.io/osgeo/gdal'; - const DOCKER_CONTAINER_TAG = Env.get(Env.Gdal.DockerContainerTag) ?? 'ubuntu-small-3.7.0'; - const userInfo = os.userInfo(); - const credentials = await this.getCredentials(); - return [ - 'run', - // Config the container to be run as the current user - '--user', - `${userInfo.uid}:${userInfo.gid}`, - - ...this.getMounts(), - ...credentials, - - // Docker container - '-i', - `${DOCKER_CONTAINER}:${DOCKER_CONTAINER_TAG}`, - ]; - } - - /** Provide redacted argument string for logging which removes sensitive information */ - maskArgs(args: string[]): string[] { - const cred = this.credentials; - if (cred == null) return args; - - return args.map((c) => c.replace(cred.secretAccessKey, '****').replace(cred.sessionToken, '****')); - } - - async run(cmd: string, args: string[], log: LogType): Promise<{ stdout: string; stderr: string }> { - const dockerArgs = await this.getDockerArgs(); - log.debug( - { - mounts: this.mounts, - cmd, - docker: this.maskArgs(dockerArgs).join(' '), - gdalArgs: args.slice(0, 50).join(' '), - }, - 'StartGdal:Docker', - ); - return super.run('docker', [...dockerArgs, cmd, ...args], log); - } -} diff --git a/packages/cli/src/gdal/gdal.local.ts b/packages/cli/src/gdal/gdal.local.ts deleted file mode 100644 index 8a54b5a67..000000000 --- a/packages/cli/src/gdal/gdal.local.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { LogType } from '@basemaps/shared'; -import { GdalCommand } from './gdal.command.js'; - -export class GdalLocal extends GdalCommand { - async env(): Promise> { - if (this.credentials == null) { - return process.env; - } - if (this.credentials.needsRefresh()) { - await this.credentials.refreshPromise(); - } - return { - ...process.env, - AWS_ACCESS_KEY_ID: this.credentials.accessKeyId, - AWS_SECRET_ACCESS_KEY: this.credentials.secretAccessKey, - AWS_SESSION_TOKEN: this.credentials.sessionToken, - }; - } - - async run(cmd: string, args: string[], log: LogType): Promise<{ stdout: string; stderr: string }> { - log.debug({ cmd, gdalArgs: args.slice(0, 50).join(' ') }, 'StartGdal:Local'); - return super.run(cmd, args, log); - } -} diff --git a/packages/cli/src/gdal/gdal.progress.ts b/packages/cli/src/gdal/gdal.progress.ts deleted file mode 100644 index 117e8efe8..000000000 --- a/packages/cli/src/gdal/gdal.progress.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { EventEmitter } from 'events'; - -/** - * Emit a "progress" event every time a "." is recorded in the output - */ -export class GdalProgressParser extends EventEmitter { - // Progress starts with "Input file size is .., ..\n" - waitNewLine = true; - dotCount = 0; - byteCount = 0; - - /** Reset the progress counter */ - reset(): void { - this.waitNewLine = true; - this.dotCount = 0; - this.byteCount = 0; - } - - get progress(): number { - return this.dotCount * (100 / 31); - } - - data(data: Buffer): void { - const str = data.toString('utf8'); - this.byteCount += str.length; - // In theory only a small amount of output bytes should be recorded - if (this.byteCount > 1024) { - throw new Error('Too much data: ' + str); - } - if (str === '0') { - this.waitNewLine = false; - return; - } - - if (this.waitNewLine) { - const newLine = str.indexOf('\n'); - if (newLine > -1) { - this.waitNewLine = false; - return this.data(Buffer.from(str.substr(newLine + 1))); - } - return; - } - - const bytes = str.split(''); - for (const byte of bytes) { - if (byte === '.') { - this.dotCount++; - this.emit('progress', this.progress); - } - } - } -} diff --git a/packages/cli/src/gdal/gdal.ts b/packages/cli/src/gdal/gdal.ts deleted file mode 100644 index f0bfffa68..000000000 --- a/packages/cli/src/gdal/gdal.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { Env, LogType } from '@basemaps/shared'; -import { GdalCommand } from './gdal.command.js'; -import { GdalDocker } from './gdal.docker.js'; -import { GdalLocal } from './gdal.local.js'; - -export class Gdal { - /** - * Create a new GdalCommand instance ready to run commands - * - * This could be a local or docker container depending on environment variables - * @see Env.Gdal.UseDocker - */ - static create(): GdalCommand { - if (Env.get(Env.Gdal.UseDocker)) return new GdalDocker(); - return new GdalLocal(); - } - - /** - * Run a `gdal_translate --version` to extract the current gdal version - * - * @example "GDAL 2.4.2, released 2019/06/28" - * @example "GDAL 3.2.0dev-69b0c4ec4174fde36c609a4aac6f4281424021b3, released 2020/06/26" - */ - static async version(logger: LogType): Promise { - const gdal = Gdal.create(); - const { stdout } = await gdal.run('gdal_translate', ['--version'], logger); - return stdout; - } -} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts deleted file mode 100644 index 7ebe65037..000000000 --- a/packages/cli/src/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -// export * from './cli/basemaps/tileset.updater'; -export { CogBuilder } from './cog/builder.js'; -export { CogJobFactory } from './cog/job.factory.js'; -export { Gdal } from './gdal/gdal.js'; -export { GdalCogBuilder } from './gdal/gdal.cog.js'; -export { GdalCogBuilderOptions } from './gdal/gdal.config.js'; diff --git a/yarn.lock b/yarn.lock index 145290e20..ee229d95f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1449,13 +1449,6 @@ dependencies: "@octokit/types" "^6.0.3" -"@octokit/auth-token@^3.0.0": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-3.0.1.tgz#88bc2baf5d706cb258474e722a720a8365dff2ec" - integrity sha512-/USkK4cioY209wXRpund6HZzHo9GmjakpV9ycOkpMcMxMk7QVcVFVyCMtzvXYiHsB2crgDgrtNYSELYFBXhhaA== - dependencies: - "@octokit/types" "^7.0.0" - "@octokit/core@^3.2.3": version "3.4.0" resolved "https://registry.yarnpkg.com/@octokit/core/-/core-3.4.0.tgz#b48aa27d755b339fe7550548b340dcc2b513b742" @@ -1469,19 +1462,6 @@ before-after-hook "^2.2.0" universal-user-agent "^6.0.0" -"@octokit/core@^4.0.5": - version "4.0.5" - resolved "https://registry.yarnpkg.com/@octokit/core/-/core-4.0.5.tgz#589e68c0a35d2afdcd41dafceab072c2fbc6ab5f" - integrity sha512-4R3HeHTYVHCfzSAi0C6pbGXV8UDI5Rk+k3G7kLVNckswN9mvpOzW9oENfjfH3nEmzg8y3AmKmzs8Sg6pLCeOCA== - dependencies: - "@octokit/auth-token" "^3.0.0" - "@octokit/graphql" "^5.0.0" - "@octokit/request" "^6.0.0" - "@octokit/request-error" "^3.0.0" - "@octokit/types" "^7.0.0" - before-after-hook "^2.2.0" - universal-user-agent "^6.0.0" - "@octokit/endpoint@^6.0.1": version "6.0.11" resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-6.0.11.tgz#082adc2aebca6dcefa1fb383f5efb3ed081949d1" @@ -1491,15 +1471,6 @@ is-plain-object "^5.0.0" universal-user-agent "^6.0.0" -"@octokit/endpoint@^7.0.0": - version "7.0.2" - resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-7.0.2.tgz#11ee868406ba7bb1642e61bbe676d641f79f02be" - integrity sha512-8/AUACfE9vpRpehE6ZLfEtzkibe5nfsSwFZVMsG8qabqRt1M81qZYUFRZa1B8w8lP6cdfDJfRq9HWS+MbmR7tw== - dependencies: - "@octokit/types" "^7.0.0" - is-plain-object "^5.0.0" - universal-user-agent "^6.0.0" - "@octokit/graphql@^4.5.8": version "4.6.1" resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-4.6.1.tgz#f975486a46c94b7dbe58a0ca751935edc7e32cc9" @@ -1509,20 +1480,6 @@ "@octokit/types" "^6.0.3" universal-user-agent "^6.0.0" -"@octokit/graphql@^5.0.0": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-5.0.1.tgz#a06982514ad131fb6fbb9da968653b2233fade9b" - integrity sha512-sxmnewSwAixkP1TrLdE6yRG53eEhHhDTYUykUwdV9x8f91WcbhunIHk9x1PZLALdBZKRPUO2HRcm4kezZ79HoA== - dependencies: - "@octokit/request" "^6.0.0" - "@octokit/types" "^7.0.0" - universal-user-agent "^6.0.0" - -"@octokit/openapi-types@^13.11.0": - version "13.12.0" - resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-13.12.0.tgz#cd49f28127ee06ee3edc6f2b5f5648c7332f6014" - integrity sha512-1QYzZrwnn3rTQE7ZoSxXrO8lhu0aIbac1c+qIPOPEaVXBWSaUyLV1x9yt4uDQOwmu6u5ywVS8OJgs+ErDLf6vQ== - "@octokit/openapi-types@^6.0.0": version "6.0.0" resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-6.0.0.tgz#7da8d7d5a72d3282c1a3ff9f951c8133a707480d" @@ -1562,15 +1519,6 @@ deprecation "^2.0.0" once "^1.4.0" -"@octokit/request-error@^3.0.0": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-3.0.1.tgz#3fd747913c06ab2195e52004a521889dadb4b295" - integrity sha512-ym4Bp0HTP7F3VFssV88WD1ZyCIRoE8H35pXSKwLeMizcdZAYc/t6N9X9Yr9n6t3aG9IH75XDnZ6UeZph0vHMWQ== - dependencies: - "@octokit/types" "^7.0.0" - deprecation "^2.0.0" - once "^1.4.0" - "@octokit/request@^5.3.0", "@octokit/request@^5.4.12": version "5.4.15" resolved "https://registry.yarnpkg.com/@octokit/request/-/request-5.4.15.tgz#829da413dc7dd3aa5e2cdbb1c7d0ebe1f146a128" @@ -1583,18 +1531,6 @@ node-fetch "^2.6.1" universal-user-agent "^6.0.0" -"@octokit/request@^6.0.0": - version "6.2.1" - resolved "https://registry.yarnpkg.com/@octokit/request/-/request-6.2.1.tgz#3ceeb22dab09a29595d96594b6720fc14495cf4e" - integrity sha512-gYKRCia3cpajRzDSU+3pt1q2OcuC6PK8PmFIyxZDWCzRXRSIBH8jXjFJ8ZceoygBIm0KsEUg4x1+XcYBz7dHPQ== - dependencies: - "@octokit/endpoint" "^7.0.0" - "@octokit/request-error" "^3.0.0" - "@octokit/types" "^7.0.0" - is-plain-object "^5.0.0" - node-fetch "^2.6.7" - universal-user-agent "^6.0.0" - "@octokit/rest@^18.1.0": version "18.5.2" resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.5.2.tgz#0369e554b7076e3749005147be94c661c7a5a74b" @@ -1612,13 +1548,6 @@ dependencies: "@octokit/openapi-types" "^6.0.0" -"@octokit/types@^7.0.0": - version "7.5.0" - resolved "https://registry.yarnpkg.com/@octokit/types/-/types-7.5.0.tgz#85646021bd618467b7cc465d9734b3f2878c9fae" - integrity sha512-aHm+olfIZjQpzoODpl+RCZzchKOrdSLJs+yfI7pMMcmB19Li6vidgx0DwUDO/Ic4Q3fq/lOjJORVCcLZefcrJw== - dependencies: - "@octokit/openapi-types" "^13.11.0" - "@popperjs/core@^2.9.0": version "2.11.6" resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.6.tgz#cee20bd55e68a1720bdab363ecf0c821ded4cd45" @@ -4375,11 +4304,6 @@ get-port@^5.1.1: resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193" integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ== -get-port@^6.1.2: - version "6.1.2" - resolved "https://registry.yarnpkg.com/get-port/-/get-port-6.1.2.tgz#c1228abb67ba0e17fb346da33b15187833b9c08a" - integrity sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw== - get-stdin@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-4.0.1.tgz#b968c6b0a04384324902e8bf1a5df32579a450fe" @@ -5954,7 +5878,7 @@ node-domexception@^1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@^2.6.1: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==