diff --git a/packages/bathymetry/src/bathy.maker.ts b/packages/bathymetry/src/bathy.maker.ts index 129e59549..ea6da17ee 100644 --- a/packages/bathymetry/src/bathy.maker.ts +++ b/packages/bathymetry/src/bathy.maker.ts @@ -1,4 +1,4 @@ -import { Gdal } from '@basemaps/cli'; +import { Gdal } from '@basemaps/cli/build/gdal/gdal.js'; import { GdalCommand } from '@basemaps/cli/build/gdal/gdal.command.js'; import { Bounds, Epsg, Tile, TileMatrixSet } from '@basemaps/geo'; import { fsa, LogType, s3ToVsis3 } from '@basemaps/shared'; diff --git a/packages/cli/package.json b/packages/cli/package.json index e9135addd..338dd5d40 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -43,7 +43,6 @@ "@basemaps/geo": "^6.44.0", "@basemaps/server": "^6.46.0", "@basemaps/shared": "^6.46.0", - "@basemaps/sprites": "^6.41.0", "@basemaps/tiler": "^6.44.0", "@basemaps/tiler-sharp": "^6.45.0", "@chunkd/fs": "^10.0.2", @@ -52,12 +51,10 @@ "@cotar/core": "^5.4.0", "@cotar/tar": "^5.4.1", "@linzjs/geojson": "^6.43.0", - "@octokit/core": "^4.0.5", "@rushstack/ts-command-line": "^4.3.13", "ansi-colors": "^4.1.1", "deep-diff": "^1.0.2", "flatgeobuf": "^3.23.1", - "get-port": "^6.1.2", "node-fetch": "^3.2.3", "p-limit": "^4.0.0", "pretty-json-log": "^1.0.0", @@ -66,7 +63,6 @@ }, "devDependencies": { "@types/deep-diff": "^1.0.1", - "@types/proj4": "^2.5.2", "@types/sharp": "^0.31.0" }, "publishConfig": { diff --git a/packages/cli/src/cli/base.cli.ts b/packages/cli/src/cli/base.cli.ts deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/cli/src/cli/cogify/__tests__/batch.job.test.ts b/packages/cli/src/cli/cogify/__tests__/batch.job.test.ts deleted file mode 100644 index 347ad4fa4..000000000 --- a/packages/cli/src/cli/cogify/__tests__/batch.job.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { LogConfig } from '@basemaps/shared'; -import o from 'ospec'; -import { CogJob } from '../../../cog/types.js'; -import { BatchJob, extractResolutionFromName } from '../batch.job.js'; - -o.spec('action.batch', () => { - o('extractResolutionFromName', () => { - o(extractResolutionFromName('2013')).equals(-1); - o(extractResolutionFromName('new_zealand_sentinel_2018-19_10m')).equals(10000); - o(extractResolutionFromName('abc2017def_1.00m')).equals(1000); - o(extractResolutionFromName('wellington_urban_2017_0.10m')).equals(100); - o(extractResolutionFromName('wellington_urban_2017_0-10m')).equals(100); - o(extractResolutionFromName('wellington_urban_2017_1.00m')).equals(1000); - o(extractResolutionFromName('wellington_urban_2017_0.025m')).equals(25); - }); - - o('should create valid jobNames', () => { - const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' } as CogJob; - o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-1x-0'); - - fakeJob.name = 'ōtorohanga_urban_2021_0.1m_RGB'; - o(BatchJob.id(fakeJob, ['0'])).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-1x-0'); - - o(BatchJob.id(fakeJob)).equals('01FHRPYJ5FV1XAARZAC4T4K6MC-5294acface81c107-'); - }); - - o('should truncate job names to 128 characters', () => { - const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', name: 'geographx_nz_texture_shade_2012_8-0m' } as CogJob; - - o( - BatchJob.id(fakeJob, [ - 'this is a really long file name', - 'it should over flow 128 characters', - 'so it should be truncated at some point.tiff', - ]), - ).equals( - '01FHRPYJ5FV1XAARZAC4T4K6MC-9af5e139bbb3e502-3x-it should over flow 128 characters_so it should be truncated at some point.tiff_t', - ); - }); - - o.spec('ChunkJobs', () => { - const fakeGsd = 0.9; - const ChunkJobSmall = 4097; - const ChunkJobMiddle = 8193; - const ChunkJobLarge = 16385; - const fakeFiles = [ - { name: '1-2-0', width: ChunkJobSmall * fakeGsd - 1 }, // Small Job 20 - { name: '1-2-1', width: ChunkJobSmall * fakeGsd - 1 }, // Small Job 40 - { name: '1-2-2', width: ChunkJobLarge * fakeGsd + 1 }, // Single Job - { name: '1-2-3', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 90 - { name: '1-2-4', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 140 - { name: '1-2-5', width: ChunkJobLarge * fakeGsd + 0.1 }, // Single Job - { name: '1-2-6', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 190 - { name: '1-2-7', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 390 - { name: '1-2-8', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 590 - { name: '1-2-9', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 790 - { name: '1-2-10', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 990 - { name: '1-2-11', width: ChunkJobLarge * fakeGsd - 1 }, // Large Job 1090 - { name: '1-2-12', width: ChunkJobSmall * fakeGsd - 1 }, // Small Job 20 - { name: '1-2-13', width: ChunkJobLarge * fakeGsd + 1 }, // Single Job - { name: '1-2-14', width: ChunkJobMiddle * fakeGsd - 1 }, // Middle Job 70 - ]; - - const fakeJob = { id: '01FHRPYJ5FV1XAARZAC4T4K6MC', output: { files: fakeFiles, gsd: fakeGsd } } as CogJob; - o('should prepare valid chunk jobs', async () => { - o(BatchJob.getJobs(fakeJob, new Set(), LogConfig.get())).deepEquals([ - [fakeFiles[0].name, fakeFiles[1].name, fakeFiles[2].name], - [fakeFiles[3].name, fakeFiles[4].name, fakeFiles[5].name], - [ - fakeFiles[6].name, - fakeFiles[7].name, - fakeFiles[8].name, - fakeFiles[9].name, - fakeFiles[10].name, - fakeFiles[11].name, - fakeFiles[12].name, - fakeFiles[13].name, - ], - [fakeFiles[14].name], - ]); - }); - - o('should skip tiffs that exist', () => { - const existing = new Set([ - `${fakeFiles[0].name}.tiff`, - `${fakeFiles[1].name}.tiff`, - `${fakeFiles[13].name}.tiff`, - ]); - o(BatchJob.getJobs(fakeJob, existing, LogConfig.get())).deepEquals([ - [fakeFiles[2].name], // First single Job - [fakeFiles[3].name, fakeFiles[4].name, fakeFiles[5].name], // Second single Job - [ - fakeFiles[6].name, - fakeFiles[7].name, - fakeFiles[8].name, - fakeFiles[9].name, - fakeFiles[10].name, - fakeFiles[11].name, - fakeFiles[12].name, - fakeFiles[14].name, - ], - ]); - }); - }); -}); diff --git a/packages/cli/src/cli/cogify/__tests__/semver.test.ts b/packages/cli/src/cli/cogify/__tests__/semver.test.ts deleted file mode 100644 index 3670c8d71..000000000 --- a/packages/cli/src/cli/cogify/__tests__/semver.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -import o from 'ospec'; -import { SemVer } from '../semver.util.js'; -o.spec('VersionCompare', () => { - o.spec('toNumber', () => { - o('should make correct numbers', () => { - o(SemVer.toNumber('0.0.1')).equals(1); - o(SemVer.toNumber('0.1.0')).equals(1024); - o(SemVer.toNumber('0.1.1')).equals(1025); - o(SemVer.toNumber('0.1.1023')).equals(2047); - o(SemVer.toNumber('1.0.0')).equals(1048576); - }); - - o('should not overflow', () => { - o(SemVer.toNumber('1023.1023.1023') > 0).equals(true); - }); - - o('should support up to three dots', () => { - o(SemVer.toNumber('')).equals(0); - o(SemVer.toNumber('1')).equals(1048576); - o(SemVer.toNumber('1.0')).equals(1048576); - o(SemVer.toNumber('1.0.0')).equals(1048576); - o(SemVer.toNumber('2')).equals(2097152); - o(SemVer.toNumber('')).equals(0); - }); - - o('should be getting bigger', () => { - o(SemVer.toNumber('1.0.0') > SemVer.toNumber('0.0.0')).equals(true); - o(SemVer.toNumber('1.0.1') > SemVer.toNumber('1.0.0')).equals(true); - o(SemVer.toNumber('1.1.1') > SemVer.toNumber('1.1.0')).equals(true); - o(SemVer.toNumber('2.1.1') > SemVer.toNumber('1.1.0')).equals(true); - }); - - o('should support release candidates', () => { - o(SemVer.toNumber('0.0.1-rc1')).equals(1); - o(SemVer.toNumber('1.0.0-rc2')).equals(SemVer.toNumber('1.0.0')); - }); - }); - - o.spec('fromNumber', () => { - o('should create versions', () => { - o(SemVer.fromNumber(1)).equals('0.0.1'); - o(SemVer.fromNumber(1023)).equals('0.0.1023'); - o(SemVer.fromNumber(1024)).equals('0.1.0'); - o(SemVer.fromNumber(1025)).equals('0.1.1'); - o(SemVer.fromNumber(1048576)).equals('1.0.0'); - o(SemVer.fromNumber(2097152)).equals('2.0.0'); - }); - - o('should round trip', () => { - for (const num of ['0.1.0', '1.2.3', '3.2.1']) { - o(SemVer.fromNumber(SemVer.toNumber(num))).equals(num)(`Should roundtrip ${num}`); - } - }); - }); - - o.spec('compare', () => { - o('should compare versions', () => { - o(SemVer.compare('0.0.0', '0.0.1')).equals(-1); - o(SemVer.compare('0.0.1', '0.0.0')).equals(1); - o(SemVer.compare('0.0.0', '0.0.0')).equals(0); - }); - - o('should sort versions', () => { - const versions = ['1.0.0', '2.0.0', '0.0.1', '1.2.3']; - versions.sort(SemVer.compare); - o(versions).deepEquals(['0.0.1', '1.0.0', '1.2.3', '2.0.0']); - }); - }); -}); diff --git a/packages/cli/src/cli/cogify/action.cog.ts b/packages/cli/src/cli/cogify/action.cog.ts deleted file mode 100644 index 54b934229..000000000 --- a/packages/cli/src/cli/cogify/action.cog.ts +++ /dev/null @@ -1,247 +0,0 @@ -import { Env, fsa, LogConfig, LoggerFatalError, LogType } from '@basemaps/shared'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerListParameter, - CommandLineIntegerParameter, - CommandLineStringListParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; -import { createReadStream, promises as fs } from 'fs'; -import { FeatureCollection } from 'geojson'; -import { buildCogForName } from '../../cog/cog.js'; -import { CogStacJob } from '../../cog/cog.stac.job.js'; -import { CogVrt } from '../../cog/cog.vrt.js'; -import { Cutline } from '../../cog/cutline.js'; -import { CogJob } from '../../cog/types.js'; -import { Gdal } from '../../gdal/gdal.js'; -import { makeTempFolder, makeTiffFolder } from '../folder.js'; -import path from 'path'; -import { prepareUrl } from '../util.js'; -import pLimit from 'p-limit'; - -const DefaultConcurrency = 2; - -export class CommandCogCreate extends CommandLineAction { - private job?: CommandLineStringParameter; - private name?: CommandLineStringListParameter; - private commit?: CommandLineFlagParameter; - private cogIndex?: CommandLineIntegerListParameter; - private concurrency?: CommandLineIntegerParameter; - - public constructor() { - super({ - actionName: 'cog', - summary: 'create a COG', - documentation: 'Create a COG for the specified cog name', - }); - } - - parseName(name: string): string[] { - // Name could both be a string or a array type of string - try { - return JSON.parse(name); - } catch (e) { - return [name]; - } - } - - getNames(job: CogJob): Set | null { - const output: Set = new Set(); - const { files } = job.output; - const batchIndex = Env.getNumber(Env.BatchIndex, -1); - if (batchIndex > -1) { - const { name } = files[batchIndex]; - if (name == null) { - throw new LoggerFatalError( - { cogIndex: batchIndex, tileMax: files.length - 1 }, - 'Failed to find cog name from batch index', - ); - } - output.add(name); - return output; - } - - const cogIndex = this.cogIndex?.values; - if (cogIndex != null && cogIndex.length > 0) { - for (const i of cogIndex) { - if (i < 0 || i >= files.length) { - throw new LoggerFatalError({ cogIndex: i, tileMax: files.length - 1 }, 'Failed to find cog name from index'); - } - const { name } = files[i]; - output.add(name); - } - - return output; - } - - const names = this.name?.values; - if (names != null) { - for (const nameStr of names) { - const nameArr = this.parseName(nameStr); - for (const name of nameArr) { - if (!files.find((r) => r.name === name)) - throw new LoggerFatalError( - { name, names: files.map((r) => r.name).join(', ') }, - 'Name does not exist inside job', - ); - output.add(name); - } - } - } - - return output; - } - - async onExecute(): Promise { - const jobLocation = this.job?.value; - if (jobLocation == null) throw new Error('Missing job name'); - - const isCommit = this.commit?.value ?? false; - const job = await CogStacJob.load(jobLocation); - - const logger = LogConfig.get().child({ - correlationId: job.id, - imageryName: job.name, - tileMatrix: job.tileMatrix.identifier, - }); - - LogConfig.set(logger); - logger.info('CogCreate:Start'); - - const gdalVersion = await Gdal.version(logger); - logger.info({ gdalVersion }, 'CogCreate:GdalVersion'); - - const names = this.getNames(job); - if (names == null || names.size === 0) return; - - const tmpFolder = await makeTempFolder(`basemaps-${job.id}-${CliId}`); - - const Q = pLimit(this.concurrency?.value ?? DefaultConcurrency); - - try { - Promise.all( - Array.from(names).map((name) => - Q(async () => { - const tiffJob = await CogStacJob.load(jobLocation); - await this.processTiff(tiffJob, name, tmpFolder, isCommit, logger.child({ tiffName: name })); - }), - ), - ); - } catch (e) { - // Ensure the error is thrown - throw e; - } finally { - // Cleanup! - await fs.rm(tmpFolder, { recursive: true }); - } - } - - async processTiff( - job: CogStacJob, - tiffName: string, - tmpFolder: string, - isCommit: boolean, - logger: LogType, - ): Promise { - const tiffFolder = await makeTiffFolder(tmpFolder, tiffName); - const targetPath = job.getJobPath(`${tiffName}.tiff`); - fsa.configure(job.output.location); - - const outputExists = await fsa.exists(targetPath); - logger.info({ targetPath, outputExists }, 'CogCreate:CheckExists'); - // Output file exists don't try and overwrite it - if (outputExists) { - logger.warn({ targetPath }, 'CogCreate:OutputExists'); - await this.checkJobStatus(job, logger); - return; - } - - let cutlineJson: FeatureCollection | undefined; - if (job.output.cutline != null) { - const cutlinePath = job.getJobPath('cutline.geojson.gz'); - logger.info({ path: cutlinePath }, 'CogCreate:UsingCutLine'); - cutlineJson = await Cutline.loadCutline(cutlinePath); - } else { - logger.warn('CutLine:Skip'); - } - const cutline = new Cutline(job.tileMatrix, cutlineJson, job.output.cutline?.blend); - - const tmpVrtPath = await CogVrt.buildVrt(tiffFolder, job, cutline, tiffName, logger); - - if (tmpVrtPath == null) { - logger.warn('CogCreate:NoMatchingSourceImagery'); - return; - } - - const tmpTiff = fsa.join(tiffFolder, `${tiffName}.tiff`); - - await buildCogForName(job, tiffName, tmpVrtPath, tmpTiff, logger, isCommit); - logger.info({ target: targetPath }, 'CogCreate:StoreTiff'); - if (isCommit) { - await fsa.write(targetPath, createReadStream(tmpTiff)); - await this.checkJobStatus(job, logger); - } else { - logger.warn('DryRun:Done'); - } - } - - /** Check to see how many tiffs are remaining in the job */ - async checkJobStatus(job: CogStacJob, logger: LogType): Promise { - const basePath = job.getJobPath(); - const expectedTiffs = new Set(); - const jobSize = job.output.files.length; - for (const file of job.output.files) expectedTiffs.add(`${file.name}.tiff`); - - for await (const fileName of fsa.list(basePath)) { - const basename = path.basename(fileName); - // Look for tile tiffs only - if (!basename.includes('-') || !basename.endsWith('.tiff')) continue; - expectedTiffs.delete(basename); - } - - if (expectedTiffs.size === 0) { - const url = await prepareUrl(job); - logger.info({ tiffCount: jobSize, tiffTotal: jobSize, url }, 'CogCreate:JobComplete'); - } else { - logger.info({ tiffCount: jobSize, tiffRemaining: expectedTiffs.size }, 'CogCreate:JobProgress'); - } - } - - protected onDefineParameters(): void { - this.job = this.defineStringParameter({ - argumentName: 'JOB', - parameterLongName: '--job', - description: 'Job config source to access', - required: true, - }); - - this.name = this.defineStringListParameter({ - argumentName: 'NAME', - parameterLongName: '--name', - description: 'list of cog names to process', - required: false, - }); - - this.cogIndex = this.defineIntegerListParameter({ - argumentName: 'COG_INDEX', - parameterLongName: '--cog-index', - description: 'list of cog indexes to process', - required: false, - }); - - this.concurrency = this.defineIntegerParameter({ - argumentName: 'CONCURRENCY', - parameterLongName: '--concurrency', - description: 'Number of concurrency create cog jobs', - required: false, - }); - - this.commit = this.defineFlagParameter({ - parameterLongName: '--commit', - description: 'Begin the transformation', - required: false, - }); - } -} diff --git a/packages/cli/src/cli/cogify/action.job.ts b/packages/cli/src/cli/cogify/action.job.ts deleted file mode 100644 index a0cf111a0..000000000 --- a/packages/cli/src/cli/cogify/action.job.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { Epsg, GoogleTms, TileMatrixSets } from '@basemaps/geo'; -import { FileConfig, FileConfigPath, fsa } from '@basemaps/shared'; -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; -import { JobCreationContext } from '../../cog/cog.stac.job.js'; -import { CogJobFactory, MaxConcurrencyDefault } from '../../cog/job.factory.js'; -import { GdalCogBuilderDefaults, GdalCogBuilderResampling, GdalResamplingOptions } from '../../gdal/gdal.config.js'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; - -export class CLiInputData { - path: CommandLineStringParameter; - roleArn: CommandLineStringParameter; - externalId: CommandLineStringParameter; - - constructor(parent: CommandLineAction, prefix: string) { - this.path = parent.defineStringParameter({ - argumentName: prefix.toUpperCase(), - parameterLongName: `--${prefix}`, - description: 'Folder or S3 Bucket location to use', - required: true, - }); - - this.roleArn = parent.defineStringParameter({ - argumentName: prefix.toUpperCase() + '_ARN', - parameterLongName: `--${prefix}-role-arn`, - description: 'Role to be assumed to access the data', - required: false, - }); - - this.externalId = parent.defineStringParameter({ - argumentName: prefix.toUpperCase() + '_EXTERNAL_ID', - parameterLongName: `--${prefix}-role-external-id`, - description: 'Role external id to be assumed to access the data', - required: false, - }); - } -} - -export class CommandJobCreate extends CommandLineAction { - private source: CLiInputData; - private output: CLiInputData; - private maxConcurrency: CommandLineIntegerParameter; - private cutline: CommandLineStringParameter; - private cutlineBlend: CommandLineIntegerParameter; - private overrideId: CommandLineStringParameter; - private overrideWarpReample: CommandLineStringParameter; - private submitBatch: CommandLineFlagParameter; - private quality: CommandLineIntegerParameter; - private sourceProjection: CommandLineIntegerParameter; - private tileMatrix: CommandLineStringParameter; - private oneCog: CommandLineFlagParameter; - private fileList: CommandLineStringParameter; - - public constructor() { - super({ - actionName: 'job', - summary: 'create a list of cogs that need to be processed', - documentation: 'List a folder/bucket full of cogs and determine a optimal processing pipeline', - }); - } - - fsConfig(source: CLiInputData): FileConfig { - if (source.path.value == null) { - throw new Error('Invalid path'); - } - if (!source.path.value.startsWith('s3://')) { - return { type: 'local', path: source.path.value }; - } - if (source.roleArn.value == null) { - return { type: 's3', path: source.path.value }; - } - return { - path: source.path.value, - type: 's3', - roleArn: source.roleArn.value, - externalId: source.externalId.value, - }; - } - - async onExecute(): Promise { - const sourceLocation: FileConfig | FileConfigPath = this.fsConfig(this.source); - const outputLocation = this.fsConfig(this.output); - - fsa.configure(this.fsConfig(this.source)); - fsa.configure(this.fsConfig(this.output)); - - let cutline = undefined; - if (this.cutline?.value) { - cutline = { href: this.cutline.value, blend: this.cutlineBlend.value ?? 20 }; - } - - const tileMatrix = TileMatrixSets.find(this.tileMatrix.value ?? GoogleTms.identifier); - if (tileMatrix == null) throw new Error('Invalid target-projection'); - - let resampling: GdalCogBuilderResampling | undefined; - - if (this.overrideWarpReample?.value != null) { - const warp = GdalResamplingOptions[this.overrideWarpReample?.value]; - if (warp == null) { - throw new Error('Invalid override-warp-resampling'); - } - resampling = { - warp, - overview: 'lanczos', - }; - } - - const fileListPath = this.fileList?.value; - if (fileListPath != null) { - const fileData = await fsa.read(fileListPath); - (sourceLocation as FileConfigPath).files = fileData - .toString() - .trim() - .split('\n') - .map((fn) => sourceLocation.path + '/' + fn); - } - - const ctx: JobCreationContext = { - sourceLocation, - outputLocation, - cutline, - tileMatrix, - override: { - concurrency: this.maxConcurrency?.value ?? MaxConcurrencyDefault, - quality: this.quality?.value ?? GdalCogBuilderDefaults.quality, - id: this.overrideId?.value ?? CliId, - projection: Epsg.tryGet(this.sourceProjection?.value), - resampling, - }, - batch: this.submitBatch?.value ?? false, - oneCogCovering: this.oneCog?.value ?? false, - }; - - await CogJobFactory.create(ctx); - } - - protected onDefineParameters(): void { - this.source = new CLiInputData(this, 'source'); - this.output = new CLiInputData(this, 'output'); - - this.maxConcurrency = this.defineIntegerParameter({ - argumentName: 'MAX_CONCURRENCY', - parameterLongName: '--concurrency', - parameterShortName: '-c', - description: 'Maximum number of requests to use at one time', - defaultValue: MaxConcurrencyDefault, - required: false, - }); - - this.cutline = this.defineStringParameter({ - argumentName: 'CUTLINE', - parameterLongName: '--cutline', - description: 'use a shapefile to crop the COGs', - required: false, - }); - - this.cutlineBlend = this.defineIntegerParameter({ - argumentName: 'CUTLINE_BLEND', - parameterLongName: '--cblend', - description: 'Set a blend distance to use to blend over cutlines (in pixels)', - required: false, - }); - - this.overrideId = this.defineStringParameter({ - argumentName: 'OVERRIDE_ID', - parameterLongName: '--override-id', - description: 'create job with a pre determined job id', - required: false, - }); - - this.overrideWarpReample = this.defineStringParameter({ - argumentName: 'METHOD', - parameterLongName: '--override-warp-resampling', - description: 'Defaults to bilinear. Common options are: nearest, lanczos, cubic', - required: false, - }); - - this.submitBatch = this.defineFlagParameter({ - parameterLongName: `--batch`, - description: 'Submit the job to AWS Batch', - required: false, - }); - - this.quality = this.defineIntegerParameter({ - argumentName: 'QUALITY', - parameterLongName: '--quality', - description: 'Compression quality (0-100)', - required: false, - }); - - this.sourceProjection = this.defineIntegerParameter({ - argumentName: 'SOURCE_PROJECTION', - parameterLongName: '--source-projection', - description: 'The EPSG code of the source imagery', - required: false, - }); - - this.tileMatrix = this.defineStringParameter({ - argumentName: 'TILE_MATRIX', - parameterLongName: '--target-tile-matrix-set', - description: 'The TileMatrixSet to generate the COGS for', - defaultValue: GoogleTms.identifier, - required: false, - }); - - this.oneCog = this.defineFlagParameter({ - parameterLongName: '--one-cog', - description: 'ignore target projection window and just produce one big COG.', - required: false, - }); - - this.fileList = this.defineStringParameter({ - argumentName: 'FILE_LIST', - parameterLongName: '--filelist', - description: 'supply a list of files to use as source imagery', - required: false, - }); - } -} diff --git a/packages/cli/src/cli/cogify/action.make.cog.pr.ts b/packages/cli/src/cli/cogify/action.make.cog.pr.ts deleted file mode 100644 index 78f0be458..000000000 --- a/packages/cli/src/cli/cogify/action.make.cog.pr.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { ConfigLayer, standardizeLayerName } from '@basemaps/config'; -import { LogConfig } from '@basemaps/shared'; -import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line'; -import { MakeCogGithub } from '../github/make.cog.pr.js'; - -export enum Category { - Urban = 'Urban Aerial Photos', - Rural = 'Rural Aerial Photos', - Satellite = 'Satellite Imagery', - Event = 'Event', - Other = 'New Aerial Photos', -} - -export interface CategorySetting { - minZoom?: number; - individual?: boolean; -} - -export const DefaultCategorySetting: Record = { - [Category.Urban]: { minZoom: 14, individual: false }, - [Category.Rural]: { minZoom: 13, individual: false }, - [Category.Satellite]: { minZoom: 5, individual: false }, - [Category.Event]: { individual: true }, - [Category.Other]: { individual: true }, -}; - -export function parseCategory(category: string): Category { - const c = category.toLocaleLowerCase(); - if (c.includes('urban')) return Category.Urban; - else if (c.includes('rural')) return Category.Rural; - else if (c.includes('satellite')) return Category.Satellite; - else return Category.Other; -} - -export class CommandCogPullRequest extends CommandLineAction { - private layer: CommandLineStringParameter; - private category: CommandLineStringParameter; - private repository: CommandLineStringParameter; - private individual: CommandLineFlagParameter; - private vector: CommandLineFlagParameter; - - public constructor() { - super({ - actionName: 'cog-pr', - summary: 'create a github pull request for the import cog workflow', - documentation: 'Given a output of make-cog command and create pull request for the imagery config.', - }); - } - - protected onDefineParameters(): void { - this.layer = this.defineStringParameter({ - argumentName: 'LAYER', - parameterLongName: '--layer', - description: 'Input config layer', - required: false, - }); - this.category = this.defineStringParameter({ - argumentName: 'CATEGORY', - parameterLongName: '--category', - description: 'New Imagery Category, like Rural Aerial Photos, Urban Aerial Photos, Satellite Imagery', - required: false, - }); - this.repository = this.defineStringParameter({ - argumentName: 'REPOSITORY', - parameterLongName: '--repository', - description: 'Github repository reference', - defaultValue: 'linz/basemaps-config', - required: false, - }); - this.individual = this.defineFlagParameter({ - parameterLongName: '--individual', - description: 'Import imagery as individual layer in basemaps.', - required: false, - }); - this.vector = this.defineFlagParameter({ - parameterLongName: '--vector', - description: 'Commit changes for vector map', - required: false, - }); - } - - async onExecute(): Promise { - const logger = LogConfig.get(); - const layerStr = this.layer.value; - const category = this.category.value ? parseCategory(this.category.value) : Category.Other; - const repo = this.repository.value ?? this.repository.defaultValue; - if (layerStr == null) throw new Error('Please provide a valid input layer and urls'); - if (repo == null) throw new Error('Please provide a repository'); - let layer: ConfigLayer; - try { - layer = JSON.parse(layerStr); - } catch { - throw new Error('Please provide a valid input layer'); - } - - //Make sure the imagery name is standardized before update the config - layer.name = standardizeLayerName(layer.name); - - const git = new MakeCogGithub(layer.name, repo, logger); - if (this.vector.value) await git.updateVectorTileSet('topographic', layer); - else await git.updateRasterTileSet('aerial', layer, category, this.individual.value); - } -} diff --git a/packages/cli/src/cli/cogify/action.make.cog.ts b/packages/cli/src/cli/cogify/action.make.cog.ts deleted file mode 100644 index 4b4f1d4a5..000000000 --- a/packages/cli/src/cli/cogify/action.make.cog.ts +++ /dev/null @@ -1,243 +0,0 @@ -import { Epsg, TileMatrixSet, TileMatrixSets } from '@basemaps/geo'; -import { Env, FileConfig, fsa, LogConfig, LogType, titleizeImageryName } from '@basemaps/shared'; -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; -import { CogStacJob, JobCreationContext } from '../../cog/cog.stac.job.js'; -import { ProjectionLoader } from '../../cog/projection.loader.js'; -import * as ulid from 'ulid'; -import { getCutline } from './cutline.js'; -import { CogJobFactory } from '../../cog/job.factory.js'; -import { basename } from 'path'; -import { BatchJob } from './batch.job.js'; -import { CredentialSourceJson } from '@chunkd/source-aws-v2'; -import { ConfigLayer } from '@basemaps/config'; -import { AlignedLevel } from '../../cog/constants.js'; - -interface OutputJobs { - job: string; - names: string[]; -} - -export class CommandMakeCog extends CommandLineAction { - private imagery: CommandLineStringParameter; - private tileMatrix: CommandLineStringParameter; - private name: CommandLineStringParameter; - private target: CommandLineStringParameter; - private cutline: CommandLineStringParameter; - private blend: CommandLineIntegerParameter; - private alignedLevel: CommandLineIntegerParameter; - private maxChunkUnit: CommandLineIntegerParameter; - private output: CommandLineStringParameter; - private aws: CommandLineFlagParameter; - - public constructor() { - super({ - actionName: 'make-cog', - summary: 'Import Basemaps imagery from s3 buckets', - documentation: 'Given a valid path of raw imagery and import into basemaps', - }); - } - - protected onDefineParameters(): void { - this.imagery = this.defineStringParameter({ - argumentName: 'IMAGERY', - parameterShortName: '-i', - parameterLongName: '--imagery', - description: 'Path of source imagery to import.', - required: true, - }); - this.target = this.defineStringParameter({ - argumentName: 'BUCKET', - parameterShortName: '-t', - parameterLongName: '--target', - description: 'Target bucket for job.json', - required: true, - }); - this.name = this.defineStringParameter({ - argumentName: 'NAME', - parameterShortName: '-n', - parameterLongName: '--name', - description: 'Custom imagery name', - required: false, - }); - this.tileMatrix = this.defineStringParameter({ - argumentName: 'TILE_MATRIX', - parameterLongName: '--tile-matrix', - description: 'Target tile matrix', - required: false, - }); - this.cutline = this.defineStringParameter({ - argumentName: 'CUTLINE', - parameterLongName: '--cutline', - description: 'Path of import cutline', - required: false, - }); - this.blend = this.defineIntegerParameter({ - argumentName: 'BLEND', - parameterLongName: '--blend', - description: 'Cutline blend', - required: false, - }); - this.alignedLevel = this.defineIntegerParameter({ - argumentName: 'ALIGNED_LEVEL', - parameterLongName: '--aligned-level', - description: 'Aligned level between resolution and cog', - required: false, - }); - this.maxChunkUnit = this.defineIntegerParameter({ - argumentName: 'MAX_CHUNK_UNIT', - parameterLongName: '--max-chunk-unit', - description: 'Number of small jobs are chunked into one job', - required: false, - }); - this.output = this.defineStringParameter({ - argumentName: 'OUTPUT', - parameterShortName: '-o', - parameterLongName: '--output', - description: 'Output job.json path', - required: false, - }); - this.aws = this.defineFlagParameter({ - parameterLongName: '--aws', - description: 'Running the job on aws', - required: false, - }); - } - - async onExecute(): Promise { - const logger = LogConfig.get(); - const imagery = this.imagery.value; - let name = this.name.value === '' ? undefined : this.name.value; - if (imagery == null) throw new Error('Please provide a valid imagery source'); - await Promise.all([3791, 3790, 3789, 3788].map((code) => ProjectionLoader.load(code))); - - const source = imagery.endsWith('/') ? imagery : imagery + '/'; - logger.info({ imagery: source }, 'FindImagery'); - if (name == null) name = source.split('/').filter(Boolean).pop(); - if (name == null) throw new Error('Failed to find imagery set name'); - - let tileMatrixSets: string[] = []; - const tileMatrix = this.tileMatrix.value; - if (tileMatrix == null) throw new Error('Please provide valid tile set matrix.'); - if (tileMatrix.includes('/')) tileMatrixSets = tileMatrixSets.concat(tileMatrix.split('/')); - else tileMatrixSets.push(tileMatrix); - - const outputJobs: OutputJobs[] = []; - const configLayer: ConfigLayer = { name, title: titleizeImageryName(name) }; - const paths: string[] = []; - for (const identifier of tileMatrixSets) { - const id = ulid.ulid(); - const tileMatrix = TileMatrixSets.find(identifier); - if (tileMatrix == null) throw new Error(`Cannot find tile matrix: ${identifier}`); - logger.info({ id, tileMatrix: tileMatrix.identifier }, 'SetTileMatrix'); - const job = await this.makeCog(id, name, tileMatrix, source); - configLayer.title = job.title; - - const jobLocation = job.getJobPath('job.json'); - // Split the jobs into chunked tasks - const chunkedJobs = await this.splitJob(job, logger); - for (const names of chunkedJobs) { - outputJobs.push({ job: jobLocation, names }); - } - - // Set config layer for output - const path = jobLocation.replace('/job.json', ''); - configLayer[tileMatrix.projection.code] = path; - paths.push(path); - } - - const output = this.output.value; - if (output) { - fsa.write(fsa.join(output, 'jobs.json'), JSON.stringify(outputJobs)); - fsa.write(fsa.join(output, 'layer.json'), JSON.stringify(configLayer)); - fsa.write(fsa.join(output, 'paths.json'), JSON.stringify(paths)); - } - } - - async makeCog(id: string, imageryName: string, tileMatrix: TileMatrixSet, uri: string): Promise { - const bucket = this.target.value; - if (bucket == null && this.aws.value) throw new Error('Please provide a validate bucket for output job.json'); - let resampling; - /** Process Gebco 2193 as one cog of full extent to avoid antimeridian problems */ - if (tileMatrix.projection === Epsg.Nztm2000 && imageryName.includes('gebco')) { - resampling = { - warp: 'nearest', // GDAL doesn't like other warp settings when crossing antimeridian - overview: 'lanczos', - } as const; - } - - if (imageryName.includes('geographx')) { - resampling = { - warp: 'bilinear', - overview: 'bilinear', - } as const; - } - - // Prepare the cutline - let cutline: { href: string; blend: number } | undefined; - const cutlinePath = this.cutline.value === '' ? undefined : this.cutline.value; - const blend = this.blend.value === 0 ? undefined : this.blend.value; - if (cutlinePath && blend) cutline = { href: cutlinePath, blend }; - else if (cutlinePath) new Error('Please provide a blend for the cutline'); - else cutline = getCutline(imageryName); - if (cutline == null) throw new Error(`Cannot found default cutline from imagery name: ${imageryName}`); - - const alignedLevel = this.alignedLevel.value ?? AlignedLevel; - - const ctx: JobCreationContext = { - imageryName, - override: { - id, - projection: Epsg.Nztm2000, - resampling, - alignedLevel, - }, - outputLocation: this.aws.value - ? await this.findLocation(`s3://${bucket}/`) - : { type: 'local' as const, path: '.' }, - sourceLocation: await this.findLocation(uri), - cutline, - batch: false, // Only create the job.json in the make cog cli - tileMatrix, - oneCogCovering: false, - }; - - return await CogJobFactory.create(ctx); - } - - async splitJob(job: CogStacJob, logger: LogType): Promise { - // Get all the existing output tiffs - const existTiffs: Set = new Set(); - for await (const fileName of fsa.list(job.getJobPath())) { - if (fileName.endsWith('.tiff')) existTiffs.add(basename(fileName)); - } - - // Prepare chunk job and individual jobs based on imagery size. - const jobs = await BatchJob.getJobs(job, existTiffs, logger, this.maxChunkUnit.value); - if (jobs.length === 0) { - logger.info('NoJobs'); - return []; - } - logger.info({ jobTotal: job.output.files.length, jobLeft: jobs.length }, 'SplitJob:ChunkedJobs'); - return jobs; - } - - async findLocation(path: string): Promise { - if (path.startsWith('s3://')) { - const configPath = Env.get(Env.AwsRoleConfigPath); - if (configPath == null) throw new Error('No aws config path provided'); - const configs = await fsa.readJson(configPath); - for (const prefix of configs.prefixes) { - if (path.startsWith(prefix.prefix)) return { type: 's3', path, roleArn: prefix.roleArn }; - } - } else { - return { type: 'local', path: path }; - } - - throw new Error(`No valid role to find the path: ${path}`); - } -} diff --git a/packages/cli/src/cli/cogify/batch.job.ts b/packages/cli/src/cli/cogify/batch.job.ts deleted file mode 100644 index 273144820..000000000 --- a/packages/cli/src/cli/cogify/batch.job.ts +++ /dev/null @@ -1,227 +0,0 @@ -import { Projection, TileMatrixSet } from '@basemaps/geo'; -import { Env, fsa, LogConfig, LogType } from '@basemaps/shared'; -import Batch from 'aws-sdk/clients/batch.js'; -import { createHash } from 'crypto'; -import { basename } from 'path'; -import { CogJob } from '../../cog/types.js'; - -const JobQueue = 'CogBatchJobQueue'; -const JobDefinition = 'CogBatchJob'; -const ChunkJobMax = 2000; -const ChunkLargeUnit = 200; // Up to 10 large files in one job -const ChunkMiddleUnit = 50; // Up to 40 middle files in one job -const ChunkSmallUnit = 20; // Up to 100 small files in one job - -/** The base alignment level used by GDAL, Tiffs that are bigger or smaller than this should scale the compute resources */ -const MagicAlignmentLevel = 7; - -const ResolutionRegex = /((?:\d[\.\-])?\d+)m/; -/** - * Attempt to parse a resolution from a imagery name - * @example `wellington_urban_2017_0.10m` -> 100 - * @param name Imagery name to parse - * @returns resolution (millimeters), -1 for failure to parse - */ -export function extractResolutionFromName(name: string): number { - const matches = name.match(ResolutionRegex); - if (matches == null) return -1; - return parseFloat(matches[1].replace('-', '.')) * 1000; -} - -export class BatchJob { - static _batch: Batch; - static get batch(): Batch { - if (this._batch) return this._batch; - const region = Env.get('AWS_REGION') ?? Env.get('AWS_DEFAULT_REGION') ?? 'ap-southeast-2'; - this._batch = new Batch({ region }); - return this._batch; - } - /** - * Create a id for a job - * - * This needs to be within `[a-Z_-]` upto 128 characters log - * @param job job to process - * @param fileNames output filename - * @returns job id - */ - static id(job: CogJob, fileNames?: string[]): string { - // Job names are uncontrolled so hash the name and grab a small slice to use as a identifier - const jobName = createHash('sha256').update(job.name).digest('hex').slice(0, 16); - if (fileNames == null) return `${job.id}-${jobName}-`; - fileNames.sort((a, b) => a.localeCompare(b)); - return `${job.id}-${jobName}-${fileNames.length}x-${fileNames.join('_')}`.slice(0, 128); - } - - static async batchOne( - jobPath: string, - job: CogJob, - names: string[], - isCommit: boolean, - ): Promise<{ jobName: string; jobId: string; memory: number }> { - const jobName = BatchJob.id(job, names); - - let memory = 3900; - if (names.length === 1) { - // Calculate the larger file to provision memory if there is only one imagery in job. - const tile = TileMatrixSet.nameToTile(names[0]); - const alignmentLevels = Projection.findAlignmentLevels(job.tileMatrix, tile, job.source.gsd); - // Give 25% more memory to larger jobs - const resDiff = 1 + Math.max(alignmentLevels - MagicAlignmentLevel, 0) * 0.25; - memory *= resDiff; - } - - if (!isCommit) { - return { jobName, jobId: '', memory }; - } - - let commandStr = ['-V', 'cog', '--job', jobPath, '--commit']; - for (const name of names) commandStr = commandStr.concat(['--name', name]); - - const batchJob = await this.batch - .submitJob({ - jobName, - jobQueue: JobQueue, - jobDefinition: JobDefinition, - containerOverrides: { - memory, - command: commandStr, - }, - retryStrategy: { attempts: 3 }, - }) - .promise(); - return { jobName, jobId: batchJob.jobId, memory }; - } - - /** - * List all the current jobs in batch and their statuses - * @returns a map of JobName to if their status is "ok" (not failed) - */ - static async getCurrentJobList(job: CogJob, logger: LogType): Promise> { - const jobPrefix = BatchJob.id(job); - // For some reason AWS only lets us query one status at a time. - const allStatuses = ['SUBMITTED', 'PENDING', 'RUNNABLE', 'STARTING', 'RUNNING' /* 'SUCCEEDED' */]; - // Succeeded is not needed as we check to see if the output file exists, if it succeeds and the output file doesn't exist then something has gone wrong - const allJobs = await Promise.all( - allStatuses.map((jobStatus) => this.batch.listJobs({ jobQueue: JobQueue, jobStatus }).promise()), - ); - - const jobIds = new Set(); - - // Find all the relevant jobs that start with our job prefix - for (const status of allJobs) { - for (const job of status.jobSummaryList) { - if (!job.jobName.startsWith(jobPrefix)) continue; - jobIds.add(job.jobId); - } - } - - // Inspect all the jobs for what files are being "processed" - const tiffs = new Set(); - let allJobIds = [...jobIds]; - while (allJobIds.length > 0) { - logger.info({ jobCount: allJobIds.length }, 'JobFetch'); - const jobList = allJobIds.slice(0, 100); - allJobIds = allJobIds.slice(100); - const describedJobs = await this.batch.describeJobs({ jobs: jobList }).promise(); - if (describedJobs.jobs == null) continue; - for (const job of describedJobs.jobs) { - const jobCommand = job.container?.command; - if (jobCommand == null) continue; - - // Extract the tiff names from the job command - for (let i = 0; i < jobCommand.length; i++) { - if (jobCommand[i] === '--name') tiffs.add(jobCommand[i + 1]); - } - } - } - - return tiffs; - } - - static async batchJob(job: CogJob, commit = false, logger: LogType): Promise { - const jobPath = job.getJobPath('job.json'); - if (!jobPath.startsWith('s3://')) { - throw new Error(`AWS Batch collection.json have to be in S3, jobPath:${jobPath}`); - } - LogConfig.set(logger.child({ correlationId: job.id, imageryName: job.name })); - - fsa.configure(job.output.location); - - // Get all the existing output tiffs - const existTiffs: Set = new Set(); - for await (const fileName of fsa.list(job.getJobPath())) { - if (fileName.endsWith('.tiff')) existTiffs.add(basename(fileName)); - } - - const runningJobs = await this.getCurrentJobList(job, logger); - for (const tiffName of runningJobs) existTiffs.add(`${tiffName}.tiff`); - - // Prepare chunk job and individual jobs based on imagery size. - const jobs = await this.getJobs(job, existTiffs, logger); - - if (jobs.length === 0) { - logger.info('NoJobs'); - return; - } - - logger.info( - { - jobTotal: job.output.files.length, - jobLeft: jobs.length, - jobQueue: JobQueue, - jobDefinition: JobDefinition, - }, - 'JobSubmit', - ); - - for (const names of jobs) { - const jobStatus = await BatchJob.batchOne(jobPath, job, names, commit); - logger.info(jobStatus, 'JobSubmitted'); - } - - if (!commit) { - logger.warn('DryRun:Done'); - return; - } - } - - /** - * Prepare the jobs from job files, and chunk the small images into single - * @returns List of jobs including single job and chunk jobs. - */ - static getJobs(job: CogJob, existing: Set, log: LogType, maxChunkUnit = ChunkJobMax): string[][] { - const jobs: string[][] = []; - let chunkJob: string[] = []; - let chunkUnit = 0; // Calculate the chunkUnit based on the size - for (const file of job.output.files) { - const outputFile = `${file.name}.tiff`; - if (existing.has(outputFile)) { - log.debug({ fileName: outputFile }, 'Skip:Exists'); - continue; - } - - const imageSize = file.width / job.output.gsd; - if (imageSize > 16385) { - chunkJob.push(file.name); - chunkUnit += ChunkJobMax; - } else if (imageSize > 8193) { - chunkJob.push(file.name); - chunkUnit += ChunkLargeUnit; - } else if (imageSize > 4097) { - chunkJob.push(file.name); - chunkUnit += ChunkMiddleUnit; - } else { - chunkJob.push(file.name); - chunkUnit += ChunkSmallUnit; - } - if (chunkUnit >= maxChunkUnit) { - jobs.push(chunkJob); - chunkJob = []; - chunkUnit = 0; - } - } - if (chunkJob.length > 0) jobs.push(chunkJob); - - return jobs; - } -} diff --git a/packages/cli/src/cli/cogify/cutline.ts b/packages/cli/src/cli/cogify/cutline.ts deleted file mode 100644 index 28c0f2d4f..000000000 --- a/packages/cli/src/cli/cogify/cutline.ts +++ /dev/null @@ -1,34 +0,0 @@ -const UrbanRuralCutLine = { - href: 's3://linz-basemaps-source/cutline/2020-05-07-cutline-nz-coasts-rural-and-urban.geojson', - blend: 20, -}; - -const DefaultCutlines = [ - { - filter: 'sentinel', - cutline: { href: 's3://linz-basemaps-source/cutline/2020-05-12-cutline-nz-ci.geojson.gz', blend: 5 }, - }, - { - filter: 'chatham', - cutline: { href: 's3://linz-basemaps-source/cutline/2020-05-12-cutline-nz-ci.geojson.gz', blend: 5 }, - }, - { - filter: 'geographx', - cutline: { - href: 's3://linz-basemaps-source/cutline/2021-11-10-cutline-51153-nz-coastlines-and-islands-polygons-topo-150k.geojson.gz', - blend: 0, - }, - }, - { - filter: 'topo50', - cutline: { href: 's3://linz-basemaps-source/cutline/2020-08-10-cutline-topo50.geojson.gz', blend: 0 }, - }, - - { filter: 'satellite', cutline: UrbanRuralCutLine }, - { filter: 'urban', cutline: UrbanRuralCutLine }, - { filter: 'rural', cutline: UrbanRuralCutLine }, -]; - -export function getCutline(imageryName = ''): { href: string; blend: number } | undefined { - return DefaultCutlines.find((f) => imageryName.toLowerCase().includes(f.filter))?.cutline; -} diff --git a/packages/cli/src/cli/cogify/semver.util.ts b/packages/cli/src/cli/cogify/semver.util.ts deleted file mode 100644 index 040807310..000000000 --- a/packages/cli/src/cli/cogify/semver.util.ts +++ /dev/null @@ -1,56 +0,0 @@ -/** major.minor.patch */ -const MaxLength = 3; -/** fits into a int32 with out forcing a uint conversion */ -const EncodeBits = 10; -/** Max size for a single semver number (1024) */ -const MaxEncode = 2 ** EncodeBits - 1; - -export const SemVer = { - /** - * Encode a semver string into a number - * @example - * '0.0.1' => 1 - * 'v0.1.0' => 1024 - * @param v version to encode - */ - toNumber(v: string): number { - if (v.startsWith('v')) v = v.slice(1); - const chunks = v.split('.'); - if (chunks.length > MaxLength) throw new Error(`Failed to parse semver "${v}" too long`); - - let output = 0; - for (let i = 0; i < chunks.length; i++) { - const chunk = chunks[i]; - if (chunk === '') break; - - const num = parseInt(chunk); - if (isNaN(num)) throw new Error(`Failed to parse semver "${v}"`); - if (num > MaxEncode) throw new Error(`Failed to parse semver "${v}" ${chunk} is too large`); - - output |= num << ((MaxLength - i - 1) * EncodeBits); - } - return output; - }, - - /** - * Decoded a number encoded semver - * @example 1 => '0.0.1' - * @param v version to decode - */ - fromNumber(v: number): string { - const major = (v >> 20) & MaxEncode; - const minor = (v >> 10) & MaxEncode; - const patch = v & MaxEncode; - return `${major}.${minor}.${patch}`; - }, - - /** - * Compare two semver's - * @param vA version A - * @param vB version B - * @returns 0 if equal, >0 if vA > vB, <0 otherwise - */ - compare(vA: string, vB: string): number { - return SemVer.toNumber(vA) - SemVer.toNumber(vB); - }, -}; diff --git a/packages/cli/src/cli/config/action.import.ts b/packages/cli/src/cli/config/action.import.ts index a120b4692..0f530c08c 100644 --- a/packages/cli/src/cli/config/action.import.ts +++ b/packages/cli/src/cli/config/action.import.ts @@ -14,10 +14,10 @@ import { GoogleTms, Nztm2000QuadTms, Projection, TileMatrixSet } from '@basemaps import { Env, fsa, getDefaultConfig, LogConfig } from '@basemaps/shared'; import { CommandLineAction, CommandLineFlagParameter, CommandLineStringParameter } from '@rushstack/ts-command-line'; import fetch from 'node-fetch'; -import { CogStacJob } from '../../cog/cog.stac.job.js'; import { invalidateCache } from '../util.js'; import { Q, Updater } from './config.update.js'; import { FeatureCollection } from 'geojson'; +import { CogJobJson } from '../../cog/types.js'; const PublicUrlBase = Env.isProduction() ? 'https://basemaps.linz.govt.nz/' : 'https://dev.basemaps.linz.govt.nz/'; @@ -316,12 +316,12 @@ export class CommandImport extends CommandLineAction { return; } - _jobs: Map = new Map(); - async _loadJob(path: string): Promise { + _jobs: Map = new Map(); + async _loadJob(path: string): Promise { const existing = this._jobs.get(path); if (existing) return existing; try { - const job = await fsa.readJson(path); + const job = await fsa.readJson(path); this._jobs.set(path, job); return job; } catch { diff --git a/packages/cli/src/cli/github/github.ts b/packages/cli/src/cli/github/github.ts deleted file mode 100644 index 3ab310428..000000000 --- a/packages/cli/src/cli/github/github.ts +++ /dev/null @@ -1,118 +0,0 @@ -import { Env, LogType } from '@basemaps/shared'; -import { execFileSync } from 'child_process'; -import { Octokit } from '@octokit/core'; -import { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js'; -import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods'; - -export class Github { - repo: string; - org: string; - logger: LogType; - repoName: string; - octokit: Api; - - constructor(repo: string, logger: LogType) { - this.repo = repo; - this.logger = logger; - const [org, repoName] = repo.split('/'); - if (org == null || repoName == null) throw new Error(`Badly formatted repo name: ${repo}`); - this.org = org; - this.repoName = repoName; - - const token = Env.get(Env.GitHubToken); - if (token == null) throw new Error('Please set up github token environment variable.'); - this.octokit = restEndpointMethods(new Octokit({ auth: token })); - } - - isOk = (s: number): boolean => s >= 200 && s <= 299; - - /** - * Clone the repository - * - */ - clone(): void { - const ssh = `git@github.com:${this.repo}.git`; - this.logger.info({ repository: this.repo }, 'GitHub: Clone'); - execFileSync('git', ['clone', ssh]).toString().trim(); - } - - /** - * Get branch by name if exists, or create a new branch by name. - * - * @returns {branch} github references or the new created branch - */ - getBranch(branch: string): string { - this.logger.info({ branch }, 'GitHub: Get branch'); - try { - execFileSync('git', ['checkout', branch], { cwd: this.repoName }).toString().trim(); - this.logger.info({ branch }, 'GitHub: Branch Checkout'); - return branch; - } catch { - this.logger.info({ branch }, 'GitHub: Create New Branch'); - execFileSync('git', ['checkout', '-b', branch], { cwd: this.repoName }).toString().trim(); - return branch; - } - } - - /** - * Config github user email and user name - * - */ - configUser(): void { - const email = Env.get('GIT_USER_EMAIL') ?? 'basemaps@linz.govt.nz'; - const name = Env.get('GIT_USER_NAME') ?? 'basemaps[bot]'; - this.logger.info({ repository: this.repo }, 'GitHub: Config User Email'); - execFileSync('git', ['config', 'user.email', email], { cwd: this.repoName }).toString().trim(); - this.logger.info({ repository: this.repo }, 'GitHub: Config User Name'); - execFileSync('git', ['config', 'user.name', name], { cwd: this.repoName }).toString().trim(); - } - - /** - * Commit the changes to current branch - * - */ - add(paths: string[]): void { - this.logger.info({ repository: this.repo }, 'GitHub: Add Path'); - for (const path of paths) { - execFileSync('git', ['add', path], { cwd: this.repoName }).toString().trim(); - } - } - - /** - * Commit the changes to current branch - * - */ - commit(message: string): void { - this.logger.info({ repository: this.repo }, 'GitHub: Commit all'); - execFileSync('git', ['commit', '-am', message], { cwd: this.repoName }).toString().trim(); - } - - /** - * Push the local brach - * - */ - push(): void { - this.logger.info({ repository: this.repo }, 'GitHub: Push'); - execFileSync('git', ['push', 'origin', 'HEAD'], { cwd: this.repoName }).toString().trim(); - } - - /** - * Create pull request - * This needs to use github API to create Pull request by access token. - * - */ - async createPullRequests(branch: string, title: string, draft: boolean): Promise { - // Create pull request from the give head - const response = await this.octokit.rest.pulls.create({ - owner: this.org, - repo: this.repoName, - title, - head: branch, - base: 'master', - draft, - }); - if (!this.isOk(response.status)) throw new Error('Failed to create pull request.'); - this.logger.info({ branch, url: response.data.html_url }, 'GitHub: Create Pull Request'); - return response.data.number; - } -} diff --git a/packages/cli/src/cli/github/make.cog.pr.ts b/packages/cli/src/cli/github/make.cog.pr.ts deleted file mode 100644 index fcf662b20..000000000 --- a/packages/cli/src/cli/github/make.cog.pr.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { - ConfigId, - ConfigLayer, - ConfigPrefix, - ConfigTileSetRaster, - ConfigTileSetVector, - TileSetType, -} from '@basemaps/config'; -import { LogType, fsa } from '@basemaps/shared'; -import { Category, DefaultCategorySetting } from '../cogify/action.make.cog.pr.js'; -import { Github } from './github.js'; -import { TileSetConfigSchema } from '@basemaps/config/build/json/parse.tile.set.js'; -import { execFileSync } from 'child_process'; - -export class MakeCogGithub extends Github { - imagery: string; - _formatInstalled = false; - constructor(imagery: string, repo: string, logger: LogType) { - super(repo, logger); - this.imagery = imagery; - } - - /** - * Install the dependencies for the cloned repo - */ - npmInstall(): boolean { - this.logger.info({ repository: this.repo }, 'GitHub: Npm Install'); - execFileSync('npm', ['install', '--include=dev'], { cwd: this.repoName }); - return true; - } - - /** - * Format the config files by prettier - */ - formatConfigFile(path = './config/'): void { - if (!this._formatInstalled) this._formatInstalled = this.npmInstall(); - this.logger.info({ repository: this.repo }, 'GitHub: Prettier'); - execFileSync('npx', ['prettier', '-w', path], { cwd: this.repoName }); - } - - /** - * Prepare and create pull request for the aerial tileset config - */ - async updateRasterTileSet( - filename: string, - layer: ConfigLayer, - category: Category, - individual: boolean, - ): Promise { - const branch = `feat/bot-config-raster-${this.imagery}`; - - // Clone the basemaps-config repo and checkout branch - this.clone(); - this.configUser(); - this.getBranch(branch); - - this.logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file'); - if (individual) { - // Prepare new standalone tileset config - const tileSet: TileSetConfigSchema = { - type: TileSetType.Raster, - id: ConfigId.prefix(ConfigPrefix.TileSet, layer.name), - title: layer.title, - layers: [layer], - }; - const tileSetPath = fsa.joinAll('config', 'tileset', 'individual', `${layer.name}.json`); - const fullPath = fsa.join(this.repoName, tileSetPath); - await fsa.write(fullPath, JSON.stringify(tileSet)); - // Format the config file by prettier - this.formatConfigFile(tileSetPath); - this.add([tileSetPath]); - } else { - // Prepare new aerial tileset config - const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`); - const fullPath = fsa.join(this.repoName, tileSetPath); - const tileSet = await fsa.readJson(fullPath); - const newTileSet = await this.prepareRasterTileSetConfig(layer, tileSet, category); - // skip pull request if not an urban or rural imagery - if (newTileSet == null) return; - await fsa.write(fullPath, JSON.stringify(newTileSet)); - // Format the config file by prettier - this.formatConfigFile(tileSetPath); - this.add([tileSetPath]); - } - - // Commit and push the changes - const message = `config(raster): Add imagery ${this.imagery} to ${filename} config file.`; - this.commit(message); - this.push(); - await this.createPullRequests(branch, message, false); - } - - /** - * Set the default setting for the category - */ - setDefaultConfig(layer: ConfigLayer, category: Category): ConfigLayer { - layer.category = category; - const defaultSetting = DefaultCategorySetting[category]; - if (defaultSetting.minZoom != null && layer.minZoom != null) layer.minZoom = defaultSetting.minZoom; - return layer; - } - - /** - * Add new layer at the bottom of related category - */ - addLayer(layer: ConfigLayer, tileSet: ConfigTileSetRaster, category: Category): ConfigTileSetRaster { - for (let i = tileSet.layers.length - 1; i >= 0; i--) { - // Add new layer at the end of category - if (tileSet.layers[i].category === category) { - // Find first valid category and insert new record above that. - tileSet.layers.splice(i + 1, 0, layer); - break; - } - } - return tileSet; - } - - /** - * Prepare raster tileSet config json - */ - async prepareRasterTileSetConfig( - layer: ConfigLayer, - tileSet: ConfigTileSetRaster, - category: Category, - ): Promise { - // Reprocess existing layer - for (let i = 0; i < tileSet.layers.length; i++) { - if (tileSet.layers[i].name === layer.name) { - tileSet.layers[i] = layer; - return tileSet; - } - } - - // Set default Config if not existing layer - this.setDefaultConfig(layer, category); - - // Set layer zoom level and add to latest order - if (category === Category.Rural) { - for (let i = 0; i < tileSet.layers.length; i++) { - // Add new layer above the first Urban - if (tileSet.layers[i].category === Category.Urban) { - // Find first valid Urban and insert new record above that. - tileSet.layers.splice(i, 0, layer); - break; - } - } - } else if (category === Category.Other) { - // Add new layer at the bottom - tileSet.layers.push(layer); - } else { - this.addLayer(layer, tileSet, category); - } - - return tileSet; - } - - /** - * Prepare and create pull request for the aerial tileset config - */ - async updateVectorTileSet(filename: string, layer: ConfigLayer): Promise { - const branch = `feat/bot-config-vector-${this.imagery}`; - - // Clone the basemaps-config repo and checkout branch - this.clone(); - this.configUser(); - this.getBranch(branch); - - // Prepare new aerial tileset config - this.logger.info({ imagery: this.imagery }, 'GitHub: Get the master TileSet config file'); - const tileSetPath = fsa.joinAll('config', 'tileset', `${filename}.json`); - const fullPath = fsa.join(this.repoName, tileSetPath); - const tileSet = await fsa.readJson(fullPath); - const newTileSet = await this.prepareVectorTileSetConfig(layer, tileSet); - - // skip pull request if not an urban or rural imagery - if (newTileSet == null) return; - await fsa.write(fullPath, JSON.stringify(newTileSet)); - // Format the config file by prettier - this.formatConfigFile(tileSetPath); - - // Commit and push the changes - const message = `config(vector): Update the ${this.imagery} to ${filename} config file.`; - this.add([tileSetPath]); - this.commit(message); - this.push(); - await this.createPullRequests(branch, message, false); - } - - /** - * Prepare raster tileSet config json - */ - async prepareVectorTileSetConfig(layer: ConfigLayer, tileSet: ConfigTileSetVector): Promise { - // Reprocess existing layer - for (let i = 0; i < tileSet.layers.length; i++) { - if (tileSet.layers[i].name === layer.name) { - tileSet.layers[i] = layer; - return tileSet; - } - } - return tileSet; - } -} diff --git a/packages/cli/src/cli/index.ts b/packages/cli/src/cli/index.ts index 3b96f81fb..72def670f 100644 --- a/packages/cli/src/cli/index.ts +++ b/packages/cli/src/cli/index.ts @@ -1,18 +1,13 @@ #!/usr/bin/env node import { BaseCommandLine } from '@basemaps/shared/build/cli/base.js'; import 'source-map-support/register.js'; -import { CommandCogCreate } from './cogify/action.cog.js'; import { CommandCreateOverview } from './overview/action.create.overview.js'; -import { CommandCogPullRequest } from './cogify/action.make.cog.pr.js'; -import { CommandJobCreate } from './cogify/action.job.js'; -import { CommandMakeCog } from './cogify/action.make.cog.js'; import { CommandBundleAssets } from './config/action.bundle.assets.js'; import { CommandBundle } from './config/action.bundle.js'; import { CommandCogMapSheet } from './config/action.cog.mapsheet.js'; import { CommandImageryConfig } from './config/action.imagery.config.js'; import { CommandImport } from './config/action.import.js'; import { CommandServe } from './server/action.serve.js'; -import { CommandSprites } from './sprites/action.sprites.js'; export class BasemapsConfigCommandLine extends BaseCommandLine { constructor() { @@ -21,20 +16,16 @@ export class BasemapsConfigCommandLine extends BaseCommandLine { toolDescription: 'Basemaps config command tools', }); - this.addAction(new CommandCogCreate()); - this.addAction(new CommandJobCreate()); - this.addAction(new CommandMakeCog()); - this.addAction(new CommandCreateOverview()); - this.addAction(new CommandCogPullRequest()); - + // Basemaps Config this.addAction(new CommandBundle()); this.addAction(new CommandBundleAssets()); this.addAction(new CommandImport()); this.addAction(new CommandImageryConfig()); - this.addAction(new CommandSprites()); - this.addAction(new CommandServe()); - + // Argo this.addAction(new CommandCogMapSheet()); + this.addAction(new CommandCreateOverview()); + + this.addAction(new CommandServe()); } } diff --git a/packages/cli/src/cli/overview/action.create.overview.ts b/packages/cli/src/cli/overview/action.create.overview.ts index 9205a86f0..573cba5d1 100644 --- a/packages/cli/src/cli/overview/action.create.overview.ts +++ b/packages/cli/src/cli/overview/action.create.overview.ts @@ -12,13 +12,18 @@ import * as path from 'path'; import { resolve } from 'path'; import { CogBuilder } from '../../cog/builder.js'; import { Cutline } from '../../cog/cutline.js'; -import { filterTiff, MaxConcurrencyDefault } from '../../cog/job.factory.js'; import { createOverviewWmtsCapabilities } from './overview.wmts.js'; import { JobTiles, tile } from './tile.generator.js'; import { SimpleTimer } from './timer.js'; const DefaultMaxZoom = 15; // Limitation of maximum overview zoom level to create const MaxNumberTiles = 25000; // Limitation of maximum number of tiles we can create for overview. +export const MaxConcurrencyDefault = 50; + +export function filterTiff(a: string): boolean { + const lowerA = a.toLowerCase(); + return lowerA.endsWith('.tiff') || lowerA.endsWith('.tif'); +} export class CommandCreateOverview extends CommandLineAction { private source: CommandLineStringParameter; diff --git a/packages/cli/src/cli/sprites/action.sprites.ts b/packages/cli/src/cli/sprites/action.sprites.ts deleted file mode 100644 index dce7d08e6..000000000 --- a/packages/cli/src/cli/sprites/action.sprites.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { - CommandLineAction, - CommandLineFlagParameter, - CommandLineIntegerListParameter, - CommandLineStringListParameter, - CommandLineStringParameter, -} from '@rushstack/ts-command-line'; - -import { buildSprites } from '@basemaps/sprites'; - -export class CommandSprites extends CommandLineAction { - paths: CommandLineStringListParameter; - output: CommandLineStringParameter; - ratio: CommandLineIntegerListParameter; - retina: CommandLineFlagParameter; - - public constructor() { - super({ - actionName: 'sprites', - summary: 'Cli tool to create sprite sheet', - documentation: 'Create a sprite sheet from a folder of sprites', - }); - } - - protected onDefineParameters(): void { - this.paths = this.defineStringListParameter({ - argumentName: 'PATH', - parameterLongName: '--path', - description: 'Paths to the sprite files.', - }); - - this.output = this.defineStringParameter({ - argumentName: 'OUTPUT', - parameterLongName: '--output', - description: 'Paths of the output files', - }); - - this.ratio = this.defineIntegerListParameter({ - argumentName: 'RATIO', - parameterLongName: '--ratio', - description: 'Pixel ratio, default: "--ratio 1 --ratio 2"', - }); - - this.retina = this.defineFlagParameter({ - parameterLongName: '--retina', - description: 'Double the pixel ratios, 1x becomes 2x', - }); - } - - protected async onExecute(): Promise { - if (this.paths?.values == null || this.paths.values.length === 0) { - throw new Error('No sprite paths supplied'); - } - const ratio = [...this.ratio.values]; - const paths = [...this.paths.values]; - await buildSprites(ratio, this.retina.value, paths, this.output.value); - } -} diff --git a/packages/cli/src/cli/util.ts b/packages/cli/src/cli/util.ts index 3c03418a0..2865b6432 100644 --- a/packages/cli/src/cli/util.ts +++ b/packages/cli/src/cli/util.ts @@ -1,5 +1,4 @@ -import { Env, fsa, LogConfig } from '@basemaps/shared'; -import { Projection } from '@basemaps/geo'; +import { fsa, LogConfig } from '@basemaps/shared'; import CloudFormation from 'aws-sdk/clients/cloudformation.js'; import CloudFront from 'aws-sdk/clients/cloudfront.js'; import S3 from 'aws-sdk/clients/s3.js'; @@ -8,7 +7,6 @@ import crypto from 'crypto'; import path from 'path'; import { gzip } from 'zlib'; import { promisify } from 'util'; -import { CogStacJob } from '../cog/cog.stac.job.js'; import slugify from 'slugify'; // Cloudfront has to be defined in us-east-1 @@ -130,20 +128,6 @@ export async function uploadStaticFile( return true; } -/** - * Prepare QA urls with center location - */ -export async function prepareUrl(job: CogStacJob): Promise { - const bounds = job.output.bounds; - const center = { x: bounds.x + bounds.width / 2, y: bounds.y + bounds.height / 2 }; - const proj = Projection.get(job.tileMatrix.projection); - const centerLatLon = proj.toWgs84([center.x, center.y]).map((c) => c.toFixed(6)); - const targetZoom = Math.max(job.tileMatrix.findBestZoom(job.output.gsd) - 12, 0); - const base = Env.get(Env.PublicUrlBase); - const url = `${base}/?i=${job.id}&p=${job.tileMatrix.identifier}&debug#@${centerLatLon[1]},${centerLatLon[0]},z${targetZoom}`; - return url; -} - /** * Make a tile imagery title as imagery name * @example diff --git a/packages/cli/src/cog/__tests__/cog.test.ts b/packages/cli/src/cog/__tests__/cog.test.ts deleted file mode 100644 index 7b8181489..000000000 --- a/packages/cli/src/cog/__tests__/cog.test.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { GoogleTms } from '@basemaps/geo'; -import { LogConfig } from '@basemaps/shared'; -import { round } from '@basemaps/test/build/rounding.js'; -import o from 'ospec'; -import { GdalCogBuilder } from '../../gdal/gdal.cog.js'; -import { buildCogForName } from '../cog.js'; -import { SourceTiffTestHelper } from './source.tiff.testhelper.js'; - -LogConfig.disable(); - -o.spec('cog', () => { - o.spec('buildCogForName', () => { - const origConvert = GdalCogBuilder.prototype.convert; - - o.afterEach(() => { - GdalCogBuilder.prototype.convert = origConvert; - }); - - o('gdal_translate args', async () => { - let gdalCogBuilder: GdalCogBuilder | null = null; - let convertArgs: any = null; - const convert = function (this: any, ...args: any[]): void { - gdalCogBuilder = this; // eslint-disable-line @typescript-eslint/no-this-alias - convertArgs = args; - }; - GdalCogBuilder.prototype.convert = convert as any; - - const job = SourceTiffTestHelper.makeCogJob(); - const logger = LogConfig.get(); - - const name = '4-15-10'; - - job.output.files = [{ name, ...job.tileMatrix.tileToSourceBounds({ x: 15, y: 10, z: 4 }) }]; - - await buildCogForName(job, name, '/tmp/test.vrt', '/tmp/out-tiff', logger, true); - o(convertArgs[0].info).equals(logger.info); - - const { config } = gdalCogBuilder!; - config.bbox = round(config.bbox, 4); - config.targetRes = round(config.targetRes, 4); - - o(config).deepEquals({ - bbox: [17532819.7999, -5009377.0857, 20037508.3428, -7514065.6285], - alignmentLevels: 13, - compression: 'webp', - tileMatrix: GoogleTms, - resampling: { warp: 'bilinear', overview: 'lanczos' }, - blockSize: 512, - targetRes: 0.75, - quality: 90, - }); - o(gdalCogBuilder!.source).equals('/tmp/test.vrt'); - o(gdalCogBuilder!.target).equals('/tmp/out-tiff'); - - o(round(gdalCogBuilder!.args, 4)).deepEquals([ - '-of', - 'COG', - '-co', - 'TILING_SCHEME=GoogleMapsCompatible', - '-co', - 'NUM_THREADS=ALL_CPUS', - '--config', - 'GDAL_NUM_THREADS', - 'ALL_CPUS', - '-co', - 'BIGTIFF=YES', - '-co', - 'ADD_ALPHA=YES', - '-co', - 'BLOCKSIZE=512', - '-co', - 'WARP_RESAMPLING=bilinear', - '-co', - 'OVERVIEW_RESAMPLING=lanczos', - '-co', - 'COMPRESS=webp', - '-co', - 'ALIGNED_LEVELS=13', - '-co', - 'QUALITY=90', - '-co', - 'SPARSE_OK=YES', - '--config', - 'GDAL_DISABLE_READDIR_ON_OPEN', - 'EMPTY_DIR', - '-tr', - '0.75', - '0.75', - '-projwin', - '17532819.7999', - '-5009377.0857', - '20037508.3428', - '-7514065.6285', - '-projwin_srs', - 'EPSG:3857', - '/tmp/test.vrt', - '/tmp/out-tiff', - ]); - }); - }); -}); diff --git a/packages/cli/src/cog/__tests__/cog.vrt.test.ts b/packages/cli/src/cog/__tests__/cog.vrt.test.ts deleted file mode 100644 index e8314048e..000000000 --- a/packages/cli/src/cog/__tests__/cog.vrt.test.ts +++ /dev/null @@ -1,275 +0,0 @@ -import { EpsgCode, GoogleTms, Nztm2000Tms } from '@basemaps/geo'; -import { fsa, LogConfig } from '@basemaps/shared'; -import { qkToName } from '@basemaps/geo/build/proj/__tests__/test.util.js'; -import { round } from '@basemaps/test/build/rounding.js'; -import o from 'ospec'; -import path from 'path'; -import url from 'url'; -import { Gdal } from '../../gdal/gdal.js'; -import { CogVrt } from '../cog.vrt.js'; -import { Cutline } from '../cutline.js'; -import { SourceTiffTestHelper } from './source.tiff.testhelper.js'; -const __dirname = path.dirname(url.fileURLToPath(import.meta.url)); - -o.spec('cog.vrt', () => { - const tmpFolder = '/tmp/my-tmp-folder'; - - const job = SourceTiffTestHelper.makeCogJob(); - - const logger = LogConfig.get(); - LogConfig.disable(); - - const testDir = `${__dirname}/../../../__test.assets__`; - - const sourceBounds = SourceTiffTestHelper.tiffNztmBounds(testDir); - const [tif1, tif2] = sourceBounds; - const [tif1Path, tif2Path] = sourceBounds.map(({ name }) => name); - - let cutTiffArgs: Array> = []; - - let runSpy = o.spy(); - - const origFileOperatorWriteJson = fsa.writeJson; - const { create } = Gdal; - - let gdal: any; - - o.after(() => { - fsa.writeJson = origFileOperatorWriteJson; - Gdal.create = create; - }); - - o.beforeEach(() => { - runSpy = o.spy(); - job.output.tileMatrix = GoogleTms.identifier; - job.source.epsg = EpsgCode.Nztm2000; - job.source.gsd = 20; - gdal = { run: runSpy }; - (Gdal as any).create = (): any => gdal; - job.source.files = [tif1, tif2]; - - cutTiffArgs = []; - fsa.writeJson = ((...args: any): any => { - cutTiffArgs.push(args); - }) as any; - - job.output.cutline = undefined; - }); - - o('1 crosses, 1 outside', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson')); - const cl2 = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - cutline.clipPoly.push(...cl2.clipPoly); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, qkToName('31133322'), logger); - - o(job.source.files).deepEquals([tif1, tif2]); - o(cutline.clipPoly.length).equals(2); - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - }); - - o('not within tile', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson')); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, qkToName('3131110001'), logger); - - o(cutTiffArgs.length).equals(0); - o(vrt).equals(null); - o(runSpy.callCount).equals(0); - }); - - o('no cutline same projection', async () => { - const vrt = await CogVrt.buildVrt(tmpFolder, job, new Cutline(GoogleTms), qkToName('31'), logger); - - o(job.source.files).deepEquals([tif1, tif2]); - o(cutTiffArgs.length).equals(0); - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(runSpy.callCount).equals(2); - o(runSpy.args[0]).equals('gdalwarp'); - }); - - o('no cutline diff projection', async () => { - job.output.tileMatrix = Nztm2000Tms.identifier; //EpsgCode.Nztm2000; - const vrt = await CogVrt.buildVrt(tmpFolder, job, new Cutline(GoogleTms), qkToName('31'), logger); - - o(job.source.files).deepEquals([tif1, tif2]); - o(cutTiffArgs.length).equals(0); - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(runSpy.callCount).equals(2); - o(runSpy.args[0]).equals('gdalwarp'); - }); - - o('fully within same projection', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), -100); - - const name = qkToName('311333222321113310'); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, name, logger); - - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(cutTiffArgs.length).equals(0); - o(runSpy.callCount).equals(2); - o(runSpy.args[0]).equals('gdalwarp'); - }); - - o('intersected cutline', async () => { - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/kapiti.geojson'), 20); - job.output.cutline = { blend: 20, href: 'cutline.json' }; - - const name = qkToName('311333222321113'); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, name, logger); - - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(cutTiffArgs.length).equals(1); - o(cutTiffArgs[0][1]).deepEquals(cutline.toGeoJson()); - - o(round(cutTiffArgs[0], 6)).deepEquals([ - '/tmp/my-tmp-folder/cutline.geojson', - { - type: 'FeatureCollection', - features: [ - { - type: 'Feature', - geometry: { - type: 'MultiPolygon', - coordinates: [ - [ - [ - [19466418.186171, -4995093.077172], - [19467156.308979, -4996108.09485], - [19467854.290313, -4996091.668391], - [19468645.309917, -4995755.689377], - [19469463.193363, -4995206.362781], - [19470215.804402, -4994433.114642], - [19470856.681731, -4993478.14574], - [19471932.55678, -4992600.063654], - [19473538.532795, -4990108.677214], - [19474243.881356, -4988798.711151], - [19474243.881356, -4986828.162353], - [19472131.837905, -4986828.162353], - [19472109.508239, -4986926.945548], - [19470978.43787, -4989417.445436], - [19468800.977534, -4991497.540469], - [19467511.856865, -4993217.174782], - [19466418.186171, -4995093.077172], - ], - ], - ], - }, - properties: {}, - }, - ], - crs: { type: 'name', properties: { name: 'urn:ogc:def:crs:EPSG::3857' } }, - }, - ]); - }); - - o('1 surrounded with s3 files', async () => { - const mount = o.spy(); - const setCredentials = o.spy(); - gdal.mount = mount; - gdal.setCredentials = setCredentials; - - const vtif1 = '/vsis3' + tif1Path; - const vtif2 = '/vsis3' + tif2Path; - - job.source.location = { type: 's3', path: 's3://foo/bar', roleArn: 'a:role:string' }; - - job.source.files = job.source.files.map((o) => { - o = Object.assign({}, o); - o.name = 's3:/' + o.name; - return o; - }); - job.output.cutline = { blend: 10, href: 'cutline.json' }; - const cutline = new Cutline(GoogleTms, await Cutline.loadCutline(testDir + '/mana.geojson')); - - const vrt = await CogVrt.buildVrt(tmpFolder, job, cutline, qkToName('3131110001'), logger); - - o(vrt).equals('/tmp/my-tmp-folder/cog.vrt'); - o(cutTiffArgs.length).equals(1); - o(cutTiffArgs[0][0]).equals(tmpFolder + '/cutline.geojson'); - - o(cutline.clipPoly.length).equals(1); - - const geo = cutline.toGeoJson(); - - o(geo.type).equals('FeatureCollection'); - - const coordinates = (geo.features[0].geometry as any).coordinates; - if (geo.type === 'FeatureCollection') { - o(geo.features).deepEquals([ - { - type: 'Feature', - properties: {}, - geometry: { - type: 'MultiPolygon', - coordinates, - }, - }, - ]); - } - - o(round(coordinates, 5)).deepEquals([ - [ - [ - [19455446.57501, -5025689.98761], - [19455543.12619, -5026720.78623], - [19457346.41188, -5024885.32147], - [19456570.81047, -5023799.46087], - [19456560.03196, -5023796.37227], - [19456551.09434, -5023782.48917], - [19456540.68564, -5023777.32429], - [19456531.22726, -5023778.34575], - [19456524.31633, -5023783.03837], - [19456521.12157, -5023788.06858], - [19456521.62607, -5023792.6091], - [19456528.41956, -5023802.39913], - [19456528.99998, -5023810.66149], - [19456513.95059, -5023820.48595], - [19456497.91736, -5023822.46951], - [19456431.41331, -5023815.14488], - [19456325.95579, -5024324.4265], - [19455446.57501, -5025689.98761], - ], - ], - ]); - - o(runSpy.callCount).equals(2); - o(mount.calls.map((c: any) => c.args[0])).deepEquals([tmpFolder, ...job.source.files.map((c) => c.name)]); - - o(setCredentials.calls.map((c: any) => c.args[0].service.config.params.RoleArn)).deepEquals(['a:role:string']); - - o(round((runSpy.calls[0] as any).args)).deepEquals([ - 'gdalbuildvrt', - ['-hidenodata', '-allow_projection_difference', '-addalpha', '/tmp/my-tmp-folder/source.vrt', vtif1, vtif2], - logger, - ]); - - o(round(runSpy.args)).deepEquals([ - 'gdalwarp', - [ - '-of', - 'VRT', - '-multi', - '-wo', - 'NUM_THREADS=ALL_CPUS', - '-s_srs', - 'EPSG:2193', - '-t_srs', - 'EPSG:3857', - '-tr', - '0.75', - '0.75', - '-tap', - '-cutline', - '/tmp/my-tmp-folder/cutline.geojson', - '-cblend', - '10', - '/tmp/my-tmp-folder/source.vrt', - '/tmp/my-tmp-folder/cog.vrt', - ], - logger, - ]); - }); -}); diff --git a/packages/cli/src/cog/cog.ts b/packages/cli/src/cog/cog.ts deleted file mode 100644 index 4e72bf3d7..000000000 --- a/packages/cli/src/cog/cog.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { Bounds, Projection, TileMatrixSet } from '@basemaps/geo'; -import { Env, isConfigS3Role, LogType } from '@basemaps/shared'; -import { GdalCogBuilder } from '../gdal/gdal.cog.js'; -import { GdalCommand } from '../gdal/gdal.command.js'; -import { GdalProgressParser } from '../gdal/gdal.progress.js'; -import { CogJob } from './types.js'; -import { AwsCredentials } from '@chunkd/source-aws-v2'; - -/** - * Create a onProgress logger - * - * @param keys additional keys to log - * @param logger base logger to use - */ -export function onProgress(gdal: GdalCommand, keys: Record, logger: LogType): void { - let lastTime = Date.now(); - - gdal.parser = new GdalProgressParser(); - gdal.parser.on('progress', (p: number): void => { - logger.trace({ ...keys, progress: parseFloat(p.toFixed(2)), progressTime: Date.now() - lastTime }, 'Progress'); - lastTime = Date.now(); - }); -} - -/** - * Build a COG for a given collection of tiffs - * - * @param job the job to process - * @param name tile name of cog to generate - * @param vrtLocation Location of the source VRT file - * @param outputTiffPath Path to where the output tiff will be stored - * @param logger Logger to use - * @param execute Whether to actually execute the transformation, - */ -export async function buildCogForName( - job: CogJob, - name: string, - vrtLocation: string, - outputTiffPath: string, - logger: LogType, - execute = false, -): Promise { - const startTime = Date.now(); - - const { targetZoom, tileMatrix } = job; - - const nb = job.output.files.find((nb) => nb.name === name); - - if (nb == null) { - throw new Error("Can't find COG named " + name); - } - - const bounds = Bounds.fromJson(nb); - - const tile = TileMatrixSet.nameToTile(name); - - const blockSize = tileMatrix.tileSize * 2; // FIXME is this blockFactor always 2 - const alignmentLevels = Projection.findAlignmentLevels(tileMatrix, tile, job.source.gsd); - - const cogBuild = new GdalCogBuilder(vrtLocation, outputTiffPath, { - bbox: [bounds.x, bounds.bottom, bounds.right, bounds.y], - tileMatrix, - blockSize, - targetRes: job.output.gsd, - alignmentLevels, - resampling: job.output.resampling, - quality: job.output.quality, - }); - - onProgress(cogBuild.gdal, { name, target: 'tiff' }, logger); - - logger.info( - { - imageSize: Projection.getImagePixelWidth(tileMatrix, tile, targetZoom), - name, - tile, - alignmentLevels, - }, - 'CreateCog', - ); - - const sourceLocation = job.source.location; - // If required assume role - if (isConfigS3Role(sourceLocation)) { - const credentials = AwsCredentials.role( - sourceLocation.roleArn, - sourceLocation.externalId, - Env.getNumber(Env.AwsRoleDurationHours, 8) * 60 * 60, - ); - cogBuild.gdal.setCredentials(credentials); - } - - if (cogBuild.gdal.mount != null) { - for (const file of job.source.files) cogBuild.gdal.mount(file.name); - } - - if (execute) { - await cogBuild.convert(logger.child({ name })); - logger.info({ name, duration: Date.now() - startTime }, 'CogCreated'); - } -} diff --git a/packages/cli/src/cog/cog.vrt.ts b/packages/cli/src/cog/cog.vrt.ts deleted file mode 100644 index ab6ad041b..000000000 --- a/packages/cli/src/cog/cog.vrt.ts +++ /dev/null @@ -1,144 +0,0 @@ -import { Epsg } from '@basemaps/geo'; -import { Env, fsa, isConfigS3Role, LogType, s3ToVsis3 } from '@basemaps/shared'; -import { Gdal } from '../gdal/gdal.js'; -import { GdalCommand } from '../gdal/gdal.command.js'; -import { onProgress } from './cog.js'; -import { Cutline } from './cutline.js'; -import { CogJob } from './types.js'; -import { AwsCredentials } from '@chunkd/source-aws-v2'; - -/** - * Build the VRT for the needed source imagery - */ -async function buildPlainVrt( - job: CogJob, - sourceFiles: string[], - vrtPath: string, - gdalCommand: GdalCommand, - logger: LogType, -): Promise { - const buildOpts = ['-hidenodata', '-allow_projection_difference']; - if (job.output.addAlpha) { - buildOpts.push('-addalpha'); - } - - logger.debug({ buildOpts: buildOpts.join(' ') }, 'gdalbuildvrt'); - await gdalCommand.run('gdalbuildvrt', [...buildOpts, vrtPath, ...sourceFiles], logger); -} - -/** - * Warp the source vrt to target projection using an optional cutline - */ -async function buildWarpVrt( - job: CogJob, - sourceVrtPath: string, - gdalCommand: GdalCommand, - cogVrtPath: string, - tr: string, - logger: LogType, - cutlineTarget: string, -): Promise { - const warpOpts = [ - '-of', - 'VRT', - '-multi', - '-wo', - 'NUM_THREADS=ALL_CPUS', - '-s_srs', - Epsg.get(job.source.epsg).toEpsgString(), - '-t_srs', - job.tileMatrix.projection.toEpsgString(), - '-tr', - tr, - tr, - '-tap', - ]; - if (job.output.cutline != null) { - warpOpts.push('-cutline', cutlineTarget); - if (job.output.cutline.blend !== 0) warpOpts.push('-cblend', String(job.output.cutline.blend)); - } - if (job.output.nodata != null) { - warpOpts.push('-srcnodata', String(job.output.nodata), '-dstnodata', String(job.output.nodata)); - } - if (job.output.resampling) { - warpOpts.push('-r', job.output.resampling.warp); - } - - logger.debug({ warpOpts: warpOpts.join(' ') }, 'gdalwarp'); - await gdalCommand.run('gdalwarp', [...warpOpts, cogVrtPath, sourceVrtPath], logger); -} - -export const CogVrt = { - /** - * Build a vrt file for a COG `name` that transforms the source imagery with a cutline - * - * @param tmpFolder temporary `vrt` and `cutline.geojson` will be written here - * @param job - * @param cutline Used to filter the source imagery - * @param name COG tile to reduce vrt and cutline - * @param logger - * - * @return the path to the vrt file - */ - async buildVrt( - tmpFolder: string, - job: CogJob, - cutline: Cutline, - name: string, - logger: LogType, - ): Promise { - logger.info({ name }, 'buildCogVrt'); - - const sourceFiles = cutline.filterSourcesForName(name, job).map(s3ToVsis3); - - if (sourceFiles.length === 0) { - return null; - } - - const sourceVrtPath = fsa.join(tmpFolder, `source.vrt`); - const cogVrtPath = fsa.join(tmpFolder, `cog.vrt`); - - let cutlineTarget = ''; - - if (cutline.clipPoly.length !== 0) { - cutlineTarget = fsa.join(tmpFolder, 'cutline.geojson'); - await fsa.writeJson(cutlineTarget, cutline.toGeoJson()); - } else { - job.output.cutline = undefined; - } - - logger.info( - { - inputTotal: job.source.files.length, - outputTotal: sourceFiles.length, - cutlinePolygons: cutline.clipPoly.length, - }, - 'Tiff count', - ); - - const gdalCommand = Gdal.create(); - - const sourceLocation = job.source.location; - // If required assume role - if (isConfigS3Role(sourceLocation)) { - const credentials = AwsCredentials.role( - sourceLocation.roleArn, - sourceLocation.externalId, - Env.getNumber(Env.AwsRoleDurationHours, 8) * 60 * 60, - ); - gdalCommand.setCredentials(credentials); - } - - if (gdalCommand.mount != null) { - gdalCommand.mount(tmpFolder); - for (const file of job.source.files) gdalCommand.mount(file.name); - } - - const tr = job.output.gsd.toString(); - - onProgress(gdalCommand, { target: `vrt.${job.tileMatrix.projection.code}` }, logger); - await buildPlainVrt(job, sourceFiles, sourceVrtPath, gdalCommand, logger); - await buildWarpVrt(job, cogVrtPath, gdalCommand, sourceVrtPath, tr, logger, cutlineTarget); - return cogVrtPath; - }, -}; diff --git a/packages/cli/src/cog/job.factory.ts b/packages/cli/src/cog/job.factory.ts deleted file mode 100644 index ee31549ae..000000000 --- a/packages/cli/src/cog/job.factory.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { Bounds } from '@basemaps/geo'; -import { fsa, isConfigS3Role, isFileConfigPath, LogConfig } from '@basemaps/shared'; -import { basename } from 'path'; -import * as ulid from 'ulid'; -import { BatchJob } from '../cli/cogify/batch.job.js'; -import { CogBuilder } from '../index.js'; -import { CogStacJob, JobCreationContext } from './cog.stac.job.js'; -import { Cutline } from './cutline.js'; - -export const MaxConcurrencyDefault = 50; - -export function filterTiff(a: string): boolean { - const lowerA = a.toLowerCase(); - return lowerA.endsWith('.tiff') || lowerA.endsWith('.tif'); -} - -/** Group a file list by its zoom */ -function groupFiles(files: { name: string }[]): Record { - const fileCounts: Record = {}; - for (const f of files) { - const zoom = f.name.split('-')[0]; - fileCounts[zoom] = (fileCounts[zoom] ?? 0) + 1; - } - return fileCounts; -} - -export const CogJobFactory = { - /** - * Create a COG Job and potentially submit it to AWS Batch for processing - */ - async create(ctx: JobCreationContext): Promise { - const id = ctx.override?.id ?? ulid.ulid(); - let imageryName = ctx.imageryName; - if (imageryName == null) imageryName = basename(ctx.sourceLocation.path); - - const logger = LogConfig.get().child({ id, imageryName }); - - const { sourceLocation } = ctx; - logger.info( - { source: ctx.sourceLocation.path, sourceRole: isConfigS3Role(sourceLocation) && sourceLocation.roleArn }, - 'ListTiffs', - ); - - fsa.configure(sourceLocation); - - const tiffList = isFileConfigPath(sourceLocation) - ? sourceLocation.files - : (await fsa.toArray(fsa.list(sourceLocation.path))).filter(filterTiff); - - const tiffSource = tiffList.map((path: string) => fsa.source(path)); - - const maxConcurrency = ctx.override?.concurrency ?? MaxConcurrencyDefault; - - logger.info({ source: sourceLocation.path, tiffCount: tiffList.length }, 'LoadingTiffs'); - - const cutline = new Cutline( - ctx.tileMatrix, - ctx.cutline && (await Cutline.loadCutline(ctx.cutline.href)), - ctx.cutline?.blend, - ); - - const builder = new CogBuilder(ctx.tileMatrix, maxConcurrency, logger, ctx.override?.projection); - const metadata = await builder.build(tiffSource, cutline, ctx.override?.alignedLevel); - - if (cutline.clipPoly.length === 0) { - // no cutline needed for this imagery set - ctx.cutline = undefined; - } - - const files = metadata.files.sort(Bounds.compareArea); - if (files.length > 0) { - const bigArea = files[files.length - 1]; - const smallArea = files[0]; - logger.info( - { - tileMatrix: ctx.tileMatrix.identifier, - // Size of the biggest image - big: bigArea.width / cutline.tileMatrix.pixelScale(metadata.resZoom), - // Size of the smallest image - small: smallArea.width / cutline.tileMatrix.pixelScale(metadata.resZoom), - }, - 'Covers', - ); - } - - // Don't log bounds as it is huge - logger.info( - { - ...metadata, - tileMatrix: ctx.tileMatrix.identifier, - bounds: undefined, - files: undefined, - fileCount: files.length, - fileGroups: groupFiles(metadata.files), - }, - 'CoveringGenerated', - ); - - let addAlpha = true; - // -addalpha to vrt adds extra alpha layers even if one already exist - if (metadata.bands > 3) { - logger.info({ bandCount: metadata.bands }, 'Vrt:DetectedAlpha, Disabling -addalpha'); - addAlpha = false; - } - - const job = await CogStacJob.create({ - id, - imageryName, - metadata, - ctx, - addAlpha, - cutlinePoly: cutline.clipPoly, - }); - - if (ctx.batch) await BatchJob.batchJob(job, true, logger); - logger.info({ tileMatrix: ctx.tileMatrix.identifier, job: job.getJobPath() }, 'Done'); - - return job; - }, -}; diff --git a/packages/cli/src/gdal/__tests__/gdal.test.ts b/packages/cli/src/gdal/__tests__/gdal.test.ts deleted file mode 100644 index 3409815aa..000000000 --- a/packages/cli/src/gdal/__tests__/gdal.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import o from 'ospec'; -import { GdalCogBuilder } from '../gdal.cog.js'; -import { normalizeAwsEnv } from '../gdal.command.js'; -import { GdalCogBuilderDefaults } from '../gdal.config.js'; -import { GdalDocker } from '../gdal.docker.js'; - -o.spec('GdalCogBuilder', () => { - o('should default all options', () => { - const builder = new GdalCogBuilder('/foo', 'bar.tiff'); - - o(builder.config.bbox).equals(undefined); - o(builder.config.compression).equals('webp'); - o(builder.config.resampling).deepEquals({ warp: 'bilinear', overview: 'lanczos' }); - o(builder.config.blockSize).equals(512); - o(builder.config.alignmentLevels).equals(1); - - o(builder.config).deepEquals({ ...GdalCogBuilderDefaults, bbox: undefined }); - }); - - o('should create a docker command', () => { - const builder = new GdalCogBuilder('/foo/foo.tiff', '/bar/bar.tiff'); - - const args = builder.args; - - o(args.includes('TILING_SCHEME=GoogleMapsCompatible')).equals(true); - o(args.includes('COMPRESS=webp')).equals(true); - o(builder.args.includes('BLOCKSIZE=512')).equals(true); - - builder.config.compression = 'jpeg'; - o(builder.args.includes('COMPRESS=jpeg')).equals(true); - - builder.config.blockSize = 256; - o(builder.args.includes('BLOCKSIZE=256')).equals(true); - }); - - o('should mount folders', () => { - const gdalDocker = new GdalDocker(); - gdalDocker.mount('/foo/bar'); - o(gdalDocker.mounts).deepEquals(['/foo']); - }); - - o('should not duplicate folders', () => { - const gdalDocker = new GdalDocker(); - gdalDocker.mount('/foo/bar.html'); - gdalDocker.mount('/foo/bar.tiff'); - o(gdalDocker.mounts).deepEquals(['/foo']); - }); - - o('should ignore s3 uris', () => { - const gdalDocker = new GdalDocker(); - gdalDocker.mount('s3://foo/bar.html'); - gdalDocker.mount('s3://foo/bar.tiff'); - o(gdalDocker.mounts).deepEquals([]); - }); - - o('should normalize aws environment vars', () => { - const env = normalizeAwsEnv({ AWS_PROFILE: 'lake' }); - o(env).deepEquals({ AWS_PROFILE: 'lake', AWS_DEFAULT_PROFILE: 'lake' }); - }); - - o('should error if aws config is not sane', () => { - o(() => normalizeAwsEnv({ AWS_PROFILE: 'lake', AWS_DEFAULT_PROFILE: 'other-lake' })).throws(Error); - }); -}); diff --git a/packages/cli/src/gdal/gdal.cog.ts b/packages/cli/src/gdal/gdal.cog.ts deleted file mode 100644 index f20584cb7..000000000 --- a/packages/cli/src/gdal/gdal.cog.ts +++ /dev/null @@ -1,143 +0,0 @@ -import { GoogleTms, Nztm2000Tms } from '@basemaps/geo'; -import { LogType } from '@basemaps/shared'; -import { ChildProcessWithoutNullStreams } from 'child_process'; -import { Gdal } from './gdal.js'; -import { GdalCommand } from './gdal.command.js'; -import { GdalCogBuilderDefaults, GdalCogBuilderOptions } from './gdal.config.js'; - -/** 1% Buffer to the tiff to help prevent gaps between tiles */ -// const TiffBuffer = 1.01; - -/** - * A docker based GDAL Cog Builder - * - * This uses the new 3.1 COG Driver https://gdal.org/drivers/raster/cog.html - * - * When GDAL 3.1 is released docker could be removed from this process. - */ -export class GdalCogBuilder { - config: GdalCogBuilderOptions; - - /** - * Source file generally a .vrt - */ - source: string; - /** - * Output file - */ - target: string; - - /** - * Current running child process - */ - child: ChildProcessWithoutNullStreams | null; - /** - * Promise waiting for child process to finish - */ - promise: Promise | null; - /** When the process started */ - startTime: number; - /** Gdal process */ - gdal: GdalCommand; - - constructor(source: string, target: string, config: Partial = {}) { - this.source = source; - this.target = target; - - this.config = { - bbox: config.bbox, - alignmentLevels: config.alignmentLevels ?? GdalCogBuilderDefaults.alignmentLevels, - compression: config.compression ?? GdalCogBuilderDefaults.compression, - tileMatrix: config.tileMatrix ?? GdalCogBuilderDefaults.tileMatrix, - resampling: config.resampling ?? GdalCogBuilderDefaults.resampling, - blockSize: config.blockSize ?? GdalCogBuilderDefaults.blockSize, - targetRes: config.targetRes ?? GdalCogBuilderDefaults.targetRes, - quality: config.quality ?? GdalCogBuilderDefaults.quality, - }; - this.gdal = Gdal.create(); - this.gdal.mount?.(source); - this.gdal.mount?.(target); - } - - getBounds(): string[] { - if (this.config.bbox == null) { - return []; - } - - // TODO in theory this should be clamped to the lower right of the imagery, as there is no point generating large empty tiffs - const [ulX, ulY, lrX, lrY] = this.config.bbox; - return ['-projwin', ulX, ulY, lrX, lrY, '-projwin_srs', this.config.tileMatrix.projection.toEpsgString()].map( - String, - ); - } - - get tileMatrixFileName(): string { - const tileMatrix = this.config.tileMatrix; - // Gdal built in TileMatrixSets - if (tileMatrix.identifier === GoogleTms.identifier) return 'GoogleMapsCompatible'; - if (tileMatrix.identifier === Nztm2000Tms.identifier) return 'NZTM2000'; - - return 'https://raw.githubusercontent.com/linz/NZTM2000TileMatrixSet/master/raw/NZTM2000Quad.json'; - } - - get args(): string[] { - const tr = this.config.targetRes.toString(); - return [ - // Force output using COG Driver - '-of', - 'COG', - // Force GoogleMaps tiling - '-co', - `TILING_SCHEME=${this.tileMatrixFileName}`, - // Max CPU POWER - '-co', - 'NUM_THREADS=ALL_CPUS', - // in GDAL 3.7.x NUM_THREADS will also set GDAL_NUM_THREADS - '--config', - 'GDAL_NUM_THREADS', - 'ALL_CPUS', - // Force big tiff the extra few bytes savings of using little tiffs does not affect us - '-co', - 'BIGTIFF=YES', - // Force a alpha layer - '-co', - 'ADD_ALPHA=YES', - // User configured output block size - '-co', - `BLOCKSIZE=${this.config.blockSize}`, - // Configured resampling methods - '-co', - `WARP_RESAMPLING=${this.config.resampling.warp}`, - '-co', - `OVERVIEW_RESAMPLING=${this.config.resampling.overview}`, - // User configured compression - '-co', - `COMPRESS=${this.config.compression}`, - // Number of levels to align to web mercator - '-co', - `ALIGNED_LEVELS=${this.config.alignmentLevels}`, - // Default quality of 75 is too low for our needs - '-co', - `QUALITY=${this.config.quality}`, - // most of the imagery contains a lot of empty tiles, no need to output them - '-co', - `SPARSE_OK=YES`, - // Do not attempt to read sidecar files - '--config', - `GDAL_DISABLE_READDIR_ON_OPEN`, - `EMPTY_DIR`, - // Force a target resolution to be better than the imagery not worse - '-tr', - tr, - tr, - ...this.getBounds(), - - this.source, - this.target, - ]; - } - - async convert(log: LogType): Promise { - await this.gdal.run('gdal_translate', this.args, log); - } -} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts deleted file mode 100644 index 7ebe65037..000000000 --- a/packages/cli/src/index.ts +++ /dev/null @@ -1,6 +0,0 @@ -// export * from './cli/basemaps/tileset.updater'; -export { CogBuilder } from './cog/builder.js'; -export { CogJobFactory } from './cog/job.factory.js'; -export { Gdal } from './gdal/gdal.js'; -export { GdalCogBuilder } from './gdal/gdal.cog.js'; -export { GdalCogBuilderOptions } from './gdal/gdal.config.js'; diff --git a/yarn.lock b/yarn.lock index 145290e20..ee229d95f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1449,13 +1449,6 @@ dependencies: "@octokit/types" "^6.0.3" -"@octokit/auth-token@^3.0.0": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@octokit/auth-token/-/auth-token-3.0.1.tgz#88bc2baf5d706cb258474e722a720a8365dff2ec" - integrity sha512-/USkK4cioY209wXRpund6HZzHo9GmjakpV9ycOkpMcMxMk7QVcVFVyCMtzvXYiHsB2crgDgrtNYSELYFBXhhaA== - dependencies: - "@octokit/types" "^7.0.0" - "@octokit/core@^3.2.3": version "3.4.0" resolved "https://registry.yarnpkg.com/@octokit/core/-/core-3.4.0.tgz#b48aa27d755b339fe7550548b340dcc2b513b742" @@ -1469,19 +1462,6 @@ before-after-hook "^2.2.0" universal-user-agent "^6.0.0" -"@octokit/core@^4.0.5": - version "4.0.5" - resolved "https://registry.yarnpkg.com/@octokit/core/-/core-4.0.5.tgz#589e68c0a35d2afdcd41dafceab072c2fbc6ab5f" - integrity sha512-4R3HeHTYVHCfzSAi0C6pbGXV8UDI5Rk+k3G7kLVNckswN9mvpOzW9oENfjfH3nEmzg8y3AmKmzs8Sg6pLCeOCA== - dependencies: - "@octokit/auth-token" "^3.0.0" - "@octokit/graphql" "^5.0.0" - "@octokit/request" "^6.0.0" - "@octokit/request-error" "^3.0.0" - "@octokit/types" "^7.0.0" - before-after-hook "^2.2.0" - universal-user-agent "^6.0.0" - "@octokit/endpoint@^6.0.1": version "6.0.11" resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-6.0.11.tgz#082adc2aebca6dcefa1fb383f5efb3ed081949d1" @@ -1491,15 +1471,6 @@ is-plain-object "^5.0.0" universal-user-agent "^6.0.0" -"@octokit/endpoint@^7.0.0": - version "7.0.2" - resolved "https://registry.yarnpkg.com/@octokit/endpoint/-/endpoint-7.0.2.tgz#11ee868406ba7bb1642e61bbe676d641f79f02be" - integrity sha512-8/AUACfE9vpRpehE6ZLfEtzkibe5nfsSwFZVMsG8qabqRt1M81qZYUFRZa1B8w8lP6cdfDJfRq9HWS+MbmR7tw== - dependencies: - "@octokit/types" "^7.0.0" - is-plain-object "^5.0.0" - universal-user-agent "^6.0.0" - "@octokit/graphql@^4.5.8": version "4.6.1" resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-4.6.1.tgz#f975486a46c94b7dbe58a0ca751935edc7e32cc9" @@ -1509,20 +1480,6 @@ "@octokit/types" "^6.0.3" universal-user-agent "^6.0.0" -"@octokit/graphql@^5.0.0": - version "5.0.1" - resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-5.0.1.tgz#a06982514ad131fb6fbb9da968653b2233fade9b" - integrity sha512-sxmnewSwAixkP1TrLdE6yRG53eEhHhDTYUykUwdV9x8f91WcbhunIHk9x1PZLALdBZKRPUO2HRcm4kezZ79HoA== - dependencies: - "@octokit/request" "^6.0.0" - "@octokit/types" "^7.0.0" - universal-user-agent "^6.0.0" - -"@octokit/openapi-types@^13.11.0": - version "13.12.0" - resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-13.12.0.tgz#cd49f28127ee06ee3edc6f2b5f5648c7332f6014" - integrity sha512-1QYzZrwnn3rTQE7ZoSxXrO8lhu0aIbac1c+qIPOPEaVXBWSaUyLV1x9yt4uDQOwmu6u5ywVS8OJgs+ErDLf6vQ== - "@octokit/openapi-types@^6.0.0": version "6.0.0" resolved "https://registry.yarnpkg.com/@octokit/openapi-types/-/openapi-types-6.0.0.tgz#7da8d7d5a72d3282c1a3ff9f951c8133a707480d" @@ -1562,15 +1519,6 @@ deprecation "^2.0.0" once "^1.4.0" -"@octokit/request-error@^3.0.0": - version "3.0.1" - resolved "https://registry.yarnpkg.com/@octokit/request-error/-/request-error-3.0.1.tgz#3fd747913c06ab2195e52004a521889dadb4b295" - integrity sha512-ym4Bp0HTP7F3VFssV88WD1ZyCIRoE8H35pXSKwLeMizcdZAYc/t6N9X9Yr9n6t3aG9IH75XDnZ6UeZph0vHMWQ== - dependencies: - "@octokit/types" "^7.0.0" - deprecation "^2.0.0" - once "^1.4.0" - "@octokit/request@^5.3.0", "@octokit/request@^5.4.12": version "5.4.15" resolved "https://registry.yarnpkg.com/@octokit/request/-/request-5.4.15.tgz#829da413dc7dd3aa5e2cdbb1c7d0ebe1f146a128" @@ -1583,18 +1531,6 @@ node-fetch "^2.6.1" universal-user-agent "^6.0.0" -"@octokit/request@^6.0.0": - version "6.2.1" - resolved "https://registry.yarnpkg.com/@octokit/request/-/request-6.2.1.tgz#3ceeb22dab09a29595d96594b6720fc14495cf4e" - integrity sha512-gYKRCia3cpajRzDSU+3pt1q2OcuC6PK8PmFIyxZDWCzRXRSIBH8jXjFJ8ZceoygBIm0KsEUg4x1+XcYBz7dHPQ== - dependencies: - "@octokit/endpoint" "^7.0.0" - "@octokit/request-error" "^3.0.0" - "@octokit/types" "^7.0.0" - is-plain-object "^5.0.0" - node-fetch "^2.6.7" - universal-user-agent "^6.0.0" - "@octokit/rest@^18.1.0": version "18.5.2" resolved "https://registry.yarnpkg.com/@octokit/rest/-/rest-18.5.2.tgz#0369e554b7076e3749005147be94c661c7a5a74b" @@ -1612,13 +1548,6 @@ dependencies: "@octokit/openapi-types" "^6.0.0" -"@octokit/types@^7.0.0": - version "7.5.0" - resolved "https://registry.yarnpkg.com/@octokit/types/-/types-7.5.0.tgz#85646021bd618467b7cc465d9734b3f2878c9fae" - integrity sha512-aHm+olfIZjQpzoODpl+RCZzchKOrdSLJs+yfI7pMMcmB19Li6vidgx0DwUDO/Ic4Q3fq/lOjJORVCcLZefcrJw== - dependencies: - "@octokit/openapi-types" "^13.11.0" - "@popperjs/core@^2.9.0": version "2.11.6" resolved "https://registry.yarnpkg.com/@popperjs/core/-/core-2.11.6.tgz#cee20bd55e68a1720bdab363ecf0c821ded4cd45" @@ -4375,11 +4304,6 @@ get-port@^5.1.1: resolved "https://registry.yarnpkg.com/get-port/-/get-port-5.1.1.tgz#0469ed07563479de6efb986baf053dcd7d4e3193" integrity sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ== -get-port@^6.1.2: - version "6.1.2" - resolved "https://registry.yarnpkg.com/get-port/-/get-port-6.1.2.tgz#c1228abb67ba0e17fb346da33b15187833b9c08a" - integrity sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw== - get-stdin@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-4.0.1.tgz#b968c6b0a04384324902e8bf1a5df32579a450fe" @@ -5954,7 +5878,7 @@ node-domexception@^1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== -node-fetch@^2.6.1, node-fetch@^2.6.7: +node-fetch@^2.6.1: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==