Rever the dgal for bathmary

packages/cli/src/dgal/gdal.command.ts

@@ -0,0 +1,116 @@
+import { LogType } from '@basemaps/shared';
+import { ChildProcessWithoutNullStreams, spawn } from 'child_process';
+import { GdalProgressParser } from './gdal.progress.js';
+
+/**
+ * GDAL uses AWS_DEFAULT_PROFILE while node uses AWS_PROFILE
+ * this validates the configuration is sane
+ *
+ * @param env environment to normalize
+ */
+export function normalizeAwsEnv(env: Record<string, string | undefined>): Record<string, string | undefined> {
+  const awsProfile = env['AWS_PROFILE'];
+  const awsDefaultProfile = env['AWS_DEFAULT_PROFILE'];
+
+  if (awsProfile == null) return env;
+  if (awsDefaultProfile == null) {
+    return { ...env, AWS_DEFAULT_PROFILE: awsProfile };
+  }
+  if (awsDefaultProfile !== awsProfile) {
+    throw new Error(`$AWS_PROFILE: "${awsProfile}" and $AWS_DEFAULT_PROFILE: "${awsDefaultProfile}" are mismatched`);
+  }
+  return env;
+}
+
+export interface GdalCredentials {
+  needsRefresh(): boolean;
+  refreshPromise(): Promise<void>;
+  accessKeyId: string;
+  secretAccessKey: string;
+  sessionToken: string;
+}
+
+export abstract class GdalCommand {
+  parser?: GdalProgressParser;
+  protected child: ChildProcessWithoutNullStreams;
+  protected promise?: Promise<{ stdout: string; stderr: string }>;
+  protected startTime: number;
+
+  /** AWS Access  */
+  protected credentials?: GdalCredentials;
+
+  mount?(mount: string): void;
+  env?(): Promise<Record<string, string | undefined>>;
+
+  /** Pass AWS credentials into the container */
+  setCredentials(credentials?: GdalCredentials): void {
+    this.credentials = credentials;
+  }
+
+  /**
+   * Run a GDAL command
+   * @param cmd command to run eg "gdal_translate"
+   * @param args command arguments
+   * @param log logger to use
+   */
+  async run(cmd: string, args: string[], log: LogType): Promise<{ stdout: string; stderr: string }> {
+    if (this.promise != null) throw new Error('Cannot create multiple gdal processes, create a new GdalCommand');
+    this.parser?.reset();
+    this.startTime = Date.now();
+
+    const env = normalizeAwsEnv(this.env ? await this.env() : process.env);
+
+    const child = spawn(cmd, args, { env });
+    this.child = child;
+
+    const outputBuff: Buffer[] = [];
+    const errBuff: Buffer[] = [];
+    child.stderr.on('data', (data: Buffer) => {
+      const buf = data.toString();
+      /**
+       * Example error line
+       * `ERROR 1: TIFFReadEncodedTile:Read error at row 4294967295, col 4294967295; got 49005 bytes, expected 49152`
+       */
+      if (buf.includes('ERROR 1')) {
+        log.error({ data: buf }, 'GdalError');
+      } else {
+        log.warn({ data: buf }, 'GdalWarn');
+      }
+      errBuff.push(data);
+    });
+
+    child.stdout.on('data', (data: Buffer) => {
+      outputBuff.push(data);
+      this.parser?.data(data);
+    });
+
+    this.promise = new Promise((resolve, reject) => {
+      child.on('exit', (code: number) => {
+        const stdout = outputBuff.join('').trim();
+        const stderr = errBuff.join('').trim();
+        const duration = Date.now() - this.startTime;
+
+        if (code !== 0) {
+          log.error({ code, stdout, stderr, duration }, 'GdalFailed');
+          return reject(new Error('Failed to execute GDAL command'));
+        }
+        log.trace({ stdout, stderr, duration }, 'GdalDone');
+
+        this.promise = undefined;
+        return resolve({ stdout, stderr });
+      });
+
+      child.on('error', (error: Error) => {
+        const stdout = outputBuff.join('').trim();
+        const stderr = errBuff.join('').trim();
+        const duration = Date.now() - this.startTime;
+
+        log.error({ stdout, stderr, duration }, 'GdalFailed');
+        this.promise = undefined;
+        reject(error);
+      });
+    });
+
+    return this.promise;
+  }
+}

packages/cli/src/dgal/gdal.docker.ts

@@ -0,0 +1,94 @@
+import { Env, LogType } from '@basemaps/shared';
+import * as os from 'os';
+import * as path from 'path';
+import { GdalCommand } from './gdal.command.js';
+
+export class GdalDocker extends GdalCommand {
+  mounts: string[];
+
+  constructor() {
+    super();
+    this.mounts = [];
+  }
+
+  mount(filePath: string): void {
+    if (filePath.startsWith('s3://')) return;
+
+    const basePath = path.dirname(filePath);
+    if (this.mounts.includes(basePath)) return;
+    this.mounts.push(basePath);
+  }
+
+  private getMounts(): string[] {
+    if (this.mounts.length === 0) {
+      return [];
+    }
+    const output: string[] = [];
+    for (const mount of this.mounts) {
+      output.push('-v');
+      output.push(`${mount}:${mount}`);
+    }
+    return output;
+  }
+
+  private async getCredentials(): Promise<string[]> {
+    if (this.credentials == null) {
+      return [];
+    }
+    if (this.credentials.needsRefresh()) {
+      await this.credentials.refreshPromise();
+    }
+    return [
+      '--env',
+      `AWS_ACCESS_KEY_ID=${this.credentials.accessKeyId}`,
+      '--env',
+      `AWS_SECRET_ACCESS_KEY=${this.credentials.secretAccessKey}`,
+      '--env',
+      `AWS_SESSION_TOKEN=${this.credentials.sessionToken}`,
+    ];
+  }
+
+  /** this could contain sensitive info like AWS access keys */
+  private async getDockerArgs(): Promise<string[]> {
+    const DOCKER_CONTAINER = Env.get(Env.Gdal.DockerContainer) ?? 'ghcr.io/osgeo/gdal';
+    const DOCKER_CONTAINER_TAG = Env.get(Env.Gdal.DockerContainerTag) ?? 'ubuntu-small-3.7.0';
+    const userInfo = os.userInfo();
+    const credentials = await this.getCredentials();
+    return [
+      'run',
+      // Config the container to be run as the current user
+      '--user',
+      `${userInfo.uid}:${userInfo.gid}`,
+
+      ...this.getMounts(),
+      ...credentials,
+
+      // Docker container
+      '-i',
+      `${DOCKER_CONTAINER}:${DOCKER_CONTAINER_TAG}`,
+    ];
+  }
+
+  /** Provide redacted argument string for logging which removes sensitive information */
+  maskArgs(args: string[]): string[] {
+    const cred = this.credentials;
+    if (cred == null) return args;
+
+    return args.map((c) => c.replace(cred.secretAccessKey, '****').replace(cred.sessionToken, '****'));
+  }
+
+  async run(cmd: string, args: string[], log: LogType): Promise<{ stdout: string; stderr: string }> {
+    const dockerArgs = await this.getDockerArgs();
+    log.debug(
+      {
+        mounts: this.mounts,
+        cmd,
+        docker: this.maskArgs(dockerArgs).join(' '),
+        gdalArgs: args.slice(0, 50).join(' '),
+      },
+      'StartGdal:Docker',
+    );
+    return super.run('docker', [...dockerArgs, cmd, ...args], log);
+  }
+}
+

packages/cli/src/dgal/gdal.local.ts

@@ -0,0 +1,24 @@
+import { LogType } from '@basemaps/shared';
+import { GdalCommand } from './gdal.command.js';
+
+export class GdalLocal extends GdalCommand {
+  async env(): Promise<Record<string, string | undefined>> {
+    if (this.credentials == null) {
+      return process.env;
+    }
+    if (this.credentials.needsRefresh()) {
+      await this.credentials.refreshPromise();
+    }
+    return {
+      ...process.env,
+      AWS_ACCESS_KEY_ID: this.credentials.accessKeyId,
+      AWS_SECRET_ACCESS_KEY: this.credentials.secretAccessKey,
+      AWS_SESSION_TOKEN: this.credentials.sessionToken,
+    };
+  }
+
+  async run(cmd: string, args: string[], log: LogType): Promise<{ stdout: string; stderr: string }> {
+    log.debug({ cmd, gdalArgs: args.slice(0, 50).join(' ') }, 'StartGdal:Local');
+    return super.run(cmd, args, log);
+  }
+}

packages/cli/src/dgal/gdal.progress.ts

@@ -0,0 +1,52 @@
+import { EventEmitter } from 'events';
+
+/**
+ * Emit a "progress" event every time a "." is recorded in the output
+ */
+export class GdalProgressParser extends EventEmitter {
+  // Progress starts with "Input file size is .., ..\n"
+  waitNewLine = true;
+  dotCount = 0;
+  byteCount = 0;
+
+  /** Reset the progress counter */
+  reset(): void {
+    this.waitNewLine = true;
+    this.dotCount = 0;
+    this.byteCount = 0;
+  }
+
+  get progress(): number {
+    return this.dotCount * (100 / 31);
+  }
+
+  data(data: Buffer): void {
+    const str = data.toString('utf8');
+    this.byteCount += str.length;
+    // In theory only a small amount of output bytes should be recorded
+    if (this.byteCount > 1024) {
+      throw new Error('Too much data: ' + str);
+    }
+    if (str === '0') {
+      this.waitNewLine = false;
+      return;
+    }
+
+    if (this.waitNewLine) {
+      const newLine = str.indexOf('\n');
+      if (newLine > -1) {
+        this.waitNewLine = false;
+        return this.data(Buffer.from(str.substr(newLine + 1)));
+      }
+      return;
+    }
+
+    const bytes = str.split('');
+    for (const byte of bytes) {
+      if (byte === '.') {
+        this.dotCount++;
+        this.emit('progress', this.progress);
+      }
+    }
+  }
+}

packages/cli/src/dgal/gdal.ts

@@ -0,0 +1,29 @@
+import { Env, LogType } from '@basemaps/shared';
+import { GdalCommand } from './gdal.command.js';
+import { GdalDocker } from './gdal.docker.js';
+import { GdalLocal } from './gdal.local.js';
+
+export class Gdal {
+  /**
+   * Create a new GdalCommand instance ready to run commands
+   *
+   * This could be a local or docker container depending on environment variables
+   * @see Env.Gdal.UseDocker
+   */
+  static create(): GdalCommand {
+    if (Env.get(Env.Gdal.UseDocker)) return new GdalDocker();
+    return new GdalLocal();
+  }
+
+  /**
+   * Run a `gdal_translate --version` to extract the current gdal version
+   *
+   * @example "GDAL 2.4.2, released 2019/06/28"
+   * @example "GDAL 3.2.0dev-69b0c4ec4174fde36c609a4aac6f4281424021b3, released 2020/06/26"
+   */
+  static async version(logger: LogType): Promise<string> {
+    const gdal = Gdal.create();
+    const { stdout } = await gdal.run('gdal_translate', ['--version'], logger);
+    return stdout;
+  }
+}