fix: add missing files

packages/shared/src/dynamo/__tests__/config.dynamo.test.ts

@@ -0,0 +1,129 @@
+import { ConfigImagery, ConfigTileSet } from '@basemaps/config';
+import DynamoDB from 'aws-sdk/clients/dynamodb.js';
+import o from 'ospec';
+import sinon from 'sinon';
+import { ConfigDynamoCached } from '../dynamo.config.cached.js';
+import { ConfigProviderDynamo } from '../dynamo.config.js';
+
+const sandbox = sinon.createSandbox();
+
+class FakeDynamoDb {
+  values: Map<string, Record<string, unknown>> = new Map();
+  get: unknown[] = [];
+  getAll: { RequestItems: { Foo: { Keys: { id: { S: string } }[] } } }[] = [];
+  getItem(req: any): unknown {
+    this.get.push(req);
+    const reqId = req.Key.id.S;
+    const val = this.values.get(reqId);
+    return {
+      promise(): Promise<unknown> {
+        if (val) return Promise.resolve({ Item: DynamoDB.Converter.marshall(val) });
+        return Promise.resolve(null);
+      },
+    };
+  }
+
+  batchGetItem(req: any): unknown {
+    this.getAll.push(req);
+    const keys = req.RequestItems.Foo.Keys.map((c: any) => DynamoDB.Converter.unmarshall(c).id);
+    const output = keys.map((c: string) => this.values.get(c)).filter((f: unknown) => f != null);
+    return {
+      promise(): Promise<unknown> {
+        if (output.length === 0) return Promise.resolve({ Responses: {} });
+        return Promise.resolve({ Responses: { Foo: output.map((c: any) => DynamoDB.Converter.marshall(c)) } });
+      },
+    };
+  }
+}
+
+o.spec('ConfigDynamo', () => {
+  let provider: ConfigProviderDynamo;
+  let fakeDynamo: FakeDynamoDb;
+
+  o.beforeEach(() => {
+    provider = new ConfigProviderDynamo('Foo');
+    fakeDynamo = new FakeDynamoDb();
+    provider.dynamo = fakeDynamo as any;
+  });
+
+  o.afterEach(() => sandbox.restore());
+
+  o('should not get if missing', async () => {
+    const ret = await provider.TileSet.get('ts_abc123');
+
+    o(fakeDynamo.get).deepEquals([{ Key: { id: { S: 'ts_abc123' } }, TableName: 'Foo' }]);
+    o(ret).equals(null);
+  });
+
+  o('should get', async () => {
+    fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' });
+    const ret = await provider.TileSet.get('ts_abc123');
+
+    o(fakeDynamo.get).deepEquals([{ Key: { id: { S: 'ts_abc123' } }, TableName: 'Foo' }]);
+    o(ret).deepEquals({ id: 'ts_abc123' } as ConfigTileSet);
+  });
+
+  o('should get-all', async () => {
+    fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' });
+    fakeDynamo.values.set('ts_abc456', { id: 'ts_abc456' });
+    const ret = await provider.TileSet.getAll(new Set(fakeDynamo.values.keys()));
+
+    o(fakeDynamo.getAll[0].RequestItems.Foo.Keys).deepEquals([{ id: { S: 'ts_abc123' } }, { id: { S: 'ts_abc456' } }]);
+    o([...ret.values()]).deepEquals([...fakeDynamo.values.values()] as any);
+  });
+
+  o('should throw without prefix', async () => {
+    fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' });
+    const ret = await provider.TileSet.get('abc123').catch((e) => e);
+
+    o(fakeDynamo.get).deepEquals([]);
+    o(String(ret)).deepEquals('Error: Trying to query "abc123" expected prefix of ts');
+  });
+
+  o('should get-all partial', async () => {
+    fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' });
+    const ret = await provider.TileSet.getAll(new Set(['ts_abc123', 'ts_abc456']));
+    o(fakeDynamo.getAll[0].RequestItems.Foo.Keys).deepEquals([{ id: { S: 'ts_abc123' } }, { id: { S: 'ts_abc456' } }]);
+    o([...ret.values()]).deepEquals([...fakeDynamo.values.values()] as any);
+  });
+
+  o('should throw if on wrong prefix', async () => {
+    const ret = await provider.TileSet.get('im_abc123').catch((e) => e);
+    o(fakeDynamo.get).deepEquals([]);
+    o(String(ret)).deepEquals('Error: Trying to query "im_abc123" expected prefix of ts');
+  });
+
+  o('should throw on prefixed and un-prefixed', async () => {
+    fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' });
+
+    const ret = provider.TileSet.getAll(new Set(['abc123', 'ts_abc123']));
+    const err = await ret.then(() => null).catch((e) => e);
+    o(String(err)).equals('Error: Trying to query "abc123" expected prefix of ts');
+    o(fakeDynamo.getAll).deepEquals([]);
+  });
+
+  o.spec('DynamoCached', () => {
+    o('should get-all with cache', async () => {
+      fakeDynamo.values.set('im_abc123', { id: 'im_abc123' });
+      fakeDynamo.values.set('im_abc456', { id: 'im_abc456' });
+
+      const cached = provider.Imagery as ConfigDynamoCached<ConfigImagery>;
+      cached.cache.set('im_abc123', { id: 'im_abc123' } as ConfigImagery);
+      const ret = await provider.Imagery.getAll(new Set(['im_abc123', 'im_abc456']));
+
+      o(fakeDynamo.getAll[0].RequestItems.Foo.Keys).deepEquals([{ id: { S: 'im_abc456' } }]);
+      o([...ret.values()]).deepEquals([...fakeDynamo.values.values()] as any);
+    });
+
+    o('should get with cache', async () => {
+      fakeDynamo.values.set('im_abc123', { id: 'im_abc123' });
+
+      const cached = provider.Imagery as ConfigDynamoCached<ConfigImagery>;
+      cached.cache.set('im_abc123', { id: 'im_abc123' } as ConfigImagery);
+      const ret = await provider.Imagery.get('im_abc123');
+
+      o(fakeDynamo.get).deepEquals([]);
+      o(ret).deepEquals({ id: 'im_abc123' } as ConfigImagery);
+    });
+  });
+});

packages/shared/src/dynamo/__tests__/config.imagery.test.ts

@@ -0,0 +1,82 @@
+import { ConfigId, ConfigImagery, ConfigPrefix, getAllImagery } from '@basemaps/config';
+import { Epsg } from '@basemaps/geo';
+import DynamoDB from 'aws-sdk/clients/dynamodb.js';
+import o from 'ospec';
+import sinon from 'sinon';
+import { ConfigProviderDynamo } from '../dynamo.config.js';
+
+const sandbox = sinon.createSandbox();
+
+o.spec('ConfigProvider.Imagery', () => {
+  const provider = new ConfigProviderDynamo('Foo');
+
+  o.afterEach(() => sandbox.restore());
+
+  const item: ConfigImagery = { id: 'im_foo', name: 'abc' } as any;
+
+  o('isWriteable', () => {
+    o(provider.Imagery.isWriteable()).equals(true);
+    // Validate the typing works
+    if (provider.Imagery.isWriteable()) o(typeof provider.Imagery.put).equals('function');
+  });
+
+  o('prefix', () => {
+    o(ConfigId.prefix(ConfigPrefix.Imagery, '1234')).equals('im_1234');
+    o(ConfigId.prefix(ConfigPrefix.Imagery, 'im_1234')).equals('im_1234');
+    o(ConfigId.prefix(ConfigPrefix.TileSet, '1234')).equals('ts_1234');
+    o(ConfigId.prefix(ConfigPrefix.TileSet, '')).equals('');
+  });
+
+  o('unprefix', () => {
+    o(ConfigId.unprefix(ConfigPrefix.Imagery, 'im_1234')).equals('1234');
+    o(ConfigId.unprefix(ConfigPrefix.Imagery, 'ts_1234')).equals('ts_1234');
+    o(ConfigId.unprefix(ConfigPrefix.Imagery, '1234')).equals('1234');
+  });
+
+  o('is', () => {
+    o(provider.Imagery.is(item)).equals(true);
+    o(provider.Imagery.is({ id: 'ts_foo' } as any)).equals(false);
+    if (provider.Imagery.is(item)) {
+      o(item.name).equals('abc'); // tests compiler
+    }
+  });
+
+  o('Should get all Imagery', async () => {
+    const items = new Map();
+    items.set('im_foo1', item);
+    items.set('im_foo2', item);
+    items.set('im_foo4', item);
+    const get = sandbox.stub(provider.Imagery, 'getAll').resolves(items);
+
+    const layers = [{ [3857]: 'foo1' }, { [3857]: 'im_foo2' }, { [2193]: 'foo3', [3857]: 'im_foo4' }] as any;
+
+    const result = await getAllImagery(provider, layers, [Epsg.Google]);
+    o(get.callCount).equals(1);
+    o([...get.firstCall.firstArg.keys()]).deepEquals(['im_foo1', 'im_foo2', 'im_foo4']);
+    o(result.get('im_foo1')).deepEquals(item);
+    o(result.get('im_foo2')).deepEquals(item);
+    o(result.get('im_foo3')).equals(undefined);
+    o(result.get('im_foo4')).deepEquals(item);
+  });
+
+  o('should handle unprocessed keys', async () => {
+    const bulk = sandbox.stub(provider.dynamo, 'batchGetItem').callsFake((req: any) => {
+      const keys = req.RequestItems[provider.tableName].Keys;
+      return {
+        promise() {
+          // Only return one element and label the rest as unprocessed
+          const ret = keys.slice(0, 1);
+          const rest = keys.slice(1);
+          const output: DynamoDB.BatchGetItemOutput = { Responses: { [provider.tableName]: ret } };
+          if (rest.length > 0) output.UnprocessedKeys = { [provider.tableName]: { Keys: rest } };
+          return Promise.resolve(output);
+        },
+      } as any;
+    });
+    const result = await provider.Provider.getAll(new Set(['pv_1234', 'pv_2345']));
+
+    o(bulk.callCount).equals(2);
+    o(result.get('pv_1234')?.id).equals('pv_1234');
+    o(result.get('pv_2345')?.id).equals('pv_2345');
+  });
+});

packages/shared/src/dynamo/__tests__/config.provider.test.ts

@@ -0,0 +1,12 @@
+import o from 'ospec';
+import { ConfigProviderDynamo } from '../dynamo.config.js';
+
+o.spec('ConfigProviderDynamo', () => {
+  const Config = new ConfigProviderDynamo('table');
+
+  o.spec('id', () => {
+    o('should create ids', () => {
+      o(Config.Provider.id('linz')).equals('pv_linz');
+    });
+  });
+});

packages/shared/src/dynamo/dynamo.config.base.ts

@@ -0,0 +1,83 @@
+import { BaseConfig, BaseConfigWriteableObject, BasemapsConfigObject, ConfigPrefix } from '@basemaps/config';
+import DynamoDB from 'aws-sdk/clients/dynamodb.js';
+import { ConfigProviderDynamo } from './dynamo.config.js';
+
+export type IdQuery = { id: { S: string } };
+function toId(id: string): IdQuery {
+  return { id: { S: id } };
+}
+
+export class ConfigDynamoBase<T extends BaseConfig = BaseConfig> extends BasemapsConfigObject<T> {
+  cfg: ConfigProviderDynamo;
+
+  constructor(cfg: ConfigProviderDynamo, prefix: ConfigPrefix) {
+    super(prefix);
+    this.cfg = cfg;
+  }
+
+  /** Ensure the ID is prefixed before querying */
+  ensureId(id: string): string {
+    if (id.startsWith(this.prefix + '_')) return id;
+    throw new Error(`Trying to query "${id}" expected prefix of ${this.prefix}`);
+  }
+
+  private get db(): DynamoDB {
+    return this.cfg.dynamo;
+  }
+
+  isWriteable(): this is BaseConfigWriteableObject<T> {
+    return true;
+  }
+
+  clone(rec: T): T {
+    return DynamoDB.Converter.unmarshall(DynamoDB.Converter.marshall(rec)) as T;
+  }
+
+  public async get(key: string): Promise<T | null> {
+    const item = await this.db
+      .getItem({ Key: { id: { S: this.ensureId(key) } }, TableName: this.cfg.tableName })
+      .promise();
+    if (item == null || item.Item == null) return null;
+    const obj = DynamoDB.Converter.unmarshall(item.Item) as BaseConfig;
+    if (this.is(obj)) return obj;
+    return null;
+  }
+
+  /** Get all records with the id */
+  public async getAll(keys: Set<string>): Promise<Map<string, T>> {
+    let mappedKeys: IdQuery[] = [];
+    for (const key of keys) mappedKeys.push(toId(this.ensureId(key)));
+
+    const output: Map<string, T> = new Map();
+
+    while (mappedKeys.length > 0) {
+      // Batch has a limit of 100 keys returned in a single get
+      const Keys = mappedKeys.length > 100 ? mappedKeys.slice(0, 100) : mappedKeys;
+      mappedKeys = mappedKeys.length > 100 ? mappedKeys.slice(100) : [];
+
+      let RequestItems: DynamoDB.BatchGetRequestMap = { [this.cfg.tableName]: { Keys } };
+      while (RequestItems != null && Object.keys(RequestItems).length > 0) {
+        const items = await this.db.batchGetItem({ RequestItems }).promise();
+
+        const metadataItems = items.Responses?.[this.cfg.tableName];
+        if (metadataItems == null) throw new Error('Failed to fetch from ' + this.cfg.tableName);
+
+        for (const row of metadataItems) {
+          const item = DynamoDB.Converter.unmarshall(row) as BaseConfig;
+          if (this.is(item)) output.set(item.id, item);
+        }
+
+        // Sometimes not all results will be returned on the first request
+        RequestItems = items.UnprocessedKeys as DynamoDB.BatchGetRequestMap;
+      }
+    }
+
+    return output;
+  }
+
+  async put(record: T): Promise<string> {
+    record.updatedAt = Date.now();
+    await this.db.putItem({ TableName: this.cfg.tableName, Item: DynamoDB.Converter.marshall(record) }).promise();
+    return record.id;
+  }
+}

packages/shared/src/dynamo/dynamo.config.cached.ts

@@ -0,0 +1,42 @@
+import { BaseConfig } from '@basemaps/config';
+import { ConfigDynamoBase } from './dynamo.config.base.js';
+
+export class ConfigDynamoCached<T extends BaseConfig> extends ConfigDynamoBase<T> {
+  cache: Map<string, T> = new Map();
+
+  public async get(id: string): Promise<T | null> {
+    const queryKey = this.ensureId(id);
+    let existing: T | null | undefined = this.cache.get(queryKey);
+    if (existing == null) {
+      existing = await super.get(queryKey);
+      if (existing == null) return null;
+      this.cache.set(queryKey, existing);
+    }
+
+    return existing;
+  }
+
+  public async getAll(ids: Set<string>): Promise<Map<string, T>> {
+    const output = new Map<string, T>();
+    const toFetch = new Set<string>();
+
+    for (const id of ids) {
+      const queryKey = this.ensureId(id);
+      const existing = this.cache.get(queryKey);
+      if (existing == null) {
+        toFetch.add(queryKey);
+      } else {
+        output.set(queryKey, existing);
+      }
+    }
+
+    if (toFetch.size > 0) {
+      const res = await super.getAll(toFetch);
+      for (const val of res.values()) {
+        output.set(val.id, val);
+        this.cache.set(val.id, val);
+      }
+    }
+    return output;
+  }
+}

packages/shared/src/dynamo/dynamo.config.ts

@@ -0,0 +1,38 @@
+import {
+  BaseConfig,
+  BasemapsConfigProvider,
+  ConfigBundle,
+  ConfigImagery,
+  ConfigPrefix,
+  ConfigProvider,
+  ConfigTileSet,
+  ConfigVectorStyle,
+} from '@basemaps/config';
+import DynamoDB from 'aws-sdk/clients/dynamodb.js';
+import { ConfigDynamoBase } from './dynamo.config.base.js';
+import { ConfigDynamoCached } from './dynamo.config.cached.js';
+
+export class ConfigProviderDynamo extends BasemapsConfigProvider {
+  Prefix = ConfigPrefix;
+
+  dynamo: DynamoDB;
+  tableName: string;
+  type = 'dynamo' as const;
+
+  Imagery = new ConfigDynamoCached<ConfigImagery>(this, ConfigPrefix.Imagery);
+  Style = new ConfigDynamoCached<ConfigVectorStyle>(this, ConfigPrefix.Style);
+  TileSet = new ConfigDynamoBase<ConfigTileSet>(this, ConfigPrefix.TileSet);
+  Provider = new ConfigDynamoCached<ConfigProvider>(this, ConfigPrefix.Provider);
+  ConfigBundle = new ConfigDynamoBase<ConfigBundle>(this, ConfigPrefix.ConfigBundle);
+
+  constructor(tableName: string) {
+    super();
+    this.dynamo = new DynamoDB({ region: 'ap-southeast-2' });
+    this.tableName = tableName;
+  }
+
+  record(): BaseConfig {
+    const now = Date.now();
+    return { id: '', name: '', updatedAt: now };
+  }
+}