diff --git a/packages/shared/src/dynamo/__tests__/config.dynamo.test.ts b/packages/shared/src/dynamo/__tests__/config.dynamo.test.ts new file mode 100644 index 000000000..bedbedb4d --- /dev/null +++ b/packages/shared/src/dynamo/__tests__/config.dynamo.test.ts @@ -0,0 +1,129 @@ +import { ConfigImagery, ConfigTileSet } from '@basemaps/config'; +import DynamoDB from 'aws-sdk/clients/dynamodb.js'; +import o from 'ospec'; +import sinon from 'sinon'; +import { ConfigDynamoCached } from '../dynamo.config.cached.js'; +import { ConfigProviderDynamo } from '../dynamo.config.js'; + +const sandbox = sinon.createSandbox(); + +class FakeDynamoDb { + values: Map> = new Map(); + get: unknown[] = []; + getAll: { RequestItems: { Foo: { Keys: { id: { S: string } }[] } } }[] = []; + getItem(req: any): unknown { + this.get.push(req); + const reqId = req.Key.id.S; + const val = this.values.get(reqId); + return { + promise(): Promise { + if (val) return Promise.resolve({ Item: DynamoDB.Converter.marshall(val) }); + return Promise.resolve(null); + }, + }; + } + + batchGetItem(req: any): unknown { + this.getAll.push(req); + const keys = req.RequestItems.Foo.Keys.map((c: any) => DynamoDB.Converter.unmarshall(c).id); + const output = keys.map((c: string) => this.values.get(c)).filter((f: unknown) => f != null); + return { + promise(): Promise { + if (output.length === 0) return Promise.resolve({ Responses: {} }); + return Promise.resolve({ Responses: { Foo: output.map((c: any) => DynamoDB.Converter.marshall(c)) } }); + }, + }; + } +} + +o.spec('ConfigDynamo', () => { + let provider: ConfigProviderDynamo; + let fakeDynamo: FakeDynamoDb; + + o.beforeEach(() => { + provider = new ConfigProviderDynamo('Foo'); + fakeDynamo = new FakeDynamoDb(); + provider.dynamo = fakeDynamo as any; + }); + + o.afterEach(() => sandbox.restore()); + + o('should not get if missing', async () => { + const ret = await provider.TileSet.get('ts_abc123'); + + o(fakeDynamo.get).deepEquals([{ Key: { id: { S: 'ts_abc123' } }, TableName: 'Foo' }]); + o(ret).equals(null); + }); + + o('should get', async () => { + fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' }); + const ret = await provider.TileSet.get('ts_abc123'); + + o(fakeDynamo.get).deepEquals([{ Key: { id: { S: 'ts_abc123' } }, TableName: 'Foo' }]); + o(ret).deepEquals({ id: 'ts_abc123' } as ConfigTileSet); + }); + + o('should get-all', async () => { + fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' }); + fakeDynamo.values.set('ts_abc456', { id: 'ts_abc456' }); + const ret = await provider.TileSet.getAll(new Set(fakeDynamo.values.keys())); + + o(fakeDynamo.getAll[0].RequestItems.Foo.Keys).deepEquals([{ id: { S: 'ts_abc123' } }, { id: { S: 'ts_abc456' } }]); + o([...ret.values()]).deepEquals([...fakeDynamo.values.values()] as any); + }); + + o('should throw without prefix', async () => { + fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' }); + const ret = await provider.TileSet.get('abc123').catch((e) => e); + + o(fakeDynamo.get).deepEquals([]); + o(String(ret)).deepEquals('Error: Trying to query "abc123" expected prefix of ts'); + }); + + o('should get-all partial', async () => { + fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' }); + const ret = await provider.TileSet.getAll(new Set(['ts_abc123', 'ts_abc456'])); + o(fakeDynamo.getAll[0].RequestItems.Foo.Keys).deepEquals([{ id: { S: 'ts_abc123' } }, { id: { S: 'ts_abc456' } }]); + o([...ret.values()]).deepEquals([...fakeDynamo.values.values()] as any); + }); + + o('should throw if on wrong prefix', async () => { + const ret = await provider.TileSet.get('im_abc123').catch((e) => e); + o(fakeDynamo.get).deepEquals([]); + o(String(ret)).deepEquals('Error: Trying to query "im_abc123" expected prefix of ts'); + }); + + o('should throw on prefixed and un-prefixed', async () => { + fakeDynamo.values.set('ts_abc123', { id: 'ts_abc123' }); + + const ret = provider.TileSet.getAll(new Set(['abc123', 'ts_abc123'])); + const err = await ret.then(() => null).catch((e) => e); + o(String(err)).equals('Error: Trying to query "abc123" expected prefix of ts'); + o(fakeDynamo.getAll).deepEquals([]); + }); + + o.spec('DynamoCached', () => { + o('should get-all with cache', async () => { + fakeDynamo.values.set('im_abc123', { id: 'im_abc123' }); + fakeDynamo.values.set('im_abc456', { id: 'im_abc456' }); + + const cached = provider.Imagery as ConfigDynamoCached; + cached.cache.set('im_abc123', { id: 'im_abc123' } as ConfigImagery); + const ret = await provider.Imagery.getAll(new Set(['im_abc123', 'im_abc456'])); + + o(fakeDynamo.getAll[0].RequestItems.Foo.Keys).deepEquals([{ id: { S: 'im_abc456' } }]); + o([...ret.values()]).deepEquals([...fakeDynamo.values.values()] as any); + }); + + o('should get with cache', async () => { + fakeDynamo.values.set('im_abc123', { id: 'im_abc123' }); + + const cached = provider.Imagery as ConfigDynamoCached; + cached.cache.set('im_abc123', { id: 'im_abc123' } as ConfigImagery); + const ret = await provider.Imagery.get('im_abc123'); + + o(fakeDynamo.get).deepEquals([]); + o(ret).deepEquals({ id: 'im_abc123' } as ConfigImagery); + }); + }); +}); diff --git a/packages/shared/src/dynamo/__tests__/config.imagery.test.ts b/packages/shared/src/dynamo/__tests__/config.imagery.test.ts new file mode 100644 index 000000000..c024b75e4 --- /dev/null +++ b/packages/shared/src/dynamo/__tests__/config.imagery.test.ts @@ -0,0 +1,82 @@ +import { ConfigId, ConfigImagery, ConfigPrefix, getAllImagery } from '@basemaps/config'; +import { Epsg } from '@basemaps/geo'; +import DynamoDB from 'aws-sdk/clients/dynamodb.js'; +import o from 'ospec'; +import sinon from 'sinon'; +import { ConfigProviderDynamo } from '../dynamo.config.js'; + +const sandbox = sinon.createSandbox(); + +o.spec('ConfigProvider.Imagery', () => { + const provider = new ConfigProviderDynamo('Foo'); + + o.afterEach(() => sandbox.restore()); + + const item: ConfigImagery = { id: 'im_foo', name: 'abc' } as any; + + o('isWriteable', () => { + o(provider.Imagery.isWriteable()).equals(true); + // Validate the typing works + if (provider.Imagery.isWriteable()) o(typeof provider.Imagery.put).equals('function'); + }); + + o('prefix', () => { + o(ConfigId.prefix(ConfigPrefix.Imagery, '1234')).equals('im_1234'); + o(ConfigId.prefix(ConfigPrefix.Imagery, 'im_1234')).equals('im_1234'); + o(ConfigId.prefix(ConfigPrefix.TileSet, '1234')).equals('ts_1234'); + o(ConfigId.prefix(ConfigPrefix.TileSet, '')).equals(''); + }); + + o('unprefix', () => { + o(ConfigId.unprefix(ConfigPrefix.Imagery, 'im_1234')).equals('1234'); + o(ConfigId.unprefix(ConfigPrefix.Imagery, 'ts_1234')).equals('ts_1234'); + o(ConfigId.unprefix(ConfigPrefix.Imagery, '1234')).equals('1234'); + }); + + o('is', () => { + o(provider.Imagery.is(item)).equals(true); + o(provider.Imagery.is({ id: 'ts_foo' } as any)).equals(false); + if (provider.Imagery.is(item)) { + o(item.name).equals('abc'); // tests compiler + } + }); + + o('Should get all Imagery', async () => { + const items = new Map(); + items.set('im_foo1', item); + items.set('im_foo2', item); + items.set('im_foo4', item); + const get = sandbox.stub(provider.Imagery, 'getAll').resolves(items); + + const layers = [{ [3857]: 'foo1' }, { [3857]: 'im_foo2' }, { [2193]: 'foo3', [3857]: 'im_foo4' }] as any; + + const result = await getAllImagery(provider, layers, [Epsg.Google]); + o(get.callCount).equals(1); + o([...get.firstCall.firstArg.keys()]).deepEquals(['im_foo1', 'im_foo2', 'im_foo4']); + o(result.get('im_foo1')).deepEquals(item); + o(result.get('im_foo2')).deepEquals(item); + o(result.get('im_foo3')).equals(undefined); + o(result.get('im_foo4')).deepEquals(item); + }); + + o('should handle unprocessed keys', async () => { + const bulk = sandbox.stub(provider.dynamo, 'batchGetItem').callsFake((req: any) => { + const keys = req.RequestItems[provider.tableName].Keys; + return { + promise() { + // Only return one element and label the rest as unprocessed + const ret = keys.slice(0, 1); + const rest = keys.slice(1); + const output: DynamoDB.BatchGetItemOutput = { Responses: { [provider.tableName]: ret } }; + if (rest.length > 0) output.UnprocessedKeys = { [provider.tableName]: { Keys: rest } }; + return Promise.resolve(output); + }, + } as any; + }); + const result = await provider.Provider.getAll(new Set(['pv_1234', 'pv_2345'])); + + o(bulk.callCount).equals(2); + o(result.get('pv_1234')?.id).equals('pv_1234'); + o(result.get('pv_2345')?.id).equals('pv_2345'); + }); +}); diff --git a/packages/shared/src/dynamo/__tests__/config.provider.test.ts b/packages/shared/src/dynamo/__tests__/config.provider.test.ts new file mode 100644 index 000000000..4075ab1d7 --- /dev/null +++ b/packages/shared/src/dynamo/__tests__/config.provider.test.ts @@ -0,0 +1,12 @@ +import o from 'ospec'; +import { ConfigProviderDynamo } from '../dynamo.config.js'; + +o.spec('ConfigProviderDynamo', () => { + const Config = new ConfigProviderDynamo('table'); + + o.spec('id', () => { + o('should create ids', () => { + o(Config.Provider.id('linz')).equals('pv_linz'); + }); + }); +}); diff --git a/packages/shared/src/dynamo/dynamo.config.base.ts b/packages/shared/src/dynamo/dynamo.config.base.ts new file mode 100644 index 000000000..898c014d3 --- /dev/null +++ b/packages/shared/src/dynamo/dynamo.config.base.ts @@ -0,0 +1,83 @@ +import { BaseConfig, BaseConfigWriteableObject, BasemapsConfigObject, ConfigPrefix } from '@basemaps/config'; +import DynamoDB from 'aws-sdk/clients/dynamodb.js'; +import { ConfigProviderDynamo } from './dynamo.config.js'; + +export type IdQuery = { id: { S: string } }; +function toId(id: string): IdQuery { + return { id: { S: id } }; +} + +export class ConfigDynamoBase extends BasemapsConfigObject { + cfg: ConfigProviderDynamo; + + constructor(cfg: ConfigProviderDynamo, prefix: ConfigPrefix) { + super(prefix); + this.cfg = cfg; + } + + /** Ensure the ID is prefixed before querying */ + ensureId(id: string): string { + if (id.startsWith(this.prefix + '_')) return id; + throw new Error(`Trying to query "${id}" expected prefix of ${this.prefix}`); + } + + private get db(): DynamoDB { + return this.cfg.dynamo; + } + + isWriteable(): this is BaseConfigWriteableObject { + return true; + } + + clone(rec: T): T { + return DynamoDB.Converter.unmarshall(DynamoDB.Converter.marshall(rec)) as T; + } + + public async get(key: string): Promise { + const item = await this.db + .getItem({ Key: { id: { S: this.ensureId(key) } }, TableName: this.cfg.tableName }) + .promise(); + if (item == null || item.Item == null) return null; + const obj = DynamoDB.Converter.unmarshall(item.Item) as BaseConfig; + if (this.is(obj)) return obj; + return null; + } + + /** Get all records with the id */ + public async getAll(keys: Set): Promise> { + let mappedKeys: IdQuery[] = []; + for (const key of keys) mappedKeys.push(toId(this.ensureId(key))); + + const output: Map = new Map(); + + while (mappedKeys.length > 0) { + // Batch has a limit of 100 keys returned in a single get + const Keys = mappedKeys.length > 100 ? mappedKeys.slice(0, 100) : mappedKeys; + mappedKeys = mappedKeys.length > 100 ? mappedKeys.slice(100) : []; + + let RequestItems: DynamoDB.BatchGetRequestMap = { [this.cfg.tableName]: { Keys } }; + while (RequestItems != null && Object.keys(RequestItems).length > 0) { + const items = await this.db.batchGetItem({ RequestItems }).promise(); + + const metadataItems = items.Responses?.[this.cfg.tableName]; + if (metadataItems == null) throw new Error('Failed to fetch from ' + this.cfg.tableName); + + for (const row of metadataItems) { + const item = DynamoDB.Converter.unmarshall(row) as BaseConfig; + if (this.is(item)) output.set(item.id, item); + } + + // Sometimes not all results will be returned on the first request + RequestItems = items.UnprocessedKeys as DynamoDB.BatchGetRequestMap; + } + } + + return output; + } + + async put(record: T): Promise { + record.updatedAt = Date.now(); + await this.db.putItem({ TableName: this.cfg.tableName, Item: DynamoDB.Converter.marshall(record) }).promise(); + return record.id; + } +} diff --git a/packages/shared/src/dynamo/dynamo.config.cached.ts b/packages/shared/src/dynamo/dynamo.config.cached.ts new file mode 100644 index 000000000..ce196e49a --- /dev/null +++ b/packages/shared/src/dynamo/dynamo.config.cached.ts @@ -0,0 +1,42 @@ +import { BaseConfig } from '@basemaps/config'; +import { ConfigDynamoBase } from './dynamo.config.base.js'; + +export class ConfigDynamoCached extends ConfigDynamoBase { + cache: Map = new Map(); + + public async get(id: string): Promise { + const queryKey = this.ensureId(id); + let existing: T | null | undefined = this.cache.get(queryKey); + if (existing == null) { + existing = await super.get(queryKey); + if (existing == null) return null; + this.cache.set(queryKey, existing); + } + + return existing; + } + + public async getAll(ids: Set): Promise> { + const output = new Map(); + const toFetch = new Set(); + + for (const id of ids) { + const queryKey = this.ensureId(id); + const existing = this.cache.get(queryKey); + if (existing == null) { + toFetch.add(queryKey); + } else { + output.set(queryKey, existing); + } + } + + if (toFetch.size > 0) { + const res = await super.getAll(toFetch); + for (const val of res.values()) { + output.set(val.id, val); + this.cache.set(val.id, val); + } + } + return output; + } +} diff --git a/packages/shared/src/dynamo/dynamo.config.ts b/packages/shared/src/dynamo/dynamo.config.ts new file mode 100644 index 000000000..2bb47fbdb --- /dev/null +++ b/packages/shared/src/dynamo/dynamo.config.ts @@ -0,0 +1,38 @@ +import { + BaseConfig, + BasemapsConfigProvider, + ConfigBundle, + ConfigImagery, + ConfigPrefix, + ConfigProvider, + ConfigTileSet, + ConfigVectorStyle, +} from '@basemaps/config'; +import DynamoDB from 'aws-sdk/clients/dynamodb.js'; +import { ConfigDynamoBase } from './dynamo.config.base.js'; +import { ConfigDynamoCached } from './dynamo.config.cached.js'; + +export class ConfigProviderDynamo extends BasemapsConfigProvider { + Prefix = ConfigPrefix; + + dynamo: DynamoDB; + tableName: string; + type = 'dynamo' as const; + + Imagery = new ConfigDynamoCached(this, ConfigPrefix.Imagery); + Style = new ConfigDynamoCached(this, ConfigPrefix.Style); + TileSet = new ConfigDynamoBase(this, ConfigPrefix.TileSet); + Provider = new ConfigDynamoCached(this, ConfigPrefix.Provider); + ConfigBundle = new ConfigDynamoBase(this, ConfigPrefix.ConfigBundle); + + constructor(tableName: string) { + super(); + this.dynamo = new DynamoDB({ region: 'ap-southeast-2' }); + this.tableName = tableName; + } + + record(): BaseConfig { + const now = Date.now(); + return { id: '', name: '', updatedAt: now }; + } +}