From d1d3e8a3fbb541bf17dba1f769fbeaa858d821ea Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 13 Sep 2023 13:46:14 +0200 Subject: [PATCH] feat: integration tests (#16) --- package.json | 10 +++ packages/core/src/data/offer.ts | 2 +- packages/core/src/queue/client.ts | 8 +- pnpm-lock.yaml | 47 ++++++++++-- stacks/DataStack.ts | 8 +- test/api.test.js | 100 ++++++++++++++++++++++++ test/helpers/bucket.js | 33 ++++++++ test/helpers/context.js | 29 +++++++ test/helpers/dealer-client.js | 30 ++++++++ test/helpers/deployment.js | 121 ++++++++++++++++++++++++++++++ test/helpers/table.js | 29 +++++++ 11 files changed, 405 insertions(+), 12 deletions(-) create mode 100644 test/api.test.js create mode 100644 test/helpers/bucket.js create mode 100644 test/helpers/context.js create mode 100644 test/helpers/dealer-client.js create mode 100644 test/helpers/deployment.js create mode 100644 test/helpers/table.js diff --git a/package.json b/package.json index e3a9e8a..c577acf 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "dev": "sst dev", "build": "sst build", "test": "npm test -w packages/core", + "test-integration": "ava --verbose --serial --timeout=300s test/*.test.js", "deploy": "sst deploy", "remove": "sst remove", "console": "sst console", @@ -16,6 +17,9 @@ "clean": "rm -rf node_modules pnpm-lock.yml packages/*/{pnpm-lock.yml,.next,out,coverage,.nyc_output,worker,dist,node_modules}" }, "devDependencies": { + "@aws-sdk/client-dynamodb": "^3.398.0", + "@aws-sdk/client-s3": "^3.398.0", + "@aws-sdk/util-dynamodb": "^3.398.0", "@ipld/dag-ucan": "3.4.0", "@sentry/serverless": "^7.52.1", "@types/git-rev-sync": "^2.0.0", @@ -26,11 +30,17 @@ "@web3-storage/filecoin-api": "^1.4.3", "@web3-storage/filecoin-client": "1.3.0", "sst": "^2.8.3", + "ava": "^5.3.0", "aws-cdk-lib": "2.72.1", "constructs": "10.1.156", + "delay": "^6.0.0", + "dotenv": "^16.3.1", "git-rev-sync": "^3.0.2", + "p-retry": "^6.0.0", + "p-wait-for": "^5.0.2", "ts-node": "^10.9.1", "typescript": "^5.0.4", + "uint8arrays": "^4.0.6", "@tsconfig/node18": "^2.0.1" }, "workspaces": [ diff --git a/packages/core/src/data/offer.ts b/packages/core/src/data/offer.ts index 488c112..36e955d 100644 --- a/packages/core/src/data/offer.ts +++ b/packages/core/src/data/offer.ts @@ -8,7 +8,7 @@ import { Deal } from './deal' // key in format of `YYYY-MM-DDTHH:MM:SS `${commitmentProof}`` function createKey (deal: DealerMessageRecord) { - return `${(new Date()).toISOString()} ${deal.aggregate.toString()}.json` + return `${(new Date(deal.insertedAt)).toISOString()} ${deal.aggregate.toString()}.json` } export const encode = { diff --git a/packages/core/src/queue/client.ts b/packages/core/src/queue/client.ts index a2245f5..25f4bd0 100644 --- a/packages/core/src/queue/client.ts +++ b/packages/core/src/queue/client.ts @@ -4,6 +4,8 @@ import { Queue, Store } from '@web3-storage/filecoin-api/types' import { connectQueue, Target } from './index.js' +const IMPLICIT_MESSAGE_GROUP_ID = '1' + export function createQueueClient (conf: Target, context: QueueContext): Queue { const queueClient = connectQueue(conf) return { @@ -31,11 +33,13 @@ export function createQueueClient (conf: Target, context: QueueContext< return { error } } - const messageGroupId = conf instanceof SQSClient ? undefined : options.messageGroupId + let messageGroupId = options.messageGroupId || IMPLICIT_MESSAGE_GROUP_ID const cmd = new SendMessageCommand({ QueueUrl: context.queueUrl, MessageBody: encodedMessage, - MessageGroupId: messageGroupId + // If SQS Client was provided, we are in testing mode, and we cannot provide message group ID + // given resource used does not support FIFO Queues with it. + MessageGroupId: conf instanceof SQSClient ? undefined : messageGroupId }) let r diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2637652..3b3e369 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,15 @@ importers: specifier: ^3.3.1 version: 3.3.1 devDependencies: + '@aws-sdk/client-dynamodb': + specifier: ^3.398.0 + version: 3.398.0 + '@aws-sdk/client-s3': + specifier: ^3.398.0 + version: 3.398.0(@aws-sdk/signature-v4-crt@3.398.0) + '@aws-sdk/util-dynamodb': + specifier: ^3.398.0 + version: 3.398.0 '@ipld/dag-ucan': specifier: 3.4.0 version: 3.4.0 @@ -42,15 +51,30 @@ importers: '@web3-storage/filecoin-client': specifier: 1.3.0 version: 1.3.0 + ava: + specifier: ^5.3.0 + version: 5.3.1 aws-cdk-lib: specifier: 2.72.1 version: 2.72.1(constructs@10.1.156) constructs: specifier: 10.1.156 version: 10.1.156 + delay: + specifier: ^6.0.0 + version: 6.0.0 + dotenv: + specifier: ^16.3.1 + version: 16.3.1 git-rev-sync: specifier: ^3.0.2 version: 3.0.2 + p-retry: + specifier: ^6.0.0 + version: 6.0.0 + p-wait-for: + specifier: ^5.0.2 + version: 5.0.2 sst: specifier: ^2.8.3 version: 2.8.3 @@ -60,6 +84,9 @@ importers: typescript: specifier: ^5.0.4 version: 5.0.4 + uint8arrays: + specifier: ^4.0.6 + version: 4.0.6 packages/core: dependencies: @@ -483,7 +510,6 @@ packages: uuid: 8.3.2 transitivePeerDependencies: - aws-crt - dev: false /@aws-sdk/client-iot-data-plane@3.398.0: resolution: {integrity: sha512-aURiQAnz/KVYdXDGN40ND67BiqdIMFpDypXGB7jOqBi+Ih3dO+htQaOPQUmkrB/5/mT1cnVgQvi4PNOiwfK1VQ==} @@ -1034,7 +1060,6 @@ packages: dependencies: mnemonist: 0.38.3 tslib: 2.6.2 - dev: false /@aws-sdk/middleware-bucket-endpoint@3.398.0: resolution: {integrity: sha512-+iDHiRofK/vIY94RWAXkSnR4rBPzc2dPHmLp+FDKywq1y708H9W7TOT37dpn+KSFeO4k2FfddFjzWBHsaeakCA==} @@ -1056,7 +1081,6 @@ packages: '@smithy/protocol-http': 2.0.5 '@smithy/types': 2.2.2 tslib: 2.6.2 - dev: false /@aws-sdk/middleware-expect-continue@3.398.0: resolution: {integrity: sha512-d6he+Qqwh1yqml9duXSv5iKJ2lS0PVrF2UEsVew2GFxfUif0E/davTZJjvWtnelbuIGcTP+wDKVVjLwBN2sN/g==} @@ -1283,7 +1307,6 @@ packages: engines: {node: '>=14.0.0'} dependencies: tslib: 2.6.2 - dev: false /@aws-sdk/util-endpoints@3.398.0: resolution: {integrity: sha512-Fy0gLYAei/Rd6BrXG4baspCnWTUSd0NdokU1pZh4KlfEAEN1i8SPPgfiO5hLk7+2inqtCmqxVJlfqbMVe9k4bw==} @@ -2918,6 +2941,10 @@ packages: resolution: {integrity: sha512-xoDlM2S4ortawSWORYqsdU+2rxdh4LRW9ytc3zmT37RIKQh6IHyKwwtKhKis9ah8ol07DCkZxPt8BBvPjC6v4g==} dev: false + /@types/retry@0.12.2: + resolution: {integrity: sha512-XISRgDJ2Tc5q4TRqvgJtzsRkFYNJzZrhTdtMoGVBttwzzQJkPnS3WWTFc7kuDRoPtPakl+T+OfdEUjYJj7Jbow==} + dev: true + /@types/send@0.17.1: resolution: {integrity: sha512-Cwo8LE/0rnvX7kIIa3QHCkcuF21c05Ayb0ZfxPiv0W8VRiZiNW/WuRupHKpqqGVGf7SUA44QSOUKaEd9lIrd/Q==} dependencies: @@ -5403,7 +5430,6 @@ packages: resolution: {integrity: sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==} dependencies: obliterator: 1.6.1 - dev: false /mnemonist@0.39.5: resolution: {integrity: sha512-FPUtkhtJ0efmEFGpU14x7jGbTB+s18LrzRL2KgoWz9YvcY3cPomz8tih01GbHwnGk/OmkOKfqd/RAQoc8Lm7DQ==} @@ -5576,7 +5602,6 @@ packages: /obliterator@1.6.1: resolution: {integrity: sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==} - dev: false /obliterator@2.0.4: resolution: {integrity: sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ==} @@ -5688,6 +5713,14 @@ packages: retry: 0.13.1 dev: false + /p-retry@6.0.0: + resolution: {integrity: sha512-6NuuXu8Upembd4sNdo4PRbs+M6aHgBTrFE6lkH0YKjVzne3cDW4gkncB98ty/bkMxLxLVNeD5bX9FyWjM7WZ+A==} + engines: {node: '>=16.17'} + dependencies: + '@types/retry': 0.12.2 + retry: 0.13.1 + dev: true + /p-timeout@5.1.0: resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==} engines: {node: '>=12'} @@ -6045,7 +6078,6 @@ packages: /retry@0.13.1: resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} engines: {node: '>= 4'} - dev: false /reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} @@ -6699,7 +6731,6 @@ packages: resolution: {integrity: sha512-4ZesjQhqOU2Ip6GPReIwN60wRxIupavL8T0Iy36BBHr2qyMrNxsPJvr7vpS4eFt8F8kSguWUPad6ZM9izs/vyw==} dependencies: multiformats: 12.1.0 - dev: false /ultron@1.1.1: resolution: {integrity: sha512-UIEXBNeYmKptWH6z8ZnqTeS8fV74zG0/eRU9VGkpzz+LIJNs8W/zM/L+7ctCkRrgbNnnR0xxw4bKOr0cW0N0Og==} diff --git a/stacks/DataStack.ts b/stacks/DataStack.ts index 3d4b51b..6b2cd69 100644 --- a/stacks/DataStack.ts +++ b/stacks/DataStack.ts @@ -22,10 +22,16 @@ export function DataStack({ stack }: StackContext) { /** * This table tracks the state of deals offered to a broker. */ - const dealTable = new Table(stack, 'deal-store', { + const dealTableName = 'deal-store' + const dealTable = new Table(stack, dealTableName, { ...dealTableProps, }) + stack.addOutputs({ + OfferBucketName: offerBucketConfig.bucketName, + DealTableName: dealTableName, + }) + return { offerBucket, dealTable diff --git a/test/api.test.js b/test/api.test.js new file mode 100644 index 0000000..d5daf2e --- /dev/null +++ b/test/api.test.js @@ -0,0 +1,100 @@ +import { test } from './helpers/context.js' + +import git from 'git-rev-sync' +import delay from 'delay' +import { toString } from 'uint8arrays/to-string' +import { Dealer } from '@web3-storage/filecoin-client' +import { randomAggregate } from '@web3-storage/filecoin-api/test' + +import { getDealerClientConfig } from './helpers/dealer-client.js' +import { pollTableItem } from './helpers/table.js' +import { pollBucketItem } from './helpers/bucket.js' +import { + getApiEndpoint, + getStage, + getDealStoreDynamoDb, + getOfferStoreBucketInfo +} from './helpers/deployment.js' +import pRetry from 'p-retry' + +test.before(t => { + t.context = { + apiEndpoint: getApiEndpoint(), + dealStoreDynamo: getDealStoreDynamoDb(), + offerStoreBucket: getOfferStoreBucketInfo() + } +}) + +test('GET /version', async t => { + const stage = getStage() + const response = await fetch(`${t.context.apiEndpoint}/version`) + t.is(response.status, 200) + + const body = await response.json() + t.is(body.env, stage) + t.is(body.commit, git.long('.')) +}) + +test('POST /', async t => { + const { + invocationConfig, + connection + } = await getDealerClientConfig(new URL(t.context.apiEndpoint)) + + // Create a random aggregate + const label = 'label' + const { pieces, aggregate } = await randomAggregate(100, 128) + const offer = pieces.map((p) => p.link) + + const res = await Dealer.dealQueue( + invocationConfig, + aggregate.link.link(), + offer, + invocationConfig.with, + label, + { connection } + ) + + t.truthy(res.out.ok) + t.falsy(res.out.error) + t.truthy(aggregate.link.link().equals(res.out.ok?.aggregate)) + + // wait for deal-store entry to exist given it is propagated with a queue message + await delay(5_000) + + /** @type {import('../packages/core/src/data/deal.js').EncodedDeal | undefined} */ + // @ts-expect-error does not automatically infer + const dealEntry = await pollTableItem( + t.context.dealStoreDynamo.client, + t.context.dealStoreDynamo.tableName, + { aggregate: res.out.ok?.aggregate?.toString() } + ) + if (!dealEntry) { + throw new Error('deal store item was not found') + } + t.is(dealEntry.aggregate, aggregate.link.link().toString()) + t.is(dealEntry.storefront, invocationConfig.with) + t.is(dealEntry.stat, 0) + t.truthy(dealEntry.insertedAt) + t.truthy(dealEntry.offer) + + // Verify offer store + // remove bucket encoding + const bucketKey = dealEntry.offer.replace('s3://', '').replace(`${t.context.offerStoreBucket.bucket}/`, '') + console.log('try to get bucket item...', bucketKey) + const bucketItem = await pollBucketItem( + t.context.offerStoreBucket.client, + t.context.offerStoreBucket.bucket, + bucketKey + ) + if (!bucketItem) { + throw new Error('offer store item was not found') + } + /** @type {import('../packages/core/src/data/offer.js').Offer} */ + const encodedOffer = JSON.parse(toString(bucketItem)) + + t.is(encodedOffer.aggregate, aggregate.link.link().toString()) + t.is(encodedOffer.tenant, invocationConfig.with) + t.truthy(encodedOffer.orderID) + t.deepEqual(encodedOffer.pieces, offer.map(o => o.toString())) +}) diff --git a/test/helpers/bucket.js b/test/helpers/bucket.js new file mode 100644 index 0000000..bb1c10b --- /dev/null +++ b/test/helpers/bucket.js @@ -0,0 +1,33 @@ +import { GetObjectCommand } from '@aws-sdk/client-s3' +import pRetry from 'p-retry' + +/** + * @param {import('@aws-sdk/client-s3').S3Client} client + * @param {string} bucketName + * @param {string} key + */ +export async function pollBucketItem (client, bucketName, key) { + const cmd = new GetObjectCommand({ + Bucket: bucketName, + Key: key + }) + + const response = await pRetry(async () => { + let r + try { + r = await client.send(cmd) + } catch (err) { + if (err?.$metadata?.httpStatusCode === 404) { + throw new Error('not found') + } + } + + return r + }, { + retries: 100, + maxTimeout: 1000, + minTimeout: 1000 + }) + + return await response?.Body?.transformToByteArray() +} diff --git a/test/helpers/context.js b/test/helpers/context.js new file mode 100644 index 0000000..853b638 --- /dev/null +++ b/test/helpers/context.js @@ -0,0 +1,29 @@ +import anyTest from 'ava' +import dotenv from 'dotenv' + +dotenv.config({ + path: '.env.local' +}) + +/** + * @typedef {object} Dynamo + * @property {import('@aws-sdk/client-dynamodb').DynamoDBClient} client + * @property {string} endpoint + * @property {string} region + * @property {string} tableName + * + * @typedef {object} Bucket + * @property {import('@aws-sdk/client-s3').S3Client} client + * @property {string} region + * @property {string} bucket + * + * @typedef {object} Context + * @property {string} apiEndpoint + * @property {Dynamo} dealStoreDynamo + * @property {Bucket} offerStoreBucket + * + * @typedef {import('ava').TestFn>} TestContextFn + */ + +// eslint-disable-next-line unicorn/prefer-export-from +export const test = /** @type {TestContextFn} */ (anyTest) diff --git a/test/helpers/dealer-client.js b/test/helpers/dealer-client.js new file mode 100644 index 0000000..20005a8 --- /dev/null +++ b/test/helpers/dealer-client.js @@ -0,0 +1,30 @@ +import * as Signer from '@ucanto/principal/ed25519' +import * as DID from '@ipld/dag-ucan/did' +import { CAR, HTTP } from '@ucanto/transport' +import { connect } from '@ucanto/client' + +/** + * + * @param {URL} url + */ +export async function getDealerClientConfig (url) { + // UCAN actors + const storefront = await Signer.generate() + const dealerService = DID.parse('did:web:staging.dealer.web3.storage') + + return { + invocationConfig: { + issuer: storefront, + with: storefront.did(), + audience: dealerService, + }, + connection: connect({ + id: dealerService, + codec: CAR.outbound, + channel: HTTP.open({ + url, + method: 'POST', + }), + }) + } +} diff --git a/test/helpers/deployment.js b/test/helpers/deployment.js new file mode 100644 index 0000000..7e02a22 --- /dev/null +++ b/test/helpers/deployment.js @@ -0,0 +1,121 @@ +import { createRequire } from 'module' +import fs from 'fs' +import path from 'path' + +import { S3Client } from '@aws-sdk/client-s3' +import { DynamoDBClient } from '@aws-sdk/client-dynamodb' + +export function getStage () { + const stage = process.env.SST_STAGE || process.env.SEED_STAGE_NAME + if (stage) { + return stage + } + + const f = fs.readFileSync(path.join( + process.cwd(), + '.sst/stage' + )) + + return f.toString() +} + +export const getStackName = () => { + const stage = getStage() + return `${stage}-dealer` +} + +export const getApiEndpoint = () => { + const stage = getStage() + + // CI/CD deployment + if (process.env.SEED_APP_NAME) { + return `https://${stage}.dealer.web3.storage` + } + + const require = createRequire(import.meta.url) + const testEnv = require(path.join( + process.cwd(), + '.sst/outputs.json' + )) + + // Get API endpoint + const id = 'ApiStack' + return testEnv[`${getStackName()}-${id}`].ApiEndpoint +} + +export const getAwsRegion = () => { + // CI/CD deployment + if (process.env.SEED_APP_NAME) { + return 'us-east-2' + } + + return 'us-west-2' +} + +export const getOfferStoreBucketInfo = () => { + const stage = getStage() + const region = getAwsRegion() + const client = new S3Client({ + region + }) + + // CI/CD deployment + if (process.env.SEED_APP_NAME) { + return { + client, + bucket: `${stage}-dealer-offer-store-0`, + region + } + } + + const require = createRequire(import.meta.url) + const testEnv = require(path.join( + process.cwd(), + '.sst/outputs.json' + )) + + // Get bucket Name + const id = 'DataStack' + return { + client, + bucket: /** @type {string} */ (testEnv[`${getStackName()}-${id}`].OfferBucketName), + region + } +} + +export const getDealStoreDynamoDb = () => { + // CI/CD deployment + if (process.env.SEED_APP_NAME) { + return getDynamoDb('deal-store') + } + + const require = createRequire(import.meta.url) + const testEnv = require(path.join( + process.cwd(), + '.sst/outputs.json' + )) + + // Get Bucket Name + const id = 'DataStack' + const tableName = testEnv[`${getStackName()}-${id}`].DealTableName + + return getDynamoDb(tableName) +} + +/** + * @param {string} tableName + */ +export const getDynamoDb = (tableName) => { + const region = getAwsRegion() + const endpoint = `https://dynamodb.${region}.amazonaws.com` + + return { + client: new DynamoDBClient({ + region, + endpoint + }), + tableName: `${getStackName()}-${tableName}`, + region, + endpoint + } +} diff --git a/test/helpers/table.js b/test/helpers/table.js new file mode 100644 index 0000000..0c5f7c7 --- /dev/null +++ b/test/helpers/table.js @@ -0,0 +1,29 @@ +import { GetItemCommand } from '@aws-sdk/client-dynamodb' +import { marshall, unmarshall } from '@aws-sdk/util-dynamodb' +import pRetry from 'p-retry' + +/** + * @param {import('@aws-sdk/client-dynamodb').DynamoDBClient} dynamo + * @param {string} tableName + * @param {object} key + */ +export async function pollTableItem (dynamo, tableName, key) { + const cmd = new GetItemCommand({ + TableName: tableName, + Key: marshall(key) + }) + + const response = await pRetry(async () => { + const r = await dynamo.send(cmd) + if (r.$metadata.httpStatusCode === 404) { + throw new Error('not found in dynamoDB yet') + } + return r + }, { + maxTimeout: 1500, + minTimeout: 1000, + retries: 100 + }) + + return response.Item && unmarshall(response.Item) +}