diff --git a/packages/lambda-analytic-cloudfront/src/__test__/analytics.test.ts b/packages/lambda-analytic-cloudfront/src/__test__/analytics.test.ts index 2ca487cc6..29e960974 100644 --- a/packages/lambda-analytic-cloudfront/src/__test__/analytics.test.ts +++ b/packages/lambda-analytic-cloudfront/src/__test__/analytics.test.ts @@ -22,7 +22,8 @@ describe('analytic lambda', () => { fsa.register('mem://', memory); memory.files.clear(); - Elastic.indexDelay = 1; + Elastic.indexDelay = 1; // do not wait between requests + Elastic.minRequestCount = 1; // index everything Elastic._client = undefined; }); @@ -111,9 +112,9 @@ describe('analytic lambda', () => { const files = [...memory.files.keys()]; assert.equal(files.length, 2); // two files one input one output - assert.ok( - files[1].startsWith(`mem://cache/errors-${new Date().toISOString().slice(0, 12)}`), - // ${shortDate.slice(0, 4)}/${shortDate.slice(5, 7)}/${shortDate}.ndjson.gz`, - ); + assert.ok(files[1].startsWith(`mem://cache/errors-${new Date().toISOString().slice(0, 12)}`)); + + const data = await fsa.read(new URL(files[1])); + assert.ok(data.toString().includes(JSON.stringify('Hello'))); }); }); diff --git a/packages/lambda-analytic-cloudfront/src/elastic.ts b/packages/lambda-analytic-cloudfront/src/elastic.ts index a3f287c8b..5da2877e4 100644 --- a/packages/lambda-analytic-cloudfront/src/elastic.ts +++ b/packages/lambda-analytic-cloudfront/src/elastic.ts @@ -8,6 +8,15 @@ export class ElasticClient { /** Between index requests delay this amount */ indexDelay: number = 200; + /** + * Do not index analytics for buckets that contain less than this number of total requests + * + * @example + * `1` - drop all requests where total requests <= 1 + * + */ + minRequestCount: number = 1; + get client(): Client { if (this._client != null) return this._client; @@ -44,7 +53,7 @@ export class ElasticClient { const ret = await client.bulk({ operations }); if (ret.errors) { - errors.push(ret); + errors.push({ prefix, errors: ret.errors }); throw new Error('Failed to index: ' + prefix); } // Give it a little bit of time to index @@ -53,7 +62,8 @@ export class ElasticClient { } for (const rec of combined) { - if (rec.total < 1) { + // skip over roll ups that are less than + if (rec.total <= this.minRequestCount) { skipHits++; continue; } diff --git a/packages/lambda-analytic-cloudfront/src/handler.ts b/packages/lambda-analytic-cloudfront/src/handler.ts index aec59e716..e67edcfa7 100644 --- a/packages/lambda-analytic-cloudfront/src/handler.ts +++ b/packages/lambda-analytic-cloudfront/src/handler.ts @@ -138,7 +138,7 @@ export async function handler(): Promise { if (Elastic.errors.length > 0) { const errorLocation = new URL(`./errors-${new Date().toISOString()}.json`, CacheLocation); logger.fatal({ errorLocation: errorLocation.href }, 'log:index:failed'); - await fsa.write(errorLocation, JSON.stringify(rets)); + await fsa.write(errorLocation, JSON.stringify(Elastic.errors)); } let failed = false;