refactor: validate errors are written into s3

linz · Dec 13, 2024 · f163ff0 · f163ff0
1 parent 517dd7b
commit f163ff0
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 8 deletions.
diff --git a/packages/lambda-analytic-cloudfront/src/__test__/analytics.test.ts b/packages/lambda-analytic-cloudfront/src/__test__/analytics.test.ts
@@ -22,7 +22,8 @@ describe('analytic lambda', () => {
     fsa.register('mem://', memory);
     memory.files.clear();
 
-    Elastic.indexDelay = 1;
+    Elastic.indexDelay = 1; // do not wait between requests
+    Elastic.minRequestCount = 1; // index everything
     Elastic._client = undefined;
   });
 
@@ -111,9 +112,9 @@ describe('analytic lambda', () => {
     const files = [...memory.files.keys()];
     assert.equal(files.length, 2); // two files one input one output
 
-    assert.ok(
-      files[1].startsWith(`mem://cache/errors-${new Date().toISOString().slice(0, 12)}`),
-      //  ${shortDate.slice(0, 4)}/${shortDate.slice(5, 7)}/${shortDate}.ndjson.gz`,
-    );
+    assert.ok(files[1].startsWith(`mem://cache/errors-${new Date().toISOString().slice(0, 12)}`));
+
+    const data = await fsa.read(new URL(files[1]));
+    assert.ok(data.toString().includes(JSON.stringify('Hello')));
   });
 });
diff --git a/packages/lambda-analytic-cloudfront/src/elastic.ts b/packages/lambda-analytic-cloudfront/src/elastic.ts
@@ -8,6 +8,15 @@ export class ElasticClient {
   /** Between index requests delay this amount */
   indexDelay: number = 200;
 
+  /**
+   * Do not index analytics for buckets that contain less than this number of total requests
+   *
+   * @example
+   * `1` - drop all requests where total requests <= 1
+   *
+   */
+  minRequestCount: number = 1;
+
   get client(): Client {
     if (this._client != null) return this._client;
 
@@ -44,7 +53,7 @@ export class ElasticClient {
       const ret = await client.bulk({ operations });
 
       if (ret.errors) {
-        errors.push(ret);
+        errors.push({ prefix, errors: ret.errors });
         throw new Error('Failed to index: ' + prefix);
       }
       // Give it a little bit of time to index
@@ -53,7 +62,8 @@ export class ElasticClient {
     }
 
     for (const rec of combined) {
-      if (rec.total < 1) {
+      // skip over roll ups that are less than
+      if (rec.total <= this.minRequestCount) {
         skipHits++;
         continue;
       }

diff --git a/packages/lambda-analytic-cloudfront/src/handler.ts b/packages/lambda-analytic-cloudfront/src/handler.ts
@@ -138,7 +138,7 @@ export async function handler(): Promise<void> {
     if (Elastic.errors.length > 0) {
       const errorLocation = new URL(`./errors-${new Date().toISOString()}.json`, CacheLocation);
       logger.fatal({ errorLocation: errorLocation.href }, 'log:index:failed');
-      await fsa.write(errorLocation, JSON.stringify(rets));
+      await fsa.write(errorLocation, JSON.stringify(Elastic.errors));
     }
 
     let failed = false;