Merge pull request #24 from donflopez/OPEX-527

[OPEX-527] Increase bach limit logic to handle 1k logs

package.json

@@ -1,6 +1,6 @@
 {
   "name": "auth0-log-extension-tools",
-  "version": "1.3.6",
+  "version": "1.3.7",
   "description": "A set of tools for logging",
   "main": "src/index.js",
   "dependencies": {

src/processor.js

@@ -175,8 +175,8 @@ LogsProcessor.prototype.run = function(handler) {
     const getNextLimit = () => {
       var limit = batchSize;
       limit -= logsBatch.length;
-      if (limit > 100) {
-        limit = 100;
+      if (limit > 1000) {
+        limit = 1000;
       }
       return limit;
     };

tests/helpers/mocks.js

@@ -27,10 +27,10 @@ module.exports.logs = (options = {}) =>
     const query = querystring.parse(uri);
     const logs = [];
     const from = (query.from) ? parseInt(query.from, 10) : 0;
-    const take = (query.take) ? parseInt(query.take, 10) : 100;
+    const take = parseInt(query.take || options.take, 10) || 100;
 
     for (let i = from + 1; i <= from + take; i += 1) {
-      if (i <= 500) {
+      if (i <= 500 || take > 100) {
         logs.push({ _id: '' + i, date: (options.outdated) ? new Date('1999-10-10') : new Date(), type: options.type });
       }
     }

tests/lib/processor.tests.js

@@ -76,6 +76,19 @@ describe('LogsProcessor', () => {
         });
     });
 
+    it('should process higher limit logs and send response', () => {
+      helpers.mocks.logs({ take: 1000 });
+
+      const processor = createProcessor();
+      return processor.run((logs, cb) => setTimeout(() => cb()))
+        .then((result) => {
+          expect(result).to.be.an('object');
+          expect(result.status).to.be.an('object');
+          expect(result.status.logsProcessed).to.equal(1000);
+          expect(result.checkpoint).to.equal('1000');
+        });
+    });
+
     it('should process logs and done by timelimit', () => {
       helpers.mocks.logs({ times: 2 });
 

tests/lib/stream.tests.js

@@ -113,6 +113,23 @@ describe('LogsApiStream', () => {
       logger.next();
     });
 
+    it('should done reading logs with a higher log limit', (done) => {
+      helpers.mocks.logs({ take: 1000 });
+      // helpers.mocks.logs({ empty: true });
+
+      const logger = createStream();
+      logger.on('data', () => logger.done());
+      logger.on('end', () => {
+        logger.batchSaved();
+        expect(logger.status).to.be.an('object');
+        expect(logger.status.logsProcessed).to.equal(1000);
+        expect(logger.lastCheckpoint).to.equal('1000');
+        done();
+      });
+
+      logger.next();
+    });
+
     it('should done reading logs, if ratelimit reached', (done) => {
       helpers.mocks.logs({ limit: 0 });