fixed scan idx and page close detection (#221)

crawlers/custom/utils.js

@@ -119,9 +119,10 @@ export const runAxeScan = async (
   });
 };
 
-export const processPage = async (
-  page,
-  {
+export const processPage = async (page, processPageParams) => {
+  // make sure to update processPageParams' scannedIdx
+  processPageParams.scannedIdx += 1;
+  const {
     scannedIdx,
     needsReviewItems,
     blacklistedPatterns,
@@ -130,8 +131,7 @@ export const processPage = async (
     intermediateScreenshotsPath,
     urlsCrawled,
     randomToken,
-  },
-) => {
+  } = processPageParams;
   try {
     await page.waitForLoadState('networkidle', { timeout: 10000 });
     await page.waitForLoadState('domcontentloaded');
@@ -168,7 +168,6 @@ export const processPage = async (
     y: window.scrollY,
   }));
 
-  scannedIdx += 1;
   const pageImagePath = await screenshotFullPage(page, intermediateScreenshotsPath, scannedIdx);
 
   guiInfoLog(guiInfoStatusTypes.SCANNED, {

crawlers/runCustom.js

@@ -1,5 +1,4 @@
 /* eslint-env browser */
-
 import { chromium } from 'playwright';
 import { createCrawleeSubFolders } from './commonCrawlerFunc.js';
 import { cleanUp } from '../utils.js';
@@ -63,7 +62,20 @@ const runCustom = async (
     const page = await context.newPage();
     await page.goto(url);
 
-    await Promise.all(pageClosePromises);
+    // to execute and wait for all pages to close
+    // idea is for promise to be pending until page.on('close') detected
+    const allPagesClosedPromise = async promises =>
+      Promise.all(promises)
+        // necessary to recheck as during time of execution, more pages added
+        .then(() => {
+          if (Object.keys(pagesDict).length > 0) {
+            return allPagesClosedPromise(promises);
+          }
+
+          return Promise.resolve(true);
+        });
+
+    await allPagesClosedPromise(pageClosePromises);
   } catch (error) {
     log('PLAYWRIGHT EXECUTION ERROR', error);
     process.exit(1);