From 1ff4f0328c9f5f9ec04b4d9afc1d85f6e345f074 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andr=C3=A9=20Santos?= <andrefs@andrefs.com>
Date: Mon, 7 Aug 2023 20:25:11 +0100
Subject: [PATCH] wip

---
 src/models/Domain.ts | 17 +----------------
 1 file changed, 1 insertion(+), 16 deletions(-)

diff --git a/src/models/Domain.ts b/src/models/Domain.ts
index 14601d7..91b1910 100644
--- a/src/models/Domain.ts
+++ b/src/models/Domain.ts
@@ -483,22 +483,7 @@ schema.statics.domainsToCrawl2 = async function* (wId, domLimit, resLimit) {
       }
       pathSkip += pathLimit;
 
-      // filter out head resources that don't match the white/black lists
-      const heads = [];
-      for (const p of paths) {
-        if (
-          proc.params.whiteList &&
-          matchesOne(p.head.url, proc.params.whiteList)
-        ) {
-          heads.push(p.head);
-        } else if (
-          proc.params.blackList &&
-          !matchesOne(p.head.url, proc.params.blackList)
-        ) {
-          heads.push(p.head);
-        }
-      }
-      const origins = new Set<string>(heads.map((h) => h.domain));
+      const origins = new Set<string>(paths.map((p) => p.head.domain));
       const domains = await Domain.lockForCrawl(
         wId,
         Array.from(origins).slice(0, 20)