wip

andrefs · Aug 7, 2023 · 1ff4f03 · 1ff4f03
1 parent 0352ec1
commit 1ff4f03
Showing 1 changed file with 1 addition and 16 deletions.
diff --git a/src/models/Domain.ts b/src/models/Domain.ts
@@ -483,22 +483,7 @@ schema.statics.domainsToCrawl2 = async function* (wId, domLimit, resLimit) {
       }
       pathSkip += pathLimit;
 
-      // filter out head resources that don't match the white/black lists
-      const heads = [];
-      for (const p of paths) {
-        if (
-          proc.params.whiteList &&
-          matchesOne(p.head.url, proc.params.whiteList)
-        ) {
-          heads.push(p.head);
-        } else if (
-          proc.params.blackList &&
-          !matchesOne(p.head.url, proc.params.blackList)
-        ) {
-          heads.push(p.head);
-        }
-      }
-      const origins = new Set<string>(heads.map((h) => h.domain));
+      const origins = new Set<string>(paths.map((p) => p.head.domain));
       const domains = await Domain.lockForCrawl(
         wId,
         Array.from(origins).slice(0, 20)