From 5701fc9b7046391f43091534119284be833c42d7 Mon Sep 17 00:00:00 2001 From: SukkaW Date: Fri, 3 Nov 2023 11:08:36 +0800 Subject: [PATCH] Perf: speed up tldts --- Build/build-internal-cdn-rules.js | 2 +- Build/build-phishing-domainset.js | 2 +- Build/build-redirect-module.js | 2 +- Build/build-reject-domainset.js | 2 +- Build/lib/get-gorhill-publicsuffix.js | 5 +++-- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Build/build-internal-cdn-rules.js b/Build/build-internal-cdn-rules.js index 574aaa1e8..77bb556ce 100644 --- a/Build/build-internal-cdn-rules.js +++ b/Build/build-internal-cdn-rules.js @@ -35,7 +35,7 @@ const buildInternalCDNDomains = task(__filename, async () => { */ const processLocalDomainSet = async (domainSetPath) => { for await (const line of readFileByLine(domainSetPath)) { - const parsed = tldts.parse(line, { allowPrivateDomains: true }); + const parsed = tldts.parse(line, { allowPrivateDomains: true, detectIp: false }); if (parsed.isIp) continue; if (parsed.isIcann || parsed.isPrivate) { if (parsed.domain) { diff --git a/Build/build-phishing-domainset.js b/Build/build-phishing-domainset.js index 4c972d8c1..757ba6b61 100644 --- a/Build/build-phishing-domainset.js +++ b/Build/build-phishing-domainset.js @@ -142,7 +142,7 @@ const buildPhishingDomainSet = task(__filename, async () => { } if (domainCountMap[apexDomain] < 5) { - const subdomain = tldts.getSubdomain(line); + const subdomain = tldts.getSubdomain(line, { detectIp: false }); if (subdomain?.includes('.')) { domainCountMap[apexDomain] += 1.5; } diff --git a/Build/build-redirect-module.js b/Build/build-redirect-module.js index 5bdb5f313..22c86cf01 100644 --- a/Build/build-redirect-module.js +++ b/Build/build-redirect-module.js @@ -74,7 +74,7 @@ const REDIRECT = /** @type {const} */ ([ ]); const buildRedirectModule = task(__filename, async () => { - const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from)))).filter(Boolean); + const domains = Array.from(new Set(REDIRECT.map(([from]) => tldts.getHostname(from, { detectIp: false })))).filter(Boolean); return compareAndWriteFile( [ diff --git a/Build/build-reject-domainset.js b/Build/build-reject-domainset.js index 9de264da7..8866a5b22 100644 --- a/Build/build-reject-domainset.js +++ b/Build/build-reject-domainset.js @@ -179,7 +179,7 @@ const buildRejectDomainSet = task(__filename, async () => { '* Collect reject domain stats', () => Object.entries( dudupedDominArray.reduce((acc, cur) => { - const suffix = tldts.getDomain(cur, { allowPrivateDomains: false }); + const suffix = tldts.getDomain(cur, { allowPrivateDomains: false, detectIp: false }); if (suffix) { acc[suffix] = (acc[suffix] ?? 0) + 1; } diff --git a/Build/lib/get-gorhill-publicsuffix.js b/Build/lib/get-gorhill-publicsuffix.js index f2a9ad691..71e2abc01 100644 --- a/Build/lib/get-gorhill-publicsuffix.js +++ b/Build/lib/get-gorhill-publicsuffix.js @@ -1,10 +1,11 @@ const { toASCII } = require('punycode/'); const fs = require('fs'); const path = require('path'); +const { traceAsync } = require('./trace-runner'); const publicSuffixPath = path.resolve(__dirname, '../../node_modules/.cache/public_suffix_list_dat.txt'); -const getGorhillPublicSuffix = async () => { +const getGorhillPublicSuffix = () => traceAsync('create gorhill public suffix instance', async () => { const customFetch = async (url) => { const buf = await fs.promises.readFile(url); return { @@ -26,7 +27,7 @@ const getGorhillPublicSuffix = async () => { await gorhill.enableWASM({ customFetch }); return gorhill; -}; +}); /** @type {Promise | null} */ let gorhillPublicSuffixPromise = null;