feat(sync) Detect new brand: Weiimi and Detect devices for exist bran…

…ds (matomo-org#7751) feat(device) detect new brand Weiimi: X16 Pro (20200808A1) feat(device) detect brand OPPO: A3 Pro 5G (CPH2665) feat(device) detect brand Google: Pixel 3a (G020E), Pixel 4a (G025J) feat(device) detect brand HTC: U11 (G020E) feat(device) detect brand MTC: DV9157 feat(device) detect brand SUNWIND: SUN-LED43XS301, SUN-LED32XS311, SUN-LED32XS305 feat(device) detect brand Starwind: SW-LED32SG305, SW-LED32SG304 feat(device) detect brand Huawei: Honor 200 Pro (2PZC100) feat(device) detect brand Ace: Buzz 5 Prime, Buzz 5 Pro, Buzz 5, Buzz 5 Note, Buzz 5 Pro+ feat(device) detect brand Zuum: Stellar P7, Stellar P6 feat(device) detect brand POCO: X2 (MZB8745IN, MZB8741IN) feat(device) detect brand Google: Pixel 5a 5G (G1F8F) feat(device) detect brand Scoole: SL-LED55S01T2SU feat(device) detect brand Telefunken: TF-LED32S08T2S feat(device) detect brand ZTE: Blade A75 (Z2359) --- feat(sync) Adds detection for Spark, Total Browser, improves detection for Belva Browser, Phoenix and rename MIUI Browser to Mi Browser (matomo-org#7741) Rename MIUI Browser to Mi Browser Adds detection for Total Browser Adds client hints test for Mi Browser Adds detection for Spark Improves detection for Belva Browser Improves detection for Phoenix Adds detection for The Epoch Times app Escape dots --- feat(sync) Improves version detection for Opera and Opera Mobile and improves browser version detection for client hints (matomo-org#7065) Adds detection for Wave Browser and improves version detection for client hints Improves version detection for Opera Mobile Improves version detection for Opera Add client hints fixture for Coc Coc Remove Wave Browser Update version for Brave browser Update version for Wavebox Fix version for Crow Browser Fix version for Vewd Browser Fix Catsxp version Fix version detection for DuckDuckGo Privacy Browser Improves version detection for Mi Browser --- feat(sync) Adds detection for various bots (matomo-org#7739) Add another user agent for Qwantify Add test for PagePeeker Add another test for SemrushBot Improves DuckDuckBot Adds detection for DuckAssistBot Adds detection for RedekenBot Adds detection for semaltbot Adds detection for MakeMerryBot Adds detection for Timpibot Add generic bot test Adds detection for ValidBot Adds detection for NameProtect Adds detection for CLASSLA-web Add generic bot test Improves detection for generic bots Move heritrix at the bottom Fix Arquivo.pt test Adds detection for Domain Codex Adds detection for Swisscows Favicons Adds detection for leak.info Adds detection for Workona Adds detection for Bloglines Improves detection for generic bots Adds detection for Marginalia Adds detection for VU Server Health Scanner Improves detection for generic bots Adds detection for Functionize Adds detection for Prerender --- chore: generate indexes
sanchezzzhak · Aug 1, 2024 · bb09655 · bb09655
1 parent 66d995c
commit bb09655
Show file tree

Hide file tree

Showing 51 changed files with 3,058 additions and 2,023 deletions.
diff --git a/README.md b/README.md
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "node-device-detector",
-  "version": "2.1.2",
+  "version": "2.1.3",
   "description": "Nodejs device detector (port matomo-org/device-detector)",
   "main": "index.js",
   "scripts": {

diff --git a/parser/client/browser-families.js b/parser/client/browser-families.js
@@ -1,6 +1,6 @@
 // prettier-ignore
 module.exports = {
-  'Android Browser': ['AN', 'MU'],
+  'Android Browser': ['AN'],
   'BlackBerry Browser': ['BB'],
   'Baidu': ['BD', 'BS'],
   'Amiga': ['AV', 'AW'],
@@ -33,7 +33,7 @@ module.exports = {
   'N3', 'GD', 'O9', 'Q3', 'F7', 'K2', 'P5', 'H5', 'V3',
   'K3', 'Q4', 'G2', 'R2', 'WX', 'XP', '3I', 'BG', 'R0',
   'JO', 'OL', 'GN', 'W4', 'QI', 'E1', 'RI', '8B', '5B',
-  'K4', 'WK',
+  'K4', 'WK', 'T3', 'K5', 'MU',
   ],
   'Firefox': [
   'FF', 'BI', 'BF', 'BH', 'BN', 'C0', 'CU', 'EI', 'F1',

diff --git a/parser/client/browser-short.js b/parser/client/browser-short.js
@@ -347,7 +347,7 @@ module.exports = {
   'M3': 'Midori Lite',
   'M6': 'MixerBox AI',
   'MO': 'Mobicip',
-  'MU': 'MIUI Browser',
+  'MU': 'Mi Browser',
   'MS': 'Mobile Silk',
   'MK': 'Mogok Browser',
   'M8': 'Motorola Internet Browser',
@@ -524,6 +524,7 @@ module.exports = {
   'LE': 'Smart Lenovo Browser',
   'OZ': 'Smooz',
   'SN': 'Snowshoe',
+  'K5': 'Spark',
   'B1': 'Spectre Browser',
   'S2': 'Splash',
   'SI': 'Sputnik Browser',
@@ -563,6 +564,7 @@ module.exports = {
   'TC': 'TUC Mini Browser',
   'TU': 'Tungsten',
   'TG': 'ToGate',
+  'T3': 'Total Browser',
   'TS': 'TweakStyle',
   'TV': 'TV Bro',
   'U0': 'U Browser',

diff --git a/parser/client/browser.js b/parser/client/browser.js
@@ -104,15 +104,10 @@ class Browser extends ClientAbstractParser {
           version = data.version;
         }
 
-        if ('DuckDuckGo Privacy Browser' === name) {
-          version = '';
-        }
-
         if ('Vewd Browser' === name) {
           engine = data.engine;
           engineVersion = data.engine_version;
         }
-
         // If client hints report Chromium, but user agent detects a Chromium based browser, we favor this instead
         if (data.name && 'Chromium' === name && 'Chromium' !== data.name) {
           name = data.name;
@@ -125,18 +120,27 @@ class Browser extends ClientAbstractParser {
           name = data.name;
           short = data.short_name;
         }
-
+        // If user agent detects another browser, but the family matches, we use the detected engine from user agent
         if (name !== data.name && family === this.buildFamily(data.short_name)) {
           engine = data.engine;
           engineVersion = data.engine_version;
         }
-
+        // If the browser name matches the client hints then browser engine overwrite
         if (name === data.name) {
           engine = data.engine;
           engineVersion = data.engine_version;
-          if (data.version && data.version.indexOf(version) === 0 && helper.versionCompare(version, data.version) < 0) {
-            version = data.version;
-          }
+        }
+        // In case the user agent reports a more detailed version, we try to use this instead
+        if (data.version && data.version.indexOf(version) === 0 && helper.versionCompare(version, data.version) < 0) {
+          version = data.version;
+        }
+        // If DDG Private browser then set version empty string
+        if ('DuckDuckGo Privacy Browser' === name) {
+          version = '';
+        }
+        // If client hints report the following browsers, we use the version from useragent
+        if (data.version && ['MU', 'OM', 'OP'].indexOf(short) !== -1) {
+          version = data.version;
         }
       }
     } else if (data !== null) {

diff --git a/parser/device/brand-short.js b/parser/device/brand-short.js
@@ -1764,6 +1764,7 @@ module.exports = {
   'WBL': 'We. by Loewe.',
   'WCP': 'WeChip',
   'WM': 'Weimei',
+  'WM1': 'Weiimi',
   'WE': 'WellcoM',
   'W6': 'WELLINGTON',
   'WD': 'Western Digital',

diff --git a/regexes/bots.yml b/regexes/bots.yml
@@ -513,9 +513,17 @@
     url: 'http://moz.com/'
 
 - regex: 'DuckDuck(?:Go-Favicons-)?Bot'
-  name: 'DuckDuckGo Bot'
+  name: 'DuckDuckBot'
   category: 'Search bot'
-  url: 'https://duckduckgo.com/duckduckbot'
+  url: 'https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot/'
+  producer:
+    name: 'DuckDuckGo'
+    url: 'https://duckduckgo.com/'
+
+- regex: 'DuckAssistBot'
+  name: 'DuckAssistBot'
+  category: 'Search bot'
+  url: 'https://duckduckgo.com/duckduckgo-help-pages/results/duckassistbot/'
   producer:
     name: 'DuckDuckGo'
     url: 'https://duckduckgo.com/'
@@ -829,14 +837,6 @@
     name: 'Google Inc.'
     url: 'https://www.google.com/'
 
-- regex: 'heritrix'
-  name: 'Heritrix'
-  category: 'Crawler'
-  url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
-  producer:
-    name: 'The Internet Archive'
-    url: 'https://archive.org'
-
 - regex: 'HubSpot '
   name: 'HubSpot'
   category: 'Crawler'
@@ -2187,6 +2187,11 @@
 
 - regex: 'PagePeeker'
   name: 'PagePeeker'
+  category: 'Crawler'
+  url: 'https://pagepeeker.com/robots/'
+  producer:
+    name: 'PAGEPEEKER SRL'
+    url: 'https://pagepeeker.com/'
 
 - regex: 'WebThumbnail'
   name: 'WebThumbnail'
@@ -3178,6 +3183,14 @@
     name: 'Jožef Stefan Institute'
     url: 'https://www.ijs.si/ijsw/JSI'
 
+- regex: 'CLASSLA'
+  name: 'CLASSLA-web'
+  category: 'Crawler'
+  url: 'https://www.clarin.si/info/classla-web-crawler/'
+  producer:
+    name: 'Jožef Stefan Institute'
+    url: 'https://www.ijs.si/ijsw/JSI'
+
 - regex: 'dnt-policy@eff\.org'
   name: 'EFF Do Not Track Verifier'
   category: 'Crawler'
@@ -4533,8 +4546,130 @@
     name: 'Meins und Vogel GmbH'
     url: 'https://muv.com/'
 
+- regex: 'RedekenBot'
+  name: 'RedekenBot'
+  category: 'Crawler'
+  url: 'https://www.redeken.com/en/help/bot.html'
+  producer:
+    name: 'Redeken'
+    url: 'https://www.redeken.com/'
+
+- regex: 'semaltbot'
+  name: 'semaltbot'
+  category: 'Crawler'
+  url: 'https://semalt.net/'
+  producer:
+    name: 'Semalt LP'
+    url: 'https://semalt.net/'
+
+- regex: 'MakeMerryBot'
+  name: 'MakeMerryBot'
+  category: 'Crawler'
+  url: 'https://makemerry.app/bots'
+
+- regex: 'Timpibot'
+  name: 'Timpibot'
+  category: 'Crawler'
+  url: 'https://timpi.io/'
+  producer:
+    name: 'Timpi Inc.'
+    url: 'https://timpi.io/'
+
+- regex: 'Validbot'
+  name: 'ValidBot'
+  category: 'Crawler'
+  url: 'https://www.validbot.com/'
+  producer:
+    name: 'Jake Olefsky LLC'
+    url: 'https://www.validbot.com/'
+
+- regex: 'NPBot'
+  name: 'NameProtectBot'
+  category: 'Crawler'
+  url: 'https://www.cscglobal.com/cscglobal/home/'
+  producer:
+    name: 'NameProtect, Inc.'
+    url: 'https://www.cscglobal.com/'
+
+- regex: 'domaincodex\.com'
+  name: 'Domain Codex'
+  category: 'Crawler'
+  url: 'https://www.domaincodex.com/'
+  producer:
+    name: 'Erie Data Systems, LLC'
+    url: 'https://www.eriedatasys.com/'
+
+- regex: 'Swisscows Favicons'
+  name: 'Swisscows Favicons'
+  category: 'Crawler'
+  url: 'https://swisscows.com/'
+  producer:
+    name: 'Swisscows AG'
+    url: 'https://swisscows.com/'
+
+- regex: 'leak\.info'
+  name: 'leak.info'
+  category: 'Crawler'
+  url: 'http://www.leak.info/'
+
+- regex: 'workona'
+  name: 'Workona'
+  category: 'Crawler'
+  url: 'https://workona.com/'
+  producer:
+    name: 'Workona, Inc.'
+    url: 'https://workona.com/'
+
+- regex: 'Bloglines'
+  name: 'Bloglines'
+  category: 'Crawler'
+  url: 'https://web.archive.org/web/20140309033202/http://www.bloglines.com/'
+  producer:
+    name: 'Reply!, Inc.'
+    url: 'https://www.reply.com/'
+
+- regex: 'heritrix'
+  name: 'Heritrix'
+  category: 'Crawler'
+  url: 'https://webarchive.jira.com/wiki/display/Heritrix/Heritrix'
+  producer:
+    name: 'The Internet Archive'
+    url: 'https://archive.org'
+
+- regex: 'search\.marginalia\.nu'
+  name: 'Marginalia'
+  category: 'Crawler'
+  url: 'https://www.marginalia.nu/marginalia-search/for-webmasters/'
+  producer:
+    name: 'Marginalia'
+    url: 'https://www.marginalia.nu/'
+
+- regex: 'vu-server-health-scanner/[\d.]+'
+  name: 'VU Server Health Scanner'
+  category: 'Security Checker'
+  url: 'https://130.37.198.75/index.html'
+  producer:
+    name: 'VU Amsterdam'
+    url: 'https://vu.nl/en'
+
+- regex: 'Functionize'
+  name: 'Functionize'
+  category: 'Crawler'
+  url: 'https://www.functionize.com/'
+  producer:
+    name: 'Functionize, Inc.'
+    url: 'https://www.functionize.com/'
+
+- regex: 'Prerender'
+  name: 'Prerender'
+  category: 'Crawler'
+  url: 'https://docs.prerender.io/docs/33-overview-of-prerender-crawlers'
+  producer:
+    name: 'saas.group Inc.'
+    url: 'https://saas.group/'
+
 # Generic bots
-- regex: 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?: Build|Plus| CM62| HD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherweb|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|url|Zeus|ZmEu)$'
+- regex: 'nuhk|grub-client|Download Demon|SearchExpress|Microsoft URL Control|borg|altavista|dataminr\.com|teoma|oegp|http%20client|htdig|mogimogi|larbin|scrubby|searchsight|semanticdiscovery|snappy|vortex(?!(?: Build|Plus| CM62| HD65))|zeal(?!ot)|dataparksearch|findlinks|BrowserMob|URL2PNG|ZooShot|GomezA|Google SketchUp|Read%20Later|7Siters|centuryb\.o\.t9|InterNaetBoten|EasyBib AutoCite|Bidtellect|tomnomnom/meg|cortex|Re-re Studio|adreview|AHC/|NameOfAgent|Request-Promise|ALittle Client|Hello,? world|wp_is_mobile|0xAbyssalDoesntExist|Anarchy99|^revolt|nvd0rz|xfa1|Hakai|gbrmss|fuck-your-hp|IDBTE4M CODE87|Antoine|Insomania|Hells-Net|b3astmode|Linux Gnu \(cow\)|Test Certificate Info|iplabel|Magellan|TheSafex?Internetx?Search|Searcherx?web|kirkland-signature|LinkChain|survey-security-dot-txt|infrawatch|Time/|r00ts3c-owned-you|nvdorz|Root Slut|NiggaBalls|BotPoke|GlobalWebSearch|xx032_bo9vs83_2a|sslshed|geckotrail|^xenu|^(?:chrome|firefox|Abcd|Dark|KvshClient|Node.js|Report Runner|url|Zeus|ZmEu)$'
   name: 'Generic Bot'
 
 # Generic detections