add --allow-subdomains, add test, fixes #11

fffunction · Jan 25, 2017 · 807e12b · 807e12b
1 parent c40d189
commit 807e12b
Show file tree

Hide file tree

Showing 7 changed files with 1,109 additions and 6 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,9 +1,6 @@
 language: node_js
 node_js:
   - stable
-  - --lts
-  - 7
-  - 6
-  - 5
-  - 4
+  - lts/boron
+  - lts/argon
 after_success: npm run coveralls
diff --git a/backstop.json b/backstop.json
@@ -6,7 +6,318 @@
       "height": 900
     }
   ],
-  "scenarios": [],
+  "scenarios": [
+    {
+      "label": "/",
+      "url": "https://badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://revoked.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://sha256.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://1000-sans.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://rsa8192.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://mixed-script.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://very.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://mixed.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://mixed-favicon.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "http://http-password.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "http://http-login.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "http://http-dynamic-login.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "http://http-credit-card.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://cbc.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://mozilla-old.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://mozilla-intermediate.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://mozilla-modern.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://dh1024.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://dh2048.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://dh-small-subgroup.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://dh-composite.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://static-rsa.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://hsts.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://upgrade.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://preloaded-hsts.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://https-everywhere.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "http://http.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://spoofed-favicon.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://pinning-test.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://long-extended-subdomain-name-containing-many-letters-and-dashes.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    },
+    {
+      "label": "/",
+      "url": "https://longextendedsubdomainnamewithoutdashesinordertotestwordwrapping.badssl.com/",
+      "selectors": [
+        "document"
+      ],
+      "hideSelectors": [
+        "iframe"
+      ]
+    }
+  ],
   "paths": {
     "bitmaps_reference": "backstop_data/bitmaps_reference",
     "bitmaps_test": "backstop_data/bitmaps_test",

diff --git a/index.js b/index.js
@@ -16,6 +16,8 @@ const cli = meow(`
       --ignore-ssl-errors  Treat any certificate as valid (e.g. self-signed
                             or expired)
       --debug              Logs out errors produced while crawling
+      --allow-subdomains   Allow crawling links found to subdomains of the
+                            current domain
 
     Examples
       $ backstop-crawl http://localhost

diff --git a/lib/crawl.js b/lib/crawl.js
@@ -48,6 +48,11 @@ module.exports = function crawl (url, flags) {
         });
     }
 
+    if (flags.allowSubdomains) {
+        crawler.scanSubdomains = true;
+    }
+
+
     // Skip this small blacklist of extensions
     crawler.addFetchCondition(queueItem => !queueItem.path.match(EXT_BLACKLIST));