From 017234231f4844599f8e7fd5990d3066041d4983 Mon Sep 17 00:00:00 2001
From: Bart Veneman <bartveneman@users.noreply.github.com>
Date: Sat, 23 Mar 2019 23:01:08 +0100
Subject: [PATCH] init

---
 .gitignore                        |   2 +
 .travis.yml                       |   4 +
 license                           |  21 ++++
 package.json                      |  37 +++++++
 readme.md                         |  94 ++++++++++++++++++
 src/index.js                      |  85 +++++++++++++++++
 test/css-in-js.html               |  38 ++++++++
 test/fixture.css                  |   3 +
 test/index.js                     | 154 ++++++++++++++++++++++++++++++
 test/js-create-link-element.html  |  19 ++++
 test/js-create-style-element.html |  18 ++++
 test/kitchen-sink.html            |  35 +++++++
 test/snapshots/index.js.md        |  33 +++++++
 test/snapshots/index.js.snap      | Bin 0 -> 197 bytes
 14 files changed, 543 insertions(+)
 create mode 100644 .gitignore
 create mode 100644 .travis.yml
 create mode 100644 license
 create mode 100644 package.json
 create mode 100644 readme.md
 create mode 100644 src/index.js
 create mode 100644 test/css-in-js.html
 create mode 100644 test/fixture.css
 create mode 100644 test/index.js
 create mode 100644 test/js-create-link-element.html
 create mode 100644 test/js-create-style-element.html
 create mode 100644 test/kitchen-sink.html
 create mode 100644 test/snapshots/index.js.md
 create mode 100644 test/snapshots/index.js.snap

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..25c8fdb
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+node_modules
+package-lock.json
\ No newline at end of file
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..e6175ef
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,4 @@
+language: node_js
+node_js:
+  - '8'
+  - '10'
diff --git a/license b/license
new file mode 100644
index 0000000..d81d966
--- /dev/null
+++ b/license
@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Bart Veneman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..4f5650d
--- /dev/null
+++ b/package.json
@@ -0,0 +1,37 @@
+{
+  "name": "distill-css",
+  "description": "PACKAGE_DESCRIPTION",
+  "version": "0.1.0",
+  "homepage": "https://www.projectwallace.com/oss",
+  "repository": "PACKAGE_REPOSITORY",
+  "issues": "PACKAGE_ISSUES",
+  "license": "MIT",
+  "author": "Bart Veneman",
+  "keywords": [
+    "PACKAGE_KEYWORD"
+  ],
+  "scripts": {
+    "test": "xo && ava test"
+  },
+  "files": [
+    "src"
+  ],
+  "main": "src/index.js",
+  "engines": {
+    "node": ">=8.0"
+  },
+  "xo": {
+    "prettier": true
+  },
+  "devDependencies": {
+    "ava": "^1.3.1",
+    "chromium": "^2.1.0",
+    "create-test-server": "^2.4.0",
+    "prettier": "^1.16.4",
+    "puppeteer-core": "^1.13.0",
+    "xo": "^0.24.0"
+  },
+  "dependencies": {
+    "puppeteer": "^1.13.0"
+  }
+}
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..f4ecfd1
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,94 @@
+<div align="center">
+	<h1>extract-css-core</h1>
+	<p>Extract all CSS from a given url, both server side and client side rendered.</p>
+</div>
+
+[![NPM Version](https://img.shields.io/npm/v/extract-css-core.svg)](https://www.npmjs.com/package/extract-css-core)
+[![Build Status](https://travis-ci.org/bartveneman/extract-css-core.svg?branch=master)](https://travis-ci.org/bartveneman/extract-css-core)
+[![Known Vulnerabilities](https://snyk.io/test/github/bartveneman/extract-css-core/badge.svg)](https://snyk.io/test/github/bartveneman/extract-css-core)
+[![Weekly downloads](https://img.shields.io/npm/dw/extract-css-core.svg)](https://www.npmjs.com/package/extract-css-core)
+![Dependencies Status](https://img.shields.io/david/bartveneman/extract-css-core.svg)
+![Dependencies Status](https://img.shields.io/david/dev/bartveneman/extract-css-core.svg)
+[![XO code style](https://img.shields.io/badge/code_style-XO-5ed9c7.svg)](https://github.com/sindresorhus/xo)
+[![Project: Wallace](https://img.shields.io/badge/Project-Wallace-29c87d.svg)](https://www.projectwallace.com/oss)
+
+## Problem, solution and shortcomings
+
+### Problem
+
+Existing packages like [get-css](https://github.com/cssstats/cssstats/tree/master/packages/get-css) look at a server-generated piece of HTML and get all the `<link>` and `<style>` tags from it. This works fine for 100% server rendered pages, but apps that employ style injection with JavaScript will not be covered.
+
+### Solution
+
+This module uses an instance of Chromium to render a page. This has the benefit that most of the styles can be rendered, even when generated by JavaScript. The [Puppeteer CSSCoverage API](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#coveragestartcsscoverageoptions) is the power behind finding most of the CSS.
+
+### Shortcomings
+
+Currently, there is no solution to get the CSS from modules that use [Styled Components](https://www.styled-components.com) or something similar. Any help resolving this issue will be very much appreciated.
+
+## Installation
+
+```sh
+npm install extract-css-core
+
+# Or with Yarn
+yarn add extract-css-core
+```
+
+## Usage
+
+```js
+const extractCss = require("extract-css-core");
+
+extractCss("http://www.projectwallace.com").then(css => console.log(css));
+```
+
+## API
+
+### extractCss(url, [options])
+
+Extract CSS from a page. Returns a Promise that contains the CSS as a single String.
+
+### Options
+
+Type: `Object`
+
+#### debug
+
+Type: `Boolean`
+Default: `false`
+
+Set to `true` if you want a Chromium window to open as it works to get all the CSS from the page.
+
+#### waitUntil
+
+Type: `String`
+Default: `networkidle2`
+
+Can be any value as provided by the [Puppeteer docs](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#pagegotourl-options).
+
+#### customBrowser
+
+Type: `Object`
+Default: `null`
+
+This is useful if you want to run extract-css on AWS Lambda for example.
+
+##### executablePath
+
+Type: `String`
+Default: `null`
+
+Pass in the executable path for a custom Chromium instance.
+
+##### customPuppeteer
+
+Type: `Object`
+Default: `null`
+
+You probably want to provide [puppeteer-core](https://www.npmjs.com/package/puppeteer-core) for a custom browser instead of [puppeteer](https://www.npmjs.com/package/puppeteer) which brings it's own Chromium instance.
+
+## Related
+
+- [extract-css lambda](https://github.com/bartveneman/extract-css) - Extract CSS running as a serverless function
+- [get-css](https://github.com/cssstats/cssstats/tree/master/packages/get-css) - The original get-css
diff --git a/src/index.js b/src/index.js
new file mode 100644
index 0000000..db22f66
--- /dev/null
+++ b/src/index.js
@@ -0,0 +1,85 @@
+const puppeteer = require("puppeteer");
+
+function InvalidUrlError({ url, statusCode, statusText }) {
+  this.name = "InvalidUrlError";
+  this.message = `There was an error retrieving CSS from ${url}.\n\tHTTP status code: ${statusCode} (${statusText})`;
+}
+
+InvalidUrlError.prototype = Error.prototype;
+
+module.exports = async (
+  url,
+  { debug = false, waitUntil = "networkidle2", customBrowser = {} } = {}
+) => {
+  const browserOptions = {
+    headless: debug !== true,
+    puppeteer
+  };
+
+  // Replace the puppeteer instance if a custom one is provided
+  // This also means that the executablePath needs to be set to
+  // a custom path where some chromium instance is running.
+  if (
+    customBrowser &&
+    customBrowser.executablePath &&
+    customBrowser.customPuppeteer
+  ) {
+    browserOptions.executablePath = customBrowser.executablePath;
+    browserOptions.puppeteer = customBrowser.customPuppeteer;
+  }
+
+  // Setup a browser instance
+  const browser = await browserOptions.puppeteer.launch(browserOptions);
+
+  // Create a new page and navigate to it
+  const page = await browser.newPage();
+
+  // Start CSS coverage. This is the meat and bones of this module
+  await page.coverage.startCSSCoverage();
+  const response = await page.goto(url, { waitUntil });
+
+  // Make sure that we only try to extract CSS from valid pages.
+  // Bail out if the response is an invalid request (400, 500)
+  if (response.status() >= 400) {
+    await browser.close(); // Don't leave any resources behind
+
+    return Promise.reject(
+      new InvalidUrlError({
+        url,
+        statusCode: response.status(),
+        statusText: response.statusText()
+      })
+    );
+  }
+
+  // Coverage contains a lot of <style> and <link> CSS,
+  // but not all...
+  const coverage = await page.coverage.stopCSSCoverage();
+
+  // Fetch all <style> tags from the page, because the coverage
+  // API may have missed some JS-generated <style> tags.
+  // Some of them *were* already caught by the coverage API,
+  // but they will be removed later on to prevent duplicates.
+  const styleTagsCss = (await page.$$eval("style", styles => {
+    // Get the text inside each <style> tag and trim() the
+    // results to prevent all the inside-html indentation
+    // clogging up the results and making it look
+    // bigger than it actually is
+    return styles.map(style => style.innerHTML.trim());
+  })).join("");
+
+  await browser.close();
+
+  // Turn the coverage Array into a single string of CSS
+  const coverageCss = coverage
+    // Filter out the <style> tags that were found in the coverage
+    // report since we've conducted our own search for them.
+    // A coverage CSS item with the same url as the url of the page
+    // we requested is an indication that this was a <style> tag
+    .filter(styles => styles.url !== url)
+    // The `text` property contains the actual CSS
+    .map(({ text }) => text)
+    .join("");
+
+  return Promise.resolve(coverageCss + styleTagsCss);
+};
diff --git a/test/css-in-js.html b/test/css-in-js.html
new file mode 100644
index 0000000..5381479
--- /dev/null
+++ b/test/css-in-js.html
@@ -0,0 +1,38 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <title>CodePen - CSS-in-JS example for extract-css-core unit tests</title>
+
+    <style>
+      html {
+        color: #f00;
+      }
+    </style>
+  </head>
+
+  <body translate="no">
+    <div id="app"></div>
+
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/react/16.8.4/umd/react.production.min.js"></script>
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/react-dom/16.8.4/umd/react-dom.production.min.js"></script>
+    <script src="https://unpkg.com/styled-components/dist/styled-components.min.js"></script>
+
+    <script id="rendered-js">
+      const Title = styled.h1`
+        color: blue;
+        font-family: sans-serif;
+        font-size: 3em;
+      `;
+
+      const App = () => {
+        return React.createElement(Title, null, "Title");
+      };
+
+      ReactDOM.render(
+        React.createElement(App, null),
+        document.querySelector("#app")
+      );
+    </script>
+  </body>
+</html>
diff --git a/test/fixture.css b/test/fixture.css
new file mode 100644
index 0000000..b82e370
--- /dev/null
+++ b/test/fixture.css
@@ -0,0 +1,3 @@
+body {
+  color: teal;
+}
diff --git a/test/index.js b/test/index.js
new file mode 100644
index 0000000..752bff2
--- /dev/null
+++ b/test/index.js
@@ -0,0 +1,154 @@
+const test = require("ava");
+const createTestServer = require("create-test-server");
+const { readFileSync } = require("fs");
+const { resolve } = require("path");
+const extractCss = require("..");
+
+let server;
+const expected = readFileSync(resolve(__dirname, "fixture.css"), "utf8");
+
+test.before(async () => {
+  server = await createTestServer();
+
+  server.get("/fixture.css", (req, res) => {
+    res.send(expected);
+  });
+});
+
+test.after(async () => {
+  await server.close();
+});
+
+test("it fetches css from a page with CSS in a server generated <link> inside the <head>", async t => {
+  const url = "/server-link-head";
+  server.get(url, (req, res) => {
+    res.send(`
+			<!doctype>
+			<html>
+				<head>
+					<link rel="stylesheet" href="fixture.css" />
+				</head>
+			</html>
+    `);
+  });
+
+  const actual = await extractCss(server.url + url);
+
+  t.is(actual, expected);
+});
+
+test("it fetches css from a page with CSS in server generated <style> inside the <head>", async t => {
+  const url = "/server-style-head";
+  server.get(url, (req, res) => {
+    res.send(`
+			<!doctype>
+			<html>
+				<head>
+					<style>${expected.trim()}</style>
+				</head>
+			</html>
+    `);
+  });
+
+  const actual = await extractCss(server.url + url);
+
+  t.is(actual, expected.trim());
+});
+
+test("it finds JS generated <link /> CSS", async t => {
+  const path = "/js-generated-link";
+  const cssInJsExampleHtml = readFileSync(
+    resolve(__dirname, "js-create-link-element.html"),
+    "utf8"
+  );
+
+  server.get(path, (req, res) => {
+    res.send(cssInJsExampleHtml);
+  });
+
+  const actual = await extractCss(server.url + path);
+
+  t.is(actual, expected);
+});
+
+test("it finds JS generated <style /> CSS", async t => {
+  const url = "/js-generated-js-style-tag";
+  const cssInJsExampleHtml = readFileSync(
+    resolve(__dirname, "js-create-style-element.html"),
+    "utf8"
+  );
+  server.get(url, (req, res) => {
+    res.send(cssInJsExampleHtml);
+  });
+
+  const actual = await extractCss(server.url + url, { waitUntil: "load" });
+  const expected = `body { color: teal; }`;
+
+  t.is(actual, expected);
+});
+
+test("it combines server generated <link> and <style> tags with client side created <link> and <style> tags", async t => {
+  const path = "/kitchen-sink";
+  const kitchenSinkExample = readFileSync(
+    resolve(__dirname, "kitchen-sink.html"),
+    "utf8"
+  );
+  server.get(path, (req, res) => {
+    res.send(kitchenSinkExample);
+  });
+
+  const actual = await extractCss(server.url + path);
+
+  t.snapshot(actual);
+});
+
+test("it rejects if the url has an HTTP error status", async t => {
+  const urlWith404 = server.url + "/404-page";
+  await t.throwsAsync(extractCss(urlWith404), {
+    message: `There was an error retrieving CSS from ${urlWith404}.\n\tHTTP status code: 404 (Not Found)`
+  });
+});
+
+test("it rejects on an invalid url", async t => {
+  await t.throwsAsync(extractCss("site.example"));
+});
+
+test("it accepts a custom Chrome and Puppeteer version for use on AWS", async t => {
+  const chromium = require("chromium");
+  const puppeteerCore = require("puppeteer-core");
+
+  const path = "/custom-chrome";
+  const kitchenSinkExample = readFileSync(
+    resolve(__dirname, "kitchen-sink.html"),
+    "utf8"
+  );
+  server.get(path, (req, res) => {
+    res.send(kitchenSinkExample);
+  });
+
+  const actual = await extractCss(server.url + path, {
+    customBrowser: {
+      executablePath: chromium.path,
+      customPuppeteer: puppeteerCore
+    }
+  });
+
+  t.snapshot(actual);
+});
+
+// TODO: write coverage for dynamically inserted style tags
+test.skip("it finds CSS-in-JS (styled components)", async t => {
+  const path = "/css-in-js";
+  const cssInJsExampleHtml = readFileSync(
+    resolve(__dirname, "css-in-js.html"),
+    "utf8"
+  );
+  server.get(path, (req, res) => {
+    res.send(cssInJsExampleHtml);
+  });
+
+  const actual = await extractCss(server.url + path);
+  const expected = "lots of JS generated classNames and styles";
+
+  t.is(actual, expected);
+});
diff --git a/test/js-create-link-element.html b/test/js-create-link-element.html
new file mode 100644
index 0000000..7996477
--- /dev/null
+++ b/test/js-create-link-element.html
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+    <title>Document</title>
+  </head>
+  <body>
+    <h1>Title</h1>
+
+    <script>
+      var style = document.createElement("link");
+      style.href = "fixture.css";
+      style.rel = "stylesheet";
+      document.head.appendChild(style);
+    </script>
+  </body>
+</html>
diff --git a/test/js-create-style-element.html b/test/js-create-style-element.html
new file mode 100644
index 0000000..edd6fa2
--- /dev/null
+++ b/test/js-create-style-element.html
@@ -0,0 +1,18 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+    <title>Document</title>
+  </head>
+  <body>
+    <h1>Title</h1>
+
+    <script>
+      var style = document.createElement("style");
+      style.textContent = "body { color: teal; }";
+      document.body.appendChild(style);
+    </script>
+  </body>
+</html>
diff --git a/test/kitchen-sink.html b/test/kitchen-sink.html
new file mode 100644
index 0000000..d8d42d8
--- /dev/null
+++ b/test/kitchen-sink.html
@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+    <title>Document</title>
+
+    <!-- Server generated link -->
+    <link rel="stylesheet" href="fixture.css" />
+
+    <!-- Server generated style -->
+    <style>
+      .server-style {
+        counter-increment: 2;
+      }
+    </style>
+  </head>
+  <body>
+    <h1>Title</h1>
+
+    <script>
+      // Client generated style
+      var style = document.createElement("style");
+      style.textContent = ".js-style { counter-increment: 3; }";
+      document.body.appendChild(style);
+
+      // Client generated link
+      var style = document.createElement("link");
+      style.href = "fixture.css";
+      style.rel = "stylesheet";
+      document.head.appendChild(style);
+    </script>
+  </body>
+</html>
diff --git a/test/snapshots/index.js.md b/test/snapshots/index.js.md
new file mode 100644
index 0000000..14c1072
--- /dev/null
+++ b/test/snapshots/index.js.md
@@ -0,0 +1,33 @@
+# Snapshot report for `test/index.js`
+
+The actual snapshot is saved in `index.js.snap`.
+
+Generated by [AVA](https://ava.li).
+
+## it combines server generated <link> and <style> tags with client side created <link> and <style> tags
+
+> Snapshot 1
+
+    `body {␊
+      color: teal;␊
+    }␊
+    body {␊
+      color: teal;␊
+    }␊
+    .server-style {␊
+            counter-increment: 2;␊
+          }.js-style { counter-increment: 3; }`
+
+## it accepts a custom Chrome and Puppeteer version for use on AWS
+
+> Snapshot 1
+
+    `body {␊
+      color: teal;␊
+    }␊
+    body {␊
+      color: teal;␊
+    }␊
+    .server-style {␊
+            counter-increment: 2;␊
+          }.js-style { counter-increment: 3; }`
diff --git a/test/snapshots/index.js.snap b/test/snapshots/index.js.snap
new file mode 100644
index 0000000000000000000000000000000000000000..22a9c45590b22363d232d646c278765fbcafc096
GIT binary patch
literal 197
zcmV;$06PCcRzV<BZeeh9Xm4~Nb}|YA02C_<UT@@|-p>i6jow=5`yY!400000000BC
zVqjokVmSYharr$j8<*nWvj6TFmds~l0D<X1Jm;Q_lGJbWM7uQ}H~rq-+{y?R)dMPE
zW?%=aVq|3yWa?l@%1^0OsOC~oNY2m6FS1f7NlnbL=BnkwC9YSTT2z)=q+484nUe}u
z00GJQrFkVl@yxvBqSV~fyb>z~BWtK^tzK3!OanHB#?}h8gVb37%8Xz}XaN8KHgi~U

literal 0
HcmV?d00001