diff --git a/docs/docs/telemetry.md b/docs/docs/telemetry.md new file mode 100644 index 0000000000000..b5b71e052c686 --- /dev/null +++ b/docs/docs/telemetry.md @@ -0,0 +1,49 @@ +--- +title: Telemetry +--- + +Gatsby contains a telemetry feature that collects anonymous usage information that is used to help improve Gatsby for all users. +The Gatsby user base is growing very rapidly. It's important that our small team and the greater community will better understand the usage patterns, so we can best decide how to design future features and prioritize current work. + +You will be notified when installing Gatsby and when running it for the first time. + +## How to opt-out + +Users may always opt-out from the telemetry with `gatsby telemetry --disable` or setting the environment variable `GATSBY_TELEMETRY_DISABLED` to `1` + +## Why? + +**Anonymous** aggregate user analytics allow us to prioritize fixes and features based on how and when people use Gatsby. +Since much of Gatsby’s function revolves around community plugins and starters, we want to collect information on usage +and reliability so that we can ensure a high-quality ecosystem. + +This raises a question: how will we use this telemetry data to improve the ecosystem? Some examples are helpful: + +- We will be able to understand which plugins are typically used together. This will enable us to surface this information in our public plugin library and build more relevant starters and tutorials based on this data. +- We will be able to surface popularity of different starters in the starter showcase. +- We will be able to get more detail on the types of errors users are running into in _every_ build stage (e.g. development, build, etc.). This will let us improve the quality of our tool and better focus our time on solving more common, frustrating issues. +- We will be able to surface reliability of different plugins and starters, and detect which of these tend to error more frequently. We can use this data to surface quality metrics and improve the quality of our plugins and starters. +- We will be able to see timings for different build stages to guide us in where we should focus optimization work. + +## What do we track? + +We track general usage details, including command invocation, build process status updates, performance measurements, and errors. +We use these metrics to better understand the usage patterns. These metrics will directly allow us to better decide how to design future features and prioritize current work. + +Specifically, we collect the following information for _all_ telemetry events: + +- Timestamp of the occurrence +- Command invoked (e.g. `build` or `develop`) +- Gatsby machine ID. This is generated with UUID and stored in global gatsby config at ~/.config/gatsby/config.json. +- Unique session ID. This is generated on each run with UUID. +- One-way hash of the current working directory or a hash of the git remote +- General OS level information (operating system, version, CPU architecture, and whether the command is run inside a CI) +- Current Gatsby version + +The access to the raw data is highly controlled, and we cannot identify individual users from the dataset. It is anonymized and untraceable back to the user. + +## What about sensitive data? (e.g. secrets) + +We perform additional steps to ensure that secure data (e.g. environment variables used to store secrets for the build process) **do not** make their way into our analytics. [We strip logs, error messages, etc.](https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-telemetry/src/sanitize-error.js) of this sensitive data to ensure we _never_ gain access to this sensitive data. + +You can view all the information that is sent by Gatsby’s telemetry by setting the environment variable `GATSBY_TELEMETRY_DEBUG`to `1` to print the telemetry data instead of sending it over. diff --git a/packages/gatsby-cli/package.json b/packages/gatsby-cli/package.json index 6f0f6aa873b4a..6dbc1de2d750b 100644 --- a/packages/gatsby-cli/package.json +++ b/packages/gatsby-cli/package.json @@ -17,12 +17,14 @@ "convert-hrtime": "^2.0.0", "core-js": "^2.5.0", "envinfo": "^5.8.1", + "gatsby-telemetry": "^0.0.1", "execa": "^0.8.0", "fs-exists-cached": "^1.0.0", "fs-extra": "^4.0.1", "hosted-git-info": "^2.6.0", "lodash": "^4.17.10", "meant": "^1.0.1", + "node-fetch": "2.3.0", "opentracing": "^0.14.3", "pretty-error": "^2.1.1", "resolve-cwd": "^2.0.0", @@ -30,6 +32,7 @@ "stack-trace": "^0.0.10", "update-notifier": "^2.3.0", "yargs": "^12.0.5", + "uuid": "3.3.2", "yurnalist": "^1.0.2" }, "devDependencies": { diff --git a/packages/gatsby-cli/src/create-cli.js b/packages/gatsby-cli/src/create-cli.js index afefea37487cd..38c2dac3aac15 100644 --- a/packages/gatsby-cli/src/create-cli.js +++ b/packages/gatsby-cli/src/create-cli.js @@ -5,6 +5,11 @@ const report = require(`./reporter`) const didYouMean = require(`./did-you-mean`) const envinfo = require(`envinfo`) const existsSync = require(`fs-exists-cached`).sync +const { + trackCli, + setDefaultTags, + setTelemetryEnabled, +} = require(`gatsby-telemetry`) const handlerP = fn => (...args) => { Promise.resolve(fn(...args)).then( @@ -40,6 +45,11 @@ function buildLocalCommands(cli, isLocalSite) { `gatsby`, `package.json` )) + try { + setDefaultTags({ installedGatsbyVersion: packageInfo.version }) + } catch (e) { + // ignore + } majorVersion = parseInt(packageInfo.version.split(`.`)[0], 10) } catch (err) { /* ignore */ @@ -311,6 +321,15 @@ module.exports = argv => { buildLocalCommands(cli, isLocalSite) + try { + const { version } = require(`../package.json`) + setDefaultTags({ gatsbyCliVersion: version }) + } catch (e) { + // ignore + } + + trackCli(argv) + return cli .command({ command: `new [rootPath] [starter]`, @@ -322,6 +341,26 @@ module.exports = argv => { } ), }) + .command({ + command: `telemetry`, + desc: `Enable or disable Gatsby anonymous analytics collection.`, + builder: yargs => + yargs + .option(`enable`, { + type: `boolean`, + description: `Enable telemetry (default)`, + }) + .option(`disable`, { + type: `boolean`, + description: `Disable telemetry`, + }), + + handler: handlerP(({ enable, disable }) => { + const enabled = enable || !disable + setTelemetryEnabled(enabled) + report.log(`Telemetry collection ${enabled ? `enabled` : `disabled`}`) + }), + }) .wrap(cli.terminalWidth()) .demandCommand(1, `Pass --help to see all available commands and options.`) .strict() diff --git a/packages/gatsby-cli/src/init-starter.js b/packages/gatsby-cli/src/init-starter.js index be0f88c276ac3..5c5742aba86fd 100644 --- a/packages/gatsby-cli/src/init-starter.js +++ b/packages/gatsby-cli/src/init-starter.js @@ -7,7 +7,7 @@ const sysPath = require(`path`) const report = require(`./reporter`) const url = require(`url`) const existsSync = require(`fs-exists-cached`).sync - +const { trackCli, trackError } = require(`gatsby-telemetry`) const spawn = (cmd: string, options: any) => { const [file, ...args] = cmd.split(/\s+/) return execa(file, args, { stdio: `inherit`, ...options }) @@ -147,6 +147,7 @@ module.exports = async (starter: string, options: InitOptions = {}) => { const urlObject = url.parse(rootPath) if (urlObject.protocol && urlObject.host) { + trackError(`NEW_PROJECT_NAME_MISSING`) report.panic( `It looks like you forgot to add a name for your new project. Try running instead "gatsby new new-gatsby-project ${rootPath}"` ) @@ -154,11 +155,14 @@ module.exports = async (starter: string, options: InitOptions = {}) => { } if (existsSync(sysPath.join(rootPath, `package.json`))) { + trackError(`NEW_PROJECT_IS_NPM_PROJECT`) report.panic(`Directory ${rootPath} is already an npm project`) return } const hostedInfo = hostedGitInfo.fromUrl(starter) + + trackCli(`NEW_PROJECT`, { starterName: starter }) if (hostedInfo) await clone(hostedInfo, rootPath) else await copy(starter, rootPath) } diff --git a/packages/gatsby-cli/src/reporter/index.js b/packages/gatsby-cli/src/reporter/index.js index 0367664c57c7b..6d578a3360518 100644 --- a/packages/gatsby-cli/src/reporter/index.js +++ b/packages/gatsby-cli/src/reporter/index.js @@ -5,6 +5,7 @@ const { stripIndent } = require(`common-tags`) const convertHrtime = require(`convert-hrtime`) const tracer = require(`opentracing`).globalTracer() const { getErrorFormatter } = require(`./errors`) +const { trackError } = require(`gatsby-telemetry`) const VERBOSE = process.env.gatsby_log_level === `verbose` @@ -47,11 +48,13 @@ module.exports = Object.assign(reporter, { */ panic(...args) { this.error(...args) + trackError(`GENERAL_PANIC`, { error: args }) process.exit(1) }, panicOnBuild(...args) { this.error(...args) + trackError(`BUILD_PANIC`, { error: args }) if (process.env.gatsby_executing_command === `build`) { process.exit(1) } diff --git a/packages/gatsby-telemetry/.babelrc b/packages/gatsby-telemetry/.babelrc new file mode 100644 index 0000000000000..9f5de61e0d79e --- /dev/null +++ b/packages/gatsby-telemetry/.babelrc @@ -0,0 +1,5 @@ +{ + "presets": [ + ["babel-preset-gatsby-package"] + ] +} diff --git a/packages/gatsby-telemetry/.gitignore b/packages/gatsby-telemetry/.gitignore new file mode 100644 index 0000000000000..626c4f31eeeba --- /dev/null +++ b/packages/gatsby-telemetry/.gitignore @@ -0,0 +1,2 @@ +/lib +/node_modules diff --git a/packages/gatsby-telemetry/CHANGELOG.md b/packages/gatsby-telemetry/CHANGELOG.md new file mode 100644 index 0000000000000..420e6f23d0e37 --- /dev/null +++ b/packages/gatsby-telemetry/CHANGELOG.md @@ -0,0 +1 @@ +# Change Log diff --git a/packages/gatsby-telemetry/README.md b/packages/gatsby-telemetry/README.md new file mode 100644 index 0000000000000..4a94726ab51b8 --- /dev/null +++ b/packages/gatsby-telemetry/README.md @@ -0,0 +1,38 @@ +# gatsby-telemetry + +Check out: [gatsby.dev/telemetry](https://gatsby.dev/telemetry) + +## API + +### trackCli(type, tags) + +Capture an event of type `type` and decorate the generated event with these tags (note: allowed tags are filtered on server side) + +### trackError(type, tags) + +Capture an error of type `type`. The exception maybe passed in tags and it will be sanitize to anonymize the contents. + +### trackBuildError(type, tags) + +Capture an build error of type `type`. The exception maybe passed in tags and it will be sanitize to anonymize the contents. + +### setDefaultTags(tags) + +Set additional tags to be included in all future events. + +### decorateEvent(type, tags) + +Attach additional tags to the next event generated of type `type`. + +### setTelemetryEnabled(enabled) + +Enable or disable the telemetry collection. + +### expressMiddleware(type) + +Returns a debounced events tracker for collecting general activity information for incoming requests. + +## ENV Variables + +- Set `GATSBY_TELEMETRY_DEBUG` to `1` to print the telemetry data instead of sending it over +- Set `GATSBY_TELEMETRY_DISABLED` to `1` to opt out of all telemetry diff --git a/packages/gatsby-telemetry/package.json b/packages/gatsby-telemetry/package.json new file mode 100644 index 0000000000000..7cf165f5c9ef0 --- /dev/null +++ b/packages/gatsby-telemetry/package.json @@ -0,0 +1,54 @@ +{ + "name": "gatsby-telemetry", + "description": "Gatsby Telemetry", + "version": "0.0.1", + "author": "Jarmo Isotalo ", + "bugs": { + "url": "https://github.com/gatsbyjs/gatsby/issues" + }, + "dependencies": { + "@babel/code-frame": "^7.0.0", + "@babel/runtime": "^7.0.0", + "bluebird": "^3.5.0", + "ci-info": "2.0.0", + "configstore": "4.0.0", + "envinfo": "^5.8.1", + "node-fetch": "2.3.0", + "resolve-cwd": "^2.0.0", + "source-map": "^0.5.7", + "stack-trace": "^0.0.10", + "stack-utils": "1.0.2", + "uuid": "3.3.2" + }, + "devDependencies": { + "@babel/cli": "^7.0.0", + "@babel/core": "^7.0.0", + "babel-jest": "^24.0.0", + "babel-preset-gatsby-package": "^0.1.3", + "cross-env": "^5.1.4", + "jest": "^24.0.0", + "jest-cli": "^24.0.0", + "jest-junit": "^6.1.0" + }, + "files": [ + "lib", + "src/postinstall.js" + ], + "homepage": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-telemetry#readme", + "keywords": [ + "telemetry" + ], + "license": "MIT", + "main": "lib/index.js", + "repository": "https://github.com/gatsbyjs/gatsby/tree/master/packages/gatsby-telemetry", + "scripts": { + "build": "babel src --out-dir lib --ignore **/__tests__", + "prepare": "cross-env NODE_ENV=production npm run build", + "jest": "jest", + "postinstall": "node src/postinstall.js", + "watch": "babel -w src --out-dir lib --ignore **/__tests__" + }, + "yargs": { + "boolean-negation": false + } +} diff --git a/packages/gatsby-telemetry/src/__tests__/is-truthy.js b/packages/gatsby-telemetry/src/__tests__/is-truthy.js new file mode 100644 index 0000000000000..f380bd525c1cc --- /dev/null +++ b/packages/gatsby-telemetry/src/__tests__/is-truthy.js @@ -0,0 +1,22 @@ +const isTruthy = require(`../is-truthy`) + +describe(`isTruthy`, () => { + it(`handles Booleans`, () => { + expect(isTruthy(true)).toBe(true) + expect(isTruthy(false)).toBe(false) + }) + it(`handles true or false strings `, () => { + expect(isTruthy(`true`)).toBe(true) + expect(isTruthy(`false`)).toBe(false) + expect(isTruthy(`TRUE`)).toBe(true) + expect(isTruthy(`FALSE`)).toBe(false) + }) + it(`handles numbers`, () => { + expect(isTruthy(`1`)).toBe(true) + expect(isTruthy(`0`)).toBe(false) + expect(isTruthy(`-1`)).toBe(false) + }) + it(`defaults to false`, () => { + expect(isTruthy(`blah`)).toBe(false) + }) +}) diff --git a/packages/gatsby-telemetry/src/__tests__/sanitize-error.js b/packages/gatsby-telemetry/src/__tests__/sanitize-error.js new file mode 100644 index 0000000000000..0a24e7b1aee23 --- /dev/null +++ b/packages/gatsby-telemetry/src/__tests__/sanitize-error.js @@ -0,0 +1,62 @@ +const sanitize = require(`../sanitize-error`) + +describe(`sanitize errors`, () => { + it(`Removes env, output and converts buffers to strings from execa output`, () => { + const tags = { + error: [ + { + error: null, + cmd: `git commit -m"test"`, + file: `/bin/sh`, + args: [`/bin/sh`, `-c`, `git commit -m"test`], + options: { + cwd: `here`, + shell: true, + envPairs: [`VERSION=1.2.3`], + stdio: [{}, {}, {}], // pipes + }, + envPairs: [`VERSION=1.2.3`], + + stderr: Buffer.from(`this is a test`), + stdout: Buffer.from(`this is a test`), + }, + ], + } + + const error = tags.error[0] + expect(error).toBeDefined() + expect(error.envPairs).toBeDefined() + expect(error.options).toBeDefined() + + expect(typeof error.stdout).toEqual(`object`) + + sanitize(tags) + expect(typeof error.stdout).toEqual(`string`) + + expect(error).toBeDefined() + expect(error.envPairs).toBeUndefined() + expect(error.options).toBeUndefined() + }) + + it(`Sanitizes current path from error stracktraces`, () => { + const errormessage = `this is a test` + let e + try { + throw new Error(errormessage) + } catch (error) { + e = error + } + expect(e).toBeDefined() + expect(e.message).toEqual(errormessage) + expect(e.stack).toBeDefined() + const localPathRegex = new RegExp( + process.cwd().replace(/[-[/{}()*+?.\\^$|]/g, `\\$&`) + ) + expect(localPathRegex.test(e.stack)).toBeTruthy() + const tags = { error: [e] } + + sanitize(tags) + + expect(localPathRegex.test(e.stack)).toBeFalsy() + }) +}) diff --git a/packages/gatsby-telemetry/src/__tests__/telemetry.js b/packages/gatsby-telemetry/src/__tests__/telemetry.js new file mode 100644 index 0000000000000..2539b8630c191 --- /dev/null +++ b/packages/gatsby-telemetry/src/__tests__/telemetry.js @@ -0,0 +1,17 @@ +jest.mock(`../event-storage`) +const eventStore = require(`../event-storage`) +const Telemetry = require(`../telemetry`) + +let telemetry +beforeEach(() => { + eventStore.mockReset() + telemetry = new Telemetry() +}) + +describe(`Telemetry`, () => { + it(`Adds event to store`, () => { + telemetry.buildAndStoreEvent(`demo`, {}) + expect(eventStore).toHaveBeenCalledTimes(1) + expect(eventStore.mock.instances[0].addEvent).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/gatsby-telemetry/src/event-storage.js b/packages/gatsby-telemetry/src/event-storage.js new file mode 100644 index 0000000000000..f3ad90a4c1365 --- /dev/null +++ b/packages/gatsby-telemetry/src/event-storage.js @@ -0,0 +1,63 @@ +const Store = require(`./store`) +const fetch = require(`node-fetch`) + +const isTruthy = require(`./is-truthy`) + +/* The events data collection is a spooled process that + * buffers events to a local fs based buffer + * which then is asynchronously flushed to the server. + * This both increases the fault tolerancy and allows collection + * to continue even when working offline. + */ +module.exports = class EventStorage { + store = new Store() + debugEvents = isTruthy(process.env.GATSBY_TELEMETRY_DEBUG) + disabled = isTruthy(process.env.GATSBY_TELEMETRY_DISABLED) + + addEvent(event) { + if (this.disabled) { + return + } + + if (this.debugEvents) { + console.error(`Captured event:`, JSON.stringify(event)) + } else { + this.store.appendToBuffer(JSON.stringify(event) + `\n`) + } + } + + async sendEvents() { + return this.store.startFlushEvents(async eventsData => { + const events = eventsData + .split(`\n`) + .filter(e => e && e.length > 2) // drop empty lines + .map(e => JSON.parse(e)) + + return this.submitEvents(events) + }) + } + + async submitEvents(events) { + try { + const res = await fetch(`https://analytics.gatsbyjs.com/events`, { + method: `POST`, + headers: { "content-type": `application/json` }, + body: JSON.stringify(events), + }) + return res.ok + } catch (e) { + return false + } + } + + getConfig(key) { + if (key) { + return this.store.getConfig(key) + } + return this.store.getConfig() + } + + updateConfig(...conf) { + return this.store.updateConfig(...conf) + } +} diff --git a/packages/gatsby-telemetry/src/flush.js b/packages/gatsby-telemetry/src/flush.js new file mode 100644 index 0000000000000..060e8e68872f1 --- /dev/null +++ b/packages/gatsby-telemetry/src/flush.js @@ -0,0 +1,12 @@ +const { join } = require(`path`) +const { fork } = require(`child_process`) + +module.exports = async () => { + // Submit events on background w/o blocking the main process + // nor relying on it's lifecycle + const forked = fork(join(__dirname, `send.js`), { + detached: true, + stdio: `ignore`, + }) + forked.unref() +} diff --git a/packages/gatsby-telemetry/src/index.js b/packages/gatsby-telemetry/src/index.js new file mode 100644 index 0000000000000..a301d77e4cfa3 --- /dev/null +++ b/packages/gatsby-telemetry/src/index.js @@ -0,0 +1,34 @@ +const Telemetry = require(`./telemetry`) +const flush = require(`./flush`) + +const instance = new Telemetry() + +process.on(`exit`, flush) + +// For longrunning commands we want to occasinally flush the data +// The data is also sent on exit. +const interval = 10 * 60 * 1000 // 10 min +const tick = _ => { + flush() + .catch(console.error) + .then(_ => setTimeout(tick, interval)) +} +setTimeout(tick, interval) + +module.exports = { + trackCli: (input, tags) => instance.captureEvent(input, tags), + trackError: (input, tags) => instance.captureError(input, tags), + trackBuildError: (input, tags) => instance.captureBuildError(input, tags), + setDefaultTags: tags => instance.decorateAll(tags), + decorateEvent: (event, tags) => instance.decorateNextEvent(event, tags), + setTelemetryEnabled: enabled => instance.setTelemetryEnabled(enabled), + + expressMiddleware: source => (req, res, next) => { + try { + instance.trackActivity(`${source}_ACTIVE`) + } catch (e) { + // ignore + } + next() + }, +} diff --git a/packages/gatsby-telemetry/src/is-truthy.js b/packages/gatsby-telemetry/src/is-truthy.js new file mode 100644 index 0000000000000..c76eef39cd56b --- /dev/null +++ b/packages/gatsby-telemetry/src/is-truthy.js @@ -0,0 +1,23 @@ +// Returns true for `true`, true, positive numbers +// Returns false for `false`, false, 0, negative integers and anything else +const isTruthy = value => { + // Return if Boolean + if (typeof value === `boolean`) return value + + // Return false if null or undefined + if (value === undefined || value === null) return false + + // If the String is true or false + if (value.toLowerCase() === `true`) return true + if (value.toLowerCase() === `false`) return false + + // Now check if it's a number + const number = parseInt(value, 10) + if (isNaN(number)) return false + if (number > 0) return true + + // Default to false + return false +} + +module.exports = isTruthy diff --git a/packages/gatsby-telemetry/src/postinstall.js b/packages/gatsby-telemetry/src/postinstall.js new file mode 100644 index 0000000000000..9489510fec7d6 --- /dev/null +++ b/packages/gatsby-telemetry/src/postinstall.js @@ -0,0 +1,16 @@ +let enabled = false +try { + const Configstore = require(`configstore`) + const config = new Configstore(`gatsby`, {}, { globalConfigPath: true }) + enabled = config.get(`telemetry.enabled`) + if (enabled === undefined) { + console.log( + `Gatsby has started collecting anonymous usage analytics to help improve Gatsby for all users.\n` + + `If you'd like to opt-out, you can use \`gatsby telemetry --disable\`\n` + + `To learn more, checkout http://gatsby.dev/telemetry` + ) + enabled = config.set(`telemetry.enabled`, true) + } +} catch (e) { + // ignore +} diff --git a/packages/gatsby-telemetry/src/sanitize-error.js b/packages/gatsby-telemetry/src/sanitize-error.js new file mode 100644 index 0000000000000..bbf30fdae3d5f --- /dev/null +++ b/packages/gatsby-telemetry/src/sanitize-error.js @@ -0,0 +1,28 @@ +const StackUtils = require(`stack-utils`) + +module.exports = tags => { + if (!tags) return + const { error } = tags + if (error) { + try { + ;[].concat(error).forEach(e => { + ;[`envPairs`, `options`, `output`].forEach(f => delete e[f]) + // These may be buffers + if (e.stderr) e.stderr = String(e.stderr) + if (e.stdout) e.stdout = String(e.stdout) + let { stack } = e + if (stack) { + const stackUtils = new StackUtils({ + cwd: process.cwd(), + internals: StackUtils.nodeInternals(), + }) + stack = stackUtils.clean(stack) + e.stack = stack + } + return e + }) + } catch (err) { + // ignore + } + } +} diff --git a/packages/gatsby-telemetry/src/send.js b/packages/gatsby-telemetry/src/send.js new file mode 100644 index 0000000000000..87ef881717dc3 --- /dev/null +++ b/packages/gatsby-telemetry/src/send.js @@ -0,0 +1,10 @@ +const Telemetry = require(`./telemetry`) +const instance = new Telemetry() + +const flush = _ => { + instance.sendEvents().catch(e => { + // ignore + }) +} + +flush() diff --git a/packages/gatsby-telemetry/src/store.js b/packages/gatsby-telemetry/src/store.js new file mode 100644 index 0000000000000..591e7365a1139 --- /dev/null +++ b/packages/gatsby-telemetry/src/store.js @@ -0,0 +1,95 @@ +const { homedir } = require(`os`) +const path = require(`path`) +const { + appendFileSync, + readFileSync, + renameSync, + existsSync, + unlinkSync, +} = require(`fs`) +const { ensureDirSync } = require(`fs-extra`) +const Configstore = require(`configstore`) + +module.exports = class Store { + config = new Configstore(`gatsby`, {}, { globalConfigPath: true }) + + constructor() {} + + getConfig(key) { + if (key) { + return this.config.get(key) + } + return this.config.all + } + + updateConfig(...fields) { + this.config.set(...fields) + } + + appendToBuffer(event) { + const bufferPath = this.getBufferFilePath() + try { + appendFileSync(bufferPath, event, `utf8`) + } catch (e) { + //ignore + } + } + + getConfigPath() { + const configPath = path.join(homedir(), `.config/gatsby`) + try { + ensureDirSync(configPath) + } catch (e) { + //ignore + } + return configPath + } + + getBufferFilePath() { + const configPath = this.getConfigPath() + return path.join(configPath, `events.json`) + } + + async startFlushEvents(flushOperation) { + const now = Date.now() + const filePath = this.getBufferFilePath() + try { + if (!existsSync(filePath)) { + return + } + } catch (e) { + // ignore + return + } + const newPath = `${filePath}-${now}` + + try { + renameSync(filePath, newPath) + } catch (e) { + // ignore + return + } + let contents + try { + contents = readFileSync(newPath, `utf8`) + unlinkSync(newPath) + } catch (e) { + //ignore + return + } + + // There is still a chance process dies while sending data and some events are lost + // This will be ok for now, however + let success = false + try { + success = await flushOperation(contents) + } catch (e) { + // ignore + } finally { + // if sending fails, we write the data back to the log + if (!success) { + this.appendToBuffer(contents) + } + } + } +} diff --git a/packages/gatsby-telemetry/src/telemetry.js b/packages/gatsby-telemetry/src/telemetry.js new file mode 100644 index 0000000000000..9ae02544fcdf2 --- /dev/null +++ b/packages/gatsby-telemetry/src/telemetry.js @@ -0,0 +1,195 @@ +const { createHash } = require(`crypto`) +const uuid = require(`uuid/v1`) +const EventStorage = require(`./event-storage`) +const sanitizeError = require(`./sanitize-error`) +const ci = require(`ci-info`) +const os = require(`os`) +const { basename } = require(`path`) +const { execSync } = require(`child_process`) + +module.exports = class AnalyticsTracker { + store = new EventStorage() + debouncer = {} + metadataCache = {} + defaultTags = {} + osInfo // lazy + trackingEnabled // lazy + componentVersion + sessionId = uuid() + constructor() { + try { + this.componentVersion = require(`../package.json`).version + } catch (e) { + // ignore + } + } + + captureEvent(type = ``, tags = {}) { + if (!this.isTrackingEnabled()) { + return + } + if (Array.isArray(type) && type.length > 2) { + type = type[2].toUpperCase() + } + const decoration = this.metadataCache[type] + delete this.metadataCache[type] + const eventType = `CLI_COMMAND_${type}` + this.buildAndStoreEvent(eventType, Object.assign(tags, decoration)) + } + + captureError(type, tags = {}) { + if (!this.isTrackingEnabled()) { + return + } + const decoration = this.metadataCache[type] + delete this.metadataCache[type] + const eventType = `CLI_ERROR_${type}` + sanitizeError(tags) + + // JSON.stringify won't work for Errors w/o some trickery: + let { error } = tags + if (error) { + error = error.map(e => + JSON.parse(JSON.stringify(e, Object.getOwnPropertyNames(e))) + ) + } + tags.error = JSON.stringify(error) + + this.buildAndStoreEvent(eventType, Object.assign(tags, decoration)) + } + + captureBuildError(type, tags = {}) { + if (!this.isTrackingEnabled()) { + return + } + const decoration = this.metadataCache[type] + delete this.metadataCache[type] + const eventType = `BUILD_ERROR_${type}` + sanitizeError(tags) + tags.error = JSON.stringify(tags.error) + + this.buildAndStoreEvent(eventType, Object.assign(tags, decoration)) + } + + buildAndStoreEvent(eventType, tags) { + const event = { + ...this.defaultTags, + ...tags, // The schema must include these + eventType, + sessionId: this.sessionId, + time: new Date(), + machineId: this.getMachineId(), + repositoryId: this.getRepoId(), + componentId: `gatsby-cli`, + osInformation: this.getOsInfo(), + componentVersion: this.componentVersion, + } + this.store.addEvent(event) + } + + getMachineId() { + // Cache the result + if (this.machineId) { + return this.machineId + } + let machineId = this.store.getConfig(`telemetry.machineId`) + if (!machineId) { + machineId = uuid() + this.store.updateConfig(`telemetry.machineId`, machineId) + } + this.machineId = machineId + return machineId + } + + isTrackingEnabled() { + // Cache the result + if (this.trackingEnabled !== undefined) { + return this.trackingEnabled + } + let enabled = this.store.getConfig(`telemetry.enabled`) + if (enabled === undefined || enabled === null) { + console.log( + `Gatsby has started collecting anonymous usage analytics to help improve Gatsby for all users.\n` + + `If you'd like to opt-out, you can use \`gatsby telemetry --disable\`\n` + + `To learn more, checkout http://gatsby.dev/telemetry` + ) + enabled = true + this.store.updateConfig(`telemetry.enabled`, enabled) + } + this.trackingEnabled = enabled + return enabled + } + + getRepoId() { + // we may live multiple levels in git repo + let prefix = `pwd:` + let repo = basename(process.cwd()) + try { + const originBuffer = execSync( + `git config --local --get remote.origin.url`, + { timeout: 1000, stdio: `pipe` } + ) + repo = String(originBuffer).trim() + prefix = `git:` + } catch (e) { + // ignore + } + const hash = createHash(`sha256`) + hash.update(repo) + return prefix + hash.digest(`hex`) + } + + getOsInfo() { + if (this.osInfo) { + return this.osInfo + } + const cpus = os.cpus() + const osInfo = { + nodeVersion: process.version, + platform: os.platform(), + release: os.release(), + cpus: cpus && cpus.length > 0 && cpus[0].model, + arch: os.arch(), + ci: ci.isCI, + ciName: (ci.isCI && ci.name) || undefined, + } + this.osInfo = osInfo + return osInfo + } + + trackActivity(source) { + if (!this.isTrackingEnabled()) { + return + } + // debounce by sending only the first event whithin a rolling window + const now = Date.now() + const last = this.debouncer[source] || 0 + const debounceTime = 5 * 1000 // 5 sec + + if (now - last > debounceTime) { + this.captureEvent(source) + } + this.debouncer[source] = now + } + + decorateNextEvent(event, obj) { + const cached = this.metadataCache[event] || {} + this.metadataCache[event] = Object.assign(cached, obj) + } + + decorateAll(tags) { + this.defaultTags = Object.assign(this.defaultTags, tags) + } + + setTelemetryEnabled(enabled) { + this.trackingEnabled = enabled + this.store.updateConfig(`telemetry.enabled`, enabled) + } + + async sendEvents() { + if (!this.isTrackingEnabled()) { + return Promise.resolve() + } + return this.store.sendEvents() + } +} diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index a9e6f6b5294a1..85a32566166d3 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -68,6 +68,7 @@ "gatsby-link": "^2.0.16", "gatsby-plugin-page-creator": "^2.0.11", "gatsby-react-router-scroll": "^2.0.6", + "gatsby-telemetry": "^0.0.1", "glob": "^7.1.1", "graphql": "^14.1.1", "graphql-compose": "^6.0.3", diff --git a/packages/gatsby/src/bootstrap/index.js b/packages/gatsby/src/bootstrap/index.js index 119aa8ed72607..75ca0ba50fe65 100644 --- a/packages/gatsby/src/bootstrap/index.js +++ b/packages/gatsby/src/bootstrap/index.js @@ -9,6 +9,7 @@ const del = require(`del`) const path = require(`path`) const convertHrtime = require(`convert-hrtime`) const Promise = require(`bluebird`) +const telemetry = require(`gatsby-telemetry`) const apiRunnerNode = require(`../utils/api-runner-node`) const getBrowserslist = require(`../utils/browserslist`) @@ -112,6 +113,10 @@ module.exports = async (args: BootstrapArgs) => { const flattenedPlugins = await loadPlugins(config, program.directory) activity.end() + telemetry.decorateEvent(`BUILD_END`, { + plugins: flattenedPlugins.map(p => `${p.name}@${p.version}`), + }) + // onPreInit activity = report.activityTimer(`onPreInit`, { parentSpan: bootstrapSpan, diff --git a/packages/gatsby/src/commands/build-html.js b/packages/gatsby/src/commands/build-html.js index 2a1548bc77240..06a8510dafdb4 100644 --- a/packages/gatsby/src/commands/build-html.js +++ b/packages/gatsby/src/commands/build-html.js @@ -7,6 +7,7 @@ const webpackConfig = require(`../utils/webpack.config`) const { store } = require(`../redux`) const { createErrorFromString } = require(`gatsby-cli/lib/reporter/errors`) const renderHTMLQueue = require(`../utils/html-renderer-queue`) +const telemetry = require(`gatsby-telemetry`) module.exports = async (program: any, activity: any) => { const { directory } = program @@ -14,6 +15,9 @@ module.exports = async (program: any, activity: any) => { debug(`generating static HTML`) // Reduce pages objects to an array of paths. const pages = Array.from(store.getState().pages.values(), page => page.path) + telemetry.decorateEvent(`BUILD_END`, { + siteMeasurements: { pagesCount: pages.length }, + }) // Static site generation. const compilerConfig = await webpackConfig( diff --git a/packages/gatsby/src/commands/build.js b/packages/gatsby/src/commands/build.js index 1150b538a4639..ced4aa023cefc 100644 --- a/packages/gatsby/src/commands/build.js +++ b/packages/gatsby/src/commands/build.js @@ -9,6 +9,8 @@ const { copyStaticDir } = require(`../utils/get-static-dir`) const { initTracer, stopTracer } = require(`../utils/tracer`) const chalk = require(`chalk`) const tracer = require(`opentracing`).globalTracer() +const signalExit = require(`signal-exit`) +const telemetry = require(`gatsby-telemetry`) function reportFailure(msg, err: Error) { report.log(``) @@ -26,6 +28,11 @@ type BuildArgs = { module.exports = async function build(program: BuildArgs) { initTracer(program.openTracingConfigFile) + telemetry.trackCli(`BUILD_START`) + signalExit(() => { + telemetry.trackCli(`BUILD_END`) + }) + const buildSpan = tracer.startSpan(`build`) buildSpan.setTag(`directory`, program.directory) @@ -82,6 +89,5 @@ module.exports = async function build(program: BuildArgs) { report.info(`Done building in ${process.uptime()} sec`) buildSpan.finish() - await stopTracer() } diff --git a/packages/gatsby/src/commands/develop.js b/packages/gatsby/src/commands/develop.js index 45e882478e045..e78807c2e3885 100644 --- a/packages/gatsby/src/commands/develop.js +++ b/packages/gatsby/src/commands/develop.js @@ -31,6 +31,7 @@ const getSslCert = require(`../utils/get-ssl-cert`) const slash = require(`slash`) const { initTracer } = require(`../utils/tracer`) const apiRunnerNode = require(`../utils/api-runner-node`) +const telemetry = require(`gatsby-telemetry`) // const isInteractive = process.stdout.isTTY @@ -48,6 +49,7 @@ const rlInterface = rl.createInterface({ // Quit immediately on hearing ctrl-c rlInterface.on(`SIGINT`, () => { + telemetry.trackCli(`DEVELOP_STOP`) process.exit() }) @@ -88,6 +90,7 @@ async function startServer(program) { * Set up the express app. **/ const app = express() + app.use(telemetry.expressMiddleware(`DEVELOP`)) app.use( require(`webpack-hot-middleware`)(compiler, { log: false, @@ -254,6 +257,7 @@ async function startServer(program) { module.exports = async (program: any) => { initTracer(program.openTracingConfigFile) + telemetry.trackCli(`DEVELOP_START`) const detect = require(`detect-port`) const port = diff --git a/packages/gatsby/src/commands/serve.js b/packages/gatsby/src/commands/serve.js index 9ed5fe0a74d36..2d52b37ffe76d 100644 --- a/packages/gatsby/src/commands/serve.js +++ b/packages/gatsby/src/commands/serve.js @@ -10,6 +10,8 @@ const preferDefault = require(`../bootstrap/prefer-default`) const chalk = require(`chalk`) const { match: reachMatch } = require(`@reach/router/lib/utils`) +const telemetry = require(`gatsby-telemetry`) + const getPages = directory => fs .readFile(path.join(directory, `.cache`, `pages.json`)) @@ -41,6 +43,7 @@ const clientOnlyPathsRouter = (pages, options) => { } module.exports = async program => { + telemetry.trackCli(`SERVE_START`) let { prefixPaths, port, open, host } = program port = typeof port === `string` ? parseInt(port, 10) : port @@ -56,6 +59,9 @@ module.exports = async program => { const app = express() const router = express.Router() + + app.use(telemetry.expressMiddleware(`SERVE`)) + router.use(compression()) router.use(express.static(`public`)) router.use(clientOnlyPathsRouter(pages, { root })) @@ -86,5 +92,8 @@ module.exports = async program => { } }) - signalExit(() => server.close()) + signalExit(() => { + telemetry.trackCli(`SERVE_STOP`) + server.close() + }) } diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index 3404dbba57f3b..2a28109e6b8c6 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -17,6 +17,7 @@ const { } = require(`../schema/types/type-builders`) const { emitter } = require(`../redux`) const { getNonGatsbyCodeFrame } = require(`./stack-trace-utils`) +const { trackBuildError, decorateEvent } = require(`gatsby-telemetry`) // Bind action creators per plugin so we can auto-add // metadata to actions they create. @@ -188,7 +189,16 @@ const runAPI = (plugin, api, args) => { callback(err, val) apiFinished = true } - gatsbyNode[api](...apiCallArgs, cb) + + try { + gatsbyNode[api](...apiCallArgs, cb) + } catch (e) { + trackBuildError(api, { + error: e, + pluginName: `${plugin.name}@${plugin.version}`, + }) + throw e + } }) } else { const result = gatsbyNode[api](...apiCallArgs) @@ -326,6 +336,9 @@ module.exports = async (api, args = {}, pluginSource) => return new Promise(resolve => { resolve(runAPI(plugin, api, { ...args, parentSpan: apiSpan })) }).catch(err => { + decorateEvent(`BUILD_PANIC`, { + pluginName: `${plugin.name}@${plugin.version}`, + }) reporter.panicOnBuild(`${pluginName} returned an error`, err) return null }) diff --git a/yarn.lock b/yarn.lock index 92d42f47e5113..2e65623acf266 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5085,14 +5085,15 @@ chrome-trace-event@^1.0.0: dependencies: tslib "^1.9.0" -ci-info@^1.5.0: - version "1.5.1" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-1.5.1.tgz#17e8eb5de6f8b2b6038f0cbb714d410bfa9f3030" - -ci-info@^2.0.0: +ci-info@2.0.0, ci-info@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-2.0.0.tgz#67a9e964be31a51e15e5010d58e6f12834002f46" +ci-info@^1.5.0: + version "1.6.0" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-1.6.0.tgz#2ca20dbb9ceb32d4524a683303313f0304b1e497" + integrity sha512-vsGdkwSCDpWmP80ncATX7iea5DWQemg1UgCW5J8tqjU3lYw4FBYuj89J0CTVomA7BEfvSZd84GmHko+MxFQU2A== + cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" resolved "https://registry.yarnpkg.com/cipher-base/-/cipher-base-1.0.4.tgz#8760e4ecc272f4c363532f926d874aae2c1397de" @@ -5503,6 +5504,18 @@ config-chain@^1.1.11: ini "^1.3.4" proto-list "~1.2.1" +configstore@4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/configstore/-/configstore-4.0.0.tgz#5933311e95d3687efb592c528b922d9262d227e7" + integrity sha512-CmquAXFBocrzaSM8mtGPMM/HiWmyIpr4CcJl/rgY2uCObZ/S7cKU0silxslqJejl+t/T9HS8E0PUNQD81JGUEQ== + dependencies: + dot-prop "^4.1.0" + graceful-fs "^4.1.2" + make-dir "^1.0.0" + unique-string "^1.0.0" + write-file-atomic "^2.0.0" + xdg-basedir "^3.0.0" + configstore@^3.0.0, configstore@^3.1.0: version "3.1.2" resolved "https://registry.yarnpkg.com/configstore/-/configstore-3.1.2.tgz#c6f25defaeef26df12dd33414b001fe81a543f8f" @@ -12754,6 +12767,11 @@ node-fetch@2.1.2: version "2.1.2" resolved "http://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz#ab884e8e7e57e38a944753cec706f788d1768bb5" +node-fetch@2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.3.0.tgz#1a1d940bbfb916a1d3e0219f037e89e71f8c5fa5" + integrity sha512-MOd8pV3fxENbryESLgVIeaGKrdl+uaYhCSSVkjeOb/31/njTpcis5aWfdqgNlHIrKOLRbMnfPINPOML2CIFeXA== + node-fetch@^1.0.1, node-fetch@^1.7.3: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" @@ -16536,6 +16554,11 @@ stack-trace@^0.0.10: version "0.0.10" resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" +stack-utils@1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-1.0.2.tgz#33eba3897788558bebfc2db059dc158ec36cebb8" + integrity sha512-MTX+MeG5U994cazkjd/9KNAapsHnibjMLnfXodlkXw76JEea0UiNzrqidzo1emMwk7w5Qhc9jd4Bn9TBb1MFwA== + stack-utils@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/stack-utils/-/stack-utils-1.0.1.tgz#d4f33ab54e8e38778b0ca5cfd3b3afb12db68620" @@ -18007,7 +18030,7 @@ utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" -uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0, uuid@^3.2.1, uuid@^3.3.2: +uuid@3.3.2, uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0, uuid@^3.2.1, uuid@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131"